Initial curated CertScope source from fe2bbe3ca5c98bf64be26a09c6c58ee805a7f87e
This commit is contained in:
Public Source Publisher committed
commit
7348faf6c0
65 files changed
+12509
No files matched your search
@@ -0,0 +1,293 @@
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { analyzeHeaders, type HeaderMap } from "./headers";
|
||||
import { createDeadline, remainingMs, timeoutError, withDeadline } from "./deadline";
|
||||
import { InspectionInputError, InspectionPolicyError, InspectionResolutionError, normalizeHost, resolvePublicHostDetailed, validateResolvedAddresses } from "./policy";
|
||||
import { CERTSCOPE_VERSION } from "../version";
|
||||
|
||||
const HEADER_NAMES = [
|
||||
"strict-transport-security",
|
||||
"content-security-policy",
|
||||
"x-content-type-options",
|
||||
"referrer-policy",
|
||||
"x-frame-options",
|
||||
"server",
|
||||
"x-powered-by",
|
||||
];
|
||||
export const HTTP_TIMEOUT_MS = 5_000;
|
||||
export const HTTP_TOTAL_TIMEOUT_MS = 15_000;
|
||||
export const DEFAULT_MAX_REDIRECTS = 3;
|
||||
export const MAX_ADDRESS_ATTEMPTS = 6;
|
||||
export const HTTPS_DOWNGRADE_ERROR = "HTTPS redirect downgrade is not allowed.";
|
||||
export const REDIRECT_LOOP_ERROR = "Redirect loop detected.";
|
||||
|
||||
export type HttpHop = Record<string, unknown>;
|
||||
type Requester = (url: URL, deadline: number) => Promise<HttpHop>;
|
||||
export type CandidateRequester = (url: URL, address: string, deadline: number) => Promise<HttpHop>;
|
||||
export interface HttpRequestOptions {
|
||||
resolve?: typeof resolvePublicHostDetailed;
|
||||
request?: CandidateRequester;
|
||||
}
|
||||
|
||||
function safeUrl(value: string | URL) {
|
||||
const text = typeof value === "string" ? value : value.toString();
|
||||
try {
|
||||
const parsed = new URL(text);
|
||||
parsed.username = "";
|
||||
parsed.password = "";
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return text.replace(/(\/\/)[^/\s@]+@/g, "$1[redacted]@");
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizedHop(hop: HttpHop): HttpHop {
|
||||
return typeof hop.location === "string" ? { ...hop, location: safeUrl(hop.location) } : hop;
|
||||
}
|
||||
|
||||
export function validateProbeUrl(url: URL) {
|
||||
if (!["http:", "https:"].includes(url.protocol)) throw new InspectionPolicyError("Redirect used an unsupported protocol.");
|
||||
if (url.username || url.password) throw new InspectionPolicyError("Redirect URLs with credentials are not allowed.");
|
||||
if (url.port && url.port !== (url.protocol === "https:" ? "443" : "80")) {
|
||||
throw new InspectionPolicyError("Redirect to a non-default port is not allowed.");
|
||||
}
|
||||
normalizeHost(url.hostname);
|
||||
}
|
||||
|
||||
function errorCode(error: unknown) {
|
||||
return typeof error === "object" && error !== null && "code" in error
|
||||
? String((error as { code?: unknown }).code ?? "")
|
||||
: "";
|
||||
}
|
||||
|
||||
function transportError(error: unknown) {
|
||||
if (error instanceof InspectionInputError || error instanceof InspectionPolicyError || error instanceof InspectionResolutionError) return error.message;
|
||||
if (errorCode(error) === "ETIMEOUT") return "HTTP request timed out.";
|
||||
const code = errorCode(error);
|
||||
return code ? "HTTP request failed (" + code + ")." : "HTTP request failed.";
|
||||
}
|
||||
|
||||
function requestCandidate(url: URL, address: string, deadline: number): Promise<HttpHop> {
|
||||
const budget = Math.min(HTTP_TIMEOUT_MS, remainingMs(deadline));
|
||||
if (!budget) return Promise.reject(timeoutError("HTTP request timed out."));
|
||||
const host = normalizeHost(url.hostname);
|
||||
const client = url.protocol === "https:" ? https : http;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
// The request constructor can throw before a timer exists; the callback
|
||||
// therefore needs an intentionally late-bound timer reference.
|
||||
// eslint-disable-next-line prefer-const
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const finish = (callback: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
callback();
|
||||
};
|
||||
|
||||
let request: ReturnType<typeof client.request>;
|
||||
try {
|
||||
request = client.request({
|
||||
hostname: address,
|
||||
port: url.protocol === "https:" ? 443 : 80,
|
||||
path: url.pathname + url.search,
|
||||
method: "GET",
|
||||
servername: url.protocol === "https:" ? host : undefined,
|
||||
rejectUnauthorized: url.protocol === "https:" ? true : undefined,
|
||||
headers: { host, "user-agent": "CertScope/" + CERTSCOPE_VERSION },
|
||||
timeout: budget,
|
||||
}, (response) => {
|
||||
if (settled) {
|
||||
response.destroy();
|
||||
return;
|
||||
}
|
||||
const headers = Object.fromEntries(
|
||||
HEADER_NAMES.map((name) => [name, response.headers[name] ?? null]),
|
||||
) as HeaderMap;
|
||||
finish(() => {
|
||||
response.destroy();
|
||||
resolve({
|
||||
url: url.toString(),
|
||||
address,
|
||||
status: response.statusCode ?? null,
|
||||
location: response.headers.location ?? null,
|
||||
headers,
|
||||
headerPosture: analyzeHeaders(headers),
|
||||
});
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
finish(() => reject(error));
|
||||
return;
|
||||
}
|
||||
|
||||
const abortOnTimeout = () => finish(() => {
|
||||
request.destroy();
|
||||
reject(timeoutError("HTTP request timed out."));
|
||||
});
|
||||
timer = setTimeout(abortOnTimeout, budget);
|
||||
request.once("timeout", abortOnTimeout);
|
||||
request.once("error", (error) => finish(() => reject(error)));
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
export async function requestOnce(url: URL, deadline = createDeadline(HTTP_TIMEOUT_MS), options: HttpRequestOptions = {}): Promise<HttpHop> {
|
||||
validateProbeUrl(url);
|
||||
const resolution = await (options.resolve ?? resolvePublicHostDetailed)(url.hostname, deadline);
|
||||
const addresses = validateResolvedAddresses(resolution.addresses);
|
||||
const request = options.request ?? requestCandidate;
|
||||
const attempts: Array<Record<string, unknown>> = [];
|
||||
for (const address of addresses.slice(0, MAX_ADDRESS_ATTEMPTS)) {
|
||||
if (!remainingMs(deadline)) break;
|
||||
try {
|
||||
const result = await request(url, address, deadline);
|
||||
return { ...result, address: result.address ?? address, addresses, resolution: resolution._meta, attempts };
|
||||
} catch (error) {
|
||||
attempts.push({ address, error: transportError(error) });
|
||||
}
|
||||
}
|
||||
const message = remainingMs(deadline) ? "HTTP request failed for all resolved addresses." : "HTTP probe deadline exhausted.";
|
||||
return { url: safeUrl(url), addresses, attempts, error: message, resolution: resolution._meta };
|
||||
}
|
||||
|
||||
async function safeRequest(url: URL, requester: Requester, deadline: number, markPolicyBlock = false): Promise<HttpHop> {
|
||||
try {
|
||||
validateProbeUrl(url);
|
||||
if (!remainingMs(deadline)) return { url: safeUrl(url), error: "HTTP probe deadline exhausted." };
|
||||
return await withDeadline(requester(url, deadline), deadline, () => undefined, "HTTP request timed out.");
|
||||
} catch (error) {
|
||||
return {
|
||||
url: safeUrl(url),
|
||||
error: transportError(error),
|
||||
...(markPolicyBlock && (error instanceof InspectionInputError || error instanceof InspectionPolicyError)
|
||||
? { code: "REDIRECT_TARGET_BLOCKED" }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function normalizedUrlKey(url: URL) {
|
||||
const host = normalizeHost(url.hostname);
|
||||
const port = url.port && url.port !== (url.protocol === "https:" ? "443" : "80") ? ":" + url.port : "";
|
||||
return url.protocol.toLowerCase() + "//" + host + port + (url.pathname || "/") + url.search;
|
||||
}
|
||||
|
||||
export async function followRedirectChain(
|
||||
start: URL,
|
||||
requester: Requester = requestOnce,
|
||||
maxRedirects = DEFAULT_MAX_REDIRECTS,
|
||||
deadline = createDeadline(HTTP_TOTAL_TIMEOUT_MS),
|
||||
): Promise<HttpHop[]> {
|
||||
const chain: HttpHop[] = [];
|
||||
const redirectLimit = Math.max(0, Math.floor(maxRedirects));
|
||||
let current = start;
|
||||
const visited = new Set<string>();
|
||||
|
||||
for (let index = 0; index <= redirectLimit; index++) {
|
||||
let key: string;
|
||||
try {
|
||||
key = normalizedUrlKey(current);
|
||||
} catch (error) {
|
||||
chain.push({ url: safeUrl(current), error: error instanceof Error ? error.message : "Invalid redirect target." });
|
||||
break;
|
||||
}
|
||||
if (visited.has(key)) {
|
||||
chain.push({ url: current.toString(), error: REDIRECT_LOOP_ERROR, code: "REDIRECT_LOOP" });
|
||||
break;
|
||||
}
|
||||
visited.add(key);
|
||||
|
||||
const hop = await safeRequest(current, requester, deadline, index > 0);
|
||||
chain.push(sanitizedHop(hop));
|
||||
if ("error" in hop) break;
|
||||
|
||||
const status = Number(hop.status ?? 0);
|
||||
const location = typeof hop.location === "string" ? hop.location : null;
|
||||
if (status < 300 || status >= 400 || !location) break;
|
||||
if (index === redirectLimit) {
|
||||
chain.push({ url: current.toString(), error: "Redirect limit reached", code: "REDIRECT_LIMIT" });
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
const next = new URL(location, current);
|
||||
validateProbeUrl(next);
|
||||
if (current.protocol === "https:" && next.protocol === "http:") {
|
||||
chain.push({ url: next.toString(), error: HTTPS_DOWNGRADE_ERROR, code: "HTTPS_DOWNGRADE", from: current.toString() });
|
||||
break;
|
||||
}
|
||||
if (visited.has(normalizedUrlKey(next))) {
|
||||
chain.push({ url: next.toString(), error: REDIRECT_LOOP_ERROR, code: "REDIRECT_LOOP", from: current.toString() });
|
||||
break;
|
||||
}
|
||||
current = next;
|
||||
} catch (error) {
|
||||
chain.push({
|
||||
url: safeUrl(location),
|
||||
error: error instanceof Error ? error.message : "Invalid redirect target.",
|
||||
code: error instanceof InspectionInputError || error instanceof InspectionPolicyError ? "REDIRECT_TARGET_BLOCKED" : "INVALID_REDIRECT",
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return chain;
|
||||
}
|
||||
|
||||
function errorEvidence(hop: HttpHop, probe: string) {
|
||||
return "error" in hop ? { probe, message: String(hop.error) } : null;
|
||||
}
|
||||
|
||||
export async function collectHttp(host: string): Promise<Record<string, unknown>> {
|
||||
const normalizedHost = normalizeHost(host);
|
||||
const deadline = createDeadline(HTTP_TOTAL_TIMEOUT_MS);
|
||||
const rawHttpResult = await safeRequest(new URL("http://" + normalizedHost + "/"), requestOnce, deadline);
|
||||
const httpResult = sanitizedHop(rawHttpResult);
|
||||
let httpRedirectsToHttps = false;
|
||||
let httpRedirectTarget: HttpHop | null = null;
|
||||
|
||||
if (!("error" in rawHttpResult)) {
|
||||
const status = Number(rawHttpResult.status ?? 0);
|
||||
const location = typeof rawHttpResult.location === "string" ? rawHttpResult.location : null;
|
||||
if (status >= 300 && status < 400 && location) {
|
||||
try {
|
||||
const target = new URL(location, new URL("http://" + normalizedHost + "/"));
|
||||
validateProbeUrl(target);
|
||||
const resolution = await resolvePublicHostDetailed(target.hostname, deadline);
|
||||
httpRedirectsToHttps = target.protocol === "https:";
|
||||
httpRedirectTarget = { url: target.toString(), addresses: resolution.addresses, resolution: resolution._meta, validated: true };
|
||||
} catch (error) {
|
||||
httpRedirectTarget = {
|
||||
url: safeUrl(location),
|
||||
error: error instanceof Error ? error.message : "Invalid redirect target.",
|
||||
code: error instanceof InspectionInputError || error instanceof InspectionPolicyError ? "REDIRECT_TARGET_BLOCKED" : "INVALID_REDIRECT",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const httpsChain = await followRedirectChain(new URL("https://" + normalizedHost + "/"), requestOnce, DEFAULT_MAX_REDIRECTS, deadline);
|
||||
const finalHop = httpsChain.at(-1) && !("error" in httpsChain.at(-1)!) ? httpsChain.at(-1)! : null;
|
||||
const downgrade = httpsChain.some((hop) => hop.code === "HTTPS_DOWNGRADE");
|
||||
const evidenceErrors = [
|
||||
errorEvidence(httpResult, "HTTP"),
|
||||
...(httpRedirectTarget && "error" in httpRedirectTarget
|
||||
? [{ probe: "HTTP redirect", message: String(httpRedirectTarget.error) }]
|
||||
: []),
|
||||
...httpsChain.map((hop) => errorEvidence(hop, "HTTPS")),
|
||||
].filter((item): item is { probe: string; message: string } => Boolean(item));
|
||||
const hasSuccessfulObservation = ![httpResult, ...httpsChain].every((hop) => "error" in hop);
|
||||
const status = evidenceErrors.length ? (hasSuccessfulObservation ? "partial" : "failed") : "ok";
|
||||
|
||||
return {
|
||||
http: httpResult,
|
||||
httpRedirectsToHttps,
|
||||
httpRedirectTarget,
|
||||
httpsChain,
|
||||
httpsDowngrade: downgrade,
|
||||
finalUrl: finalHop && typeof finalHop.url === "string" ? finalHop.url : null,
|
||||
_meta: { status, errors: evidenceErrors },
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user