From 5336d146f2e74aa85759ba8d58f5af6cea627eb2 Mon Sep 17 00:00:00 2001 From: cje <12779999+yesnet0@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:43:54 -0700 Subject: [PATCH 1/2] Retry 503 with Retry-After, default concurrency 4, 45s request timeout lookup.disclose.io sheds work with 503 lookup-overloaded when its ~8 resolver slots are full. dio-lookup retried only 429, so a shed host came back as an error row. Lookups can take ~30s, so give up no sooner than 45s, and default to 4 workers so a single scan does not occupy every slot. --- README.md | 4 ++-- dio-lookup.test.ts | 22 +++++++++++++++++----- dio-lookup.ts | 26 +++++++++++++++++--------- package.json | 2 +- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c298b7b..d7cb1c7 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ chmod +x dio-lookup && sudo mv dio-lookup /usr/local/bin/ dio-lookup [options] [asset ...] cat hosts.txt | dio-lookup [options] - -c, --concurrency N parallel requests (default 5) + -c, --concurrency N parallel requests (default 4) -k, --key KEY API key (raises rate limits); or set DIO_API_KEY --api URL API endpoint (default https://lookup.disclose.io/api/lookup) --full emit the full LookupResult instead of the compact summary @@ -95,7 +95,7 @@ nuclei -u example.com -jsonl | jq -r '.host' | sort -u | dio-lookup ## Notes - **Free and anonymous.** A free API key only raises rate limits — request one by emailing [hello@disclose.io](mailto:hello@disclose.io). Pass it with `-k` or `DIO_API_KEY`. -- Honors the API's `Retry-After` on 429 and retries transient failures. +- Honors the API's `Retry-After` on 429 (rate limit) and 503 (service at capacity), and retries transient failures. Requests time out at 45s, above the slowest lookups. - `--full` emits the complete `LookupResult` (attribution, contacts, resolution chain, data sources) — see the [OpenAPI spec](https://lookup.disclose.io/openapi.yaml). A [disclose.io](https://disclose.io) project. MIT licensed. diff --git a/dio-lookup.test.ts b/dio-lookup.test.ts index ecacf5a..8947222 100644 --- a/dio-lookup.test.ts +++ b/dio-lookup.test.ts @@ -31,7 +31,7 @@ afterEach(() => { servers.forEach((s) => s.stop(true)); servers = []; }); describe("parseArgs", () => { test("defaults", () => { const o = parseArgs([]) as any; - expect(o.concurrency).toBe(5); + expect(o.concurrency).toBe(4); expect(o.full).toBe(false); expect(o.inputs).toEqual([]); expect(o.api).toContain("lookup.disclose.io"); @@ -48,10 +48,10 @@ describe("parseArgs", () => { expect(o.full).toBe(true); expect(o.nuclei).toBe(true); }); - test("non-positive or garbage concurrency falls back to the default (5)", () => { - // `0 || 5` -> 5 (zero is falsy); NaN || 5 -> 5. Either way a sane worker count. - expect((parseArgs(["-c", "0"]) as any).concurrency).toBe(5); - expect((parseArgs(["-c", "nope"]) as any).concurrency).toBe(5); + test("non-positive or garbage concurrency falls back to the default (4)", () => { + // `0 || 4` -> 4 (zero is falsy); NaN || 4 -> 4. Either way a sane worker count. + expect((parseArgs(["-c", "0"]) as any).concurrency).toBe(4); + expect((parseArgs(["-c", "nope"]) as any).concurrency).toBe(4); expect((parseArgs(["-c", "8"]) as any).concurrency).toBe(8); }); test("--version / --help are recognized", () => { @@ -153,6 +153,18 @@ describe("lookupOne resilience", () => { expect(out.organization).toBe("Cloudflare"); }, 10_000); + test("honors a 503 (service at capacity) then succeeds on retry", async () => { + let n = 0; + const api = serve(() => { + n++; + if (n === 1) return new Response(JSON.stringify({ errorCode: "lookup-overloaded" }), { status: 503, headers: { "retry-after": "1" } }); + return Response.json(RECORDED); + }); + const out = (await lookupOne("cloudflare.com", opts({ api }))) as any; + expect(n).toBe(2); + expect(out.organization).toBe("Cloudflare"); + }, 10_000); + test("returns an {error} object after exhausting retries", async () => { const out = (await lookupOne("x", opts({ api: "http://127.0.0.1:1/nope" }))) as any; expect(out.error).toBeTruthy(); diff --git a/dio-lookup.ts b/dio-lookup.ts index c85c3f3..cd837d2 100644 --- a/dio-lookup.ts +++ b/dio-lookup.ts @@ -5,7 +5,7 @@ * * Built for recon pipelines: * subfinder -d example.com | httpx -silent | dio-lookup - * cat hosts.txt | dio-lookup --concurrency 8 > contacts.jsonl + * cat hosts.txt | dio-lookup --concurrency 4 > contacts.jsonl * dio-lookup cloudflare.com npm:express gh:facebook/react * * Each input asset (domain, IP, ASN, URL, email, package, repo, container, @@ -13,7 +13,9 @@ * object on stdout. Anonymous + free; an optional API key raises rate limits. */ -const VERSION = '0.2.0'; +const VERSION = '0.2.1'; +const DEFAULT_CONCURRENCY = 4; +const REQUEST_TIMEOUT_MS = 45_000; const DEFAULT_API = 'https://lookup.disclose.io/api/lookup'; interface Options { @@ -26,12 +28,12 @@ interface Options { } export function parseArgs(argv: string[]): Options | { help: true } | { version: true } { - const o: Options = { concurrency: 5, api: DEFAULT_API, full: false, nuclei: false, inputs: [] }; + const o: Options = { concurrency: DEFAULT_CONCURRENCY, api: DEFAULT_API, full: false, nuclei: false, inputs: [] }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '-h' || a === '--help') return { help: true }; if (a === '-V' || a === '--version') return { version: true }; - else if (a === '-c' || a === '--concurrency') o.concurrency = Math.max(1, parseInt(argv[++i] ?? '5', 10) || 5); + else if (a === '-c' || a === '--concurrency') o.concurrency = Math.max(1, parseInt(argv[++i] ?? '', 10) || DEFAULT_CONCURRENCY); else if (a === '--api') o.api = argv[++i] ?? DEFAULT_API; else if (a === '-k' || a === '--key') o.apiKey = argv[++i]; else if (a === '--full') o.full = true; @@ -49,7 +51,8 @@ USAGE cat hosts.txt | dio-lookup [options] OPTIONS - -c, --concurrency N parallel requests (default 5) + -c, --concurrency N parallel requests (default 4; the service runs ~8 lookups + at once for everyone, so more mostly earns 503s) -k, --key KEY API key (raises rate limits); or set DIO_API_KEY --api URL API endpoint (default ${DEFAULT_API}) --full emit the full LookupResult instead of the compact summary @@ -64,7 +67,7 @@ OUTPUT EXAMPLES dio-lookup cloudflare.com - subfinder -d example.com | httpx -silent | dio-lookup -c 8 > contacts.jsonl + subfinder -d example.com | httpx -silent | dio-lookup -c 4 > contacts.jsonl echo npm:express | dio-lookup --full | jq . nuclei -u example.com -jsonl | dio-lookup --nuclei # enrich each scanned host @@ -121,11 +124,16 @@ export async function lookupOne(input: string, o: Options): Promise Date: Thu, 8 Oct 2026 22:07:13 -0700 Subject: [PATCH 2/2] Jitter 429/503 retries by up to 1s Workers shed in the same instant otherwise retry in the same instant. --- dio-lookup.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dio-lookup.ts b/dio-lookup.ts index cd837d2..5e86e1d 100644 --- a/dio-lookup.ts +++ b/dio-lookup.ts @@ -135,7 +135,8 @@ export async function lookupOne(input: string, o: Options): Promise