From 356fb11c63902a98b3b6f173e27ddabf6faa8749 Mon Sep 17 00:00:00 2001 From: i Date: Sun, 6 Sep 2026 14:14:58 -0400 Subject: [PATCH 1/4] Add SEC EDGAR API checkpoint --- README.md | 3 +- Sec.idric | 1 + checkpoints/sec/README.md | 52 +++++++++++++++++ checkpoints/sec/check | 74 ++++++++++++++++++++++++ checkpoints/sec/idric/Sec.idric | 99 +++++++++++++++++++++++++++++++++ 5 files changed, 228 insertions(+), 1 deletion(-) create mode 120000 Sec.idric create mode 100644 checkpoints/sec/README.md create mode 100755 checkpoints/sec/check create mode 100644 checkpoints/sec/idric/Sec.idric diff --git a/README.md b/README.md index 8dbb14f..7680b30 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This repository is the consolidation point for the CLI/API-access programs that - `Guardian.idric` — Guardian API checkpoint. - `Nyt.idric` — New York Times API checkpoint. - `Reddit.idric` — Reddit Data API checkpoint, with a synthetic fixture and manual receipt. +- `Sec.idric` — SEC EDGAR public-data checkpoint: submissions, XBRL facts/frames, ticker maps, and bulk archives. - `Stripe.idric` — Stripe API checkpoint; first slice is pinned, read-only Balance access with a synthetic fixture and manual receipt. - `Reuters.idric` — Reuters GraphQL checkpoint. - `Wayback.idric` — Internet Archive Wayback/CDX checkpoint. @@ -28,6 +29,6 @@ Where these clients need networking, ICU/Idric-Net remains the intended transpor ## Tests -`make test` runs the existing Amazon and AbeBooks smoke tests. Reddit and Stripe have separate manual compiler checkpoints under `checkpoints/reddit/check` and `checkpoints/stripe/check`; they are not part of `make test` while named Idriç holes remain. +`make test` runs the existing Amazon and AbeBooks smoke tests. Reddit, SEC, and Stripe have separate manual compiler checkpoints under `checkpoints/reddit/check`, `checkpoints/sec/check`, and `checkpoints/stripe/check`; they are not part of `make test` while named Idriç holes remain. See `PROVENANCE.md` for the source branches copied into this repository. diff --git a/Sec.idric b/Sec.idric new file mode 120000 index 0000000..2ffbfcb --- /dev/null +++ b/Sec.idric @@ -0,0 +1 @@ +checkpoints/sec/idric/Sec.idric \ No newline at end of file diff --git a/checkpoints/sec/README.md b/checkpoints/sec/README.md new file mode 100644 index 0000000..61b0566 --- /dev/null +++ b/checkpoints/sec/README.md @@ -0,0 +1,52 @@ +# SEC EDGAR checkpoint + +Read-only command-line access to the SEC's documented public EDGAR data surfaces. + +## Endpoints + +The first slice exposes: + +- `submissions CIK10` — current filing history and filer metadata. +- `companyfacts CIK10` — all standard-taxonomy XBRL facts for one filer. +- `companyconcept CIK10 TAXONOMY TAG` — one standard XBRL concept for one filer. +- `frame TAXONOMY TAG UNIT PERIOD` — one XBRL fact across reporting entities for a calendar frame. +- `tickers` — company ticker / CIK / name associations. +- `ticker-exchanges` — company / CIK / ticker / exchange associations. +- `mutual-fund-tickers` — fund CIK / series / class / ticker associations. +- `bulk-submissions` — nightly submissions archive. +- `bulk-companyfacts` — nightly XBRL company-facts archive. + +CIKs passed to the `data.sec.gov` endpoints are the SEC's 10-digit zero-padded form, without the `CIK` prefix. For Apple, for example, use `0000320193`. + +## URL-only use + +`url` prints the endpoint without performing a request: + +```text +sec url submissions 0000320193 +sec url companyfacts 0000320193 +sec url companyconcept 0000320193 us-gaap AssetsCurrent +sec url frame us-gaap AccountsPayableCurrent USD CY2025Q4I +sec url tickers +sec url bulk-submissions +``` + +## Network use + +The public data APIs require no API key. SEC automated-access policy does require a declared User-Agent identifying the requester and a contact address. The checkpoint therefore requires `SEC_USER_AGENT` before any network request, for example: + +```text +SEC_USER_AGENT='Example Research example@example.org' sec submissions 0000320193 +``` + +The actual HTTP operation remains a named ICU/Idric-Net hole: `?icu_get_with_user_agent_and_forward_stdout`. Do not replace it with curl or another transport merely to make the checkpoint look complete. + +## Scope + +This checkpoint intentionally does **not** implement EDGAR Next filer-management or filing-submission APIs. Those are authenticated write-capable interfaces with filer/user tokens and should be added as a separate authenticated slice rather than conflated with public EDGAR data access. + +## Sources + +- SEC, EDGAR Application Programming Interfaces (APIs): https://www.sec.gov/search-filings/edgar-application-programming-interfaces +- SEC, Accessing EDGAR Data: https://www.sec.gov/search-filings/edgar-search-assistance/accessing-edgar-data +- SEC, Developer Resources: https://www.sec.gov/about/developer-resources diff --git a/checkpoints/sec/check b/checkpoints/sec/check new file mode 100755 index 0000000..01a2d25 --- /dev/null +++ b/checkpoints/sec/check @@ -0,0 +1,74 @@ +#!/usr/bin/env ysh + +# Manual SEC EDGAR checkpoint runner. +# URL literals are checked without networking; Idriç typechecking is optional. + +set -u + +HERE=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +IDRIC=${IDRIC:-idris2} +SOURCE="$HERE/idric/Sec.idric" + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT HUP INT TERM + +status=0 + +available() { + case "$1" in + */*) test -x "$1" ;; + *) command -v "$1" >/dev/null 2>&1 ;; + esac +} + +pass() { + printf 'PASS\t%s\n' "$1" +} + +fail() { + printf 'FAIL\t%s\n' "$1" + status=1 +} + +skip() { + printf 'SKIP\t%s\n' "$1" +} + +expect_source() { + label=$1 + text=$2 + if grep -Fq -- "$text" "$SOURCE"; then + pass "$label" + else + fail "$label" + fi +} + +expect_source 'url/submissions' '/submissions/CIK' +expect_source 'url/companyfacts' '/api/xbrl/companyfacts/CIK' +expect_source 'url/companyconcept' '/api/xbrl/companyconcept/CIK' +expect_source 'url/frame' '/api/xbrl/frames/' +expect_source 'url/tickers' '/files/company_tickers.json' +expect_source 'url/ticker-exchanges' '/files/company_tickers_exchange.json' +expect_source 'url/mutual-fund-tickers' '/files/company_tickers_mf.json' +expect_source 'url/bulk-submissions' '/Archives/edgar/daily-index/bulkdata/submissions.zip' +expect_source 'url/bulk-companyfacts' '/Archives/edgar/daily-index/xbrl/companyfacts.zip' +expect_source 'policy/user-agent' 'SEC_USER_AGENT' + +if ! available "$IDRIC"; then + skip "idric (not found: $IDRIC)" + exit "$status" +fi + +if "$IDRIC" --check "$SOURCE" >"$TMP/idric" 2>"$TMP/idric.err"; then + pass 'idric/check' +else + fail 'idric/check' + for path in "$TMP/idric" "$TMP/idric.err"; do + if test -s "$path"; then + sed 's/^/ /' "$path" + fi + done +fi + +exit "$status" diff --git a/checkpoints/sec/idric/Sec.idric b/checkpoints/sec/idric/Sec.idric new file mode 100644 index 0000000..cd9c5d5 --- /dev/null +++ b/checkpoints/sec/idric/Sec.idric @@ -0,0 +1,99 @@ +module Sec + +import System + + +data_api : String +data_api = "https://data.sec.gov" + +sec_site : String +sec_site = "https://www.sec.gov" + +submissions_bulk_url : String +submissions_bulk_url = sec_site ++ "/Archives/edgar/daily-index/bulkdata/submissions.zip" + +companyfacts_bulk_url : String +companyfacts_bulk_url = sec_site ++ "/Archives/edgar/daily-index/xbrl/companyfacts.zip" + +tickers_url : String +tickers_url = sec_site ++ "/files/company_tickers.json" + +ticker_exchanges_url : String +ticker_exchanges_url = sec_site ++ "/files/company_tickers_exchange.json" + +mutual_fund_tickers_url : String +mutual_fund_tickers_url = sec_site ++ "/files/company_tickers_mf.json" + +usage : String +usage = "usage: sec [url] {submissions CIK10 | companyfacts CIK10 | companyconcept CIK10 TAXONOMY TAG | frame TAXONOMY TAG UNIT PERIOD | tickers | ticker-exchanges | mutual-fund-tickers | bulk-submissions | bulk-companyfacts}" + + +-- Public EDGAR data does not require an API key, but automated access must +-- identify itself. Keep this header seam explicit at the ICU/Idric-Net +-- boundary rather than quietly substituting curl or another HTTP client. +environment_value : String → IO (Maybe String) +environment_value name = ?read_environment_value + +declared_icu_get : String → String → IO () +declared_icu_get user_agent url = ?icu_get_with_user_agent_and_forward_stdout + + +submissions_url : String → String +submissions_url cik = + data_api ++ "/submissions/CIK" ++ cik ++ ".json" + +companyfacts_url : String → String +companyfacts_url cik = + data_api ++ "/api/xbrl/companyfacts/CIK" ++ cik ++ ".json" + +companyconcept_url : String → String → String → String +companyconcept_url cik taxonomy tag = + data_api ++ "/api/xbrl/companyconcept/CIK" ++ cik ++ "/" ++ taxonomy ++ "/" ++ tag ++ ".json" + +frame_url : String → String → String → String → String +frame_url taxonomy tag unit period = + data_api ++ "/api/xbrl/frames/" ++ taxonomy ++ "/" ++ tag ++ "/" ++ unit ++ "/" ++ period ++ ".json" + + +endpoint_url : List String → Either String String +endpoint_url ["submissions", cik] = Right (submissions_url cik) +endpoint_url ["companyfacts", cik] = Right (companyfacts_url cik) +endpoint_url ["companyconcept", cik, taxonomy, tag] = Right (companyconcept_url cik taxonomy tag) +endpoint_url ["frame", taxonomy, tag, unit, period] = Right (frame_url taxonomy tag unit period) +endpoint_url ["tickers"] = Right tickers_url +endpoint_url ["ticker-exchanges"] = Right ticker_exchanges_url +endpoint_url ["mutual-fund-tickers"] = Right mutual_fund_tickers_url +endpoint_url ["bulk-submissions"] = Right submissions_bulk_url +endpoint_url ["bulk-companyfacts"] = Right companyfacts_bulk_url +endpoint_url _ = Left usage + + +fetch : String → IO () +fetch url = do + maybe_user_agent ← environment_value "SEC_USER_AGENT" + case maybe_user_agent of + Just user_agent => declared_icu_get user_agent url + Nothing => do + putStrLn "sec: missing SEC_USER_AGENT (use a declared organization/name and contact email)" + exitFailure + + +main : IO () +main = do + arguments ← getArgs + case arguments of + _ :: "url" :: rest => + case endpoint_url rest of + Left problem => do + putStrLn problem + exitFailure + Right url => putStrLn url + _ :: rest => + case endpoint_url rest of + Left problem => do + putStrLn problem + exitFailure + Right url => fetch url + [] => do + putStrLn usage + exitFailure From cb85abe4703e9f0372c84ddf434f94ca2d281eef Mon Sep 17 00:00:00 2001 From: i Date: Sun, 6 Sep 2026 16:14:27 -0400 Subject: [PATCH 2/4] Reject empty SEC user agent --- checkpoints/sec/idric/Sec.idric | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/checkpoints/sec/idric/Sec.idric b/checkpoints/sec/idric/Sec.idric index cd9c5d5..93d1fae 100644 --- a/checkpoints/sec/idric/Sec.idric +++ b/checkpoints/sec/idric/Sec.idric @@ -68,14 +68,19 @@ endpoint_url ["bulk-companyfacts"] = Right companyfacts_bulk_url endpoint_url _ = Left usage +missing_user_agent : IO () +missing_user_agent = do + putStrLn "sec: missing or empty SEC_USER_AGENT (use a declared organization/name and contact email)" + exitFailure + + fetch : String → IO () fetch url = do maybe_user_agent ← environment_value "SEC_USER_AGENT" case maybe_user_agent of + Just "" => missing_user_agent Just user_agent => declared_icu_get user_agent url - Nothing => do - putStrLn "sec: missing SEC_USER_AGENT (use a declared organization/name and contact email)" - exitFailure + Nothing => missing_user_agent main : IO () From 5c3560b8825455c62151ced8aef97df4a2aad7bb Mon Sep 17 00:00:00 2001 From: i Date: Sun, 6 Sep 2026 16:14:35 -0400 Subject: [PATCH 3/4] Strengthen SEC policy checkpoint --- checkpoints/sec/check | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/checkpoints/sec/check b/checkpoints/sec/check index 01a2d25..32b469b 100755 --- a/checkpoints/sec/check +++ b/checkpoints/sec/check @@ -53,7 +53,9 @@ expect_source 'url/ticker-exchanges' '/files/company_tickers_exchange.json' expect_source 'url/mutual-fund-tickers' '/files/company_tickers_mf.json' expect_source 'url/bulk-submissions' '/Archives/edgar/daily-index/bulkdata/submissions.zip' expect_source 'url/bulk-companyfacts' '/Archives/edgar/daily-index/xbrl/companyfacts.zip' -expect_source 'policy/user-agent' 'SEC_USER_AGENT' +expect_source 'policy/user-agent' 'environment_value "SEC_USER_AGENT"' +expect_source 'policy/user-agent-nonempty' 'Just "" => missing_user_agent' +expect_source 'transport/icu-idric-net' '?icu_get_with_user_agent_and_forward_stdout' if ! available "$IDRIC"; then skip "idric (not found: $IDRIC)" From 70c24e3c0a31e29eca7e2f88be0de3943b910bea Mon Sep 17 00:00:00 2001 From: i Date: Sun, 6 Sep 2026 16:14:39 -0400 Subject: [PATCH 4/4] Run SEC checkpoint in CI --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 151193c..cc4d15e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,8 @@ jobs: - name: Install JSON tool run: sudo apt-get update && sudo apt-get install -y jq - name: Shell syntax - run: bash -n bin/az bin/abe test/az-test.sh test/abe-test.sh + run: bash -n bin/az bin/abe test/az-test.sh test/abe-test.sh checkpoints/sec/check - name: Smoke tests run: make test + - name: SEC checkpoint + run: bash checkpoints/sec/check