From 858e76c29056b222c5f1c0fd6bc665e82208b9bd Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Mon, 26 Oct 2026 02:39:46 -0400 Subject: [PATCH] feat(updater): shared calendar dates, ticker and new-fund checks, guard keeps numeric weights SPDR adopts the shared updater blocks it has a path for (holdings-guard Q24, calendar-dates, fund-selection Q27 Q28; no N-PORT or Yahoo window) and publishes a bare holdings ticker only for a U.S. listing. --- README.md | 8 +- scripts/update-data.test.ts | 88 +++++++++++- scripts/update-data.ts | 280 +++++++++++++++++++++++++++++------- 3 files changed, 316 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index dcbf4c1b..b86ae68e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha - `ytd` / `tr1y` - official YTD and 1-year returns -> *YTD Return*, *TR 1Y* - `cagr3y` / `cagr5y` / `cagr10y` - published annualized 3Y/5Y/10Y figures -> *CAGR 3Y/5Y/10Y* - `tr3y` / `tr5y` / `tr10y` - cumulative 3Y/5Y/10Y figures `(1 + CAGR)^n - 1` -> *TR 3Y/5Y/10Y* -- `siAnn` - since-inception annualized -> *SI Ann.*; `null` for funds with less than one year of history at the performance date +- `siAnn` - since-inception annualized -> *SI Ann.*; `null` for funds with less than one year of history at the performance date, and every 1, 3, 5 or 10-year figure is `null` before that anniversary of the inception (calendar date, inclusive) - `dividendYield` - official SSGA Fund Dividend Yield, else an indicated yield (latest distribution x payments per year / price; semi-annual = 2), an estimate derived from SSGA data. HECO, XITK and XSW publish an official `0.00%`, which is kept as `0` (a real zero, not a missing value) - `dividendYieldBasis` - code of the definition behind `dividendYield`, `null` exactly when `dividendYield` is `null` (table below) - `secYield` - official 30-day SEC yield when published; unavailable otherwise, never `0` @@ -83,11 +83,11 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha Expense ratio: `terValue` / `ter` are the NET expense ratio (after waivers; the single published figure when there is no waiver), `terGrossValue` / `terGross` the gross ratio. Both come from the product-data workbook ("Gross Expense Ratio", "Net Expense Ratio"), with the fund finder expense ratio as the gross fallback. The `TER` filter uses the net value. -Sources and scope: SPDR has no Yahoo Finance and no SEC EDGAR source. Everything comes from SSGA's own files, so there is no `SKIP_YAHOO`, `EDGAR_FALLBACK` or `SEC_UA` control. Yields are the official 30-day SEC yield and Fund Dividend Yield from SSGA's product-data workbook. When the product-data workbook or the dividend-distribution feed fails or comes back empty, every published fund is kept exactly as published and reported failed; only new funds are built. Identifier columns of the holdings workbooks (Ticker, Identifier, SEDOL) are kept as published text, so CUSIP `037833100` and A-share ticker `002594` keep their leading zeros. +Sources and scope: SPDR has no Yahoo Finance and no SEC EDGAR source. Everything comes from SSGA's own files, so there is no `SKIP_YAHOO`, `EDGAR_FALLBACK` or `SEC_UA` control. Yields are the official 30-day SEC yield and Fund Dividend Yield from SSGA's product-data workbook. When the product-data workbook or the dividend-distribution feed fails or comes back empty, every published fund is kept exactly as published and reported failed; only new funds are built. Identifier columns of the holdings workbooks (Ticker, Identifier, SEDOL) are kept as published text, so CUSIP `037833100` keeps its leading zero. A holdings ticker is published bare only for a U.S. listing: SSGA's Ticker of a line traded in another currency is the local code (ACA is Arcosa in USD and Credit Agricole in EUR), so a line whose Local Currency is not USD gets `-` and is keyed by its Identifier; futures, options and cash lines get `-` too. Dates (`31-Aug-2026`, `Aug 31 2026`, `08/31/2026`) are read as plain calendar text, never through the local time zone. Commodity trusts such as GLD and GLDM hold metal and publish no holdings workbook. That is valid: they get `meta.json`, NAV, returns and history, with holdings `status: "empty"` (`holdingsStatus` in `index.json`). A fund that published holdings before is never emptied by a transient 404. -Consistency: each fund is fetched and computed completely in memory, then written once (pages, then `meta.json`, then stale pages are removed; every JSON file is written through a temp file and renamed). If any required file of a fund fails to download, that fund keeps its previous `meta.json`, pages and `index.json` row together, so the workflow never publishes a new return next to stale files. A fund without `funds//meta.json` has `dataFile: null` in `index.json`. A rerun with identical upstream data changes nothing, and `generatedAt` moves only when content moved. Newly discovered funds are announced as `NEW FUNDS: A, B` in the run output and in the job summary. Every request times out after 45 s (headers and body) and is retried per `MAX_RETRIES`; the run stops starting new funds after 25 minutes and still writes the index; it exits non-zero when every selected fund failed. +Consistency: each fund is fetched and computed completely in memory, then written once (pages, then `meta.json`, then stale pages are removed; every JSON file is written through a temp file and renamed). If any required file of a fund fails to download, that fund keeps its previous `meta.json`, pages and `index.json` row together, so the workflow never publishes a new return next to stale files. A fund without `funds//meta.json` has `dataFile: null` in `index.json`. A rerun with identical upstream data changes nothing, and `generatedAt` moves only when content moved. `NEW FUNDS: A, B` (run output and job summary) names a fund when it is published for the first time, also when its first read failed or it was not selected and a later run publishes it; `NEW FUNDS (not published yet): ...` names new catalog funds that have no data yet. Every request times out after 45 s (headers and body) and is retried per `MAX_RETRIES`; the run stops starting new funds after 25 minutes and still writes the index; it exits non-zero when every selected fund failed. Returns come from SSGA's own NAV series: `PERFORMANCE_*` filters use month-end NAV returns and `TOTAL_RETURN_*` filters use the separate quarter-end series (3Y/5Y/10Y are annualized CAGR). `DIVIDEND_YIELD` and `SEC_YIELD` filters use the same `metrics` values. There are no market-price or Yahoo estimates in this feed, and no ticker exclusions. Funds not selected for a successful update keep their prior published metadata and data files. `TICKERS` combines with the other filters using AND logic; it does not override them. @@ -104,7 +104,7 @@ Every control is in `scripts/update-data.config.json`; the table shows the shipp | `TER` | `:` | Net expense ratio range in % (strict `min:max`; gross when no waiver is published) | | `DIVIDEND_YIELD` | `:` | Dividend yield range in % (strict `min:max`); official Fund Dividend Yield, else the indicated yield from the latest distribution; funds without a value fail a bounded range | | `SEC_YIELD` | `:` | Official 30-day SEC yield range in % (strict `min:max`); funds without a published value fail a bounded range | -| `TICKERS` | empty (all) | Space-, comma- or semicolon-separated ticker allowlist, e.g. `SPY SPYG SPYD SDY XLK`; a ticker that is not in the SPDR catalog is an error | +| `TICKERS` | empty (all) | Space-, comma- or semicolon-separated ticker allowlist, e.g. `SPY SPYG SPYD SDY XLK`; a ticker that is neither in the live SPDR catalog nor a published fund is an error before any other request (nothing is written) | | `HOLDINGS_PAGE_SIZE` | `250` | Rows in each generated current-holdings JSON page | | `HISTORY_PAGE_SIZE` | `1000` | Rows in each generated daily-history JSON page | | `HISTORY_RANGE` | `max` | Window for the generated NAV and premium/discount history: `max`, `Ny` or `Nmo` counted back from the newest row (for example `5y`, `18mo`); SSGA serves the full workbook, so the window trims the generated pages, not the download; older rows already published are kept, a shorter window never deletes history | diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index c8067e93..e9f4c3d3 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -44,6 +44,9 @@ import { setSoftDeadlineMs, sheetToTable, toIsoDate, + withListedTickers, + HOLDINGS_GUARD_RELAX, + holdingsWeightCount, } from './update-data'; // --------------------------------------------------------------------------- @@ -284,6 +287,17 @@ describe('parsing', () => { expect(sheet.table.rows.map((r) => r[6])).toEqual(['180135563', '58400000']); }); + test('holdings tickers: a bare symbol only for a U.S. listing; local codes, futures, options and cash get -', () => { + const headers = ['Name', 'Ticker', 'Identifier', 'Weight', 'Local Currency', 'Number of Contracts']; + const rows = [ + ['ARCOSA INC', 'ACA', '039653100', '1', 'USD', '-'], ['CREDIT AGRICOLE SA', 'ACA', '4262335', '1', 'EUR', '-'], ['TAIWAN SEMICONDUCTOR', '2330', '6889106', '1', 'TWD', '-'], + ['S+P500 EMINI FUT DEC26', 'ESZ6', 'ESZ6', '1', 'USD', '4'], ['MICROSOFT CORP NOV26 515 PUT', 'MSFT261120P00515000', 'X', '1', 'USD', '2'], ['U.S. Dollar', 'CASH_USD', 'CASH_USD', '1', 'USD', '-'], + ]; + expect(withListedTickers({ headers, rows }).rows.map((row) => row[1])).toEqual(['ACA', '-', '-', '-', '-', '-']); + // a fund of SPDR funds publishes no currency: its tickers stay, its futures do not + expect(withListedTickers({ headers: ['Name', 'Ticker', 'Weight'], rows: [['STATE STREET SPDR PORTFOLIO S+', 'SPYM', '50'], ['AUST 10Y BOND FUT DEC26', 'XMZ6', '1']] }).rows.map((row) => row[1])).toEqual(['SPYM', '-']); + }); + test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { const headers = ['Name', 'Ticker', 'Weight']; const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }]; @@ -341,9 +355,12 @@ describe('parsing 2', () => { expect(() => parseProductDataSheet(buildXlsx([['not', 'a', 'header', 'row']]))).toThrow(); }); - test('dates: SSGA formats normalize to ISO, garbage becomes null', () => { - for (const ok of ['Aug 31 2026', '08/31/2026', '2026-08-31']) expect(toIsoDate(ok)).toBe('2026-08-31'); - for (const bad of ['Feb 30 2026', '', null, '-']) expect(toIsoDate(bad)).toBeNull(); + test('dates: SSGA formats normalize to ISO in every time zone, garbage becomes null', () => { + for (const tz of ['Asia/Tokyo', 'America/Los_Angeles', 'UTC']) { + process.env.TZ = tz; + for (const ok of ['Aug 31 2026', '08/31/2026', '2026-08-31', 'Aug 31, 2026', 'August 31 2026', '31-Aug-2026', '08/31/26']) expect(toIsoDate(ok)).toBe('2026-08-31'); + for (const bad of ['Feb 30 2026', '', null, '-']) expect(toIsoDate(bad)).toBeNull(); + } }); }); @@ -435,6 +452,9 @@ describe('metrics 2', () => { expect(deriveCatalogMetrics({ ...base, inceptionDate: 'Feb 10 2026' }, 10, null).siAnn).toBeNull(); expect(deriveCatalogMetrics({ ...base, inceptionDate: 'Aug 31 2025' }, 10, null).siAnn).toBe(4.2); expect(deriveCatalogMetrics({ ...base, inceptionDate: 'Sep 01 2025' }, 10, null).siAnn).toBeNull(); + // every horizon the fund is too young for is null, not only since inception (calendar anniversary) + const young2 = deriveCatalogMetrics({ ...base, yr1: 3, yr3: 4, yr5: 5, inceptionDate: 'Aug 31 2024' }, 10, null); + expect([young2.tr1y, young2.cagr3y, young2.tr3y, young2.cagr5y, young2.siAnn]).toEqual([3, null, null, null, 4.2]); }); test('returnsBasis and performanceAsOf travel together at the end of every metrics object', () => { @@ -717,7 +737,7 @@ describe('pipeline 3', () => { test('unknown TICKERS is an error that writes nothing; all selected funds failing is an error', async () => { const dir = tempRoot(); installMock(); - await expect(runMain({ TICKERS: 'XXXX' })).rejects.toThrow(/unknown ticker/); + await expect(runMain({ TICKERS: 'XXXX' })).rejects.toThrow('TICKERS: XXXX not in the SPDR catalog or published feed; nothing was fetched or written'); expect(existsSync(join(dir, 'index.json'))).toBe(false); installMock({ failNavhist: new Set(['SPY', 'XLK', 'GLD']) }); await expect(runMain({ CONCURRENCY: '3' })).rejects.toThrow(/all 3 selected funds failed/); @@ -794,6 +814,66 @@ function statSyncMtime(path: string): number { const settled = (p: Promise, ms = 3000) => Promise.race([p.then((v) => (typeof v === 'string' ? v : 'resolved'), () => 'rejected'), new Promise((r) => setTimeout(() => r('hung'), ms))]); +describe('pipeline 4', () => { + test('a relaxed guard still keeps numeric published weights from an all-dash sheet (Q24)', async () => { + const dir = tempRoot(); + installMock(); + await runMain(); + const before = listFiles(dir).map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`); + const good = globalThis.fetch; + const dashes = holdingsSheet('SPY').map((r, i) => (i === 4 ? r.map((cell, c) => (c === 2 ? '-' : cell)) : r)); + globalThis.fetch = (async (input: unknown, init?: RequestInit) => (String(input).includes('holdings-daily-us-en-spy') + ? new Response(new Uint8Array(buildXlsx(dashes))) + : good(input as string, init))) as unknown as typeof fetch; + HOLDINGS_GUARD_RELAX.SPY = ['numeric', 'sum']; + const lines: string[] = []; + try { + await runMain({ TICKERS: 'SPY,XLK' }, lines); + } finally { + delete HOLDINGS_GUARD_RELAX.SPY; + } + expect(listFiles(dir).map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`)).toEqual(before); + expect(lines.join('\n')).toMatch(/SPY +failed .*reason=holdings guard: no row has a numeric weight/); + expect(holdingsWeightCount(['Name', 'Weight'], [['A', '-'], ['B', '2']])).toBe(1); + }); + + test('TICKERS outside the live catalog stops the run before any fund request; a stale catalog-only row is dropped (Q27)', async () => { + const dir = tempRoot(); + installMock(); + await runMain(); + const index = readJson(dir, 'index.json'); + index.funds.push({ ...row(dir, 'XLK'), ticker: 'NOPE', dataFile: null }); + writeFileSync(join(dir, 'index.json'), JSON.stringify(index)); + const before = listFiles(dir).map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`); + const requests: string[] = []; + const good = globalThis.fetch; + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { requests.push(String(input)); return good(input as string, init); }) as unknown as typeof fetch; + await expect(runMain({ TICKERS: 'NOPE' })).rejects.toThrow('TICKERS: NOPE not in the SPDR catalog or published feed; nothing was fetched or written'); + expect(requests.filter((url) => !url.includes('fundfinder'))).toEqual([]); + expect(listFiles(dir).map((f) => `${f}:${readFileSync(join(dir, f), 'utf8')}`)).toEqual(before); + await runMain({ TICKERS: 'SPY' }); + expect(readJson(dir, 'index.json').funds.map((f: any) => f.ticker)).toEqual(['GLD', 'SPY', 'XLK']); + }); + + test('NEW FUNDS announces a fund on its first successful publish, a new catalog-only fund as not published yet (Q28)', async () => { + const dir = tempRoot(); + installMock({ tickers: ['SPY', 'XLK'] }); + await runMain(); + installMock(); + process.env.GITHUB_STEP_SUMMARY = join(dir, '..', `${dir.split('/').pop()}-summary.md`); + dirs.push(process.env.GITHUB_STEP_SUMMARY); + const announced = async (env: Record): Promise => { + const lines: string[] = []; + await runMain(env, lines); + return lines.filter((line) => /^NEW FUNDS/.test(line)); + }; + expect(await announced({ TICKERS: 'SPY' })).toEqual(['NEW FUNDS (not published yet): GLD']); + expect(readFileSync(process.env.GITHUB_STEP_SUMMARY, 'utf8')).toContain('### NEW FUNDS\n\nNEW FUNDS (not published yet): GLD\n'); + expect(await announced({ TICKERS: 'SPY' })).toEqual([]); + expect(await announced({ TICKERS: 'GLD' })).toEqual(['NEW FUNDS: GLD']); + }); +}); + describe('network', () => { test('a stalled connection (headers never arrive) is aborted by the timeout', async () => { setRequestTimeoutMs(100); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 6fcf06fe..22e8743e 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -201,7 +201,7 @@ export function setApiRoot(root: URL): void { const HOLDINGS_PAGE_SIZE_FALLBACK = 250; /** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */ -const HOLDINGS_GUARD_RELAX: Record = {}; +export const HOLDINGS_GUARD_RELAX: Record = {}; /** Funds the issuer closed, each confirmed from an SEC filing or the issuer notice (`source`): never fetched or guarded, last published data kept, index row marked closed. */ export const CLOSED_FUNDS: ClosedFund[] = []; const HISTORY_PAGE_SIZE_FALLBACK = 1000; @@ -424,9 +424,9 @@ export function normalizeHistoryRange(raw: string): string { const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; function historyDate(text: string): number | null { - const match = /^(\d{1,2})-([A-Za-z]{3})-(\d{4})$/.exec(text.trim()); - const month = match ? MONTHS.indexOf(match[2].toLowerCase()) : -1; - return match && month >= 0 ? Date.UTC(Number(match[3]), month, Number(match[1])) : null; + // SSGA workbook dates (`31-Aug-2026`) as calendar text, never the local time zone (shared calendar-dates block) + const iso = calendarIso(text); + return iso ? Date.UTC(Number(iso.slice(0, 4)), Number(iso.slice(5, 7)) - 1, Number(iso.slice(8, 10))) : null; } /** Keeps only rows inside the HISTORY_RANGE window ending at the newest dated row; undated rows are kept. */ @@ -965,16 +965,7 @@ export const RETURNS_BASIS = /** Converts SSGA dates ("Aug 31 2026", "08/31/2026", "2026-08-31") to ISO YYYY-MM-DD; null when unknown or invalid. */ export function toIsoDate(value: unknown): string | null { - const text = typeof value === 'string' ? value.trim() : ''; - let year: number, month: number, day: number; - let match: RegExpMatchArray | null; - if ((match = text.match(/^(\d{4})-(\d{2})-(\d{2})$/))) [year, month, day] = [Number(match[1]), Number(match[2]), Number(match[3])]; - else if ((match = text.match(/^([A-Za-z]{3})[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4})$/))) [year, month, day] = [Number(match[3]), MONTHS.indexOf(match[1].toLowerCase()) + 1, Number(match[2])]; - else if ((match = text.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/))) [year, month, day] = [Number(match[3]), Number(match[1]), Number(match[2])]; - else return null; - const date = new Date(Date.UTC(year, month - 1, day)); - if (month < 1 || date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day) return null; - return date.toISOString().slice(0, 10); + return typeof value === 'string' ? calendarIso(value) : null; } export type DividendYieldBasis = 'official-other' | 'indicated'; @@ -1012,20 +1003,17 @@ export function deriveCatalogMetrics( ): JsonRecord { const numberOrNull = (value: unknown): number | null => typeof value === 'number' && Number.isFinite(value) ? value : null; - const cagr1 = numberOrNull(monthEnd.yr1); - const cagr3 = numberOrNull(monthEnd.yr3); - const cagr5 = numberOrNull(monthEnd.yr5); - const cagr10 = numberOrNull(monthEnd.yr10); - // Since-inception annualized only for funds with >= 1 year of history at the performance date. - const inception = toIsoDate(monthEnd.inceptionDate); - const performanceDate = toIsoDate(monthEnd.asOfDate); - const underOneYear = (() => { - if (!inception || !performanceDate) return false; // unknown dates: keep the published figure - const limit = new Date(`${inception}T00:00:00Z`); - limit.setUTCFullYear(limit.getUTCFullYear() + 1); - return limit.toISOString().slice(0, 10) > performanceDate; - })(); - const siAnn = underOneYear ? null : numberOrNull(monthEnd.sinceInception); + // The shared "younger than one year" rule: no horizon the fund is too young for at the performance date (calendar + // anniversary of its inception; unknown dates keep the published figure). + const official = withoutYoungReturns( + { yr1: numberOrNull(monthEnd.yr1), yr3: numberOrNull(monthEnd.yr3), yr5: numberOrNull(monthEnd.yr5), yr10: numberOrNull(monthEnd.yr10), sinceInception: numberOrNull(monthEnd.sinceInception) }, + monthEnd.inceptionDate, monthEnd.asOfDate, { yr1: 1, yr3: 3, yr5: 5, yr10: 10, sinceInception: 1 }, + ); + const cagr1 = official.yr1; + const cagr3 = official.yr3; + const cagr5 = official.yr5; + const cagr10 = official.yr10; + const siAnn = official.sinceInception; const indicatedDividendYield = indicatedYield(distribution, navValue); const officialDividendYield = productData?.fundDividendYield?.value ?? null; const dividendYieldIsOfficial = officialDividendYield !== null; @@ -1476,15 +1464,56 @@ export function distributionsWorksheet( }; } +/** + * Holdings tickers, a bare symbol only for a U.S. listing (open-items etfs, brand ticker hygiene): SSGA's Ticker of a line + * traded in another currency is the local code (ACA is Arcosa in USD and Credit Agricole in EUR, TEL, 2330), so a line whose + * Local Currency is not USD gets '-' and is keyed by its Identifier, never merged with the U.S. company of that symbol. + * Futures, options (a Number of Contracts) and cash lines (CASH_USD) get '-' too. A sheet without Local Currency (a fund of + * SPDR funds) keeps its tickers. + */ +export function withListedTickers(table: SheetTable): SheetTable { + const ticker = table.headers.indexOf('Ticker'); + if (ticker < 0) return table; + const currency = table.headers.indexOf('Local Currency'); + const contracts = table.headers.indexOf('Number of Contracts'); + const name = table.headers.indexOf('Name'); + const rows = table.rows.map((row) => { + const symbol = cleanText(row[ticker] ?? ''); + if (!symbol || symbol === '-') return row; + const local = currency < 0 ? '' : cleanText(row[currency] ?? '').toUpperCase(); + const derivative = (contracts >= 0 && !['', '-'].includes(cleanText(row[contracts] ?? ''))) || (name >= 0 && /\bFUT(URES?)?\b/i.test(row[name] ?? '')); + const listed = !derivative && !/^CASH_/i.test(symbol) && (!local || local === '-' || local === 'USD'); + return listed ? row : row.map((cell, i) => (i === ticker ? '-' : cell)); + }); + return { headers: table.headers, rows }; +} + /** * The shared holdingsGuard plus what only SPDR needs: the UI reads holdings cells by the exact header text Name and * Weight, so a workbook that answered 200 must carry that header row. A renamed column (Weight (%), which the shared * guard accepts for other brands) or a shifted sheet whose header row was not found (a new fund's 0-row result would * otherwise be published as status "empty") is broken. Commodity trusts without holdings answer 404 and never get here. */ -export function spdrHoldingsProblem(table: SheetTable, previousRows: number | null | undefined, relax: ReadonlyArray): string | null { +export function spdrHoldingsProblem(table: SheetTable, previousRows: number | null | undefined, relax: ReadonlyArray, previousWeights: number | null = null): string | null { if (!table.headers.includes('Name') || !table.headers.includes('Weight')) return `no Name and Weight header row (headers: ${table.headers.join(', ') || 'none'})`; - return holdingsGuard(table.headers, table.rows, previousRows, relax); + return holdingsGuard(table.headers, table.rows, previousRows, relax, previousWeights); +} + +/** The published sheet of a fund (the pages its meta.json lists), empty when nothing is published. */ +async function readPublishedSheet(fundDir: URL, meta: JsonRecord | null, kind: 'holdings' | 'history'): Promise<{ headers: string[]; rows: JsonRecord[] }> { + const pages = Array.isArray(meta?.[kind]?.pages) ? (meta?.[kind]?.pages as unknown[]) : []; + let headers: string[] = []; + const rows: JsonRecord[] = []; + for (const page of pages) { + try { + const data = JSON.parse(await readFile(decodeURIComponent(new URL(String(page), fundDir).pathname), 'utf8')) as JsonRecord; + if (Array.isArray(data.headers)) headers = data.headers.map(String); + if (Array.isArray(data.rows)) rows.push(...data.rows); + } catch { + // a missing page: keep what the other pages give + } + } + return { headers, rows }; } async function processFund( @@ -1516,10 +1545,13 @@ async function processFund( } else { holdingsBytes = new Uint8Array(await holdingsResponse.arrayBuffer()); holdings = sheetToTable(parseXlsxSheet(holdingsBytes, loadSharedStrings(holdingsBytes)), 'holdings'); + holdings.table = withListedTickers(holdings.table); // Freshly fetched holdings that look broken (no weight column, non-numeric weights, a bad weight sum, half the rows gone) // never replace the published ones: the fund fails here, before anything of it is written, and keeps its previous state. const previousMeta = await readPreviousMeta(); - const holdingsProblem = spdrHoldingsProblem(holdings.table, previousMeta?.holdings?.totalRows, HOLDINGS_GUARD_RELAX[ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []); + const published = previousMeta ? await readPublishedSheet(fundDir, previousMeta, 'holdings') : null; + const publishedWeights = published && published.rows.length ? holdingsWeightCount(published.headers, published.rows) : null; + const holdingsProblem = spdrHoldingsProblem(holdings.table, previousMeta?.holdings?.totalRows, HOLDINGS_GUARD_RELAX[ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? [], publishedWeights); if (holdingsProblem) throw new Error(`holdings guard: ${holdingsProblem}`); } @@ -1680,6 +1712,10 @@ export async function main(env: Record = process.env catalogSource = 'previous index.json'; } if (!catalog.length) throw new Error('No SPDR funds discovered and no previous catalog to fall back to'); + // Q27: a TICKERS entry outside the live catalog and the published funds stops the run before any other request + const live = catalogSource === 'ssga.com fund finder' ? catalog.map((fund) => fund.ticker) : null; + const previousFunds: JsonRecord[] = Array.isArray(previousIndex?.funds) ? previousIndex.funds : []; + assertKnownTickers(config.tickers.length ? config.tickers : null, knownTickers(live, previousFunds), 'the SPDR catalog or published feed'); const distributions = await (async () => { try { @@ -1720,19 +1756,7 @@ export async function main(env: Record = process.env const fileExists = (path: string): Promise => access(path).then(() => true, () => false); const metaPath = (ticker: string): string => decodeURIComponent(new URL(`funds/${ticker}/meta.json`, API_ROOT).pathname); - // A closed fund whose row is published is a known ticker even after the live catalog dropped it (the run then writes only its closed mark). - const publishedTickers = new Set((previousIndex?.funds || []).map((row: JsonRecord) => String(row.ticker))); - const unknownTickers = config.tickers.filter((ticker) => !catalog.some((fund) => fund.ticker === ticker) && !(closedFund(CLOSED_FUNDS, ticker) && publishedTickers.has(ticker))); - if (unknownTickers.length) throw new Error(`TICKERS: unknown ticker(s) ${unknownTickers.join(', ')} (not in the SPDR catalog)`); - - const previousRows = new Map((previousIndex?.funds || []).map((row: JsonRecord) => [row.ticker, row])); - if (previousRows.size) { - const newFunds = catalog.map((fund) => fund.ticker).filter((ticker) => !previousRows.has(ticker)); - if (newFunds.length) { - console.log(`NEW FUNDS: ${newFunds.join(', ')}`); - if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `### NEW FUNDS\n\n${newFunds.join(', ')}\n`, 'utf8'); - } - } + const previousRows = new Map(previousFunds.map((row: JsonRecord) => [row.ticker, row])); // Every filter is applied before the cursor, so a bounded run counts only funds that pass the filters. // Return filters exclude funds with null for a bounded range (matchesRange treats null as no match). @@ -1874,6 +1898,10 @@ export async function main(env: Record = process.env indexPayload.generatedAt = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); } const indexChanged = await writeIfChanged(INDEX_FILE, indexPayload); + // Q28: a fund is announced on its first successful publish (a catalog-only row is not published), new catalog-only rows as pending + const newFunds = newFundsLines(newFundsReport(previousFunds, funds)); + for (const line of newFunds) console.log(line); + if (newFunds.length && process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `### NEW FUNDS\n\n${newFunds.join('\n')}\n`, 'utf8'); // A TICKERS run must never overwrite the cursor of the real bounded rotation. if (config.maxFetches > 0 && taken > 0 && !config.tickers.length) { @@ -1986,10 +2014,12 @@ if (import.meta.main) { * An empty sheet passes only when nothing was published before. Otherwise a Name and a weight column (Weight, * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund - * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'. + * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'. No relax lets + * a sheet without a single numeric weight replace a published sheet that had numeric weights (`previousWeights`: how many + * rows of the published sheet carry a numeric weight, see holdingsWeightCount; open-items etfs Q24). * Rows are arrays in header order or objects keyed by the header text. A fund in the brand's CLOSED_FUNDS never reaches the guard. */ -export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArray, previousRows: number | null | undefined, relax: ReadonlyArray = []): string | null { +export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArray, previousRows: number | null | undefined, relax: ReadonlyArray = [], previousWeights: number | null = null): string | null { const previous = Number(previousRows) > 0 ? Number(previousRows) : 0; if (!rows.length) return previous ? `no holdings rows (${previous} published)` : null; const names = headers.map((header) => String(header).trim().toLowerCase()); @@ -1998,20 +2028,38 @@ export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArra if (weightAt < 0) weightAt = names.indexOf('weight (%)'); if (weightAt < 0) weightAt = names.indexOf('market weight'); if (nameAt < 0 || weightAt < 0) return `no ${nameAt < 0 ? 'Name' : 'weight'} column (headers: ${headers.join(', ') || 'none'})`; - let numeric = 0; + const weights = holdingsGuardWeights(headers, rows, weightAt); + const numeric = weights.count; + const sum = weights.sum; + if (numeric === 0 && Number(previousWeights) > 0) return `no row has a numeric weight, the published sheet had ${Number(previousWeights)}`; + if (relax.indexOf('numeric') < 0 && numeric < rows.length * 0.8) return `only ${numeric} of ${rows.length} rows have a numeric weight`; + if (relax.indexOf('sum') < 0 && (sum < 50 || sum > 150)) return `weights add up to ${sum.toFixed(2)}, outside 50..150`; + if (relax.indexOf('count') < 0 && previous && rows.length < previous * 0.5) return `${rows.length} rows, less than half of the ${previous} published`; + return null; +} + +/** How many rows carry a numeric weight (Weight, Weight (%) or Market Weight column), 0 without such a column: pass it for the published sheet as holdingsGuard's previousWeights. */ +export function holdingsWeightCount(headers: ReadonlyArray, rows: ReadonlyArray): number { + const names = headers.map((header) => String(header).trim().toLowerCase()); + let weightAt = names.indexOf('weight'); + if (weightAt < 0) weightAt = names.indexOf('weight (%)'); + if (weightAt < 0) weightAt = names.indexOf('market weight'); + return weightAt < 0 ? 0 : holdingsGuardWeights(headers, rows, weightAt).count; +} + +/** Count and sum of the numeric weights in column weightAt; rows are arrays in header order or objects keyed by the header text. */ +function holdingsGuardWeights(headers: ReadonlyArray, rows: ReadonlyArray, weightAt: number): { count: number; sum: number } { + let count = 0; let sum = 0; for (let i = 0; i < rows.length; i++) { const row: any = rows[i]; const weight = holdingsGuardNumber(Array.isArray(row) ? row[weightAt] : row && typeof row === 'object' ? row[headers[weightAt]] : undefined); if (weight !== null) { - numeric++; + count++; sum += weight; } } - if (relax.indexOf('numeric') < 0 && numeric < rows.length * 0.8) return `only ${numeric} of ${rows.length} rows have a numeric weight`; - if (relax.indexOf('sum') < 0 && (sum < 50 || sum > 150)) return `weights add up to ${sum.toFixed(2)}, outside 50..150`; - if (relax.indexOf('count') < 0 && previous && rows.length < previous * 0.5) return `${rows.length} rows, less than half of the ${previous} published`; - return null; + return { count, sum }; } /** A weight cell as a number: %, $, commas and spaces are ignored, (x) is negative; null when it is no plain number (a zero-width character inside counts as broken). */ @@ -2063,3 +2111,131 @@ export function withoutClosedFunds(queue: ReadonlyArray, tickerOf: (item: return open; } // + +// shared block of .claude/tools/etfs/etf-std/calendar-dates.block.ts, applied by apply-block.ts: never edit it in one repo +/** + * A calendar date as ISO `YYYY-MM-DD`, or null. Pure text work, never `Date.parse` or local getters, so a run in Tokyo + * publishes the same day as CI (open-items etfs ETF-H4). Accepts ISO (`2026-10-24`, with a time the date part is kept as + * written), `2026/10/24`, U.S. `10/24/2026` and `10-24-26` (two-digit years up to 69 are 20xx), month names in either order + * (`Oct 24, 2026`, `October 24 2026`, `Sept. 30, 2026`, `24 Oct 2026`, `24-Oct-26`), an optional weekday or `as of` in front. + * A day that does not exist (`02/30/2026`) is null. + */ +export function calendarIso(value: unknown): string | null { + const text = String(value === null || value === undefined ? '' : value).replace(/[\s ]+/g, ' ').trim() + .replace(/^as of:? /i, '').replace(/^(mon|tue|wed|thu|fri|sat|sun)[a-z]*\.?,? /i, ''); + const year = (raw: string): number => (raw.length === 2 ? (Number(raw) <= 69 ? 2000 : 1900) + Number(raw) : Number(raw)); + const month = (raw: string): number => { + const word = raw.toLowerCase(); + const names = ['january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december']; + for (let i = 0; i < names.length; i++) if (word.length >= 3 && names[i].indexOf(word) === 0) return i + 1; + return 0; + }; + let y = 0, m = 0, d = 0, match: RegExpExecArray | null; + if ((match = /^(\d{4})-(\d{1,2})-(\d{1,2})(?:[T ].*)?$/.exec(text)) || (match = /^(\d{4})\/(\d{1,2})\/(\d{1,2})$/.exec(text))) { + y = Number(match[1]); m = Number(match[2]); d = Number(match[3]); + } else if ((match = /^(\d{1,2})[/-](\d{1,2})[/-](\d{4}|\d{2})$/.exec(text))) { + y = year(match[3]); m = Number(match[1]); d = Number(match[2]); + } else if ((match = /^([A-Za-z]{3,9})\.? (\d{1,2})(?:st|nd|rd|th)?,? (\d{4})$/.exec(text))) { + y = Number(match[3]); m = month(match[1]); d = Number(match[2]); + } else if ((match = /^(\d{1,2})(?:st|nd|rd|th)?[ -]([A-Za-z]{3,9})\.?,?[ -](\d{4}|\d{2})$/.exec(text))) { + y = year(match[3]); m = month(match[2]); d = Number(match[1]); + } else { + return null; + } + const days = [31, y % 4 === 0 && (y % 100 !== 0 || y % 400 === 0) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + if (y < 1900 || y > 2999 || m < 1 || m > 12 || d < 1 || d > days[m - 1]) return null; + return `${y}-${m < 10 ? '0' : ''}${m}-${d < 10 ? '0' : ''}${d}`; +} + +/** + * Is the fund at least `years` old on `asOf`? Calendar anniversary, inclusive: a fund started 2025-10-24 is one year old on + * 2026-10-24 (a 29 February start on 1 March). Unknown or unreadable dates answer true, so a value is never dropped for want + * of a date. The shared "younger than one year" rule: since-inception returns are annualized only from one year on. + */ +export function fundAgeAtLeast(inception: unknown, asOf: unknown, years: number): boolean { + const start = calendarIso(inception), end = calendarIso(asOf); + if (!start || !end) return true; + return end >= `${Number(start.slice(0, 4)) + years}${start.slice(4)}`; +} + +/** + * The returns with every horizon the fund is too young for set to null: `horizons` maps a key of `returns` to the years it + * needs (`{ yr1: 1, cagr3y: 3, siAnn: 1 }`). Keys that are absent or already null stay as they are; the object is copied. + */ +export function withoutYoungReturns(returns: T, inception: unknown, asOf: unknown, horizons: Record): T { + const out: Record = { ...(returns as Record) }; + const keys = Object.keys(horizons); + for (let i = 0; i < keys.length; i++) { + if (out[keys[i]] !== undefined && out[keys[i]] !== null && !fundAgeAtLeast(inception, asOf, horizons[keys[i]])) out[keys[i]] = null; + } + return out as T; +} +// + +// shared block of .claude/tools/etfs/etf-std/fund-selection.block.ts, applied by apply-block.ts: never edit it in one repo +/** An index row as far as these helpers read it: the ticker and whether it has published data (`dataFile`, null for a catalog-only row). */ +export type FundSelectionRow = { ticker?: unknown; dataFile?: unknown }; + +function fundSelectionTicker(row: FundSelectionRow): string { + return String(row.ticker === null || row.ticker === undefined ? '' : row.ticker).trim().toUpperCase(); +} + +/** + * The tickers a TICKERS entry may name: the live catalog plus every fund with published data (closed funds included). When + * the live catalog could not be read (`live` null) every previous index row counts, since nothing tells a real catalog-only + * row from a stale one. A catalog-only row from an earlier mistyped TICKERS run is therefore not "known" (open-items etfs Q27). + */ +export function knownTickers(live: ReadonlyArray | null, previous: ReadonlyArray): string[] { + const out: string[] = []; + const add = (ticker: string): void => { if (ticker && out.indexOf(ticker) < 0) out.push(ticker); }; + if (live) for (let i = 0; i < live.length; i++) add(String(live[i]).trim().toUpperCase()); + for (let i = 0; i < previous.length; i++) if (!live || previous[i].dataFile) add(fundSelectionTicker(previous[i])); + return out; +} + +/** Throws before any fund request when TICKERS names a fund outside `known` (see knownTickers): a clear error, nothing fetched or written. */ +export function assertKnownTickers(selected: ReadonlyArray | null, known: ReadonlyArray, where: string): void { + const unknown = (selected || []).map((ticker) => String(ticker).trim().toUpperCase()).filter((ticker) => ticker && known.indexOf(ticker) < 0); + if (unknown.length) throw new Error(`TICKERS: ${unknown.join(', ')} not in ${where}; nothing was fetched or written`); +} + +/** + * The index rows without stale catalog-only rows: a row with no published data whose ticker the live catalog no longer lists + * (an earlier mistyped TICKERS entry, a fund withdrawn before launch). Rows with data always stay; nothing is dropped when the + * live catalog could not be read (`live` null). + */ +export function withoutStalePlaceholders(rows: ReadonlyArray, live: ReadonlyArray | null): T[] { + if (!live) return rows.slice(); + const listed = live.map((ticker) => String(ticker).trim().toUpperCase()); + return rows.filter((row) => Boolean(row.dataFile) || listed.indexOf(fundSelectionTicker(row)) >= 0); +} + +/** + * New funds of a run (open-items etfs Q28): `published` - first published in this run (the previous index had no row for it, + * or only a catalog-only one), so a fund whose first read failed is announced when it finally succeeds; `pending` - new in the + * catalog this run but not published yet. Nothing is reported on a first run (empty previous index). + */ +export function newFundsReport(previous: ReadonlyArray, next: ReadonlyArray): { published: string[]; pending: string[] } { + const report = { published: [] as string[], pending: [] as string[] }; + if (!previous.length) return report; + const before = new Map(); + for (let i = 0; i < previous.length; i++) before.set(fundSelectionTicker(previous[i]), Boolean(previous[i].dataFile)); + for (let i = 0; i < next.length; i++) { + const ticker = fundSelectionTicker(next[i]); + if (!ticker || before.get(ticker)) continue; + if (next[i].dataFile) report.published.push(ticker); + else if (!before.has(ticker)) report.pending.push(ticker); + } + report.published.sort(); + report.pending.sort(); + return report; +} + +/** The log and step-summary lines of a newFundsReport: `NEW FUNDS: A, B` and `NEW FUNDS (not published yet): C`; none when empty. */ +export function newFundsLines(report: { published: ReadonlyArray; pending: ReadonlyArray }): string[] { + const lines: string[] = []; + if (report.published.length) lines.push(`NEW FUNDS: ${report.published.join(', ')}`); + if (report.pending.length) lines.push(`NEW FUNDS (not published yet): ${report.pending.join(', ')}`); + return lines; +} +//