diff --git a/architecture/artifact_index.py b/architecture/artifact_index.py index 7cb920cf6..de5364867 100644 --- a/architecture/artifact_index.py +++ b/architecture/artifact_index.py @@ -30,6 +30,8 @@ "Authority / canonical impact", "Dependencies or supersession", "Modified UTC", ) PUBLIC_INDICES = tuple(HEADER.index(column) for column in PUBLIC_COLUMNS) +# Every public column precedes the private trailing Source/Notes suffix. +ROW_MINIMUM_CELLS = max(PUBLIC_INDICES) + 1 ROLES = frozenset(("source", "review", "kernel", "computation", "alignment", "teaching-source")) PIN_KEYS = frozenset(( "capture_sha256", "metadata_sha256", "policy_sha256", @@ -230,14 +232,18 @@ def _native_rows(native, source, intervals): rows = _bounded_list(block.get("rowData"), stop - start, stop - start) for offset, row in enumerate(rows): _object(row) - cells = _bounded_list(row.get("values"), len(HEADER), len(HEADER)) + position = start + offset + # formattedValue reads elide trailing empty cells. Nonheader rows may + # omit only the trailing private Source/Notes cells; the header may not. + minimum = len(HEADER) if position == 0 else ROW_MINIMUM_CELLS + cells = _bounded_list(row.get("values"), minimum, len(HEADER)) values = [] for cell in cells: _object(cell) value = cell.get("formattedValue", "") _require(type(value) is str and len(value.encode("utf-8")) <= CELL_LIMIT) values.append(value) - position = start + offset + values.extend([""] * (len(HEADER) - len(values))) if position == 0: _require(values == list(HEADER)) header_count += 1 diff --git a/docs/plans/2026-10-08-artifact-index-contract.md b/docs/plans/2026-10-08-artifact-index-contract.md index c1e95087a..09788a526 100644 --- a/docs/plans/2026-10-08-artifact-index-contract.md +++ b/docs/plans/2026-10-08-artifact-index-contract.md @@ -27,7 +27,7 @@ Capture ID is a nonempty ASCII identifier matching `[A-Za-z0-9][A-Za-z0-9_.:-]{0 Stdin limit16MiB, decoded capture limit4MiB, JSON nesting at most64 container levels (a scalar leaf does not add a level), at most4096 nonheader native data rows plus the one header and4096 graph IDs/bindings/policy IDs, each formatted cell at most8192 UTF-8 bytes. Artifact and graph IDs use the same ASCII identifier pattern and192-character bound as capture_id. Reject duplicate object keys, nonfinite numbers, multiple JSON documents, malformed UTF-8/base64/noncanonical base64, booleans used as integers, overflow, empty or unknown envelope fields, and mismatching pins. Unknown native annotations stay private and inert; no unknown native field is projected. -Native response is an object containing spreadsheetId and one sheet. The sheet properties must identify the declared worksheet/title. Grid properties must declare rowCount positive <=10000000 and columnCount exactly12. Its data list must have exactly one grid block for each declared range in the same order. Missing startRow/startColumn means0; supplied offsets are exact nonnegative integers and must equal the range's0-based position/column0. Every block has exactly its declared number of rowData rows, every row exactly12 values. Each cell is an object; missing formattedValue means empty string, supplied formattedValue must be a string within the bound. Unknown annotations remain inert. Blocks may not overlap and must be inside declared grid bounds. Exactly one row0 contains the exact12-column header; all other rows have a nonempty valid artifact ID, with no duplicate ID even on unselected rows. Nonselected row values and private columns never enter output. Selected IDs must each exist exactly once; unknown/duplicate/invalid policy IDs refuse. +Native response is an object containing spreadsheetId and one sheet. The sheet properties must identify the declared worksheet/title. Grid properties must declare rowCount positive <=10000000 and columnCount exactly12. Its data list must have exactly one grid block for each declared range in the same order. Missing startRow/startColumn means0; supplied offsets are exact nonnegative integers and must equal the range's0-based position/column0. Every block has exactly its declared number of rowData rows. The header row has exactly12 values; every other row has10 to12 values, because a formattedValue read omits trailing empty cells, and only the trailing private Source/Notes cells may be absent. An absent trailing cell reads as an empty string and is never projected. Rows with fewer than10 or more than12 values refuse. Each cell is an object; missing formattedValue means empty string, supplied formattedValue must be a string within the bound. Unknown annotations remain inert. Blocks may not overlap and must be inside declared grid bounds. Exactly one row0 contains the exact12-column header; all other rows have a nonempty valid artifact ID, with no duplicate ID even on unselected rows. Nonselected row values and private columns never enter output. Selected IDs must each exist exactly once; unknown/duplicate/invalid policy IDs refuse. Policy columns must equal the fixed eight public columns above, in that exact order. Policy schema_version exact1. artifact_ids is a nonempty unique list of valid IDs; its order is preserved for identity hashing but output rows sort by artifact ID. graph_nodes are valid unique IDs. Bindings require known graph ID and selected artifact ID. Roles are a nonempty unique list drawn from source, review, kernel, computation, alignment, teaching-source; no inferred acceptance or title join. Duplicate node/artifact pairs refuse, even with different roles. Explicit valid bindings sort by node_id, artifact_id; their roles sort lexically. Unknown mappings refuse; absent mappings remain unmapped. diff --git a/tests/fixtures/artifact_index_p2_trailing_suffix.json b/tests/fixtures/artifact_index_p2_trailing_suffix.json new file mode 100644 index 000000000..06a3d93d3 --- /dev/null +++ b/tests/fixtures/artifact_index_p2_trailing_suffix.json @@ -0,0 +1 @@ +{"bindings":[{"artifact_id":"TEST-P2-ARTIFACT","node_id":"TEST-P2-NODE","roles":["source"]}],"capture":{"capture_id":"TEST-P2-TRAILING-SUFFIX","captured_at":"2026-10-21T00:00:00Z","raw_base64":"eyJzaGVldHMiOlt7ImRhdGEiOlt7InJvd0RhdGEiOlt7InZhbHVlcyI6W3siZm9ybWF0dGVkVmFsdWUiOiJBcnRpZmFjdCBJRCJ9LHsiZm9ybWF0dGVkVmFsdWUiOiJUaXRsZSJ9LHsiZm9ybWF0dGVkVmFsdWUiOiJPcmcifSx7ImZvcm1hdHRlZFZhbHVlIjoiQ2xhc3MifSx7ImZvcm1hdHRlZFZhbHVlIjoiUHJpb3JpdHkifSx7ImZvcm1hdHRlZFZhbHVlIjoiVG9waWNzIC8gb2JqZWN0IHRhZ3MifSx7ImZvcm1hdHRlZFZhbHVlIjoiU3RhdHVzIn0seyJmb3JtYXR0ZWRWYWx1ZSI6IkF1dGhvcml0eSAvIGNhbm9uaWNhbCBpbXBhY3QifSx7ImZvcm1hdHRlZFZhbHVlIjoiRGVwZW5kZW5jaWVzIG9yIHN1cGVyc2Vzc2lvbiJ9LHsiZm9ybWF0dGVkVmFsdWUiOiJNb2RpZmllZCBVVEMifSx7ImZvcm1hdHRlZFZhbHVlIjoiU291cmNlIn0seyJmb3JtYXR0ZWRWYWx1ZSI6Ik5vdGVzIn1dfSx7InZhbHVlcyI6W3siZm9ybWF0dGVkVmFsdWUiOiJURVNULVAyLUFSVElGQUNUIn0seyJmb3JtYXR0ZWRWYWx1ZSI6IlRFU1Qgc291cmNlIHJlZmVyZW5jZSJ9LHsiZm9ybWF0dGVkVmFsdWUiOiJURVNUIHByb3ZpZGVyIn0seyJmb3JtYXR0ZWRWYWx1ZSI6IlNPVVJDRV9SRUZFUkVOQ0UifSx7ImZvcm1hdHRlZFZhbHVlIjoiVEVTVF9QUklWQVRFX1BSSU9SSVRZX0NBTkFSWSJ9LHsiZm9ybWF0dGVkVmFsdWUiOiJURVNUX1BSSVZBVEVfVE9QSUNTX0NBTkFSWSJ9LHsiZm9ybWF0dGVkVmFsdWUiOiJSRUdJU1RFUkVEX1NPVVJDRV9PTkxZIC8gUFVCTElDX1BST0pFQ1RJT05fTk9UX0FETUlUVEVEIn0seyJmb3JtYXR0ZWRWYWx1ZSI6IlRFU1Qgb25seTsgc2NpZW50aWZpYyBlZmZlY3QgTk9ORSJ9LHsiZm9ybWF0dGVkVmFsdWUiOiJURVNUIHNvdXJjZSBwaW47IG5vIHByb29mIGFjY2VwdGFuY2UifSx7ImZvcm1hdHRlZFZhbHVlIjoiMjAyNi0xMC0yMVQwMDowMDowMFoifV19XX1dLCJwcm9wZXJ0aWVzIjp7ImdyaWRQcm9wZXJ0aWVzIjp7ImNvbHVtbkNvdW50IjoxMiwicm93Q291bnQiOjJ9LCJzaGVldElkIjoxNywidGl0bGUiOiJBcnRpZmFjdCBJbmRleCJ9fV0sInNwcmVhZHNoZWV0SWQiOiJURVNULVNZTlRIRVRJQy1TSEVFVCJ9","source":{"cell_fields":"formattedValue","ranges":["'Artifact Index'!A1:L2"],"spreadsheet_id":"TEST-SYNTHETIC-SHEET","title":"Artifact Index","worksheet_id":17}},"graph_nodes":["TEST-P2-NODE","TEST-UNMAPPED-NODE"],"policy":{"artifact_ids":["TEST-P2-ARTIFACT"],"columns":["Artifact ID","Title","Org","Class","Status","Authority / canonical impact","Dependencies or supersession","Modified UTC"],"schema_version":1},"schema_version":1} diff --git a/tests/test_architecture_artifact_index.py b/tests/test_architecture_artifact_index.py index 46fc4f736..a1ecc0c92 100644 --- a/tests/test_architecture_artifact_index.py +++ b/tests/test_architecture_artifact_index.py @@ -635,7 +635,8 @@ def test_native_sheet_block_row_and_cell_shapes_and_cardinalities_are_enforced(s elif fault == "extra_rows": blocks[1]["rowData"].append(copy.deepcopy(blocks[1]["rowData"][0])) elif fault == "short_cells": - blocks[2]["rowData"][0]["values"].pop() + # Nine cells: public Modified UTC (J) is missing as well as K/L. + del blocks[2]["rowData"][0]["values"][9:] elif fault == "extra_cells": blocks[2]["rowData"][0]["values"].append({}) elif fault == "cell_not_object": @@ -652,6 +653,98 @@ def test_native_sheet_block_row_and_cell_shapes_and_cardinalities_are_enforced(s result = self.refuse(packet) self.assertNotIn(b"PRIVATE_BAD_CELL", result.stderr) + def test_nonheader_rows_may_omit_only_trailing_private_source_and_notes_cells(self): + # formattedValue reads elide trailing empty cells, so blank private K/L + # may be absent. Absent cells read as empty strings and never project. + for block_index, row_index in ((1, 0), (1, 1), (2, 0)): + for length in (10, 11, 12): + with self.subTest(block=block_index, row=row_index, cells=length): + packet = copy.deepcopy(self.packet) + native = get_native(packet) + del native["sheets"][0]["data"][block_index]["rowData"][row_index]["values"][length:] + replace_native(packet, native) + self.accept(packet) + packet = copy.deepcopy(self.packet) + native = get_native(packet) + for block in native["sheets"][0]["data"][1:]: + for native_row in block["rowData"]: + del native_row["values"][10:] + replace_native(packet, native) + self.accept(packet) + + def test_nonheader_rows_outside_ten_to_twelve_cells_and_short_headers_refuse(self): + for block_index, row_index in ((1, 0), (1, 1), (2, 0)): + for length in (0, 1, 8, 9): + with self.subTest(block=block_index, row=row_index, cells=length): + packet = copy.deepcopy(self.packet) + native = get_native(packet) + del native["sheets"][0]["data"][block_index]["rowData"][row_index]["values"][length:] + replace_native(packet, native) + self.refuse(packet) + for extra in (1, 2): + with self.subTest(block=block_index, row=row_index, cells=12 + extra): + packet = copy.deepcopy(self.packet) + native = get_native(packet) + native["sheets"][0]["data"][block_index]["rowData"][row_index]["values"].extend([{}] * extra) + replace_native(packet, native) + self.refuse(packet) + for length in (9, 10, 11, 13): + with self.subTest(header_cells=length): + packet = copy.deepcopy(self.packet) + native = get_native(packet) + header = native["sheets"][0]["data"][0]["rowData"][0]["values"] + if length > 12: + header.append({"formattedValue": "Extra"}) + else: + del header[length:] + replace_native(packet, native) + self.refuse(packet) + # A short row still refuses bad private cells that are present. + for value in ("PRIVATE_BAD_CELL", None, ["PRIVATE_ARRAY_CELL"]): + with self.subTest(short_row_private_cell=value): + packet = copy.deepcopy(self.packet) + native = get_native(packet) + cells = native["sheets"][0]["data"][2]["rowData"][0]["values"] + del cells[11:] + cells[10] = value if type(value) is str else {"formattedValue": value} + replace_native(packet, native) + result = self.refuse(packet) + self.assertNotIn(b"PRIVATE_BAD_CELL", result.stderr) + self.assertNotIn(b"PRIVATE_ARRAY_CELL", result.stderr) + + def test_p2_synthetic_trailing_suffix_fixture_projects_with_its_independent_pins(self): + # Synthetic TEST fixture and pins published in main#327 comment 6325876189 + # (2082 bytes, SHA256 0a49c41e...a18bdc): header 12 cells, data row 10. + fixture = (ROOT / "tests" / "fixtures" / "artifact_index_p2_trailing_suffix.json").read_bytes() + self.assertEqual(digest(fixture), "0a49c41eda400a2e71a20ad73a33877432e5adae784399d90a998de8d9a18bdc") + packet = json.loads(fixture) + expected = { + "bindings_sha256": "2b296a32b0037e49a722ca88cce47e678d2472c3e2cf5cd805900fd4aba0ba65", + "capture_sha256": "77929c4fcb3d815fb8a9f262a1b6e71f22e85fb0d12da5b743717f220820117e", + "graph_nodes_sha256": "221139c5f0571ac60640e8e2f591de16263cd0ab02805910070c728afcca1d31", + "metadata_sha256": "3aab18af5822ae5ca6fa1f98a22b72d2d69b742b5365ced6e0b2ba3cbac30539", + "policy_sha256": "9be75008ad5ad069787fa4e61723541de612660eb5674998ee45c2e059e198ee", + } + self.assertEqual(pins_for(packet), expected) + rows = get_native(packet)["sheets"][0]["data"][0]["rowData"] + self.assertEqual([len(native_row["values"]) for native_row in rows], [12, 10]) + result = self.invoke(packet, expected=expected) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, b"") + projected = json.loads(result.stdout) + self.assertEqual(projected, self.expected_projection(packet, expected)) + self.assertEqual(result.stdout, canonical(projected) + b"\n") + self.assertEqual([value["artifact_id"] for value in projected["rows"]], ["TEST-P2-ARTIFACT"]) + self.assertEqual(projected["rows"][0]["source_row"], 2) + self.assertEqual(projected["custody"], "unknown") + self.assertIs(projected["scientific_status_authority"], False) + self.assertEqual(projected["bindings"], [{"artifact_id": "TEST-P2-ARTIFACT", + "node_id": "TEST-P2-NODE", "roles": ["source"]}]) + self.assertEqual(projected["unmapped_nodes"], ["TEST-UNMAPPED-NODE"]) + for canary in (b"TEST_PRIVATE_PRIORITY_CANARY", b"TEST_PRIVATE_TOPICS_CANARY"): + self.assertNotIn(canary, result.stdout + result.stderr) + self.assertNotIn(packet["capture"]["raw_base64"].encode("ascii"), result.stdout + result.stderr) + def test_all_formatted_cells_are_strings_including_private_and_unselected_cells(self): for block_index, row_index, column in ((1, 0, 1), (1, 1, 10), (1, 1, 11), (2, 0, 3)): for value in (None, True, 3, 3.0, ["PRIVATE_ARRAY_CELL"], {"PRIVATE_OBJECT_CELL": "value"}):