diff --git a/research/finance_program_v1/README.md b/research/finance_program_v1/README.md index 5d95ef3..f705054 100644 --- a/research/finance_program_v1/README.md +++ b/research/finance_program_v1/README.md @@ -58,3 +58,12 @@ 这是开发验证集、单 seed，仅 8/12 类有支持，且来源可能共享表；不是独立 Test 或跨领域泛化结论。教育与钢铁数据集不在本轮结果内。该更新只替换结果展示和学生公开轨迹，不修改实验代码、权重或训练设置。 真实数据集、gold、权重、凭证和实验服务器地址均不上传。clone/import 不会启动任何训练、GPU 或 API 作业。复现说明见 [REPRODUCE.md](REPRODUCE.md)。 + +## 发布依赖修复（2026-10-19） + +补回历史批次审计模块与源代码/官方模型哈希清单、教师录制调用回放助手，以及评估旧后端的SHA清单工具。未改变模型、实验协议、训练参数、结果数字或公开轨迹。 + +`pool_metadata.py` 位于 `nontraining/vendor/` 和 `evaluation/runtime/vendor/`，不是仓库根目录模块。通过 `program_loop` 导入时会初始化对应的vendor路径；若只使用该工具，请按 [REPRODUCE.md](REPRODUCE.md) 设置明确的导入路径。新增隔离进程回归覆盖批次入口、完整合成回放和请求篡改拒绝。 + +**代码依赖补齐不等于发布了私有数据/历史执行目录。** 历史 `teacher/collect.py`、`teacher/detached.py`、评估watcher/legacy环境身份仍要求经授权的原始配置、manifest、ledger或数据；不得直接运行它们来启动付费请求/GPU。公开CPU测试不需要这些私有资产。 + diff --git a/research/finance_program_v1/REPRODUCE.md b/research/finance_program_v1/REPRODUCE.md index 469d19f..22658ef 100644 --- a/research/finance_program_v1/REPRODUCE.md +++ b/research/finance_program_v1/REPRODUCE.md @@ -44,3 +44,32 @@ The prior partial-validation result tables and standalone partial trajectory fil ## Interpretation Val422 is a development set with eight supported classes out of twelve, one seed and potentially correlated table sources. It is not held-out Test or independent generalization. Supported-class and fixed-twelve-class Macro-F1 are reported separately; legal termination is not business correctness. Historical non-training interfaces are not interchangeable with this matched F66 comparison. + +## Publication dependency repair (2026-10-19) + +The public nontraining audit now includes `reviews/independent_audit_01.py`, its exact historical source SHA inventory, the public official-model file-hash inventory, and the two omitted source files needed by that inventory. This preserves the historical 121-file/109-core-file signature; it does not fabricate a historical run or ship model weights. `batch_audit.py` imports and a real-controller recorded-I/O replay with synthetic source data are covered by fresh isolated-process tests, including tampered-request rejection. + +Additional restored code: `teacher/pilot_audit.py` contains only the recorded retry/I-O replay helpers used by `detached.py`, not the private-gold historical quality aggregator. The public teacher runtime resolves to the already included `evaluation/runtime` when no controlled local `teacher/runtime` is present. `evaluation/legacy/tools/freeze_revision.py` restores the original pure SHA inventory helper, not the private revision manifests or a new deployment authorization. Legacy environment-identity functions still require their controlled snapshot layout; this code snapshot is not a turnkey historical deployment. + +### Import paths and CPU checks + +From `research/finance_program_v1/nontraining`: + +```powershell +python -B -c "import batch_audit; import program_loop; import pool_metadata; print(pool_metadata.__file__)" +``` + +If using only the standalone metadata helper in your own Python program, add the explicit snapshot directory before importing it: + +```python +import sys +from pathlib import Path +snapshot = Path("research/finance_program_v1/nontraining").resolve() +sys.path.insert(0, str(snapshot / "vendor")) +from pool_metadata import PoolMetadata +``` + +Do not copy files between different historical snapshots or rely on another checkout's PYTHONPATH. Run the teacher synthetic suite from `teacher/` with `python -B -m pytest tests -q -p no:cacheprovider`. Run the publication checks from this directory with `python -B -m unittest discover -s publication_tests -v`; dependency checks use `-I` fresh processes so private/local import paths cannot mask missing modules. + +`python -B -m reviews.independent_audit_01 --help` and `python -B batch_audit.py --help` are safe CPU-only entrypoint checks. Actual audit commands need complete, authorized, unredacted recorded runs and the matching study/model manifests; redacted public trajectories cannot substitute for them. Cloning/importing/testing does not authorize or start new model downloads, API calls, training, or GPU work. + diff --git a/research/finance_program_v1/evaluation/legacy/tools/__init__.py b/research/finance_program_v1/evaluation/legacy/tools/__init__.py new file mode 100644 index 0000000..7a23289 --- /dev/null +++ b/research/finance_program_v1/evaluation/legacy/tools/__init__.py @@ -0,0 +1 @@ +"""Historical SHA-inventory helpers; private execution assets are not included.""" diff --git a/research/finance_program_v1/evaluation/legacy/tools/freeze_revision.py b/research/finance_program_v1/evaluation/legacy/tools/freeze_revision.py new file mode 100644 index 0000000..e1960b8 --- /dev/null +++ b/research/finance_program_v1/evaluation/legacy/tools/freeze_revision.py @@ -0,0 +1,177 @@ +"""Pure SHA inventories. No model acceptance, transport, launch, or approval. + +Create: python -B -m tools.freeze_revision --root . --output reviews/revision-identity +Verify: python -B -m tools.freeze_revision --root . --output reviews/revision-identity --verify +The output directory is created exclusively; a failed/old freeze is never reused. +""" +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath +import re + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_IDENTITY = ROOT / 'reviews/revision-identity' +SOURCE_DIRS = ('program_loop', 'vendor', 'tools', 'tests', 'tests_batch') +CORE_DIRS = ('program_loop', 'vendor', 'tools') +SOURCE_FILES = ('reviews/independent_audit_01.py', 'reviews/remote_gate.py') +EXECUTION_FILES = ('data/study.json', 'data/study-manifest.json', 'reviews/trusted-model-manifest-01.json') +BASELINE_FILES = {'source_manifest_sha256': 'reviews/baseline-source-manifest-01.json', + 'execution_manifest_sha256': 'reviews/baseline-execution-manifest-01.json'} + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def signature(entries): + return hashlib.sha256(''.join(f"{e['path']}\t{e['bytes']}\t{e['sha256']}\n" + for e in sorted(entries, key=lambda e: e['path'])).encode()).hexdigest() + + +def safe_path(root, name): + if (not isinstance(name, str) or not name or '\\' in name or ':' in name or + PurePosixPath(name).is_absolute() or PurePosixPath(name).as_posix() != name or + any(p in ('', '.', '..') or p.endswith((' ', '.')) for p in name.split('/')) or + any(ord(c) < 32 for c in name)): + raise ValueError(f'unsafe inventory path: {name!r}') + path = root + for part in name.split('/'): + path = path / part + if path.is_symlink() or (hasattr(path, 'is_junction') and path.is_junction()): + raise ValueError(f'symlink inventory path: {name}') + # Canonical relative components and the ancestor link checks above prevent + # escaping root without re-resolving the full Windows path for every file. + return path + + +def inventory(root, names): + entries = [] + seen = set() + for name in sorted(names): + path = safe_path(root, name) + if name.casefold() in seen: + raise ValueError('duplicate inventory path') + seen.add(name.casefold()) + if not path.is_file(): + raise ValueError(f'missing inventory file: {name}') + entries.append({'path': name, 'bytes': path.stat().st_size, 'sha256': sha(path)}) + return entries + + +def source_names(root): + names = [p.name for p in root.glob('*.py')] + def walk(directory): + for path in directory.iterdir(): + relative = path.relative_to(root).as_posix() + safe_path(root, relative) + if path.name == '__pycache__': + continue + if path.is_dir(): + walk(path) + elif path.is_file(): + names.append(relative) + else: + raise ValueError(f'nonregular inventory entry: {relative}') + for name in SOURCE_DIRS: + directory = safe_path(root, name) + if not directory.is_dir(): + raise ValueError(f'missing source directory: {name}') + walk(directory) + return names + list(SOURCE_FILES) + + +def manifests(root): + root = Path(root) + if root.is_symlink() or (hasattr(root, 'is_junction') and root.is_junction()): + raise ValueError('symlink root') + provenance = {key: sha(safe_path(root, name)) for key, name in BASELINE_FILES.items()} + source = inventory(root, source_names(root)) + execution = sorted(source + inventory(root, EXECUTION_FILES), key=lambda e: e['path']) + shared = {'schema': 1, 'baseline_provenance': provenance} + return ({**shared, 'kind': 'source', 'files': source, 'signature_sha256': signature(source)}, + {**shared, 'kind': 'execution', 'files': execution, 'signature_sha256': signature(execution), + 'source_signature_sha256': signature(source), 'study_sha256': sha(safe_path(root, 'data/study.json'))}) + + +def environment_identity(root=ROOT): + source, execution = manifests(root) + return identity_from_manifests(source, execution) + + +def identity_from_manifests(source, execution): + return {'revision_identity': {'schema': 1, 'source_signature_sha256': source['signature_sha256'], + 'execution_signature_sha256': execution['signature_sha256'], + 'study_sha256': execution['study_sha256'], 'baseline_provenance': source['baseline_provenance']}, + 'core_source_sha256': {e['path']: e['sha256'] for e in source['files'] if e['path'].split('/')[0] in CORE_DIRS}} + + +def check_entries(root, manifest): + entries = manifest.get('files') + if not isinstance(entries, list) or not entries: + raise ValueError('empty or invalid manifest inventory') + for entry in entries: + if (not isinstance(entry, dict) or set(entry) != {'path', 'bytes', 'sha256'} or + type(entry['bytes']) is not int or entry['bytes'] < 0 or + not isinstance(entry['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', entry['sha256'])): + raise ValueError('invalid inventory entry') + actual = inventory(root, [e['path'] for e in entries]) + if actual != entries or signature(entries) != manifest.get('signature_sha256'): + raise ValueError('inventory bytes/hash/order/signature mismatch') + + +def no_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate manifest key') + result[key] = value + return result + + +def verify_revision(root=ROOT, output=None): + root = Path(root); output = Path(output) if output is not None else DEFAULT_IDENTITY + expected = manifests(root) + for name, wanted in zip(('source.json', 'execution.json'), expected): + # Also reject linked manifest directories, not just linked final files. + for parent in (output, *output.parents): + if parent.is_symlink() or (hasattr(parent, 'is_junction') and parent.is_junction()): + raise ValueError('symlink manifest directory') + path = safe_path(output, name) + actual = json.loads(path.read_text(encoding='utf-8'), object_pairs_hook=no_duplicate_keys) + check_entries(root, actual) + if actual != wanted: + raise ValueError(f'exact {name} inventory or provenance mismatch') + return identity_from_manifests(*expected) + + +def freeze(root=ROOT, output=DEFAULT_IDENTITY): + root = Path(root); output = Path(output) + source, execution = manifests(root) + for parent in (output, *output.parents): + if parent.is_symlink() or (hasattr(parent, 'is_junction') and parent.is_junction()): + raise ValueError('symlink output path') + relative = output.resolve().relative_to(root.resolve()) if output.resolve().is_relative_to(root.resolve()) else None + if relative and (relative.parts[0] in SOURCE_DIRS or relative.as_posix() in SOURCE_FILES + EXECUTION_FILES): + raise ValueError('output would enter its own inventory') + output.mkdir() # Parent must exist; never merge or overwrite another revision. + for name, value in (('source.json', source), ('execution.json', execution)): + with (output / name).open('x', encoding='utf-8') as stream: + json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False) + stream.write('\n') + return verify_revision(root, output) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--root', type=Path, default=ROOT) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--verify', action='store_true') + args = parser.parse_args(argv) + result = verify_revision(args.root, args.output) if args.verify else freeze(args.root, args.output) + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/research/finance_program_v1/nontraining/README.md b/research/finance_program_v1/nontraining/README.md new file mode 100644 index 0000000..d368c91 --- /dev/null +++ b/research/finance_program_v1/nontraining/README.md @@ -0,0 +1,24 @@ +# Program 动态引用与事实性恢复（2026-10-11） + +这是用户批准的独立新版本，基于上一份115项冻结源码。历史实验及失败结果只读，不替换历史身份。本轮只加强显式动态参数引用约束，以及Program结束后的事实性恢复上下文。 + +Planner 优先使用 `arguments:{}`，由原binder补齐required参数；optional省略仍不注入。显式动态参数（canonical reference以`$state.`开头）必须是确切规范引用，不得手写对象、列表、null、数字或普通字符串。初始domain/field_name等保持原契约。无效计划在执行前拒绝，只计Planner尝试预算，保留原文和具体错误。 + +模型仍从全部 14 个可选 Skill 中自行选择步骤和顺序，每次 Planner 调用生成一个完整有序 Program。没有必选 Skill、固定完整流程示例、自动插入步骤或改写模型答案。可能生产者只表示静态可能性；元数据取得、输出发布、状态绑定以及最终终止，都必须由真实执行和冻结 guard 验证。`semantic_query`、`semantic_evidence` 与 `field_semantics` 保持各自契约含义，描述语义不能替代字段名语义。 + +核心入口文件： + +- `program_loop/planning_contracts.py`：根据 catalog 和共享校验规则生成无状态值的规划说明。 +- `program_loop/catalog_context.py`：把说明附加到每个 Skill 的原始契约库中，供 Planner 读取。 +- `program_loop/planner.py`：说明参数省略、合法引用、依赖顺序和条件性执行的使用规则。 +- `program_loop/validation.py`：静态依赖与显式动态引用约束，共享取值策略。 +- `program_loop/recovery_context.py`：纯事实性恢复摘要，不执行Skill、不改state。 +- `program_loop/controller.py`：Program级执行/反馈与全局预算循环；不在Skill间规划。 + +恢复消息从真实state与observation陈述实际成功/失败步骤、失败位置、尚未执行的尾部、证据失效与真实剩余预算。尾部是历史意图，不是强制路线；字段存在不等于已验证ready。最后一份Program也仍允许部分计划，代码不自动完成分类或赠送预算。合法终止仍由原terminal guard确认。 + +新增CPU测试只脚本化模型I/O，parser、binder、executor和guard用真实实现。`vendor/`、底层execution、Registry、P14池及`local_smoke.py`/`download_model.py`字节保持不变。完整变更与未改SHA由新source manifest记录，不以人工描述替代哈希证据。 + +用户已授权测试/审查门通过后在新远程目录单次运行同三条无标签输入、同9B基础模型、同生成参数和预算。无新adapter/训练、Val调策略、Test、依赖升级或Git提交推送。工程通过不等于模型成功；真实试跑结果及原始调用见交付后的`实验结果.md`与`SMOKE_README.md`，保留全部失败，不调prompt重复挑结果。 + +RED/GREEN、独立Spec/Quality审查、全CPU验收、完整diff、冻结新签名和证据ZIP在`reviews/`。两项修改同时施加，与历史三条只作描述性比较，不能分别归因或推广为准确率提升。DCG全文输入模式、池规模和训练方式不属本轮改动，不声称已实现导师全部行动指南。 diff --git a/research/finance_program_v1/nontraining/reviews/__init__.py b/research/finance_program_v1/nontraining/reviews/__init__.py new file mode 100644 index 0000000..e45a911 --- /dev/null +++ b/research/finance_program_v1/nontraining/reviews/__init__.py @@ -0,0 +1 @@ +"""Public, CPU-only historical audit helpers; no generated private receipts.""" diff --git a/research/finance_program_v1/nontraining/reviews/independent_audit_01.py b/research/finance_program_v1/nontraining/reviews/independent_audit_01.py new file mode 100644 index 0000000..9afdf2f --- /dev/null +++ b/research/finance_program_v1/nontraining/reviews/independent_audit_01.py @@ -0,0 +1,284 @@ +"""Independent CPU-only audit: recorded model I/O + exact frozen trace replay. + +Run only after the coordinator has closed all writers and verified/extracted the +new immutable snapshot. No torch/transformers import, GPU, network or model load. +""" +import argparse +from collections import Counter +import hashlib +import json +import math +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SIGNATURE = '94ae314646625e078d7b0f8ee6315bf37778e1ce81bd8435a1417f43c769da1d' +REVISION = 'c202236235762e1c871ad0ccb60c8ee5ba337b9a' +BUDGET = dict(max_programs=3, max_program_steps=10, max_skill_steps=12, max_planner_attempts=4, max_identical_calls=1) +RUN_MODE = 'caller supplied backend I/O + real frozen executor; backend evidence in calls/' + +class AuditError(AssertionError): + pass + +def require(value, message): + if not value: + raise AuditError(message) + +def read(path): + return json.loads(Path(path).read_text(encoding='utf-8')) + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + +def exact(actual, expected, label): + require(actual == expected, f'{label} mismatch') + +class RecordedBackend: + """Replays every external generation call, including retries and faults.""" + def __init__(self, records): + self.records = records + self.position = 0 + self.roles = [] + self.audit_faults = [] + + def __call__(self, messages): + try: + require(self.position < len(self.records), 'replay requested missing call') + record = self.records[self.position] + self.position += 1 + exact(record['request']['messages'], messages, f'call {self.position} request messages') + exact(record['result']['messages'], messages, f'call {self.position} result messages') + self.roles.append('planner' if 'Global Program Planner' in messages[0]['content'] else 'skill') + result = record['result'] + if result.get('error'): + kind, separator, message = result['error'].partition(': ') + require(separator and kind.isidentifier(), 'invalid recorded backend error') + # Exact original class name and str(exc), without backend execution. + raise type(kind, (Exception,), {})(message) + return result['raw_text'] + except AuditError as exc: + # Core deliberately catches generation errors. Retain audit failures + # out of band so they cannot turn into apparently legitimate traces. + self.audit_faults.append(str(exc)) + raise + + def finish(self): + require(not self.audit_faults, '; '.join(self.audit_faults)) + require(self.position == len(self.records), 'unconsumed recorded model calls') + +def replay_trace(saved, backend, registry, rows, budget, row): + from program_loop import build_program_controller, ProgramConfig + require(saved['execution_mode'] in (RUN_MODE, 'caller supplied model I/O + real executor'), 'trace execution mode invalid') + controller = build_program_controller(backend, registry, rows, ProgramConfig(**budget)) + trace = controller.run({'domain':row['domain'], 'field_name':row['metadata']['field_name'], 'source_id':row['id']}) + if saved['execution_mode'] == RUN_MODE: + trace.execution_mode = RUN_MODE + require(not backend.audit_faults, '; '.join(backend.audit_faults)) + actual = trace.to_dict() + # Includes parser raw, every message, full observations/feedback, all binding + # metadata/check-dependencies, invalidations, availability, lineage, budgets, + # state, stop reason and terminal result. No outcome-specific shortcut. + exact(actual, saved, 'complete trace replay') + first = json.loads(saved['planner_attempts'][0]['messages'][1]['content']) + require(row['metadata']['field_description'] not in json.dumps(first,ensure_ascii=False), 'initial metadata description exposed') + require(len(first['skill_library']) == 14, 'P14 library incomplete') + for attempt in saved['planner_attempts']: + combined = json.dumps(attempt['messages'],ensure_ascii=False) + require(all(r['id'] not in combined for r in rows), 'source ID exposed in planner messages') + for program in saved['programs']: + for step in program['steps']: + if step['call']['tool'] == 'GET_FIELD_METADATA' and step['status'] == 'success': + exact(step['data']['field_metadata'],row['metadata'],'actual metadata acquisition') + return actual + +def raw_steps(raw): + try: + parsed = json.loads(raw) + steps = parsed.get('program') if isinstance(parsed,dict) else None + return steps if isinstance(steps,list) else [] + except (TypeError,ValueError): + return [] + +def trace_summary(traces): + attempts, errors, execution = Counter(), Counter(), Counter() + details = [] + for trace in traces: + row_attempts = [] + for attempt in trace['planner_attempts']: + steps = raw_steps(attempt['raw_response']) + attempts[attempt['status']] += 1 + if 'error' in attempt: + errors[(attempt['status'],attempt['error'])] += 1 + row_attempts.append({'attempt':attempt['attempt'],'status':attempt['status'],'raw_steps':len(steps),'tools':[s.get('tool') if isinstance(s,dict) else None for s in steps],'error':attempt.get('error')}) + steps = [s for p in trace['programs'] for s in p['steps']] + for step in steps: + execution[(step['call']['tool'],step['status'])] += 1 + details.append({'input':trace['input'],'attempts':row_attempts,'budget':trace['budget'], + 'raw_multistep_programs':sum(len(raw_steps(a['raw_response']))>1 for a in trace['planner_attempts']), + 'accepted_multistep_programs':sum(len(p['plan']['program'])>1 for p in trace['programs']), + 'executed_multistep_programs':sum(len(p['steps'])>1 for p in trace['programs']), + 'program_status':dict(Counter(p['status'] for p in trace['programs'])), + 'actual_skills':[{'tool':s['call']['tool'],'status':s['status'],'summary':s.get('summary')} for s in steps], + 'terminated':trace['terminated'],'stop_reason':trace['stop_reason'],'final_result':trace['final_result']}) + return {'metric_scope':'unlabelled engineering smoke; no accuracy metric','samples':len(traces), + 'planner_attempts':sum(attempts.values()),'accepted_plans':sum(len(t['programs']) for t in traces), + 'attempt_status':dict(attempts),'planner_errors':[{'status':status,'reason':error,'count':count} for (status,error),count in errors.items()], + 'execution':[{'tool':tool,'status':status,'count':count} for (tool,status),count in execution.items()], + 'skill_steps':sum(execution.values()),'legal_terminations':sum(t['terminated'] for t in traces), + **{key:sum(d[key] for d in details) for key in ('raw_multistep_programs','accepted_multistep_programs','executed_multistep_programs')},'details':details} + +def call_summary(records, roles): + require(len(records)==len(roles),'model call role denominator mismatch') + peaks = [r['memory'].get('max_memory_allocated') for r in records] + return {'model_calls':len(records),'planner_calls':roles.count('planner'),'skill_model_calls':roles.count('skill'), + 'input_tokens':sum(r.get('input_tokens') or 0 for r in records),'output_tokens':sum(r.get('output_tokens') or 0 for r in records), + 'generation_seconds':sum(r.get('seconds') or 0 for r in records), + 'peak_allocated_bytes':max((p for p in peaks if p is not None),default=None), + 'backend_errors':sum(bool(r.get('error')) for r in records),'telemetry_errors':sum(bool(r.get('telemetry_errors')) for r in records), + 'output_budget_truncations':sum(bool(r['truncation_at_budget']) for r in records), + 'unknown_input_token_calls':sum(r.get('input_tokens') is None for r in records), + 'unknown_output_token_calls':sum(r.get('output_tokens') is None for r in records), + 'unknown_peak_vram_calls':sum(p is None for p in peaks)} + +def validate_call(request,result,backend): + exact(request['generation'],backend['generation'],'request generation') + exact(result['generation'],backend['generation'],'result generation') + exact(result['parameter_devices'],['cuda:0'],'parameter devices') + exact(result['parameter_dtypes'],['torch.float16'],'parameter dtypes') + exact(result['attention_implementation'],backend['attention_implementation'],'attention implementation') + require(type(result['seconds']) in (int,float) and math.isfinite(result['seconds']) and result['seconds']>=0,'generation duration invalid') + error = result['error'] + for key in ('input_tokens','output_tokens'): + value = result[key] + require(value is None or type(value) is int and value>=0, f'{key} invalid') + if not error: + require(isinstance(result['raw_text'],str),'successful generation raw text missing') + exact(result['input_devices'],['cuda:0'],'input devices') + require(result['input_tokens'] is not None and result['input_tokens']+512<=32768,'context cap violated') + require(result['output_tokens'] is not None and result['output_tokens']<=512,'output cap violated') + exact(result['truncation_at_budget'],result['output_tokens']>=512,'output budget flag') + else: + require(result['raw_text'] is None,'backend fault with raw text') + if result['output_tokens'] is None: + require(not result['truncation_at_budget'],'backend fault output evidence invalid') + else: + # Backend records tokens/flag before decode, which can itself fail. + exact(result['input_devices'],['cuda:0'],'input devices') + require(result['input_tokens'] is not None and result['input_tokens']+512<=32768,'context cap violated') + require(result['output_tokens']<=512,'output cap violated') + exact(result['truncation_at_budget'],result['output_tokens']>=512,'output budget flag') + if error.startswith('ContextBudgetExceeded:'): + require(result['input_tokens'] is not None and result['input_tokens']+512>32768,'context fault below cap') + exact(result['input_devices'],[],'context failure before tensor transfer') + exact(set(result['memory']),{'memory_allocated','memory_reserved','max_memory_allocated'},'memory telemetry keys') + require(set(result['telemetry_errors'])<=set(result['memory']),'unknown telemetry error') + for name,value in result['memory'].items(): + require((value is None and name in result['telemetry_errors']) or (type(value) is int and value>=0 and name not in result['telemetry_errors']), 'memory metric/error mismatch') + +def validate_sources(snapshot,environment): + manifest = read(ROOT/'reviews/source-manifest-01.json') + entries = manifest['files'] + require(len(entries)==121,'frozen source count mismatch') + signature = hashlib.sha256(''.join(f"{e['path']}\t{e['bytes']}\t{e['sha256']}\n" for e in sorted(entries,key=lambda e:e['path'])).encode()).hexdigest() + exact(signature,SIGNATURE,'frozen source signature') + for entry in entries: + for base in (ROOT,snapshot): + path = base/entry['path'] + require(path.is_file() and path.stat().st_size==entry['bytes'] and sha(path)==entry['sha256'],f'frozen source hash mismatch: {path}') + core = {e['path']:e['sha256'] for e in entries if e['path'].split('/')[0] in ('program_loop','vendor','tools')} + require(len(core)==109,'core manifest count must be exactly 109') + actual = {p.relative_to(snapshot).as_posix() for base in ('program_loop','vendor','tools') for p in (snapshot/base).rglob('*') if p.is_file() and '__pycache__' not in p.parts} + exact(actual,set(core),'core source inventory') + exact(environment['core_source_sha256'],core,'environment all 109 core SHA') + +def audit_snapshot(snapshot,reference_inputs,model_manifest): + snapshot = Path(snapshot).resolve() + run = snapshot/'runs/smoke-01' + inputs = read(run/'inputs.json') + exact(inputs,read(reference_inputs),'same exact historical unlabelled inputs/registry/budget') + exact(inputs['budget'],BUDGET,'fixed smoke budget') + rows,registry = inputs['rows'],inputs['registry'] + require(len(rows)==3 and len(registry)==6,'three rows and six registry entries required') + require(len({r['id'] for r in rows})==3,'source IDs not unique') + require(all(set(r)=={'id','domain','metadata'} and set(r['metadata'])=={'field_name','field_description'} for r in rows),'labels or unexpected initial fields') + env = read(run/'environment.json') + validate_sources(snapshot,env) + expected_env = {'seed':20260928,'context_cap':32768,'max_new_tokens':512,'per_generate_max_time':90.0,'whole_process_timeout_seconds':900, + 'whole_process_timeout_enforced_by':'external coordinator','inference':'FP16 cuda:0 batch1 greedy enable_thinking=False'} + for key,value in expected_env.items(): + exact(env[key],value,f'environment {key}') + require(type(env['pid']) is int and env['pid']>0,'environment PID invalid') + require(all(env['packages'].get(p) for p in ('torch','transformers','accelerate','huggingface_hub','safetensors')),'environment package versions missing') + trusted = read(ROOT/'reviews/trusted-model-manifest-01.json') + exact(read(model_manifest),trusted,'trusted model manifest identity/files/bytes/SHA') + require(trusted['repo_id']=='Qwen/Qwen3.5-9B' and trusted['revision']==REVISION and len(trusted['files'])==11 and trusted['total_bytes']==19329302129,'official pinned identity invalid') + backend = read(run/'calls/backend.json') + for key,value in {'repo_id':'Qwen/Qwen3.5-9B','revision':REVISION,'parameter_devices':['cuda:0'],'parameter_dtypes':['torch.float16'], + 'offline_options':{'local_files_only':True,'trust_remote_code':False},'config_type':'qwen3_5','model_class':'Qwen3_5ForConditionalGeneration'}.items(): + exact(backend[key],value,f'backend {key}') + gen = backend['generation'] + exact(gen,{'do_sample':False,'max_new_tokens':512,'max_time':90.0,'eos_token_id':248044,'pad_token_id':248044},'fixed generation/EOS') + paths = sorted((run/'calls').glob('call-*')) + exact([p.name for p in paths],[f'call-{i:04d}' for i in range(1,len(paths)+1)],'contiguous model call records') + records = [] + for path in paths: + request,result = read(path/'request.json'),read(path/'result.json') + validate_call(request,result,backend) + combined = json.dumps(request['messages'],ensure_ascii=False) + require(all(row['id'] not in combined for row in rows),'source ID exposed in model messages') + records.append({'request':request,'result':result}) + # Import only after validating every source byte; process must not have loaded + # another copy of program_loop before this point. + require('program_loop' not in sys.modules,'audit CLI must run in a fresh process') + sys.path.insert(0,str(snapshot)) + replay = RecordedBackend(records) + traces, row_calls = [], [] + for row in rows: + start = replay.position + traces.append(replay_trace(read(run/'samples'/(row['id']+'.json')),replay,registry,rows,inputs['budget'],row)) + row_calls.append(call_summary([r['result'] for r in records[start:replay.position]],replay.roles[start:replay.position])) + replay.finish() + require('torch' not in sys.modules and 'transformers' not in sys.modules,'CPU audit imported model runtime') + report = {**trace_summary(traces),**call_summary([r['result'] for r in records],replay.roles), + 'source_signature_sha256':SIGNATURE,'core_files_sha_matches':109,'model_revision':REVISION, + 'raw_request_trace_exact_match':True,'full_trace_replay_exact_match':True,'normalized_bindings_dependencies_verified':True, + 'initial_metadata_description_hidden':True,'source_ids_not_in_planner_messages':True,'same_unlabelled_inputs_as_previous_smoke':True, + 'all_source_ids_not_in_model_messages':True, + 'snapshot':str(snapshot),'reference_inputs_sha256':sha(reference_inputs),'trusted_model_manifest_sha256':sha(ROOT/'reviews/trusted-model-manifest-01.json')} + summary = read(run/'summary.json') + for key in ('model_calls','planner_attempts','accepted_plans','skill_steps','input_tokens','output_tokens','generation_seconds'): + exact(summary[key],report[key],f'runner summary {key}') + exact(summary['terminated_count'],report['legal_terminations'],'runner summary legal terminations') + expected_errors = [{'error':r['result']['error']} for r in records if r['result']['error']] + exact(summary['backend_errors'],expected_errors,'all backend failure denominator') + exact(summary['status'],'backend_failure' if expected_errors else 'completed','runner completion status') + expected_samples = [{'id':row['id'],'terminated':t['terminated'],'stop_reason':t['stop_reason'],'budget':t['budget'], + 'planner_errors':sum('error' in a for a in t['planner_attempts']), + 'execution_errors':sum(s['status']!='success' for p in t['programs'] for s in p['steps'])} for row,t in zip(rows,traces)] + exact(summary['samples'],expected_samples,'all runner samples including failures') + for row,detail,calls in zip(rows,report['details'],row_calls): + detail['id'] = row['id'] + detail.update(calls) + detail['execution_errors'] = sum(s['status']!='success' for s in detail['actual_skills']) + report['call_details'] = [{'call':p.name,'role':role,**{key:r['result'][key] for key in ('input_tokens','output_tokens','seconds','memory','telemetry_errors','error','truncation_at_budget')}} for p,r,role in zip(paths,records,replay.roles)] + return report + +def write_report(path,report): + with Path(path).open('x',encoding='utf-8') as stream: + json.dump(report,stream,ensure_ascii=False,indent=2,allow_nan=False) + stream.write('\n') + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--snapshot',type=Path,default=ROOT/'remote-snapshot-01') + parser.add_argument('--reference-inputs',type=Path,default=ROOT/'reviews/reference-inputs.json') + parser.add_argument('--model-manifest',type=Path) + args = parser.parse_args(argv) + target = ROOT/'reviews/independent-smoke-audit-01.json' + require(not target.exists(),'exclusive audit output already exists') + report = audit_snapshot(args.snapshot,args.reference_inputs,args.model_manifest or args.snapshot/'reviews/remote-model-manifest-01.json') + write_report(target,report) + print(json.dumps({k:v for k,v in report.items() if k not in ('details','call_details')},ensure_ascii=True,indent=2)) + +if __name__ == '__main__': + main() diff --git a/research/finance_program_v1/nontraining/reviews/source-manifest-01.json b/research/finance_program_v1/nontraining/reviews/source-manifest-01.json new file mode 100644 index 0000000..83fac3d --- /dev/null +++ b/research/finance_program_v1/nontraining/reviews/source-manifest-01.json @@ -0,0 +1,618 @@ +{ + "source_signature_sha256": "94ae314646625e078d7b0f8ee6315bf37778e1ce81bd8435a1417f43c769da1d", + "file_count": 121, + "core_file_count": 109, + "baseline_signature": "8c9a2d2b51a821942d589150d46814571c4376221945a3d607bc19d9bc9290f8", + "changed_baseline_files": [ + "tests/test_program_loop.py", + "vendor/pool_runtime.py", + "vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md" + ], + "files": [ + { + "path": "README.md", + "bytes": 3262, + "sha256": "4a95691f70cfddafe3cb50e4876a0a68cac77883489c13f5da7297d5f7564ca6" + }, + { + "path": "download_model.py", + "bytes": 6312, + "sha256": "5e435d6f87d920d58ff6d305afe61be3a98fec8d12b6f79986072a9a208a8edd" + }, + { + "path": "local_smoke.py", + "bytes": 12705, + "sha256": "30d7f733c0b506f7ac9a4870d7523aa09e485316953df8973b7e7fb049322d88" + }, + { + "path": "program_loop/__init__.py", + "bytes": 270, + "sha256": "8ebc1b9a3937a1848c199126edf97e5862741ede120a491e87a8473d6c0f7c46" + }, + { + "path": "program_loop/__main__.py", + "bytes": 48, + "sha256": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d" + }, + { + "path": "program_loop/_vendor.py", + "bytes": 305, + "sha256": "cfb187e38a0b8acd86e83476ee24f3aa02928c86fd3deb532f752ae8784bbf80" + }, + { + "path": "program_loop/catalog_context.py", + "bytes": 804, + "sha256": "d59a7d3636fa7cb92ba5fff2126a9aee2b6a2166970b20ec4a834abdd3235409" + }, + { + "path": "program_loop/cli.py", + "bytes": 4594, + "sha256": "82d3841baa287c2d5e0cf519ae2532a4fd5a4d79e89d907e29ad029c17d59335" + }, + { + "path": "program_loop/contracts.py", + "bytes": 2054, + "sha256": "884a776db1a44475943a5b08370bad9d6107cfcbe7f40380622515e85713cb51" + }, + { + "path": "program_loop/controller.py", + "bytes": 6294, + "sha256": "37de0a7bc235c7528b48e95222fd3f291fd4054d063578a46d23327bad4e6c51" + }, + { + "path": "program_loop/execution.py", + "bytes": 5211, + "sha256": "f9f61e8a3e62d827ff0ceccc57f1d08c27b63c017ca3f1506c2955433d5b5bea" + }, + { + "path": "program_loop/planner.py", + "bytes": 5519, + "sha256": "98f1510561808070bbf712f1d319a57453339909b18ca283ed76c02f045da575" + }, + { + "path": "program_loop/planning_contracts.py", + "bytes": 1492, + "sha256": "90d671bb998814249379db93c887ece2df70476d12a4e8bbc15dd6ccd78f6845" + }, + { + "path": "program_loop/recovery_context.py", + "bytes": 3147, + "sha256": "a6660c91d3e77804146b12ce1b77fb0974977bb75fd98cf81c6225f02ae48013" + }, + { + "path": "program_loop/rejection_context.py", + "bytes": 1329, + "sha256": "fd71ad424ff4a201b01f4db7276f2d958fd94eba7aaf26deb16cc54dbcb9de1f" + }, + { + "path": "program_loop/validation.py", + "bytes": 4625, + "sha256": "8ac6c4bfa272c60fb0149eab43b73c9b9b5e210859940eef316dfa00e4723e23" + }, + { + "path": "tests/test_9b_identity.py", + "bytes": 1300, + "sha256": "d350a177fc710dadf08121abe09bd0b89e2a14785e203535ea611a1bff102b5c" + }, + { + "path": "tests/test_dynamic_references.py", + "bytes": 5043, + "sha256": "588688f185f721c742e78f336ab5b42b7f7a695c87a0b8640208412372edc30e" + }, + { + "path": "tests/test_local_smoke.py", + "bytes": 20015, + "sha256": "dfdc361c3a7243c5ee78645d1d2d7bb449927633d061bfd76986fb081e6be601" + }, + { + "path": "tests/test_planning_contracts.py", + "bytes": 13708, + "sha256": "992da01d882c51a288b298db790705589e0cf16f92e9ecd4e611746c96504560" + }, + { + "path": "tests/test_program_loop.py", + "bytes": 22282, + "sha256": "18d251a2781b0bc89e9ff50fe53afe364a9e7a35fb48f19f610af987582a14fd" + }, + { + "path": "tests/test_recovery_context.py", + "bytes": 5958, + "sha256": "ba4e725f74a216831466b56f3b93616ca82dd36d0d77d690185f2e13d14de08e" + }, + { + "path": "tests/test_rejection_context.py", + "bytes": 5316, + "sha256": "48024ea991eceb98c0eb2cc9d292f9092ded253d995cbc4398b3a1dfed9bdce3" + }, + { + "path": "tests/test_source_integrity.py", + "bytes": 2617, + "sha256": "8141193bf6771e3c0a2bedbf5bb7999ba32bbaa6f74adc77fa1a18d941ea37ea" + }, + { + "path": "tests/test_task_level_prompt.py", + "bytes": 11167, + "sha256": "165fe55de77344c81d97de30eb532b7aadd326ce3f91a502025d5ab571ac4fc5" + }, + { + "path": "tools/check_local_gpu.py", + "bytes": 2809, + "sha256": "4b28ea500f7573dedc1801a8b73c8ddaa4b26d8f87e9a6624682a3aa2d8ac7f3" + }, + { + "path": "tools/verify_source.py", + "bytes": 2762, + "sha256": "c1d14bc3f5641a725d9c8d0be05391215bf696d850f3adf7e1239fbec5f02294" + }, + { + "path": "vendor/common.py", + "bytes": 5142, + "sha256": "c79a02e987a169436796f438a415d7c309c0e0cf50bb447e493cd68cb9301fa4" + }, + { + "path": "vendor/pool_availability.py", + "bytes": 6074, + "sha256": "c6fa8e23a678b4451ef78d301efb347507fb0513a9d18bfda76eeefc824d70aa" + }, + { + "path": "vendor/pool_bindings.py", + "bytes": 2822, + "sha256": "3c0c65e78f7bcdc5bb81003b455fd957dba5c4abdbfdaa20b8fddb3b155e07db" + }, + { + "path": "vendor/pool_catalog.py", + "bytes": 7159, + "sha256": "bd19db2c908d41120498eae4afc26e30726b60ddf7fbf834e77ca6251e8c8a82" + }, + { + "path": "vendor/pool_metadata.py", + "bytes": 5969, + "sha256": "0705318048785fd34edd970274145dec359269e5e30756a996eb156e168e3a8a" + }, + { + "path": "vendor/pool_runtime.py", + "bytes": 34139, + "sha256": "506f2b605a32b0a1cf1431cdd4b92f5ccf821909d20726f8aacdd672ab0a893d" + }, + { + "path": "vendor/reference_skills_63.json", + "bytes": 2232, + "sha256": "a63b6ee85f983caf9f55d537fe85d24a283a860ccb0985c8a7fce8b1466e3caf" + }, + { + "path": "vendor/runtime/code/agent/__init__.py", + "bytes": 110, + "sha256": "c2ee5b379bec6f6381e0651bee6078e38ad6ca294f497d40b6d27e583fd1492e" + }, + { + "path": "vendor/runtime/code/agent/orchestration/__init__.py", + "bytes": 946, + "sha256": "64f470fa2d85450ebbd8819a66c63064bcdbf993c829d248c64315285e7aca73" + }, + { + "path": "vendor/runtime/code/agent/orchestration/agent_loop.py", + "bytes": 39927, + "sha256": "176bc72eebd17a7d2e45276654c49cbd5b1828306e11d2579faffe143eebcade" + }, + { + "path": "vendor/runtime/code/agent/orchestration/artifact_projection.py", + "bytes": 2952, + "sha256": "d71939b3e4b04b83d33f7aedbc3173cd44043b8a55d44c149c46f35d5aa3f7d8" + }, + { + "path": "vendor/runtime/code/agent/orchestration/catalog.py", + "bytes": 5851, + "sha256": "66ceb63dd02030cecb3965247a671da64c4d29724ece9a5eb52961a0647c8e6a" + }, + { + "path": "vendor/runtime/code/agent/orchestration/category_catalog.py", + "bytes": 4738, + "sha256": "baf7b0b234822060b48dded26f00d526654749f1384c3e23f9eb07ddc8fa0a58" + }, + { + "path": "vendor/runtime/code/agent/orchestration/contracts.py", + "bytes": 3654, + "sha256": "19ed69938bdc517be14625e0ef886945dc4483faeb6a8260fdfcacaffa83f233" + }, + { + "path": "vendor/runtime/code/agent/orchestration/metadata_lookup.py", + "bytes": 2340, + "sha256": "e338e0d804caa20218637074422b08f26b16fdbdd9faff39077a1cb73beb89fc" + }, + { + "path": "vendor/runtime/code/agent/orchestration/metrics.py", + "bytes": 2007, + "sha256": "fffb69bd3e8efa149e46e3413b4056457840b9cd8f6798fd5e0871cdb7e3a0b3" + }, + { + "path": "vendor/runtime/code/agent/orchestration/model_skill_executor.py", + "bytes": 40196, + "sha256": "03a90c6c58ffad2dddc9fa5c8982045eb2a73a804f521c2c679f61e14e17f3b0" + }, + { + "path": "vendor/runtime/code/agent/orchestration/planner.py", + "bytes": 16361, + "sha256": "5be67a873f8fa54c183c9fa381def425b73cc92181fab6079506eb2296fa53d8" + }, + { + "path": "vendor/runtime/code/agent/orchestration/retrieval.py", + "bytes": 12500, + "sha256": "fbe140936695f38c6bc6838c5a9a0100ca9e6f98b6e77984e587fdbea28b6bf4" + }, + { + "path": "vendor/runtime/code/agent/orchestration/runtime.py", + "bytes": 7641, + "sha256": "887e82115154f043801719b5e292895e3eeef0d25e277748805a0c1dabc39acc" + }, + { + "path": "vendor/runtime/code/agent/orchestration/skill_loader.py", + "bytes": 6180, + "sha256": "7528bada337e98b69ef61ccf558574f5940d26fcfc2106530f8c9d0d765964f6" + }, + { + "path": "vendor/runtime/code/agent/orchestration/table_field_lookup.py", + "bytes": 3864, + "sha256": "93319c4ee044439f9e25449cec220dfaededa1c6b799816486d19402b569dc22" + }, + { + "path": "vendor/runtime/code/agent/orchestration/topk_selection.py", + "bytes": 3147, + "sha256": "a58411d1acb421bd5e5c134f4ceef7ccce76851f684a397446904b7a17b1eb4c" + }, + { + "path": "vendor/runtime/code/agent/orchestration/validator.py", + "bytes": 5283, + "sha256": "15fd2b9faca5addf49f4fdf5d816eb2ecb3723c6b800da3fd591cda93b0bedcd" + }, + { + "path": "vendor/runtime/code/script/orchestration/__init__.py", + "bytes": 72, + "sha256": "7c61249a27a9c7ad011f983e9857e3889ae54837d00faf2660fe2f72350b30d6" + }, + { + "path": "vendor/runtime/code/script/orchestration/run_agent_loop_batch.py", + "bytes": 51611, + "sha256": "6605815bf479e8fc50a3041af3c2430b3f121740bf38b8e6d497212d4425616d" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_ASSESS_CATEGORY_SUPPORT.md", + "bytes": 880, + "sha256": "85ea64633013f69d582707acf4ab1b32201cf5ebbf7e0356041119d6c08c106a" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_CHECK_EVIDENCE_SUFFICIENCY.md", + "bytes": 572, + "sha256": "f879c21470b9daeb773a80a256a07f7a6f43e38e8452347f128eca6391362f4a" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_COMPARE_CATEGORY_HYPOTHESES.md", + "bytes": 844, + "sha256": "3fa31691df6bb353b4171a8c837da229549f8f4eff827bf502f81cb2318432a5" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_DECIDE_NEXT_EVIDENCE_SOURCE.md", + "bytes": 990, + "sha256": "647a447ae584d92e67c17d2d7040ff48d14cb8d2b5039345d65eb072b5c14fd3" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_FINALIZE_CLASSIFICATION.md", + "bytes": 637, + "sha256": "cd66513c1ca4b6728bd217ae57a7ad7cd96257f61443a7af2ac1edd7e746f937" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_CONTEXT_EVIDENCE.md", + "bytes": 867, + "sha256": "b0028a313fa3f81350f07093c5389c7ec0447539119e37930f1f05bceec539d1" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_DATABASE_EVIDENCE.md", + "bytes": 835, + "sha256": "e27e9b2f57b5b1929d5286a10cc15233ab48ba40948fe520fde8853f66bc1e56" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_FIELD_EVIDENCE.md", + "bytes": 870, + "sha256": "6dfef164297cb75a02f64ebbdafecb3f0519f1a5f1992c3bb9a98157b7a7ee54" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_METADATA_EVIDENCE.md", + "bytes": 880, + "sha256": "e3a7e0de334bc39836c3a909edc79233075b0b73f7c30e879d4e087bc94aa6f2" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_STANDARD_EVIDENCE.md", + "bytes": 880, + "sha256": "eddac25de1c3dda0baf876d29a7a32ae4fab980f2ae7ce892def82af3469087a" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_FUSE_TABLE_EVIDENCE.md", + "bytes": 801, + "sha256": "24c7eb0ea3381b640465e78e41eb2ff43495e7bb3eadcc652e5587c3bd167ab4" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_IDENTIFY_MISSING_EVIDENCE.md", + "bytes": 990, + "sha256": "237695b8b5d6f9a7105902dc32a120297c364ee5f8d1c27f4ead212df110668f" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY.md", + "bytes": 586, + "sha256": "032b758517d3a8535c29d20d8391bd8d27a2ad1d5c7e9f62246578076ddb5197" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_INFER_CATEGORY_FROM_TOPK.md", + "bytes": 678, + "sha256": "7af18e11373d633fe4369338336c653b47238c214cb5f1937e0a5b30041f722f" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_INFER_GRADING_LEVEL.md", + "bytes": 588, + "sha256": "276b5e1a8b080d9aecec762762939372f40e56c8fb4b1eb92964c53073b6d603" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_FIELD_CONTEXT_CONFLICT.md", + "bytes": 908, + "sha256": "94e552edccadc03cb55a5ab8e54511a296971363bd4aafb30b3ab1385e5a18f3" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_METADATA_CONFLICT.md", + "bytes": 863, + "sha256": "643472dc5b7e7cdb38f3483857acc94d539884efede9b9ed0f4cccc79a037d0b" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_RESOLVE_STANDARD_CONFLICT.md", + "bytes": 878, + "sha256": "be9b9c9c879b964acf4b5f7317dcf90c68833a1f98f6cb8ab9f8199460066b25" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CATEGORY_WITH_STANDARD.md", + "bytes": 944, + "sha256": "deb07baab54b052a9323b475c912d7743115c96dd4b1dd0b7c4a80df3e3d0dfc" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_VERIFY_CLASSIFICATION_GRADING_CONSISTENCY.md", + "bytes": 932, + "sha256": "ff8ef17c1ce1408496c6634389ecba87f6d282d0264e4ac1ee9a8c2deb12bab1" + }, + { + "path": "vendor/runtime/skills/full_pool/evidence_reasoning_WEIGH_EVIDENCE.md", + "bytes": 874, + "sha256": "c928a03a7a8f2d8b1a28ffb01c100543000a62e4a388a0e60051a4b6815dd43f" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_ACQUIRE_MISSING_METADATA.md", + "bytes": 998, + "sha256": "cc788f17f7c63e1f7974f220eca3becfb186d3df287a35d814a5387964d20ce3" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_CHECK_METADATA_AVAILABILITY.md", + "bytes": 576, + "sha256": "fe27b6f518a98be4e88414d6180180954572f222fec37dfc4591e2b72a05ad5a" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_EXPAND_SELECTED_CONTEXT.md", + "bytes": 964, + "sha256": "323b136804a9db277d6642b71de277ae3fcd7021910645475872b1b018d79a79" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_DATABASE_CONTEXT.md", + "bytes": 913, + "sha256": "0008e4dc6b220757208cb656912ebb609c8230aaf62ce56a33f247dc5879f7cd" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_COMPLETE_TABLE_CONTEXT.md", + "bytes": 914, + "sha256": "09b1f136ca45244c603e98aba0bec6fad8997ca2e618eeaeaf7ac1c1de4fb711" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_DESCRIPTIONS.md", + "bytes": 947, + "sha256": "0b5f7d9ced8902a3a7157d6781a3b3d6bc9fb9f2d2c55da99f31977ceb5609b8" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_TYPES.md", + "bytes": 945, + "sha256": "61cfe7efeb0980d22e72c4069e70cb91d1a007c7352b528b600b7efbcfd313e5" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_CONTEXT_FIELD_VALUES.md", + "bytes": 983, + "sha256": "780305d46253a0ab132f35feee6ba420347f351326827c4be781bd4224349504" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_DATABASE_DESCRIPTION.md", + "bytes": 903, + "sha256": "37b2146b9cd669e1dc0cb0a54c0f6ee36dc0157ce1e4e753d96be08a6fdada21" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_DESCRIPTION.md", + "bytes": 951, + "sha256": "d545962070cb10fa90ee1693261d48ee08df616cad4d098bfcd98643100fce7b" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_FIELD_METADATA.md", + "bytes": 801, + "sha256": "372ce861fcca6138cda43b693a8fc32f85c0ec380b0c275c3cfe613dbe25f629" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_DESCRIPTION.md", + "bytes": 879, + "sha256": "912334b76f95c6f468cf4bab840ae0ec3a262639c59e545db63bfce866710ea0" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_GET_TABLE_FIELDS.md", + "bytes": 935, + "sha256": "99c01c5869b511754feccfd335017c7f4290683fbcc46ed6355b95f775d605b0" + }, + { + "path": "vendor/runtime/skills/full_pool/metadata_acquisition_SELECT_CONTEXT_FIELDS.md", + "bytes": 992, + "sha256": "aacb11c41507d2c2743e56a7915c9dae9d6b4e2ff3ece6fec6b3d73babeb6683" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_DISAMBIGUATE_FIELD_SEMANTICS.md", + "bytes": 626, + "sha256": "e497bddaf99da95503677a969926921b507b92ad542055dafaa88ee724a235d0" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_DATABASE_ABBREVIATION.md", + "bytes": 927, + "sha256": "2338078f06cd5276b326747fa0b80790d3c638ba369f6579e4ddcff84882eb51" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_FIELD_ABBREVIATION.md", + "bytes": 576, + "sha256": "6b84b95768e4cc77d193b31871f3b75ab17ff86f526c4000bccd6f8c423cb2a7" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_EXPAND_TABLE_ABBREVIATION.md", + "bytes": 910, + "sha256": "6ce033822d4d4f4ae7babba7bd0b44fa871476353b81fd86ac6742d266b3a426" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_CONTEXT_FIELDS.md", + "bytes": 948, + "sha256": "297bf09515bbbc0a4b9e6f80fb77c7135b7484c5c428da0868d7a0d0732717cb" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_DESCRIPTION.md", + "bytes": 925, + "sha256": "acc57509cff6330476522f8fc5a7b286b0ede3e477c9806000b54542f89070dc" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DATABASE_NAME.md", + "bytes": 880, + "sha256": "e427e888ac10d742784d390ad2bdce40d0aec7c9a7abd52b5f5497fc83b866fc" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_DOMAIN.md", + "bytes": 505, + "sha256": "6e1760ff387797cb4c171f404f744b6b6b87445969659bf8b37c8bca66de5375" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_DESCRIPTION.md", + "bytes": 681, + "sha256": "a833b94c6c7afd5e740d67674662178e9b2f9cef6e0019dd9af8bbf62b324335" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_NAME.md", + "bytes": 610, + "sha256": "99934fabad8007d155624a475b1600b5b8139152efde2a9154cd7035e42ba56e" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_TYPE.md", + "bytes": 893, + "sha256": "93500ee995113eed8ee9d6d8887d6f65f2d4aa6ac52b12697fb8e82982c1b49d" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_FIELD_VALUE.md", + "bytes": 897, + "sha256": "c0dad806fde006487b38b9d56fc0b5fde6cc2e367b8a46d13adb5c4171ed19ad" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_DESCRIPTION.md", + "bytes": 713, + "sha256": "901cef9276dd30ff1b2e35fb39bbe9138cf42bb4ded47b37fb3afe83c1945ac2" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_INTERPRET_TABLE_NAME.md", + "bytes": 857, + "sha256": "bfd2506982a98615fa40d2a2760893f7634a270a663f65a84547e4a2ec7a4186" + }, + { + "path": "vendor/runtime/skills/full_pool/semantic_interpretation_SYNTHESIZE_METADATA_SEMANTICS.md", + "bytes": 1085, + "sha256": "4d64debe0b8657d6c0960eb5e3eebad7aa5125e34e35f66deb38c050f665ec26" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_ALIGN_CONTEXT_TO_STANDARD.md", + "bytes": 935, + "sha256": "debf2fb145c10526e4952ba7cb322d8a88370ae136d1ae6f0b74b7b9d60734b0" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DATABASE_TO_STANDARD.md", + "bytes": 1007, + "sha256": "5bdb03e45720da5ff53297fae7ce476e6fd46f3af6277e8be147a87033917630" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_ALIGN_DESCRIPTION_TO_STANDARD.md", + "bytes": 982, + "sha256": "ec00677a4433b6110f529eafc28d5c0272746f186121ee563ab86eb8a3a9a6ec" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_ALIGN_FIELD_TO_STANDARD.md", + "bytes": 941, + "sha256": "19c1e2c28c4df124d9ef56501635c31443552cc3d103d727b927db78891eb53f" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_ALIGN_TABLE_TO_STANDARD.md", + "bytes": 986, + "sha256": "7fa123b49aca5ec7f2525d3697e0108aaf3d85f61c966eb9492a19b657e2a300" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_ALIGN_VALUE_TO_STANDARD.md", + "bytes": 932, + "sha256": "d71db5f6adcf269897e8c9b662cc31edc18b7a7727db7a241624e319a390588e" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_COMPARE_STANDARD_CANDIDATES.md", + "bytes": 933, + "sha256": "d5e82a8d994155a010de18c15a1923a432e2e039a0aa3016bc1a7689cdfe9fdb" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_REFINE_STANDARD_QUERY.md", + "bytes": 980, + "sha256": "51c1ead62e9d83c194ac1fc24c8f43b9879497477b79054e03384b6ded980420" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_RESEARCH_STANDARD.md", + "bytes": 908, + "sha256": "05ccd5f271a060ed497242c494c34d8ce3cac22f03e6349d0d6edfe0a4b303ac" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD.md", + "bytes": 550, + "sha256": "dcbd3841440dc7bf0ee22c02b9ba1014fff1fa6a68900f270bb8cbb40aa9074c" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_CONTEXT.md", + "bytes": 908, + "sha256": "6c1c9ad8fd7036ddf2504e30bc4f920d9754c8c3de68f2ff5263f760630d839e" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_DATABASE.md", + "bytes": 909, + "sha256": "fc38fe4bccc6637de6abe3b8434f7d069d42b6eabf1a8dc29b98cfcdd443a362" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_FIELD.md", + "bytes": 875, + "sha256": "98862fb615e97f189420455219b484ef990e0f4768f7ba4e49983e237d3998f2" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_SEARCH_STANDARD_BY_TABLE.md", + "bytes": 874, + "sha256": "ae067ae1570ad6e24e5c43a08aa4057223923b45e2593525255419144847ebd0" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_SELECT_CATEGORY_TOPK.md", + "bytes": 742, + "sha256": "c0dac66df87884c2ecd0e6dbfe2af1027b522a1653972ffa4d79e8a957a9bb74" + }, + { + "path": "vendor/runtime/skills/full_pool/standard_alignment_SUMMARIZE_STANDARD_EVIDENCE.md", + "bytes": 996, + "sha256": "ea3ee35fa9ddde244353f533a527bf25674ffb711218f74c8960862183bd664b" + }, + { + "path": "vendor/skill_reference.json", + "bytes": 20313, + "sha256": "752ed72d4ed6e987542197dfff40786ebd4487bcf6b1042cbf769d1fed2392b8" + }, + { + "path": "vendor/study.py", + "bytes": 17396, + "sha256": "d8c78d387e00b13762fb509d23e147210b0016931879c4af7089c593d6eddbb3" + } + ] +} diff --git a/research/finance_program_v1/nontraining/reviews/trusted-model-manifest-01.json b/research/finance_program_v1/nontraining/reviews/trusted-model-manifest-01.json new file mode 100644 index 0000000..fac39da --- /dev/null +++ b/research/finance_program_v1/nontraining/reviews/trusted-model-manifest-01.json @@ -0,0 +1,74 @@ +{ + "repo_id": "Qwen/Qwen3.5-9B", + "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "total_bytes": 19329302129, + "download_identity_sha256": "3087a66dc953b6aaa159fdf5286eb894d805c0c686f6d202452cf7ceb6039dea", + "files": [ + { + "path": "chat_template.jinja", + "bytes": 7756, + "expected_sha256": null, + "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715" + }, + { + "path": "config.json", + "bytes": 3126, + "expected_sha256": null, + "sha256": "d0883072e01861ed0b2d47be3c16c36a8e81c224c7ffaa310c6558fb3f932b05" + }, + { + "path": "merges.txt", + "bytes": 3353259, + "expected_sha256": null, + "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d" + }, + { + "path": "model.safetensors-00001-of-00004.safetensors", + "bytes": 5276436216, + "expected_sha256": "db6f444b43d318c92f360a13a25561a6a65b10c0631b8ed305a426dbaa6c380e", + "sha256": "db6f444b43d318c92f360a13a25561a6a65b10c0631b8ed305a426dbaa6c380e" + }, + { + "path": "model.safetensors-00002-of-00004.safetensors", + "bytes": 5335161512, + "expected_sha256": "31c7d7e2dd5d207840b31cc59083c8f4c4718959149e0358c0364052bb9a0330", + "sha256": "31c7d7e2dd5d207840b31cc59083c8f4c4718959149e0358c0364052bb9a0330" + }, + { + "path": "model.safetensors-00003-of-00004.safetensors", + "bytes": 5368717440, + "expected_sha256": "7ec36ba3a4176a44c3c0876ad80c56a2f70c84bf008d82e9501df642f17dadec", + "sha256": "7ec36ba3a4176a44c3c0876ad80c56a2f70c84bf008d82e9501df642f17dadec" + }, + { + "path": "model.safetensors-00004-of-00004.safetensors", + "bytes": 3325995712, + "expected_sha256": "b62b0c4cd7e44edee103ee8f4fe225f246d5e768e07bfd5f25b63a8aa1fdd0c6", + "sha256": "b62b0c4cd7e44edee103ee8f4fe225f246d5e768e07bfd5f25b63a8aa1fdd0c6" + }, + { + "path": "model.safetensors.index.json", + "bytes": 79657, + "expected_sha256": null, + "sha256": "26d3539b516be613f39563617cb9d33b3f83d401298125be392c80cefb8f7fe5" + }, + { + "path": "tokenizer.json", + "bytes": 12807982, + "expected_sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", + "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42" + }, + { + "path": "tokenizer_config.json", + "bytes": 16710, + "expected_sha256": null, + "sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8" + }, + { + "path": "vocab.json", + "bytes": 6722759, + "expected_sha256": null, + "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" + } + ] +} diff --git a/research/finance_program_v1/nontraining/tools/check_local_gpu.py b/research/finance_program_v1/nontraining/tools/check_local_gpu.py new file mode 100644 index 0000000..ccbf06f --- /dev/null +++ b/research/finance_program_v1/nontraining/tools/check_local_gpu.py @@ -0,0 +1,67 @@ +"""Short local CUDA arithmetic check. NOT a Planner/LLM inference benchmark.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys + + +def cuda_arithmetic(): + import torch + + if not torch.cuda.is_available(): + raise RuntimeError('PyTorch has no usable CUDA device') + torch.manual_seed(20261010) + torch.backends.cuda.matmul.allow_tf32 = False + left = torch.randn(256, 256, dtype=torch.float32) + right = torch.randn(256, 256, dtype=torch.float32) + cpu = left @ right + gpu = (left.to('cuda:0') @ right.to('cuda:0')).cpu() + torch.cuda.synchronize(0) + error = float((cpu - gpu).abs().max()) + torch.testing.assert_close(gpu, cpu, rtol=1e-4, atol=1e-4) + return {'torch': torch.__version__, 'torch_cuda': torch.version.cuda, + 'device': torch.cuda.get_device_name(0), 'device_count': torch.cuda.device_count(), + 'shape': [256, 256], 'dtype': 'float32', 'max_abs_error': error, + 'passed': True, 'test_kind': 'CUDA arithmetic, not LLM inference'} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, help='New report path; existing files are rejected') + parser.add_argument('--timeout', type=int, default=180) + parser.add_argument('--child', action='store_true', help=argparse.SUPPRESS) + args = parser.parse_args() + if args.child: + print(json.dumps(cuda_arithmetic())) + return 0 + if args.timeout < 1: + parser.error('--timeout must be positive') + if args.output is not None and args.output.exists(): + parser.error('output exists; use a new path') + report = {'utc': datetime.now(timezone.utc).isoformat(), 'python': sys.executable, + 'language_model_tested': False} + try: + child = subprocess.run([sys.executable, str(Path(__file__).resolve()), '--child'], + capture_output=True, text=True, timeout=args.timeout) + report.update({'child_exit': child.returncode, 'stderr': child.stderr}) + if child.returncode: + report.update({'passed': False, 'stdout': child.stdout}) + else: + report.update(json.loads(child.stdout)) + except (subprocess.TimeoutExpired, json.JSONDecodeError) as exc: + report.update({'passed': False, 'error': str(exc)}) + encoded = json.dumps(report, ensure_ascii=False, indent=2) + '\n' + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open('x', encoding='utf-8') as stream: + stream.write(encoded) + print(encoded, end='') + return 0 if report.get('passed') else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/research/finance_program_v1/publication_tests/test_dependency_closure.py b/research/finance_program_v1/publication_tests/test_dependency_closure.py new file mode 100644 index 0000000..dfaaaa7 --- /dev/null +++ b/research/finance_program_v1/publication_tests/test_dependency_closure.py @@ -0,0 +1,141 @@ +"""Fresh-process public entrypoint checks; no private workspace or model calls.""" +from pathlib import Path +import subprocess +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class PublicDependencyTests(unittest.TestCase): + def run_python(self, folder, code): + prefix = 'import sys; from pathlib import Path; sys.path.insert(0,str(Path.cwd())); ' + result = subprocess.run([sys.executable, '-I', '-B', '-c', prefix + code], + cwd=ROOT / folder, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_nontraining_audit_imports_without_private_checkout(self): + self.run_python('nontraining', + 'import batch_audit; from reviews.independent_audit_01 import RecordedBackend; ' + 'b=RecordedBackend([]); b.finish(); ' + 'assert "torch" not in sys.modules and "transformers" not in sys.modules') + + def test_pool_metadata_resolves_through_public_controller(self): + self.run_python('nontraining', + 'import program_loop; import pool_metadata; ' + 'assert Path(pool_metadata.__file__).resolve().parent == Path.cwd()/"vendor"') + + def test_teacher_recorded_audit_and_runtime_have_local_code(self): + self.run_python('teacher', + 'import detached, pilot_audit, runtime_identity; ' + 'assert callable(pilot_audit.replay_retry_one); ' + 'assert Path(runtime_identity.__file__).resolve().is_relative_to(Path.cwd().parent); ' + 'assert "torch" not in sys.modules') + + def test_legacy_revision_helper_has_no_private_import(self): + self.run_python('evaluation', + 'import io_utils; from tools.freeze_revision import safe_path, signature; ' + 'assert safe_path(Path.cwd(),"safe/file.json")==Path.cwd()/"safe/file.json"; ' + 'assert len(signature([]))==64') + + def test_teacher_nonterminal_trace_replays_and_rejects_tampering(self): + self.run_python('teacher', ''' +import json, copy +import detached +from program_loop import build_program_controller +from teacher_api import api_messages,digest,parse_sse,role +from pilot_audit import replay_retry_one +study={'rows':[{'id':'fixture-001','domain':'finance','metadata':{'field_name':'balance','field_description':'fixture description'}}],'registry':{'cat'+str(i):{'description':'fixture class '+str(i),'grading_level':'L1'} for i in range(1,13)},'budget':{'max_programs':3,'max_program_steps':10,'max_skill_steps':12,'max_planner_attempts':4,'max_identical_calls':1}} +c=build_program_controller(lambda m:'',study['registry'],study['rows'],study['budget'],skill_pool='F66') +allowed=[s['name'] for s in c.library]; records=[]; logical=[] +def backend(messages): + system=messages[0]['content'] + if 'Global Program Planner' in system: + value={'program':[{'tool':n,'arguments':{}} for n in ('GET_FIELD_METADATA','INTERPRET_FIELD_NAME','SELECT_CATEGORY_TOPK','INFER_CATEGORY_FROM_TOPK','INFER_GRADING_LEVEL','FINALIZE_CLASSIFICATION')]} + elif 'selected Skill SELECT_CATEGORY_TOPK.' in system: value={'option_ids':['C001','C002','C003','C004','C005'],'summary':'rank'} + elif 'selected Skill INFER_CATEGORY_FROM_TOPK.' in system: value={'option_id':'C001','summary':'choice'} + else: value={'summary':'interpret','data':{'field_semantics':'observed field'}} + raw=json.dumps(value) + body={'model':'qwen3.8-2.4t-a95b','messages':api_messages(messages),'response_format':{'type':'json_object'}} + q={'body':body,'body_sha256':digest(body),'executor_messages':messages,'role':role(messages,allowed),'credential_recorded':False} + events=[{'model':body['model'],'choices':[{'index':0,'delta':{'content':raw},'finish_reason':'stop'}],'usage':{'prompt_tokens':1,'completion_tokens':2,'total_tokens':3}}] + x=parse_sse(['data: '+json.dumps(e) for e in events]+['data: [DONE]'],body['model']) + x['response_sha256']=digest({'content':x['content'],'reasoning_content':x['reasoning_content']}) + n=len(records); records.append({'request':q,'result':x}) + logical.append({'executor_messages_sha256':digest(messages),'call_start':n,'call_end':n+1,'complete':True,'error':None}) + return raw +c=build_program_controller(backend,study['registry'],study['rows'],study['budget'],skill_pool='F66') +row=study['rows'][0]; saved=c.run({'source_id':row['id'],'domain':row['domain'],'field_name':row['metadata']['field_name']}).to_dict() +proof=replay_retry_one(study,row,saved,records,logical,allowed) +assert proof['full_trace_exact'] and proof['all_calls_consumed'] and proof['api_attempts']==4 +assert not proof['terminated'] and proof['stop_reason']=='program_budget_exhausted' +changed=copy.deepcopy(records); changed[0]['request']['body_sha256']='0'*64 +try: replay_retry_one(study,row,saved,changed,logical,allowed) +except AssertionError: pass +else: raise AssertionError('tampered teacher request accepted') +assert 'torch' not in sys.modules +''') + + def test_public_audit_manifest_binds_shipped_source(self): + self.run_python('nontraining', + 'from local_smoke import environment; from reviews.independent_audit_01 import validate_sources; ' + 'validate_sources(Path.cwd(),environment("not-a-model"))') + + def test_public_source_audit_rejects_mutation_and_missing_file(self): + self.run_python('nontraining', ''' +import shutil, tempfile +from local_smoke import environment +from reviews.independent_audit_01 import validate_sources, AuditError +with tempfile.TemporaryDirectory() as d: + snapshot=Path(d)/'snapshot'; shutil.copytree(Path.cwd(),snapshot) + environment_evidence=environment('not-a-model') + validate_sources(snapshot,environment_evidence) + target=snapshot/'vendor/pool_metadata.py'; original=target.read_bytes() + target.write_bytes(original+bytes([10])+b'# changed') + try: validate_sources(snapshot,environment_evidence) + except AuditError: pass + else: raise AssertionError('changed source accepted') + target.unlink() + try: validate_sources(snapshot,environment_evidence) + except AuditError: pass + else: raise AssertionError('missing source accepted') +''') + + def test_batch_audit_replays_real_scripted_controller(self): + self.run_python('nontraining', ''' +import json, tempfile, hashlib +from batch_runner import run_batch +from batch_audit import audit +class Scripted: + def __init__(self, model, call_dir): + self.folder=Path(call_dir); self.folder.mkdir(parents=True); self.calls=[]; self.failures=[] + def __call__(self,messages): + system=messages[0]['content'] + if 'Global Program Planner' in system: + raw=json.dumps({'program':[{'tool':n,'arguments':{}} for n in ('GET_FIELD_METADATA','INTERPRET_FIELD_NAME','SELECT_CATEGORY_TOPK','INFER_CATEGORY_FROM_TOPK','INFER_GRADING_LEVEL','FINALIZE_CLASSIFICATION')]}) + elif 'selected Skill SELECT_CATEGORY_TOPK.' in system: + raw=json.dumps({'option_ids':['C001','C002','C003','C004','C005'],'summary':'rank'}) + elif 'selected Skill INFER_CATEGORY_FROM_TOPK.' in system: + raw=json.dumps({'option_id':'C001','summary':'choice'}) + else: raw=json.dumps({'summary':'interpret','data':{'field_semantics':'observed field'}}) + folder=self.folder/('call-%04d'%(len(self.calls)+1)); folder.mkdir(); self.calls.append(str(folder)) + for name,value in [('request',{'messages':messages}),('result',{'messages':messages,'raw_text':raw,'error':None,'input_tokens':1,'output_tokens':2,'seconds':0.01})]: + (folder/(name+'.json')).write_text(json.dumps(value),encoding='utf-8') + return raw +with tempfile.TemporaryDirectory() as d: + d=Path(d) + study={'schema':1,'cohorts':{'smoke':['fixture-001'],'pilot':[],'expanded':[]},'rows':[{'id':'fixture-001','domain':'finance','metadata':{'field_name':'balance','field_description':'fixture description'}}],'registry':{'cat'+str(i):{'description':'fixture class '+str(i),'grading_level':'L1'} for i in range(1,13)},'budget':{'max_programs':3,'max_program_steps':10,'max_skill_steps':12,'max_planner_attempts':4,'max_identical_calls':1}} + p=d/'study.json'; p.write_text(json.dumps(study),encoding='utf-8') + (d/'study-manifest.json').write_text(json.dumps({'study_sha256':hashlib.sha256(p.read_bytes()).hexdigest()}),encoding='utf-8') + assert run_batch('not-a-model',d/'run','smoke',study_path=p,backend_factory=Scripted)==0 + proof=audit(d/'run',study_path=p,allow_scripted_fixture=True) + assert proof['verified_samples']==1 and proof['full_trace_exact'] and proof['all_calls_consumed'] + q=d/'run/calls/call-0001/request.json'; data=json.loads(q.read_bytes()); data['messages'][0]['content']+='changed'; q.write_text(json.dumps(data),encoding='utf-8') + try: audit(d/'run',study_path=p,allow_scripted_fixture=True) + except AssertionError: pass + else: raise AssertionError('tampered request incorrectly accepted') +''') + + +if __name__ == '__main__': + unittest.main() diff --git a/research/finance_program_v1/teacher/detached.py b/research/finance_program_v1/teacher/detached.py index f710f11..98d8a84 100644 --- a/research/finance_program_v1/teacher/detached.py +++ b/research/finance_program_v1/teacher/detached.py @@ -12,7 +12,12 @@ import time ROOT = Path(__file__).resolve().parent -sys.path[:0] = [str(ROOT/'lib'), str(ROOT/'runtime')] +# Controlled historical snapshots own runtime/; the public layout shares the +# exact F66 snapshot already shipped under evaluation/runtime, not a new copy. +RUNTIME = ROOT/'runtime' +if not RUNTIME.is_dir(): + RUNTIME = ROOT.parent/'evaluation/runtime' +sys.path[:0] = [str(ROOT/'lib'), str(RUNTIME)] from campaign_budget import PersistentBudget, RunBudget from teacher_api import TeacherBackend, save from retry_backend import RetryBackend diff --git a/research/finance_program_v1/teacher/pilot_audit.py b/research/finance_program_v1/teacher/pilot_audit.py new file mode 100644 index 0000000..2affef6 --- /dev/null +++ b/research/finance_program_v1/teacher/pilot_audit.py @@ -0,0 +1,85 @@ +"""Retry-aware recorded-I/O replay helpers; no private post-hoc gold audit.""" +import json +from pathlib import Path +from teacher_api import api_messages, digest, parse_sse, role, save +from retry_backend import retryable + + +def validate_record(record, messages, allowed, allow_fixture=False): + q, x = record['request'], record['result'] + assert q['body_sha256'] == digest(q['body']) + assert q['executor_messages'] == messages + assert q['role'] == role(messages, allowed) + if x.get('fixture'): + assert allow_fixture + return + assert q['body']['messages'] == api_messages(messages) + assert q['body']['model'] == 'qwen3.8-2.4t-a95b' + assert q['body']['response_format'] == {'type': 'json_object'} + assert q['credential_recorded'] is False and 'headers' not in q + if x['complete']: + lines = ['data: ' + json.dumps(e) for e in x['events']] + if x['stream_done']:lines.append('data: [DONE]') + rebuilt = parse_sse(lines, q['body']['model']) + for k in ('content', 'reasoning_content', 'usage', 'models', 'finish_reason', 'stream_done', 'complete'): + assert rebuilt[k] == x[k] + assert x['response_sha256'] == digest({'content': x['content'], 'reasoning_content': x['reasoning_content']}) + + +def replay_retry_one(study, row, saved, records, logical, allowed, *, allow_fixture=False): + from program_loop import build_program_controller + class Recorded: + def __init__(self):self.position = 0; self.http_position = 0; self.faults = []; self.halted = False + def __call__(self, messages): + if self.halted: + raise RuntimeError('teacher system halted; see recorded call failure') + try: + assert self.position < len(logical) + group = logical[self.position]; self.position += 1 + assert group['executor_messages_sha256'] == digest(messages) + assert group['call_start'] == self.http_position + attempts = records[group['call_start']:group['call_end']] + assert 0 <= len(attempts) <= 2 + for n, record in enumerate(attempts): + validate_record(record, messages, allowed, allow_fixture) + if n < len(attempts) - 1: + assert retryable(record['result'], record['request']['body']['model']) + self.http_position = group['call_end'] + if group['complete']: + assert attempts and attempts[-1]['result']['complete'] is True and group['error'] is None + return attempts[-1]['result']['content'] + assert group['error'] in ('teacher response incomplete; recorded and halted', 'teacher safety budget halted', 'teacher source backend failed before receipt') + assert not attempts or not attempts[-1]['result']['complete'] + except Exception: + self.faults.append('logical/HTTP recorded boundary mismatch'); raise + self.halted = True + raise RuntimeError(group['error']) + backend = Recorded() + c = build_program_controller(backend, study['registry'], study['rows'], study['budget'], skill_pool='F66') + actual = c.run({'source_id': row['id'], 'domain': row['domain'], 'field_name': row['metadata']['field_name']}).to_dict() + assert not backend.faults and backend.position == len(logical) and backend.http_position == len(records) + assert actual == saved, 'full original trace differs' + return {'full_trace_exact': True, 'all_calls_consumed': True, 'logical_calls': len(logical), + 'api_attempts': len(records), 'failed_api_attempts': sum(not r['result']['complete'] for r in records), + 'terminated': actual['terminated'], 'stop_reason': actual['stop_reason']} + + +def read_records(folder): + from collect import read + paths = sorted(folder.glob('call-*')) + assert [p.name for p in paths] == ['call-%04d' % n for n in range(1, len(paths) + 1)] + records = [] + for p in paths: + q, x = read(p / 'request.json'), read(p / 'result.json') + if x['complete']: + raw = [json.loads(line)['line'] for line in (p / 'stream.jsonl').read_text(encoding='utf-8').splitlines()] + result = parse_sse(raw, q['body']['model']) + for k in ('content', 'reasoning_content', 'usage', 'models', 'finish_reason', 'stream_done', 'events'): + assert result[k] == x[k] + records.append({'request': q, 'result': x}) + return records + + + +# The old audit_all aggregator required private historical runs and gold. +# Only the recorded-I/O replay helpers used by detached.py are published here.