Skip to content

Commit 583fe60

Browse files
hyperpolymathclaude
andcommitted
feat(extract): widen Agda extractor — add cubical + unimath (echidna#17)
Prior extractor walked only agda-stdlib (9 312 records). Added agda-cubical (HoTT / cubical type theory) and agda-unimath (univalent foundations) as additional corpus roots. Changes: - AGDA_CORPORA list enumerates (directory, label) for each clone; extractor walks whichever are present. - parse_file() now takes (corpus_root, corpus_label) so `source` fields are labelled per-library. - Accept both .agda and .lagda.md (unimath uses the literate form). - Fix multi-byte UTF-8 crash in signature-length truncation: switch from `sig_body[1:400]` (byte-index, crashes on `≡` at boundary) to `first(sig_body, 400)` (character-index, safe). - Stats block rewritten to report per-corpus files_scanned plus the list of active corpora. Rerun result: agda-stdlib 9 312 (unchanged — baseline) agda-cubical 7 224 (new) agda-unimath 3 293 (new; underperforms because unimath leans on .lagda.md literate format — follow-up to add a markdown-aware parser) Total 19 829 (2.1× prior count) Further unimath uplift blocked on literate-markdown parser; tracked under the same issue for a follow-up commit. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 76696cf commit 583fe60

1 file changed

Lines changed: 49 additions & 19 deletions

File tree

‎scripts/extract_agda.jl‎

Lines changed: 49 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,22 @@ using JSON3
4242
using Dates
4343

4444
const REPO_ROOT = dirname(dirname(abspath(@__FILE__)))
45-
const AGDA_ROOT = joinpath(REPO_ROOT, "external_corpora", "agda-stdlib")
46-
const AGDA_SRC = joinpath(AGDA_ROOT, "src")
4745
const OUTPUT_DIR = joinpath(REPO_ROOT, "training_data")
4846
const OUTPUT_FILE = joinpath(OUTPUT_DIR, "proof_states_agda.jsonl")
4947
const STATS_FILE = joinpath(OUTPUT_DIR, "stats_agda.json")
5048
const START_ID = 210000
5149

50+
# Multiple Agda corpus roots. Added 2026-04-18 (echidna#17): the
51+
# extractor previously walked only agda-stdlib. We now additionally
52+
# ingest agda-cubical (HoTT / cubical type theory) and agda-unimath
53+
# (UniMath's Agda port of univalent foundations) when present. Each
54+
# entry maps (directory → prefix used in source labels).
55+
const AGDA_CORPORA = [
56+
(joinpath(REPO_ROOT, "external_corpora", "agda-stdlib"), "agda-stdlib"),
57+
(joinpath(REPO_ROOT, "external_corpora", "agda-cubical"), "agda-cubical"),
58+
(joinpath(REPO_ROOT, "external_corpora", "agda-unimath"), "agda-unimath"),
59+
]
60+
5261
# Signature-level filters.
5362
const PROOFY = [
5463
"_≡_", "≡", "_≢_", "≢",
@@ -164,7 +173,7 @@ end
164173
165174
Pull keepable declarations out of one Agda source file.
166175
"""
167-
function parse_file(path::String)::Vector{Dict{String,Any}}
176+
function parse_file(path::String, corpus_root::String="", corpus_label::String="")::Vector{Dict{String,Any}}
168177
out = Dict{String,Any}[]
169178
local lines::Vector{String}
170179
try
@@ -222,8 +231,11 @@ function parse_file(path::String)::Vector{Dict{String,Any}}
222231

223232
sig_body = replace(join(buf, " "), r"\s+" => " ")
224233
# Bound the recorded signature length to keep JSONL rows compact.
234+
# Use `first(s, n)` for a character-count slice that is safe
235+
# against multi-byte UTF-8 (cubical / unimath use many Unicode
236+
# symbols — `sig_body[1:400]` would crash mid-glyph).
225237
if length(sig_body) > 400
226-
sig_body = sig_body[1:400] * "…"
238+
sig_body = first(sig_body, 400) * "…"
227239
end
228240

229241
# Keep any well-formed top-level signature. The PROOFY gate
@@ -239,7 +251,9 @@ function parse_file(path::String)::Vector{Dict{String,Any}}
239251
"theorem" => name,
240252
"goal" => sig_body,
241253
"context" => ctx,
242-
"source" => "agda-stdlib/" * relpath(path, AGDA_ROOT),
254+
"source" => isempty(corpus_label) ?
255+
("agda-stdlib/" * relpath(path, REPO_ROOT)) :
256+
(corpus_label * "/" * relpath(path, corpus_root)),
243257
))
244258
end
245259

@@ -279,24 +293,40 @@ function main()
279293
println("EXTRACT AGDA")
280294
println("=" ^ 60)
281295

282-
isdir(AGDA_SRC) || error("Missing $AGDA_SRC — vendor agda-stdlib first.")
283-
284-
files = String[]
285-
for (root, _, names) in walkdir(AGDA_ROOT)
286-
for n in names
287-
endswith(n, ".agda") && push!(files, joinpath(root, n))
296+
# Walk every Agda corpus that is present on disk. At least one
297+
# must exist — agda-stdlib remains the canonical minimum.
298+
corpus_file_lists = Tuple{String,String,Vector{String}}[]
299+
for (corpus_root, corpus_label) in AGDA_CORPORA
300+
isdir(corpus_root) || continue
301+
these = String[]
302+
for (root, _, names) in walkdir(corpus_root)
303+
for n in names
304+
if endswith(n, ".agda") || endswith(n, ".lagda.md")
305+
push!(these, joinpath(root, n))
306+
end
307+
end
288308
end
309+
push!(corpus_file_lists, (corpus_root, corpus_label, these))
310+
println(" $(corpus_label): $(length(these)) files under $(corpus_root)")
311+
end
312+
if isempty(corpus_file_lists)
313+
error("No Agda corpora found — vendor agda-stdlib (minimum) first.")
289314
end
290-
println("Scanning $(length(files)) .agda files under $AGDA_ROOT (incl. doc/, dev/, .github/)")
315+
total_files = sum(length(xs[3]) for xs in corpus_file_lists)
316+
println("Scanning $(total_files) Agda source files across $(length(corpus_file_lists)) corpora")
291317

292318
all_records = Dict{String,Any}[]
293319
parsed_ok = 0
294-
for f in files
295-
recs = parse_file(f)
296-
if !isempty(recs)
297-
parsed_ok += 1
298-
append!(all_records, recs)
320+
for (corpus_root, corpus_label, files) in corpus_file_lists
321+
corpus_start = length(all_records)
322+
for f in files
323+
recs = parse_file(f, corpus_root, corpus_label)
324+
if !isempty(recs)
325+
parsed_ok += 1
326+
append!(all_records, recs)
327+
end
299328
end
329+
println(" $(corpus_label): $(length(all_records) - corpus_start) declarations kept")
300330
end
301331
println("Files with keepable decls: $parsed_ok")
302332
println("Raw declarations: $(length(all_records))")
@@ -323,10 +353,10 @@ function main()
323353
stats = Dict{String,Any}(
324354
"prover" => "Agda",
325355
"total_proofs" => total,
326-
"files_scanned" => length(files),
356+
"files_scanned" => total_files,
327357
"files_with_proofs" => parsed_ok,
328358
"id_range" => [START_ID, nid - 1],
329-
"source" => "external_corpora/agda-stdlib",
359+
"corpora" => [label for (_root, label) in AGDA_CORPORA if isdir(_root)],
330360
"extraction_date" => string(today()),
331361
"extractor" => "scripts/extract_agda.jl",
332362
)

0 commit comments

Comments
 (0)