diff --git a/daslib/ast_boost.das b/daslib/ast_boost.das index 8991ef8604..a300be9e42 100644 --- a/daslib/ast_boost.das +++ b/daslib/ast_boost.das @@ -1302,7 +1302,7 @@ def private debug_expression_impl(var writer : StringBuilderWriter; expr : Expre unsafe { p8 = (reinterpret(expr)) + int(offset) } - if (tstr == "$::dasvector`ptr`Expression") { + if (tstr |> ends_with("::dasvector`ptr`Expression")) { let pv = unsafe(reinterpret(p8)) if (!empty(*pv)) { let ts = repeat(" ", tabs + 2) @@ -1316,7 +1316,7 @@ def private debug_expression_impl(var writer : StringBuilderWriter; expr : Expre } writer |> write("]") } - } elif (tstr == "$::dasvector`ptr`Variable") { + } elif (tstr |> ends_with("::dasvector`ptr`Variable")) { let pv = unsafe(reinterpret(p8)) if (!empty(*pv)) { let ts = repeat(" ", tabs + 2) diff --git a/doc/REVIEW.md b/doc/REVIEW.md index b5d3d53170..fb09af2fd3 100644 --- a/doc/REVIEW.md +++ b/doc/REVIEW.md @@ -12,7 +12,7 @@ reader sends that host and whether the host sets cookies.** **A diff that adds, to any text that reaches a built page - an authored `.rst` under `source`, a stub under `source/stdlib/handmade`, or page text a `reflections` generator writes - anything -a reader is told to fetch, run, or type - an address, a file, a command, a flag, an API a -`.das_package` manifest calls - states, in the PR body, that each exists and works at merge, and +a reader is told to fetch, run, or type - an address, a file, a command, a flag, an API, a name +a `.das_package` manifest resolves - states, in the PR body, that each exists and works at merge, and where that was checked** - the build proves the page renders, never that what it points a reader at is there. diff --git a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst index 845b1660ab..6aa693a3ca 100644 --- a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst +++ b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst @@ -1 +1 @@ -Returns ``true`` if a module with the given name is registered, ``false`` otherwise. +Returns ``true`` if a module with the given name is registered, or waits in a ``.das_module.manifest`` row for the first ``require`` that names it, ``false`` otherwise. Unlike ``typeinfo builtin_module_exists(mod)``, which takes a bare module name and loads a waiting module, it loads nothing. diff --git a/include/daScript/ast/REVIEW.md b/include/daScript/ast/REVIEW.md new file mode 100644 index 0000000000..471a307ff7 --- /dev/null +++ b/include/daScript/ast/REVIEW.md @@ -0,0 +1,11 @@ +# AST Headers Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: +`src/ast/ARCHITECTURE.md` (repo root). + +- **A diff that changes which module a bind registers into, or the name it registers under - + `vectorHomeModule`, `typeFactory>::make`, `registerVectorFunctions`, the name a + `ManagedVectorAnnotation` or `ManagedStructureAnnotation` takes (`ast_handle.h`) - bumps + `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` (repo root), in the + same change.** The JIT's DLL cache key folds the codegen version and each function's AST hash, + never the module an extern lives in, so a cached DLL binds the old name and crashes on the hit. diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 53b664c3fb..b2bf8374af 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1106,6 +1106,12 @@ namespace das DAS_API bool isValidBuiltinName ( const string & name, bool canPunkt = false ); + // src/ast/ARCHITECTURE.md sec.2 + typedef bool (*DeferredModuleLoader) ( const string & name ); + DAS_API void setDeferredModuleLoader ( DeferredModuleLoader loader ); + DAS_API DeferredModuleLoader getDeferredModuleLoader (); + DAS_API bool guardModuleAvailable ( const string & name ); + class DAS_API Module { public: Module ( const string & n = "" ); @@ -1154,6 +1160,7 @@ namespace das static Module * require ( const string & name ); static Module * requireEx ( const string & name, bool allowPromoted, const string & requireName = string(), const string & expectedFileName = string() ); static void Initialize(); + static bool InitializeDependencies ( string & notInitialized ); static void CollectFileInfo(das::vector &accesses); static void Shutdown( bool dumpHandleLeaks = true ); // Runtime-only shutdown — for standalone exes built with `daslang -exe`, diff --git a/include/daScript/ast/ast_handle.h b/include/daScript/ast/ast_handle.h index 3032233a2f..6e7356b58b 100644 --- a/include/daScript/ast/ast_handle.h +++ b/include/daScript/ast/ast_handle.h @@ -748,6 +748,14 @@ namespace das } }; + // src/ast/ARCHITECTURE.md sec.2 + __forceinline Module * vectorHomeModule ( const TypeDeclPtr & elem, const ModuleLibrary & library ) { + auto t = elem; + while ( t && t->isPointer() && t->firstType ) t = t->firstType; + if ( t && t->isHandle() && t->annotation && t->annotation->module ) return t->annotation->module; + return library.front(); + } + template struct typeFactory> { using VT = vector; @@ -759,7 +767,7 @@ namespace das ann->cppName = "das::vector<" + describeCppType(declT, CpptSubstitureRef::no, CpptSkipRef::no, CpptSkipConst::no, CpptRedundantConst::yes, ChooseSmartPtr::yes) + ">"; - auto mod = library.front(); + auto mod = vectorHomeModule(declT, library); mod->addAnnotation(ann); registerVectorFunctions>::init(mod,library, declT->canCopy(), diff --git a/include/daScript/ast/ast_serializer.h b/include/daScript/ast/ast_serializer.h index ce0466c0c5..76d94e9b56 100644 --- a/include/daScript/ast/ast_serializer.h +++ b/include/daScript/ast/ast_serializer.h @@ -258,7 +258,7 @@ namespace das { AstSerializer & serializeModule ( Module & module, bool already_exists ); static constexpr uint32_t getVersion () { - return 204; // 204: the record header stamps the source by content hash, not mtime; the policy stream carries every CodeOfPolicies field + return 205; // 205: a vector of a handled element streams under the element's module (204: the record header stamps the source by content hash; the policy stream carries every CodeOfPolicies field) } void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept; diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index 53787740b3..2eb753c387 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -40,4 +40,10 @@ DAS_API void begin_dynamic_module_recording(); DAS_API void end_dynamic_module_recording(vector & rows, bool & optOut); DAS_API void replay_native_path(const char * mod_name, const char * src, const char * dst); DAS_API void replay_dynamic_module(const char * path, const char * cpp_class, int on_error); +DAS_API void defer_dynamic_module(const char * path, const char * cpp_class, int on_error, const char * das_name); +DAS_API bool load_deferred_dynamic_module(const char * das_name); +DAS_API size_t load_all_deferred_dynamic_modules(); // the count it attempted +DAS_API bool is_dynamic_module_deferred(const char * das_name); +DAS_API void clear_deferred_dynamic_modules(); // the rows are the scan's: cleared at scan start and at shutdown +DAS_CC_API void ignore_dynamic_module_manifests(bool ignore); } diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 6dd7d6a318..2dffc9aad9 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -36,10 +36,13 @@ Option resolution before the first timer, and log lines, are not work. - **A change that can alter the machine code the JIT's DLL or split-obj cache serves back for - identical inputs bumps `LLVM_JIT_CODEGEN_VERSION`** (`daslib/llvm_jit_run.das`; what counts - as emitting: `ARCHITECTURE.md` sec.1.2). The constant folds only into those two keys; - selecting among existing generators' `[llvm_code]` arguments - the `[tune]` stamping - is not - such a change, because stamped arguments fold into the cache keys per function. + identical inputs - IR generation, target-machine setup, a `[llvm_code]` generator body, or the + call ABI the generated code binds: function signatures, the name scheme, the prologue, the + externs the install phase binds - bumps `LLVM_JIT_CODEGEN_VERSION`** + (`daslib/llvm_jit_run.das`); selecting among existing generators' `[llvm_code]` arguments, the + `[tune]` stamping, is not such a change. The caches are addressed by the AST hashes and this + constant, so such a change without the bump serves the old machine code back + (`ARCHITECTURE.md` sec.1.2). - **A diff that adds an environment or config input to a JIT cache key folds it inside `jit_env_salt` (`daslib/llvm_jit_run.das`), never directly into either JIT key - the DLL diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 7806e05c47..21f2784167 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -38,7 +38,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x76ul // 0x76: a statement after a terminator in the same block list lands in its own dead block instead of after the ret (0x75: the global-offset lookup is memory(none) and emitted at its use site - LLVM dedups and hoists it, an untaken branch never pays it; a solid-context global resolves once per function at entry (0x74: a runtime-only exe emits no register_native_path rows, and a whole-lib exe emits them once (0x73: computed goto lowers to one switch with the trap as its default, not an icmp chain (0x72: policies.fast_math defaults to the host's float flags, so a fast-math host now JITs fast-math (0x71: a CPU class row's cpu is the arch's bare baseline, so a DAS_JIT_BASELINE build enables the row's set and nothing a level implies (0x70: the wasm feature string drops +relaxed-simd and the idot family keeps only the exact extmul + extadd_pairwise lowering on wasm SIMD128 (0x6f: the first wasm idot lowering; 0x6e: the aarch64 SDOT / SMMLA tables gate on DotProd / i8mm, not the arch alone, and the force env reaches the generic exe machine (0x6d: the inline polynomial rail carries NaN: tanh selects the operand back over its ordered clamp, and the sincos quadrant / tan octant convert through llvm.fptosi.sat instead of poisoning on NaN and out-of-range (0x6c: aarch64 vector tan/exp2/log2/log/pow join the inline polynomial rail bit-exactly with the interpreter, sinh/cosh/tanh ride the exp one; 0x6b: aarch64 vector sin/cos ride the inline polynomial; 0x6a: srem/urem for 32-bit %; 0x69: every string argument of an extern is substituted, not just the ones which asked) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x77ul // 0x77: a vector of a handled element type registers into the element's module, so the externs a DLL binds by mangled name moved out of `$` (0x76: a statement after a terminator in the same block list lands in its own dead block instead of after the ret (0x75: the global-offset lookup is memory(none) and emitted at its use site - LLVM dedups and hoists it, an untaken branch never pays it; a solid-context global resolves once per function at entry (0x74: a runtime-only exe emits no register_native_path rows, and a whole-lib exe emits them once (0x73: computed goto lowers to one switch with the trap as its default, not an icmp chain (0x72: policies.fast_math defaults to the host's float flags, so a fast-math host now JITs fast-math (0x71: a CPU class row's cpu is the arch's bare baseline, so a DAS_JIT_BASELINE build enables the row's set and nothing a level implies (0x70: the wasm feature string drops +relaxed-simd and the idot family keeps only the exact extmul + extadd_pairwise lowering on wasm SIMD128 (0x6f: the first wasm idot lowering; 0x6e: the aarch64 SDOT / SMMLA tables gate on DotProd / i8mm, not the arch alone, and the force env reaches the generic exe machine (0x6d: the inline polynomial rail carries NaN: tanh selects the operand back over its ordered clamp, and the sincos quadrant / tan octant convert through llvm.fptosi.sat instead of poisoning on NaN and out-of-range (0x6c: aarch64 vector tan/exp2/log2/log/pow join the inline polynomial rail bit-exactly with the interpreter, sinh/cosh/tanh ride the exp one; 0x6b: aarch64 vector sin/cos ride the inline polynomial; 0x6a: srem/urem for 32-bit %; 0x69: every string argument of an extern is substituted, not just the ones which asked)) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/plans/review_md_splits.md b/plans/review_md_splits.md new file mode 100644 index 0000000000..03443d9cce --- /dev/null +++ b/plans/review_md_splits.md @@ -0,0 +1,28 @@ +# Checklist splits owed - a docs-only PR at the end of the startup chain + +Four rules the per-PR audits found working but hard to apply: each fuses two checks, or takes +the wrong subject. Splitting is a restructuring, so it rides its own docs-only PR (those merge +gate-free) rather than a code PR. Meaning-preserving; the dragon reads the result. + +- `utils/REVIEW.md` - the three CI-row rules take "an arm" as subject while the definition + above them says one arm may hold both assertions a CI row can run and assertions none can, + so a mixed arm fits neither rule. Make the assertion set the subject: "the load-bearing + assertions a CI row can run ship with a row that executes them on every pull request; the + ones no CI row can run ship with a compile-check row and a recorded run". Then split each + 60-word rule into a trigger sentence and a duty sentence, and drop "one arm may hold both", + which follows. +- `doc/REVIEW.md` - the fetch/run/type rule's subject and verb sit 45 words apart in two nested + dash lists. Split into the duty ("A diff that adds anything a reader is told to fetch, run, + or type to text that reaches a built page states, in the PR body, that each exists and works + at merge, and where that was checked"), then the scope sentence naming the three page kinds + and the `.. include::` arm (a diff touching only an included `.md` never surfaces this + checklist in the folder walk, so that arm needs a routing line in the opening), then the WHY. +- `modules/dasLLVM/REVIEW.md` - the module-cache flag rule fuses two checks (a child asserting + a compile-time macro line spawns with `-no-module-cache`; a test whose subject is the cache + pins its own file with `-module-cache `) behind a five-clause qualifier and a five-line + mechanism tail. One rule per check; the replay mechanism moves to `ARCHITECTURE.md`. Same + shape, weaker: the intrinsic emitter reference-cell rule fuses three obligations. +- `doc/source/stdlib/handmade/REVIEW.md` - the type-file format rule (one type-description + line, one line per member, declaration order) carries a placement duty in a subordinate + clause (a property's description is its own `function-` file). Two rules; drop "handmade + file in this folder" from the head, which the folder walk already scopes. diff --git a/skills/dynamic_modules.md b/skills/dynamic_modules.md index 4bbd5fcf15..bb5f1d0f38 100644 --- a/skills/dynamic_modules.md +++ b/skills/dynamic_modules.md @@ -100,8 +100,20 @@ of the machine, a variant picked by hardware - opts out by calling `no_manifest( `initialize`; it then runs on every start. Everything a descriptor registers is replayed, so the opt-out is only for a descriptor whose answer changes between starts. -`DAS_TRACE_MODULE_LOAD=1` prints one line per descriptor saying whether it was replayed or -compiled, and why. +A replayed C++ module loads at the first `require` naming it, not in the scan: the require +walk loads the row and runs the `initDependencies` fixed point; a module needing another +deferred one pulls the whole deferred set in. A program pays for the C++ modules it requires. +Two consequences: + +- `require ?mod x` and `typeinfo builtin_module_exists(mod)` still ask whether the build has + `mod`: a guard loads a waiting module, so a cold start and a warm one answer alike. +- A tool that enumerates the process's modules (the MCP server) runs with + `-ignore-manifest`: no manifest read or written, every descriptor compiles, every C++ module + loads on start. `has_module(name)` (`daslib/rtti`) answers loaded-or-deferred, so a sweep + gate asking what the tree has keeps its answer. + +`DAS_TRACE_MODULE_LOAD=1` prints one line per descriptor (replayed or compiled, why, the +deferred count) and one per deferred module as a require loads it. ## Adding a `.das` file to an existing module needs the same edit diff --git a/skills/internal/environment_variables.md b/skills/internal/environment_variables.md index 518a1b9f78..8eda8d5971 100644 --- a/skills/internal/environment_variables.md +++ b/skills/internal/environment_variables.md @@ -72,7 +72,7 @@ and unknown codes are harmless. The `-no-lint` command-line flag skips the lint |---|---|---| | `DAS_GC_STAGE_REPORT` | flag | Report gc_node deltas per compilation stage - the first thing to reach for on a `GC APP LEAK` at exit. | | `DAS_GC_BREAK_ON_ID` | number | Break when the gc_node with this id is allocated. Pair it with the id from a leak report. | -| `DAS_TRACE_MODULE_LOAD` | flag | Log every module as it loads, with its resolved path - the fastest way to see which of two same-named modules actually won - and one line per `.das_module` descriptor saying whether the scan replayed its manifest or compiled it, and why (`src/ast/ARCHITECTURE.md` sec.2). | +| `DAS_TRACE_MODULE_LOAD` | flag | Log every module as it loads, with its resolved path - the fastest way to see which of two same-named modules actually won - and one line per `.das_module` descriptor: replayed (with its time, shared-module load share and deferred count) or compiled, and why; a deferred module's load prints `require : loading the deferred ` (`src/ast/ARCHITECTURE.md` sec.2). | ## Ambient variables daslang reads but does not own diff --git a/skills/mcp_tools.md b/skills/mcp_tools.md index 032e92e07a..66bf1f52e3 100644 --- a/skills/mcp_tools.md +++ b/skills/mcp_tools.md @@ -72,7 +72,7 @@ The daslang MCP server (`utils/mcp/main.das`) exposes compiler diagnostics, prog **`shutdown` tool.** Shuts down the MCP server process. Claude Code auto-restarts it, picking up code changes to `.das` tool files. Tool registration changes (adding/removing tools) still require a manual MCP restart. -**Configuration.** Configure `.mcp.json` with `"command"` pointing at the daslang binary (`bin/daslang` on Windows MSVC, `build/daslang` on Linux/macOS, `bin/daslang` for the installed SDK), `"args": ["utils/mcp/main.das"]`. See `utils/mcp/README.md` for details and Claude Code permissions. +**Configuration.** Configure `.mcp.json` with `"command"` pointing at the daslang binary (`bin/daslang` on Windows MSVC, `build/daslang` on Linux/macOS, `bin/daslang` for the installed SDK), `"args": ["-ignore-manifest", "utils/mcp/main.das"]` (the flag loads every C++ module on start; the server enumerates them). See `utils/mcp/README.md` for details and Claude Code permissions. **Fresh checkouts / worktrees.** `.mcp.json`, `sgconfig.yml`, `bin/`, and the tree-sitter grammar lib are all gitignored, so a new `git worktree add` (or clone) has no daslang MCP at all. Bootstrap it with `daslang utils/mcp/setup.das -- --root ` - it configures `build/` on the cmake generator of the tree running the setup (platform default when that tree has no `build/CMakeCache.txt`), builds a worktree-local binary (+ grammar), copies the platform `sgconfig.yml`, and merges a `daslang` entry into `.mcp.json` (adds no new secrets; existing servers, including any secret env blocks, are preserved as-is). `--no-build` skips the build. Restart the session to pick it up. diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 94e87b99a0..d9ef4373ee 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -78,10 +78,56 @@ than an escaped form the reader would have to decode, and goes through a `.tmp` Recording is armed around one descriptor run: each builtin appends the arguments it actually received, in order, and `register_dynamic_module` records its call whatever the outcome and adds -the das-visible module name once the load succeeded. Replay calls the same two builtins with the -recorded rows in recorded order, so the Quiet deferral and the post-scan retry of a sibling -`DT_NEEDED` dlopen behave as on a compiled start. `no_manifest()` inside `initialize` marks the -descriptor as one that runs on every start: its manifest carries the stamp and the flag and no -rows, and is not rewritten. With `DAS_TRACE_MODULE_LOAD=1` the scan prints one line per -descriptor - `replayed N row(s)`, `compiled (), manifest written (N row(s))`, -`compiled (no_manifest)`, or why a manifest was not written. +the das-visible module name once the load succeeded. Replay registers every `np` row as +recorded. A `dm` row carrying that name is not loaded by the scan: the row waits under the name +(`defer_dynamic_module`), and the load happens at the first require that names it. The +prerequisite walk (`getPrerequisits`) finds no module under the name and asks the loader the +scan installed (`setDeferredModuleLoader`); the loader dlopens and registers the module, runs the +`initDependencies` fixed point that `Module::Initialize` runs (`Module::InitializeDependencies`) over +the grown list, and when a module reports it cannot initialize - what it needs is deferred too - +brings every deferred module in and runs the fixed point again, which is the set an eager start +has. A row whose own dlopen fails takes the same road - every deferred module comes in, the +pending retry runs - and the require then finds the module or fails as a cold start would. +The deserializer (`Program::serialize` reading, and the module-cache record header) fetches a +builtin module by name the same way: a stream written where a module had loaded lazily is read +where nothing required it yet. `Module::Initialize` takes it too: on a half-warm tree - one descriptor compiled cold, so its +module loaded on start, beside replayed ones whose rows wait - the eager fixed point fails on +the first pass, brings every deferred module in and runs once more. A tree is half-warm +whenever two processes warm it at once, which parallel AOT batches do. The +rows and the loader are the scan's: `require_dynamic_modules` clears the rows before its +walk and installs the loader, and `Module::Shutdown` clears both, so an environment that +follows sees neither the last one's rows nor its loader. A parse that no prerequisite walk +precedes - the `compile` of a string - meets a deferred module at the parser's own require +(`ast_requireModule`) and loads it there; the loader puts the parse's program back as the +bound one, since a module's builtin das part parses under a program of its own. The load runs under one gc root of its own with the thread root's nodes parked meanwhile, +because a constructor's nodes go to the active root while a builtin das module it compiles +dumps its leftovers on the thread root, and a collect stops at a node owned by another root; +after the load every module, not only the new ones, collects from that root, since a +constructor registers into modules that exist already, and the rest is swept. A `dm` row +with no name - the recording start's load failed - replays as recorded, so the Quiet deferral +and the post-scan retry of a sibling `DT_NEEDED` dlopen behave as on a compiled start. A +require guard (`require ?mod`) and `builtin_module_exists` ask whether the build has the +module (`guardModuleAvailable`): linked in, or waiting in a manifest row, which the guard +loads then - so `require ?das_metal metal/das_metal_boost` still means "on a build with +Metal", a cold start and a warm start answer alike, and `llvm`, a witness module no das file +requires unguarded, comes in through the guards `daslib/tune` places on it. A load adds +nothing to `$`: a module-cache record carries each builtin module's cumulative hash of +mangled names, and a process that loaded a different set of C++ modules would otherwise fail +every record on `$`, so a `vector` of a handled element registers into the element's +module (`vectorHomeModule`, `ast_handle.h`) whichever module builds it - a module that exists +already, when the element is another module's - and only a vector of a builtin element lands +in `$`, which every library lists first because +`ModuleLibrary::addModule` puts a module's dependencies before it. The described name of such +a vector carries that module - ``ast::dasvector`ptr`Expression``, not ``$::...`` - so code +that names one compares the part after `::` (`daslib/ast_boost`'s printer). `-ignore-manifest` +reads and writes no manifest: every descriptor compiles and every C++ module loads on start, +the form a tool that enumerates modules - the MCP server - runs under. +`no_manifest()` inside `initialize` marks the descriptor as one that runs on every start: its +manifest carries the stamp and the flag and no rows, and is not rewritten. With +`DAS_TRACE_MODULE_LOAD=1` the scan prints one line per descriptor - `replayed N row(s) in +(shared module load , deferred K)`, `compiled (), manifest written (N row(s))`, +`compiled (no_manifest)`, `compiled (manifests ignored)`, or why a manifest was not written - +and a deferred load prints `[module] require : loading the deferred `, the +fallback `[module] loading every deferred module (K)`. A replayed descriptor's time is its +manifest read plus its rows, and the second number is the share the `.shared_module` dlopen and +module constructor took. diff --git a/src/ast/REVIEW.das b/src/ast/REVIEW.das index 84ea19098d..ffefe338d1 100644 --- a/src/ast/REVIEW.das +++ b/src/ast/REVIEW.das @@ -12,6 +12,8 @@ let private SOURCE = "src/ast/ast_parse.cpp" let private READER = "trySerializeProgramModule" let private GUARD = "quietCache" let private PRINT = "logs <<" +let private CACHE_READ_PRINT = "logs << \"cache read" +let private TIMING_GUARD = "log_module_compile_time" def private count_char(line : string; ch : int) : int { var n = 0 @@ -23,35 +25,22 @@ def private count_char(line : string; ch : int) : int { return n } -// Every print inside the reader sits under a quietCache guard - on the guard's own line, or in -// the block a guard line opens. Brace depth is tracked from the reader's opening line to its -// closing brace; a guard line that opens a block guards every line until the depth drops back. -def private check_reader_prints { - let text = fread(SOURCE) - if (empty(text)) { - gate_finding(SOURCE, "cannot read the file the gate scans") - return - } - var inscope lines <- split(text, "\n") - var start = -1 - for (ln, i in lines, count()) { - if (find(ln, READER) >= 0 && find(ln, "(") >= 0 && find(ln, ";") < 0 && find(ln, "return") < 0 && find(ln, "if") < 0) { - start = i - break - } - } - if (start < 0) { - gate_finding(SOURCE, "no definition of {READER} found - the gate scans its body for module-cache diagnostics that print without a {GUARD} guard") - return - } +//! Every print matching `marker` from `start` on sits under one of `guards` - on the guard's own +//! line, or in the block a guard line opens. Brace depth is tracked from `start`; a guard line +//! that opens a block guards every line until the depth drops back. With `oneFunction` the walk +//! stops at the closing brace of the function `start` opens. +def private check_guarded_prints(lines : array; start : int; oneFunction : bool; marker : string; guards : array; fix : string) { var depth = 0 var opened = false var guardDepth = -1 for (i in range(start, length(lines))) { let ln = lines[i] - let guardLine = find(ln, GUARD) >= 0 && find(ln, "if") >= 0 - if (find(ln, PRINT) >= 0 && !guardLine && guardDepth < 0) { - gate_finding("{SOURCE}:{i + 1}", "a module-cache read diagnostic prints without a {GUARD} guard - the default cache is on unasked for an ordinary run, so this line is output every user sees; wrap it in `if ( !serializer_read->{GUARD} )`") + var guardLine = false + for (g in guards) { + guardLine ||= find(ln, g) >= 0 && find(ln, "if") >= 0 + } + if (find(ln, marker) >= 0 && !guardLine && guardDepth < 0) { + gate_finding("{SOURCE}:{i + 1}", "a module-cache read diagnostic prints without a guard - the default cache is on unasked for an ordinary run, so this line is output every user sees; {fix}") } let opens = count_char(ln, '{') let closes = count_char(ln, '}') @@ -65,12 +54,34 @@ def private check_reader_prints { if (guardDepth >= 0 && depth < guardDepth) { guardDepth = -1 } - if (opened && depth <= 0) { + if (oneFunction && opened && depth <= 0) { break } } } +def private check_reader_prints { + let text = fread(SOURCE) + if (empty(text)) { + gate_finding(SOURCE, "cannot read the file the gate scans") + return + } + var inscope lines <- split(text, "\n") + var start = -1 + for (ln, i in lines, count()) { + if (find(ln, READER) >= 0 && find(ln, "(") >= 0 && find(ln, ";") < 0 && find(ln, "return") < 0 && find(ln, "if") < 0) { + start = i + break + } + } + if (start < 0) { + gate_finding(SOURCE, "no definition of {READER} found - the gate scans its body for module-cache diagnostics that print without a {GUARD} guard") + return + } + check_guarded_prints(lines, start, true, PRINT, [GUARD], "wrap it in `if ( !serializer_read->{GUARD} )`") + check_guarded_prints(lines, 0, false, CACHE_READ_PRINT, [GUARD, TIMING_GUARD], "wrap it in `if ( !serializer_read->{GUARD} )` or `if ( policies.{TIMING_GUARD} )`") +} + [export] def main { check_reader_prints() diff --git a/src/ast/REVIEW.md b/src/ast/REVIEW.md index cffe38d4b6..66ce2fb022 100644 --- a/src/ast/REVIEW.md +++ b/src/ast/REVIEW.md @@ -4,23 +4,38 @@ `ARCHITECTURE.md`. - **Weakening `REVIEW.das` (beside this file) is a defect:** dropping its scan of the prints in - `trySerializeProgramModule` (`ast_parse.cpp`, `ARCHITECTURE.md` sec.1), or a finding text that - no longer names what failed. What the gate enforces is read from the gate itself. - -- **A diff that moves a module-cache read diagnostic out of `trySerializeProgramModule` - into - a helper it calls, or another function - extends `REVIEW.das`'s scan to the new home in the - same change.** The gate scans that one function's body, so a print moved out of it is a print - the gate no longer checks, and an ungated line there is output every user of the default cache - sees. - -- **A diff that changes the manifest format - a field added, removed, reordered or re-typed in - `read_manifest` or `write_manifest` (`dyn_modules.cpp`), or a line added beside the key lines - `stamp`, `dll`, `root`, `dasroot`, `target` and `dep` - bumps the version in `MANIFEST_HEADER` - in the same change.** A reader accepts a manifest whose first line equals `MANIFEST_HEADER`, - so without the bump an older manifest decodes the changed bytes as a wrong registration with - no diagnostic. + `trySerializeProgramModule` (`ast_parse.cpp`, `ARCHITECTURE.md` sec.1) or of the cache-read + lines elsewhere in that file, or a finding text that no longer names what failed. What the gate + enforces is read from the gate itself. + +- **A diff that adds a module-cache read diagnostic outside `trySerializeProgramModule`, or moves + one out of it - into a helper it calls, or another function - extends `REVIEW.das`'s scan to the + new home in the same change, and the line prints only when the user asked: the serializer's + `quietCache` off, or `log_module_compile_time` set.** A print the gate does not scan is a print + nobody checks, and an ungated line there is output every user of the default cache sees. + +- **A diff that changes what a manifest written by an earlier binary replays to - a field added, + removed, reordered or re-typed in `read_manifest` or `write_manifest` (`dyn_modules.cpp`), a + line added beside the key lines `stamp`, `dll`, `root`, `dasroot`, `target` and `dep`, or a + change to what an existing row registers - bumps the version in `MANIFEST_HEADER` in the same + change.** A reader accepts a manifest whose first line equals `MANIFEST_HEADER`, so without the + bump an older manifest decodes the changed bytes as a wrong registration with no diagnostic. - **A diff that gives `read_manifest` a new kind of row - one it pushes into `rows` - gives the - replay loop in `init_dyn_modules` (`dyn_modules.cpp`) a branch for it in the same change.** The loop - dispatches on one flag with `replay_native_path` as the other arm, so a kind it does not know - replays as a native path. + replay loop in `init_dyn_modules` (`dyn_modules.cpp`) a branch for it in the same change.** The + loop dispatches on `row.dynamic` and then on whether the row carries a das-visible name, so a + kind it does not know replays as a native path or waits under a name nothing requires. + +- **Removing the `setDeferredModuleLoader` call from `require_dynamic_modules` + (`dyn_modules.cpp`) is a defect.** A descriptor compiled during the scan can require a module + an earlier replay deferred, and with no loader installed that require fails. + +- **A diff that moves the `setDeferredModuleLoader` call in `require_dynamic_modules` keeps it + above the first `init_modules_for_folder` call, in the same change.** That call compiles the + descriptors, and a descriptor can require a module an earlier replay deferred. + +- **A diff that changes how `Function::getMangledName` (`ast.cpp`) forms a name, or which module + `Module::addFunction` (`ast_module.cpp`) files a builtin function under, bumps + `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` (repo root), in the + same change.** The JIT's DLL cache key folds the codegen version and each function's AST hash, + never the name an extern binds under, so a cached DLL binds the old name and crashes on the hit. diff --git a/src/ast/ast_infer_type.cpp b/src/ast/ast_infer_type.cpp index 13eabd93ae..80f5b6c633 100644 --- a/src/ast/ast_infer_type.cpp +++ b/src/ast/ast_infer_type.cpp @@ -2846,7 +2846,7 @@ namespace das { // also accepts shared das modules compiled earlier in the process auto mod = Module::requireEx(evar->name, true); reportAstChanged(); - return new ExprConstBool(expr->at, mod != nullptr); + return new ExprConstBool(expr->at, mod != nullptr || guardModuleAvailable(evar->name)); } else { error("unsupported module name subexpression ", expr->subexpr->__rtti, "", expr->at, CompilationError::invalid_typeinfo_module_subexpression); diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index 3678db73cf..38b14624b7 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -7,6 +7,7 @@ #include "daScript/daScriptModule.h" #include "daScript/misc/handle_registry.h" #include "daScript/simulate/simulate_fusion.h" +#include "daScript/ast/dyn_modules.h" #include @@ -135,16 +136,8 @@ namespace das { } } - void Module::Initialize() { - daScriptEnvironment::ensure(); - static bool atexit_registered = (atexit(daslang_atexit_audit), true); - (void)atexit_registered; - g_envTotal ++; - - if (daScriptEnvironment::getBound()->modules == nullptr) { - DAS_FATAL_ERROR("No modules founds. You should add modules before call that function."); - } - + bool Module::InitializeDependencies ( string & notInitialized ) { + notInitialized.clear(); // InitDependencies do not add new modules. vector mod_state; bool any = true; @@ -167,24 +160,56 @@ namespace das { } } } - if (!any) { - // Some modules was not initialized! - size_t i = 0; - string error = ""; - for ( auto m = daScriptEnvironment::getBound()->modules; m ; m = m->next, i++ ) { - DAS_ASSERT(mod_state.size() == i); - if (!mod_state.at(i)) { - error += " " + m->name; - } + if ( all ) return true; + size_t i = 0; + for ( auto m = daScriptEnvironment::getBound()->modules; m ; m = m->next, i++ ) { + DAS_ASSERT(mod_state.size() > i); + if (!mod_state.at(i)) { + notInitialized += " " + m->name; } - // A module that never initializes usually failed Module::require on a dependency - // whose .shared_module dlopen failed QUIETLY during the startup scan. Name those - // load failures (with their dlerror) so this doesn't read as a missing C++ module. - auto pendingNote = describe_pending_dynamic_modules(); - if ( !pendingNote.empty() ) { - error += "\nnote: these dynamic modules failed to load - an unresolved dependency may live in one of them:\n" + pendingNote; + } + // A module that never initializes usually failed Module::require on a dependency + // whose .shared_module dlopen failed QUIETLY during the startup scan. Name those + // load failures (with their dlerror) so this doesn't read as a missing C++ module. + auto pendingNote = describe_pending_dynamic_modules(); + if ( !pendingNote.empty() ) { + notInitialized += "\nnote: these dynamic modules failed to load - an unresolved dependency may live in one of them:\n" + pendingNote; + } + return false; + } + + static DeferredModuleLoader g_deferredModuleLoader = nullptr; + + void setDeferredModuleLoader ( DeferredModuleLoader loader ) { + g_deferredModuleLoader = loader; + } + + DeferredModuleLoader getDeferredModuleLoader () { + return g_deferredModuleLoader; + } + + bool guardModuleAvailable ( const string & name ) { + if ( Module::requireEx(name, false) ) return true; + return g_deferredModuleLoader && g_deferredModuleLoader(name) && Module::requireEx(name, false); + } + + void Module::Initialize() { + daScriptEnvironment::ensure(); + static bool atexit_registered = (atexit(daslang_atexit_audit), true); + (void)atexit_registered; + g_envTotal ++; + + if (daScriptEnvironment::getBound()->modules == nullptr) { + DAS_FATAL_ERROR("No modules founds. You should add modules before call that function."); + } + + string notInitialized; + if ( !InitializeDependencies(notInitialized) ) { + // a half-warm tree: a descriptor compiled cold loaded its module, its dependency's replayed row waits (ARCHITECTURE.md sec.2) + load_all_deferred_dynamic_modules(); + if ( !InitializeDependencies(notInitialized) ) { + DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); } - DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", error.c_str()); } // Collect reachable TypeDecl from thread root into module roots, sweep the rest. auto & threadRoot = gc_root::gc_get_thread_root(); @@ -229,6 +254,8 @@ namespace das { if ( dumpHandleLeaks ) handleRegistry_dumpAll(); // Free allocated structures for dynamic modules (unloads DLLs). delete daScriptEnvironment::getBound()->g_dyn_modules_resolve; + clear_deferred_dynamic_modules(); + setDeferredModuleLoader(nullptr); clearGlobalAotLibrary(); if ( resetFusion ) { diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index f978c52c31..331e1df35f 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -1,6 +1,7 @@ #include "daScript/misc/platform.h" #include "daScript/ast/ast.h" +#include "daScript/ast/dyn_modules.h" #include "daScript/ast/ast_infer_type.h" #include "daScript/ast/ast_serializer.h" #include "daScript/ast/ast_expressions.h" @@ -256,16 +257,16 @@ namespace das { // guarded optional require. Path guard (contains '/'): proceed only when // the guard's OWN file resolves — the rail for pure-das packages (nothing // C++ to guard on) and cross-package dependency witnesses. Plain-name - // guard: STRICT — proceed only when the guard module is registered (a - // linked C++ module); no target-resolvability fallback (module source - // dirs exist in every checkout regardless of build config). Otherwise — - // skip silently. Must match ast_requireModule (parser_impl.cpp). + // guard: proceed only when the build has the module (guardModuleAvailable, + // src/ast/ARCHITECTURE.md sec.2); no target-resolvability fallback (module + // source dirs exist in every checkout regardless of build config). + // Otherwise skip silently. Must match ast_requireModule (parser_impl.cpp). if ( hasReqGuard && reqGuard.find('/')!=string::npos ) { auto ginfo = access->getModuleInfo(reqGuard, fi->name); if ( ginfo.fileName.empty() || !access->getFileInfo(ginfo.fileName) ) { continue; } - } else if ( hasReqGuard && Module::requireEx(reqGuard, false)==nullptr ) { + } else if ( hasReqGuard && !guardModuleAvailable(reqGuard) ) { continue; } bool isPublic = false; @@ -433,6 +434,15 @@ namespace das { } } module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); // try native with that name AGAIN (promoted?) + if ( !module ) { + // ARCHITECTURE.md sec.2 + if ( auto loader = getDeferredModuleLoader(); loader && loader(mod) ) { + module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); + if ( log && module ) { + *log << string(tab,'\t') << " loaded deferred shared module " << mod << "\n"; + } + } + } if ( !module ) { auto it_r = find_if(req.begin(), req.end(), [&] ( const ModuleInfo & reqM ) { return reqM.moduleName == mod; @@ -580,6 +590,9 @@ namespace das { static DAS_THREAD_LOCAL(int64_t) totInfer; static DAS_THREAD_LOCAL(int64_t) totOpt; static DAS_THREAD_LOCAL(int64_t) totM; + static DAS_THREAD_LOCAL(int64_t) totCacheRead; + static DAS_THREAD_LOCAL(int64_t) cntCacheRead; + static DAS_THREAD_LOCAL(int64_t) totCacheMacroSim; // deserialization may have left the active gc root pointing at (or the old program // owning) a module root that dies with the old program — repoint around the swap so @@ -1071,7 +1084,19 @@ namespace das { program->inferPassesUsed = 0; // reset once per module; inferTypesDirty accumulates across all inferTypes legs (incl. restartInfer) program->policies = policies; // before the cache read: the reader compares the record's policies against this compile's + auto & serializer_read = daScriptEnvironment::getBound()->serializer_read; + uint64_t macroSim0 = serializer_read ? serializer_read->totMacroTime : 0; if ( trySerializeProgramModule(program, access, fileName, libGroup, logs) ) { + auto readT = get_time_usec(time0); + auto macroSimT = int64_t(serializer_read->totMacroTime - macroSim0); + *totCacheRead += readT; + *cntCacheRead += 1; + *totCacheMacroSim += macroSimT; + if ( policies.log_module_compile_time ) { + logs << "cache read took " << (readT / 1000000.) << ", " << program->thisModule->name << " (" << fileName << ")"; + if ( macroSimT ) logs << " -- macro simulate " << (macroSimT / 1000000.); + logs << "\n"; + } return program; } else { // Serialization failed and the program changed, so set it for proper GC collection on exit. @@ -1800,6 +1825,9 @@ namespace das { *totInfer = 0; *totOpt = 0; *totM = 0; + *totCacheRead = 0; + *cntCacheRead = 0; + *totCacheMacroSim = 0; daScriptEnvironment::getBound()->macroTimeTicks = 0; vector req; vector missing; @@ -1933,6 +1961,7 @@ namespace das { auto totT = get_time_usec(time0); logs << "total compile took " << (totT / 1000000.) << ", " << fileName << " -- " << res->totalFunctions << " functions\n" << "\trequire " << (preqT / 1000000.) << "\n" + << "\tcache read " << (*totCacheRead / 1000000.) << " (" << *cntCacheRead << " modules, macro simulate " << (*totCacheMacroSim / 1000000.) << ")\n" << "\tparse " << (*totParse / 1000000.) << "\n" << "\tinfer " << (*totInfer / 1000000.) << "\n" << "\toptimize " << (*totOpt / 1000000.) << "\n" diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 5ef03c5cd9..68c9858d32 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -8,7 +8,9 @@ #include // TextWriter, LOG (the env-gated scan trace) #include // hash_block64 #include // get_dasenv_trace_module_load +#include #include // tolower (case-insensitive basename normalize) +#include #include // fprintf(stderr) for the shadow-shadows-global diagnostic das::FileAccessPtr get_file_access( char * pak ); @@ -90,6 +92,12 @@ static Result run_descriptor(smart_ptr fa, const string & mod_filena static constexpr const char *MANIFEST_SUFFIX = ".das_module.manifest"; // ARCHITECTURE.md sec.2 static constexpr const char *MANIFEST_HEADER = "das_module_manifest\t2"; +static bool g_ignore_manifests = false; + +void ignore_dynamic_module_manifests(bool ignore) { + g_ignore_manifests = ignore; +} + static bool trace_scan() { static const bool on = []{ const char * e = get_dasenv_trace_module_load(); @@ -106,6 +114,7 @@ struct ManifestRead { string why; }; +#if !DAS_NO_FILEIO // the reader's helpers: a build with no file io reads no manifest static das::vector split_tabs(const string & line) { das::vector fields; size_t start = 0; @@ -119,6 +128,7 @@ static das::vector split_tabs(const string & line) { start = tab + 1; } } +#endif static string hex64(uint64_t v) { char buf[32]; @@ -159,11 +169,13 @@ static bool dep_stamp(const smart_ptr & fa, const string & path, uin return true; } +#if !DAS_NO_FILEIO static bool parse_on_error(const string & s, int & value) { // RegisterOnError: Quiet, ErrorMsg, Fail if ( s.size() != 1 || s[0] < '0' || s[0] > '2' ) return false; value = s[0] - '0'; return true; } +#endif // the inputs a descriptor's rows can depend on beyond its own bytes: its folder, the das root // (`get_das_root()`), and the cross-compile target (`get_cross_platform_name()`, read from argv) @@ -182,6 +194,10 @@ static ManifestKey manifest_key(const string & path) { static ManifestRead read_manifest(const string & file, uint32_t descSize, uint64_t descHash, const ManifestKey & key, const smart_ptr & fa) { ManifestRead res; +#if DAS_NO_FILEIO + (void)file; (void)descSize; (void)descHash; (void)key; (void)fa; + return res; +#else FILE * f = fopen(file.c_str(), "rb"); if ( !f ) { return res; @@ -272,6 +288,7 @@ static ManifestRead read_manifest(const string & file, uint32_t descSize, uint64 if ( optOut && !res.rows.empty() ) return damaged("no_manifest with rows"); res.verdict = optOut ? ManifestVerdict::OptOut : ManifestVerdict::Replay; return res; +#endif } static bool field_ok(const string & s) { @@ -309,6 +326,11 @@ static bool write_manifest(const string & file, uint32_t descSize, uint64_t desc } text += "end\t" + to_string(rows.size()) + "\n"; } +#if DAS_NO_FILEIO + (void)file; + why = "no file io"; + return false; +#else const string tmp = file + ".tmp"; FILE * f = fopen(tmp.c_str(), "wb"); if ( !f ) { why = "cannot create " + tmp; return false; } @@ -320,6 +342,7 @@ static bool write_manifest(const string & file, uint32_t descSize, uint64_t desc #endif if ( rename(tmp.c_str(), file.c_str()) != 0 ) { remove(tmp.c_str()); why = "cannot rename " + tmp; return false; } return true; +#endif } static Result init_dyn_modules(smart_ptr fa, string path, TextWriter &tout, bool debug = false) { @@ -343,27 +366,44 @@ static Result init_dyn_modules(smart_ptr fa, string path, TextWriter const uint64_t stamp = src ? hash_block64((const uint8_t *) src, len) : 0; const string manifest = path + "/" + MANIFEST_SUFFIX; const ManifestKey key = manifest_key(path); - auto mr = src ? read_manifest(manifest, len, stamp, key, fa) : ManifestRead(); + auto time0 = ref_time_ticks(); +#if DAS_NO_FILEIO + const bool useManifest = false; + const char * noManifestWhy = "no file io"; +#else + const bool useManifest = src && !g_ignore_manifests; + const char * noManifestWhy = !src ? "no source" : g_ignore_manifests ? "manifests ignored" : "no_manifest"; +#endif + auto mr = useManifest ? read_manifest(manifest, len, stamp, key, fa) : ManifestRead(); if ( mr.verdict == ManifestVerdict::Replay ) { + int64_t dllUsec = 0; + size_t deferred = 0; for ( auto & row : mr.rows ) { - if ( row.dynamic ) { - replay_dynamic_module(row.a.c_str(), row.b.c_str(), row.on_error); - } else { + if ( !row.dynamic ) { replay_native_path(row.a.c_str(), row.b.c_str(), row.c.c_str()); + } else if ( !row.c.empty() ) { + defer_dynamic_module(row.a.c_str(), row.b.c_str(), row.on_error, row.c.c_str()); + deferred ++; + } else { + auto dll0 = ref_time_ticks(); + replay_dynamic_module(row.a.c_str(), row.b.c_str(), row.on_error); + dllUsec += get_time_usec(dll0); } } if ( trace_scan() ) { - LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": replayed " << mr.rows.size() << " row(s)\n"; + LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": replayed " << mr.rows.size() << " row(s) in " + << (get_time_usec(time0) / 1000000.) << " (shared module load " << (dllUsec / 1000000.) + << ", deferred " << deferred << ")\n"; } return Result::OK; } - const bool record = src && mr.verdict != ManifestVerdict::OptOut; + const bool record = useManifest && mr.verdict != ManifestVerdict::OptOut; if ( record ) begin_dynamic_module_recording(); das::vector depFiles; auto res = run_descriptor(fa, mod_filename, path, tout, record ? &depFiles : nullptr); if ( !record ) { if ( trace_scan() ) { - LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": compiled (" << (src ? "no_manifest" : "no source") << ")\n"; + LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": compiled (" << noManifestWhy << ")\n"; } return res; } @@ -534,12 +574,68 @@ static das::string path_basename(const das::string &path) { return path.substr(slash + 1, end - slash - 1); } +static void move_all_nodes(gc_root & from, gc_root & to) { + while ( from.gc_first ) { + auto node = from.gc_first; + from.gc_unlink(node); + to.gc_link(node); + } +} + +// ARCHITECTURE.md sec.2 +static bool load_deferred_module_for_require(const string & name) { + static recursive_mutex loadMutex; + lock_guard guard(loadMutex); + // a collect walks one root: the thread root's own nodes park while the load's leftovers gather on loadRoot + auto & threadRoot = gc_root::gc_get_thread_root(); + gc_root parked, loadRoot; + move_all_nodes(threadRoot, parked); + auto boundProgram = daScriptEnvironment::getBound()->g_Program; // a module's builtin das part parses under its own program; the caller may be mid-parse + bool loaded = false; + bool grown = false; // the module set changed - the fixed point and the collect are owed whether or not `name` came in + { + gc_active_scope scope(&loadRoot); + bool wasDeferred = is_dynamic_module_deferred(name.c_str()); + loaded = load_deferred_dynamic_module(name.c_str()); + grown = loaded; + if ( !loaded && wasDeferred ) { + // its dlopen failed - a sibling it links may be deferred too; bring every row in (retries the pending ones) + load_all_deferred_dynamic_modules(); + grown = true; + loaded = Module::require(name) != nullptr; + } + if ( grown ) { + string notInitialized; + if ( !Module::InitializeDependencies(notInitialized) ) { + load_all_deferred_dynamic_modules(); + if ( !Module::InitializeDependencies(notInitialized) ) { + DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); + } + } + } + } + move_all_nodes(threadRoot, loadRoot); + if ( grown ) { + // every module: a constructor registers into existing ones too (a vector type's functions) + Module::foreach([&](Module * m) { + m->gc_collect(&loadRoot); + return true; + }); + } + loadRoot.gc_sweep(); + move_all_nodes(parked, threadRoot); + daScriptEnvironment::getBound()->g_Program = boundProgram; + return loaded; +} + bool require_dynamic_modules(FileAccessPtr file_access, const das::string &das_root, const das::string &project_root, const das::vector &load_modules, const das::vector &disabled_modules, das::TextWriter &tout) { + clear_deferred_dynamic_modules(); + setDeferredModuleLoader(&load_deferred_module_for_require); // Explicitly-disabled modules (case-insensitive on every platform) are never // loaded/registered — keeps a native-only module out of a wasm cross-compile. das_hash_set disabled_set; diff --git a/src/builtin/REVIEW.md b/src/builtin/REVIEW.md index b446a7928b..2ee28e6a25 100644 --- a/src/builtin/REVIEW.md +++ b/src/builtin/REVIEW.md @@ -5,10 +5,8 @@ - **Weakening `review_nttp.das`'s bind-flavor scan, which `REVIEW.das` runs, is a defect - fix a bind the scan reports by switching the bind.** The Inline modules are `$` (builtin), `math`, - `strings` and `jit`. In those, a plain-value bind - one returning nothing, or a value that is - neither a reference nor written into the caller's result slot - registers through - `addExternInline` or `addExternInlineEx`; in every other module it registers through - `addExtern`. + `strings` and `jit`. What the scan enforces, and which shared generic helpers it exempts, is + read from the scan itself. - **A diff that adds or changes a bind in a module the scan covers rebuilds the binary from that diff before the folder's gate runs** - the scan reads the binds compiled into the running @@ -21,11 +19,13 @@ - **Never drop a module from `review_nttp.das`'s `require` list.** The list is what sets the modules the scan covers. -- **A diff that changes what `module_builtin_ast_serialize.cpp` streams - a field added, - removed, reordered, re-typed, or given a new meaning - bumps the version `getVersion()` - returns in `include/daScript/ast/ast_serializer.h`, in the same change** - a reader accepts a - stream only when its stored version equals `getVersion()`, so without the bump an older cache - passes that check and decodes the changed bytes as something else. +- **A diff that changes the bytes a module-cache record carries or what they resolve to - a + field added, removed, reordered, re-typed or given a new meaning in + `module_builtin_ast_serialize.cpp`, or a change anywhere to which module owns a streamed + annotation, function or type - bumps the version `getVersion()` returns in + `include/daScript/ast/ast_serializer.h`, in the same change** - a reader accepts a stream only + when its stored version equals `getVersion()`, so without the bump an older cache passes that + check and decodes the changed bytes as something else. - **A diff that streams or compares a `CodeOfPolicies` field in `module_builtin_ast_serialize.cpp` outside `DAS_MODULE_CACHE_POLICY_FIELDS` is a defect - put the field on the list instead** - the @@ -46,6 +46,13 @@ A replayed start never runs the descriptor, so an effect the recorder does not see is an effect every warm start silently lacks. +- **A diff that moves a bind between modules - an `addExtern` or `addExternInline` call whose + module changes, or a builtin whose `vector` functions follow a type to another module - + bumps `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` (repo root), in + the same change.** The JIT's DLL cache key folds the codegen version and each function's AST + hash, never the module an extern lives in, so a cached DLL binds the old name and crashes on + the hit. + - **A diff that changes what `ModuleFileCache::defaultPath` folds into the module-cache key - the binary, the command line, the environment names, or which script arguments count - updates the cache-key paragraph of `ARCHITECTURE.md` in the same change.** The key is what stops a diff --git a/src/builtin/module_builtin_ast_serialize.cpp b/src/builtin/module_builtin_ast_serialize.cpp index 536056a672..d359a8edf4 100644 --- a/src/builtin/module_builtin_ast_serialize.cpp +++ b/src/builtin/module_builtin_ast_serialize.cpp @@ -1092,6 +1092,15 @@ namespace das { return *this; } + // a read no prerequisite walk precedes meets a deferred C++ module here (src/ast/ARCHITECTURE.md sec.2) + static Module * requireBuiltinModule ( const string & name ) { + auto m = Module::require(name); + if ( !m ) { + if ( auto loader = getDeferredModuleLoader(); loader && loader(name) ) m = Module::require(name); + } + return m; + } + AstSerializer & AstSerializer::operator << ( Module * & module ) { bool is_null = module == nullptr; *this << is_null; @@ -3009,7 +3018,7 @@ namespace das { string name; ser << name; if ( builtin && !promoted ) { - auto m = Module::require(name); + auto m = requireBuiltinModule(name); // a corrupted record can hand this arm a garbage name - require() // answers null, and the deref was a SIGSEGV (recoverable throw now; // the resume reparses the record in place) @@ -3221,7 +3230,7 @@ namespace das { uint64_t size_builtin = 0; ser << size_builtin; for ( uint64_t i = 0; i < size_builtin; i++ ) { string name; ser << name; - Module * m = Module::require(name); + Module * m = requireBuiltinModule(name); library.addModule(m); } diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 9d9208e13c..3a7bf26a29 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -312,6 +312,11 @@ namespace das { DAS_API void end_dynamic_module_recording ( vector &, bool & ) GENERATE_IO_STUB DAS_API void replay_native_path ( const char *, const char *, const char * ) GENERATE_IO_STUB DAS_API void replay_dynamic_module ( const char *, const char *, int ) GENERATE_IO_STUB + DAS_API void defer_dynamic_module ( const char *, const char *, int, const char * ) GENERATE_IO_STUB + DAS_API bool load_deferred_dynamic_module ( const char * ) GENERATE_IO_STUB_RET + DAS_API size_t load_all_deferred_dynamic_modules () GENERATE_IO_STUB_RET + DAS_API bool is_dynamic_module_deferred ( const char * ) GENERATE_IO_STUB_RET + DAS_API void clear_deferred_dynamic_modules () GENERATE_IO_STUB #undef GENERATE_IO_STUB #undef GENERATE_IO_STUB_RET @@ -322,6 +327,7 @@ namespace das { #include #include +#include #include #if _WIN32 #include @@ -2415,6 +2421,60 @@ namespace das { register_dynamic_module(path, cpp_class, on_error, nullptr, nullptr); } + DAS_API void retry_pending_dynamic_modules(); + + struct DeferredDynamicModule { + string path, cpp_class, das_name; + int on_error = 0; + }; + static vector g_deferred_dynamic_modules; // src/ast/ARCHITECTURE.md sec.2 + static std::recursive_mutex g_deferred_dynamic_modules_mutex; // a run-time has_module reads while another thread's compile loads + + DAS_API void defer_dynamic_module ( const char * path, const char * cpp_class, int on_error, const char * das_name ) { + lock_guard guard(g_deferred_dynamic_modules_mutex); + g_deferred_dynamic_modules.push_back({path ? path : "", cpp_class ? cpp_class : "", das_name ? das_name : "", on_error}); + } + + DAS_API bool load_deferred_dynamic_module ( const char * das_name ) { + if ( !das_name ) return false; + lock_guard guard(g_deferred_dynamic_modules_mutex); + auto it = find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), + [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }); + if ( it == g_deferred_dynamic_modules.end() ) return false; + auto dm = das::move(*it); + g_deferred_dynamic_modules.erase(it); + if ( trace_module_load() ) { + LOG(LogLevel::info) << "[module] require " << dm.das_name << ": loading the deferred " << dm.cpp_class << "\n"; + } + return register_dynamic_module(dm.path.c_str(), dm.cpp_class.c_str(), dm.on_error, nullptr, nullptr) != nullptr; + } + + DAS_API size_t load_all_deferred_dynamic_modules () { + lock_guard guard(g_deferred_dynamic_modules_mutex); + vector all; + all.swap(g_deferred_dynamic_modules); + if ( trace_module_load() && !all.empty() ) { + LOG(LogLevel::info) << "[module] loading every deferred module (" << all.size() << ")\n"; + } + for ( auto & dm : all ) { + register_dynamic_module(dm.path.c_str(), dm.cpp_class.c_str(), dm.on_error, nullptr, nullptr); + } + retry_pending_dynamic_modules(); + return all.size(); + } + + DAS_API bool is_dynamic_module_deferred ( const char * das_name ) { + if ( !das_name ) return false; + lock_guard guard(g_deferred_dynamic_modules_mutex); + return find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), + [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }) != g_deferred_dynamic_modules.end(); + } + + DAS_API void clear_deferred_dynamic_modules () { + lock_guard guard(g_deferred_dynamic_modules_mutex); + g_deferred_dynamic_modules.clear(); + } + // Re-attempt modules whose dlopen was deferred (Quiet failure during the // module-folder scan — usually a sibling-module DT_NEEDED dep not yet loaded // because directory enumeration visited the dependent before its dependency). diff --git a/src/builtin/module_builtin_rtti.cpp b/src/builtin/module_builtin_rtti.cpp index 39e1db22df..3cb2c5521f 100644 --- a/src/builtin/module_builtin_rtti.cpp +++ b/src/builtin/module_builtin_rtti.cpp @@ -11,6 +11,7 @@ #include "daScript/misc/performance_time.h" #include "daScript/ast/ast_serializer.h" +#include "daScript/ast/dyn_modules.h" #include "daScript/misc/gc_node.h" using namespace das; @@ -1187,8 +1188,9 @@ namespace das { return Module::require(name); } + // true for a module the tree has, loaded or waiting in a manifest row - this asks, it never loads bool rtti_has_module ( const char * name ) { - return Module::require(name) != nullptr; + return Module::require(name) != nullptr || is_dynamic_module_deferred(name); } void rtti_builtin_program_for_each_module ( smart_ptr_raw program, const TBlock & block, Context * context, LineInfoArg * at ) { diff --git a/src/parser/REVIEW.md b/src/parser/REVIEW.md index cfdf6910b6..4c9e5e2fb9 100644 --- a/src/parser/REVIEW.md +++ b/src/parser/REVIEW.md @@ -7,3 +7,8 @@ rule to `tree-sitter-daslang/grammar.js` (repo root), in the same change.** The editor and the MCP search tools parse with the tree-sitter grammar, not with bison, so syntax missing from `grammar.js` does not appear in code folding, outline or `grep_usage`. + +**A diff that lets `ds2_parser.ypp` or `ds2_lexer.lpp` accept new syntax also adds a section +exercising it to `modules/dasImgui/tests/test_grammar_canary.das` (repo root), in the same +change.** The canary reds only for syntax it already carries, so syntax it lacks can drift from +the tree-sitter grammar with no test to say so. diff --git a/src/parser/parser_impl.cpp b/src/parser/parser_impl.cpp index 91dc287566..7fcf78bc92 100644 --- a/src/parser/parser_impl.cpp +++ b/src/parser/parser_impl.cpp @@ -1230,17 +1230,16 @@ namespace das { if ( guard ) { // Path guard (contains '/'): availability = the guard's OWN file resolves — the rail for // pure-das packages (nothing C++ to guard on) and cross-package dependencies the target's - // resolvability can't express. Plain-name guard: STRICT — the guard module is registered - // (a linked C++ module). No target-resolvability fallback: `require ?sqlite ...`-style - // guards mean "loaded only when is linked", and module source dirs are present in - // every checkout regardless of build config. Must match the require collector's rule - // (ast_parse.cpp getAllRequireReq). + // resolvability can't express. Plain-name guard: the build has the module + // (guardModuleAvailable, src/ast/ARCHITECTURE.md sec.2). No target-resolvability + // fallback: module source dirs are present in every checkout regardless of build config. + // Must match the require collector's rule (ast_parse.cpp getAllRequireReq). bool guardAvailable; if ( guard->find('/') != string::npos ) { auto ginfo = yyextra->g_Access->getModuleInfo(*guard, yyextra->g_FileAccessStack.back()->name); guardAvailable = !ginfo.fileName.empty() && yyextra->g_Access->getFileInfo(ginfo.fileName) != nullptr; } else { - guardAvailable = Module::requireEx(*guard, false) != nullptr; + guardAvailable = guardModuleAvailable(*guard); } delete guard; if ( !guardAvailable ) { @@ -1250,7 +1249,14 @@ namespace das { } } auto info = yyextra->g_Access->getModuleInfo(*name, yyextra->g_FileAccessStack.back()->name); - if ( auto mod = yyextra->g_Program->addModule(info.moduleName) ) { + auto mod = yyextra->g_Program->addModule(info.moduleName); + if ( !mod ) { + // a parse with no prerequisite walk (compile of a string) meets a deferred module here (src/ast/ARCHITECTURE.md sec.2) + if ( auto loader = getDeferredModuleLoader(); loader && loader(info.moduleName) ) { + mod = yyextra->g_Program->addModule(info.moduleName); + } + } + if ( mod ) { yyextra->g_Program->allRequireDecl.push_back(make_tuple(mod,*name,info.fileName,pub,atName)); yyextra->g_Program->thisModule->addDependency(mod, pub); das_collect_all_keywords(mod,scanner); diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 234f725a75..6d16b91b8b 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -31,9 +31,40 @@ this document states what the folder is and why its tests take the shape they do tab leaves the manifest unwritten, and a directory sitting where the `.tmp` or the manifest goes fails the create or the rename so the start just compiles; each verdict is read from the `DAS_TRACE_MODULE_LOAD=1` line the child prints. +- `test_deferred_modules.das` - a replayed `dm` row is not loaded by the scan. The project root + holds a copy of the tree's `dasUnitTest` (descriptor and artifact), which shadows the tree's, + so the copy's manifest is the test's to make cold or warm: a cold start compiles the + descriptor, loads the module to record its name, and a `require ?UnitTest x` is taken; a + warm start defers the row and loads nothing for a program that requires nothing; the first + `require UnitTest` loads it and the program calls into it; `typeinfo builtin_module_exists` + loads it with no require naming it, while rtti `has_module` answers true and loads nothing; + a lazy start and an eager start count the same functions in `$`, so a load adds nothing + to `$`; a guard alone loads the module + and is taken, as it is with a `require UnitTest` above it, below it, or in the entry while + the guard sits in a module walked earlier; + `-ignore-manifest` compiles every descriptor, loads every C++ module on start and writes no + manifest; a manifest row hand-edited to an absent artifact makes the require bring every + deferred module in and then fail on the missing prerequisite; a module cache an eager start + wrote serves a lazy start, with `-log-compile-time` printing the reads and the startup + timeline; a dastest `--ser` stream of a test requiring the module is read by a `--deser` + child that nothing made require it, and the reader loads it; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every + deferred module in because its `initDependencies` asks for two more, and a half-warm tree - + copies of dasImgui and dasGlfw under the fixture, the imgui copy's manifest removed so its + modules load on start while glfw's row waits - initializes by bringing the rest in; a build + whose copied artifact cannot find its libraries from the copy has nothing to observe there + and the arm says so. A static host - the AOT + test binary, whose descriptors register no shared module, or a tree holding no + `.shared_module` - has nothing to observe and the test says so and returns. - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, `mc_generic_origin_*`); a case needing a macro-bearing module graph puts it here instead of writing the script inline. +- `_mc_common.das` - the spawn helpers every test here shares (the name is the folder's, since + a sweep worker holds every shared module it met under one name and `tests/linq` has a + `_common` already): the binary to spawn (`das_exe`), + the scan-trace command prefix (`trace_prefix`), the stderr-joining child run (`run_child`) + and the failure report that echoes the child's output (`report_child`). A test whose child + needs a different spawn shape - an argv spawn, an environment variable - keeps that one + helper local. ## 2. Why every case is a spawned process diff --git a/tests/module_cache/REVIEW.md b/tests/module_cache/REVIEW.md index 8b84493c83..843e9776ac 100644 --- a/tests/module_cache/REVIEW.md +++ b/tests/module_cache/REVIEW.md @@ -3,10 +3,12 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE.md`. -- **Weakening what a test in this folder checks a spawned child's output against is a defect; - an edited assertion weakens when it accepts an output the old one rejected, and re-pinning a - count or a verdict form to the child's new true output does not.** A child's output is the - only instrument a human has for what the cache and the scan served. +- **Weakening the text a test in this folder compares a spawned child's output against - an + assertion literal, or the helper that produces the compared text - is a defect; an edit + weakens when it accepts an output the old one rejected, and re-pinning a count or the fixed + words of a scan or cache line to the child's new true output does not. A timing the child + newly prints leaves the compared text; every other field stays compared.** A child's output is + the only instrument a human has for what the cache and the scan served. - **A test in this folder writes only under a directory it created for this process - its own files and its children's - and removes that directory.** diff --git a/tests/module_cache/_mc_common.das b/tests/module_cache/_mc_common.das new file mode 100644 index 0000000000..b605278509 --- /dev/null +++ b/tests/module_cache/_mc_common.das @@ -0,0 +1,41 @@ +options gen2 +options indenting = 4 + +module _mc_common shared public + +require dastest/testing_boost public + +require strings +require daslib/fio + +//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, +//! so argv[0] is the interpreter, not the test script +def das_exe() : string { + let args <- get_command_line_arguments() + return empty(args) ? "" : args[0] +} + +//! the child's environment, as a command prefix: the scan trace on, so the trace names each +//! descriptor's verdict and each shared-module load; the prefix also keeps the command from +//! opening with a quote, which cmd.exe would strip +def trace_prefix() : string { + return get_platform_name() == "windows" ? "set \"DAS_TRACE_MODULE_LOAD=1\"&& " : "DAS_TRACE_MODULE_LOAD=1 " +} + +//! stderr joins stdout: the scan trace goes through the logger +def run_child(cmd : string; var output : string&) : int { + return unsafe(popen_timeout("{cmd} 2>&1", 300.0, $(f) { + if (f != null) { + output = fread(f) + } + })) +} + +def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { + let ok = rc == 0 && find(out, marker) >= 0 + if (!ok) { + t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") + t |> failure("{phase} child output follows:\n{out}") + } + return ok +} diff --git a/tests/module_cache/test_default_cache_path.das b/tests/module_cache/test_default_cache_path.das index 71f4e4f265..5b9e9f808d 100644 --- a/tests/module_cache/test_default_cache_path.das +++ b/tests/module_cache/test_default_cache_path.das @@ -7,13 +7,7 @@ require strings require math require daslib/fio require daslib/strings_boost - -//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, -//! so argv[0] is the interpreter, not the test script -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} +require _mc_common //! the child shares dastest's cwd (argv[0] is a cwd-relative interpreter path), so the default //! cache lands in this tree's own .jitted_scripts/module_cache - the test removes what it wrote. @@ -28,15 +22,6 @@ def run(cmd : string; var output : string&) : int { let CACHE_DIR = ".jitted_scripts/module_cache" -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} - //! the cache files written for one root: `/-.dascache` def cache_files(stem : string) : int { var n = 0 diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das new file mode 100644 index 0000000000..f89928d9ff --- /dev/null +++ b/tests/module_cache/test_deferred_modules.das @@ -0,0 +1,338 @@ +options gen2 +options indenting = 4 + +require dastest/testing_boost public + +require strings +require daslib/fio +require daslib/strings_boost +require _mc_common + +//! the whole scan trace line for a module's descriptor, timing clause included +def descriptor_line(out : string; mod : string) : string { + let marker = "{mod}/.das_module: " + let at = find(out, marker) + if (at < 0) { + return "" + } + let rest = slice(out, at + length(marker)) + let eol = find(rest, "\n") + return eol < 0 ? rest : slice(rest, 0, eol) +} + +//! the `DOLLAR N` line the dollar.das child prints: the builtin module's function count +def dollar_count(out : string) : int { + let marker = "DOLLAR " + let at = find(out, marker) + if (at < 0) { + return -1 + } + let rest = slice(out, at + length(marker)) + let eol = find(rest, "\n") + return to_int(eol < 0 ? rest : slice(rest, 0, eol)) +} + +let UNIT_TEST_MODULE = "UnitTest" +let UNIT_TEST_CLASS = "Module_UnitTest" +let UNIT_TEST_FOLDER = "dasUnitTest" +let UNIT_TEST_ARTIFACT = "dasModuleUnitTest" + +//! the tree's UnitTest shared module, in whichever configuration this binary was built - absent +//! in a static build, where nothing is deferred and this test has nothing to observe +def unit_test_artifact() : string { + let dir = path_join(path_join(get_das_root(), "modules"), UNIT_TEST_FOLDER) + for (name in ["{UNIT_TEST_ARTIFACT}.shared_module", "{UNIT_TEST_ARTIFACT}_debug.shared_module"]) { + let candidate = path_join(dir, name) + if (stat(candidate).is_valid) { + return candidate + } + } + return "" +} + +let DESCRIPTOR = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n register_native_path(\"deferred_probe\", \"hello\", \"\{project_path\}/hello.das\")\n register_native_path(\"deferred_probe\", \"guarded_dep\", \"\{project_path\}/guarded_dep.das\")\n\}\n" + +//! a project root holding a copy of the tree's dasUnitTest - descriptor and artifact - so the +//! copy's manifest is the test's to make cold or warm (the copy shadows the tree's), a pure-das +//! module, and the driver scripts the arms spawn +struct Fixture { + tmp : string + manifest : string + utManifest : string + plain : string + uses : string + exists : string + hasModule : string + dollar : string + nested : string + guarded : string + guardedAfter : string + guardedBefore : string + guardedDep : string + app : string +} + +def make_fixture(tmp : string; artifact : string) : Fixture { + let modules = path_join(tmp, "modules") + let modDir = path_join(modules, "deferred_probe") + let utDir = path_join(modules, UNIT_TEST_FOLDER) + var merr : string + mkdir_rec(modDir, merr) + mkdir_rec(utDir, merr) + fwrite(path_join(modDir, ".das_module"), DESCRIPTOR) + fwrite(path_join(modDir, "hello.das"), "options gen2\nmodule hello\ndef hello_value() : int \{\n return 42\n\}\n") + fwrite(path_join(modDir, "guarded_dep.das"), "options gen2\nmodule guarded_dep\nrequire ?{UNIT_TEST_MODULE} deferred_probe/hello public\n") + let treeUt = path_join(path_join(get_das_root(), "modules"), UNIT_TEST_FOLDER) + var cerr : string + copy_file(path_join(treeUt, ".das_module"), path_join(utDir, ".das_module"), true, cerr) + copy_file(artifact, path_join(utDir, base_name(artifact)), true, cerr) + let fx = Fixture( + tmp = tmp, + manifest = path_join(modDir, ".das_module.manifest"), + utManifest = path_join(utDir, ".das_module.manifest"), + plain = path_join(tmp, "plain.das"), + uses = path_join(tmp, "uses.das"), + exists = path_join(tmp, "exists.das"), + hasModule = path_join(tmp, "has_module.das"), + dollar = path_join(tmp, "dollar.das"), + nested = path_join(tmp, "nested.das"), + guarded = path_join(tmp, "guarded.das"), + guardedAfter = path_join(tmp, "guarded_after.das"), + guardedBefore = path_join(tmp, "guarded_before.das"), + guardedDep = path_join(tmp, "guarded_dep_driver.das"), + app = path_join(tmp, "app.das") + ) + let guardLine = "require ?{UNIT_TEST_MODULE} deferred_probe/hello\n" + let guardMain = "[export]\ndef main \{\n print(\"GUARD \{typeinfo module_exists(hello)\}\\n\")\n\}\n" + fwrite(fx.plain, "options gen2\n[export]\ndef main \{\n print(\"PLAIN\\n\")\n\}\n") + fwrite(fx.uses, "options gen2\nrequire {UNIT_TEST_MODULE}\n[export]\ndef main \{\n print(\"LENGTH \{test_string_arg_length(\"hello\")\}\\n\")\n\}\n") + fwrite(fx.exists, "options gen2\n[export]\ndef main \{\n print(\"EXISTS \{typeinfo builtin_module_exists({UNIT_TEST_MODULE})\}\\n\")\n\}\n") + fwrite(fx.hasModule, "options gen2\nrequire daslib/rtti\n[export]\ndef main \{\n print(\"HAS \{has_module(\"{UNIT_TEST_MODULE}\")\}\\n\")\n\}\n") + fwrite(fx.dollar, "options gen2\nrequire daslib/rtti\n[export]\ndef main \{\n program_for_each_registered_module() $(mod) \{\n if (mod.name == \"$\") \{\n var n = 0\n module_for_each_function(mod) $(fn) \{\n n++\n \}\n print(\"DOLLAR \{n\}\\n\")\n \}\n \}\n\}\n") + fwrite(fx.nested, "options gen2\nrequire daslib/ast\n[export]\ndef main \{\n compile(\"nested\", \"require {UNIT_TEST_MODULE}\\n[export]\\ndef main \\\{\\n\\\}\\n\", CodeOfPolicies()) $(ok; _prog; errors) \{\n print(\"NESTED \{ok\}\\n\{errors\}\\n\")\n \}\n\}\n") + fwrite(fx.guarded, "options gen2\n{guardLine}{guardMain}") + fwrite(fx.guardedAfter, "options gen2\nrequire {UNIT_TEST_MODULE}\n{guardLine}{guardMain}") + fwrite(fx.guardedBefore, "options gen2\n{guardLine}require {UNIT_TEST_MODULE}\n{guardMain}") + fwrite(fx.guardedDep, "options gen2\nrequire deferred_probe/guarded_dep\nrequire {UNIT_TEST_MODULE}\n{guardMain}") + fwrite(fx.app, "options gen2\nrequire imgui_app\n[export]\ndef main \{\n print(\"APP\\n\")\n\}\n") + return fx +} + +//! a copy of one of the tree's module folders - descriptor and the artifacts that exist, in +//! either configuration's name - under the fixture's modules, where it shadows the tree's +def copy_module(fx : Fixture; folder : string; artifacts : array) : string { + let src = path_join(path_join(get_das_root(), "modules"), folder) + let dst = path_join(path_join(fx.tmp, "modules"), folder) + var merr : string + mkdir_rec(dst, merr) + var cerr : string + copy_file(path_join(src, ".das_module"), path_join(dst, ".das_module"), true, cerr) + for (name in artifacts) { + for (suffix in [".shared_module", "_debug.shared_module"]) { + let f = path_join(src, "{name}{suffix}") + if (stat(f).is_valid) { + copy_file(f, path_join(dst, "{name}{suffix}"), true, cerr) + } + } + } + dir(src) $(name) { // a library an artifact loads from beside itself + let ext = extension(name) + if (ext == ".dylib" || ext == ".so" || ext == ".dll") { + copy_file(path_join(src, name), path_join(dst, name), true, cerr) + } + } + return dst +} + +//! whether the cold run loaded the copied artifact - a build whose artifacts find their +//! libraries relative to the tree cannot load a copy, and the arms on copies say so and return +def copy_loaded(out : string; dir : string; artifact : string) : bool { + for (suffix in [".shared_module", "_debug.shared_module"]) { + if (find(out, "{path_join(dir, artifact)}{suffix} : loaded") >= 0) { + return true + } + } + return false +} + +def child_cmd(fx : Fixture; script : string; flags : string = ""; cache : string = "-no-module-cache") : string { + return "{trace_prefix()}\"{das_exe()}\" -dasroot \"{get_das_root()}\" {cache} -project_root \"{fx.tmp}\" {flags} \"{script}\"" +} + +def has_shared_modules(dir : string; names : array) : bool { + for (name in names) { + var found = false + for (suffix in [".shared_module", "_debug.shared_module"]) { + found ||= stat(path_join(dir, "{name}{suffix}")).is_valid + } + if (!found) { + return false + } + } + return true +} + +def arms_deferred(t : T?; fx : Fixture) { + t |> run("a cold start loads the module to record its name, and a guard reads it available") @(t : T?) { + var out : string + t |> success(report_child(t, "cold", run_child(child_cmd(fx, fx.guarded), out), out, "GUARD true"), "the cold start runs, and the guard takes its target") + t |> equal("compiled (no manifest), manifest written (1 row(s))", descriptor_line(out, UNIT_TEST_FOLDER), "the copied descriptor compiles and records:\n{out}") + t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the recording start loads the module:\n{out}") + } + t |> run("a warm start defers the module and loads nothing for a program that requires nothing") @(t : T?) { + var out : string + t |> success(report_child(t, "warm", run_child(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the warm start runs the program") + let line = descriptor_line(out, UNIT_TEST_FOLDER) + t |> success(line |> starts_with("replayed 1 row(s) in ") && find(line, ", deferred 1)") >= 0, "the descriptor replays its one row deferred:\n{out}") + t |> equal(-1, find(out, "{UNIT_TEST_CLASS} <- "), "nothing loads the shared module:\n{out}") + } + t |> run("the first require naming a deferred module loads it, and the program calls into it") @(t : T?) { + var out : string + t |> success(report_child(t, "uses", run_child(child_cmd(fx, fx.uses), out), out, "LENGTH 5"), "the program calls the module's function") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the trace names the require that loaded it:\n{out}") + t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads:\n{out}") + } + t |> run("builtin_module_exists loads a waiting module, with no require naming it") @(t : T?) { + var out : string + t |> success(report_child(t, "exists", run_child(child_cmd(fx, fx.exists), out), out, "EXISTS true"), "the build has the module") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the question loaded it:\n{out}") + } + t |> run("has_module answers true for a waiting module and loads nothing") @(t : T?) { + var out : string + t |> success(report_child(t, "has_module", run_child(child_cmd(fx, fx.hasModule), out), out, "HAS true"), "the build has the module") + t |> equal(-1, find(out, "{UNIT_TEST_CLASS} <- "), "nothing loads the shared module:\n{out}") + } + t |> run("a lazy start and an eager start build the same builtin module") @(t : T?) { + var lazy : string + t |> success(report_child(t, "dollar lazy", run_child(child_cmd(fx, fx.dollar), lazy), lazy, "DOLLAR "), "the lazy start counts") + var eager : string + t |> success(report_child(t, "dollar eager", run_child(child_cmd(fx, fx.dollar, "-ignore-manifest"), eager), eager, "DOLLAR "), "the eager start counts") + t |> success(dollar_count(lazy) > 0, "the count is a count:\n{lazy}") + t |> equal(dollar_count(lazy), dollar_count(eager), "a C++ module's load registers nothing into $:\nlazy:\n{lazy}\neager:\n{eager}") + } + t |> run("a compile of a string, which walks no prerequisites, meets the waiting module at its require") @(t : T?) { + var out : string + t |> success(report_child(t, "nested", run_child(child_cmd(fx, fx.nested), out), out, "NESTED true"), "the nested program compiles") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the parser's require loaded it:\n{out}") + } +} + +def arms_guards_and_caches(t : T?; fx : Fixture) { + t |> run("a require guard loads a waiting module, whatever else the compile requires") @(t : T?) { + var alone : string + t |> success(report_child(t, "guarded", run_child(child_cmd(fx, fx.guarded), alone), alone, "GUARD true"), "the guard alone takes its target") + t |> success(find(alone, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the guard loaded the module:\n{alone}") + var after : string + t |> success(report_child(t, "guarded after", run_child(child_cmd(fx, fx.guardedAfter), after), after, "GUARD true"), "a require above the guard") + var before : string + t |> success(report_child(t, "guarded before", run_child(child_cmd(fx, fx.guardedBefore), before), before, "GUARD true"), "a require below the guard") + var dep : string + t |> success(report_child(t, "guarded dep", run_child(child_cmd(fx, fx.guardedDep), dep), dep, "GUARD true"), "a guard inside a module walked before the entry's own require") + } + t |> run("-ignore-manifest compiles every descriptor, loads every C++ module on start, and writes no manifest") @(t : T?) { + remove_result(fx.manifest) + remove_result(fx.utManifest) + var out : string + t |> success(report_child(t, "ignore", run_child(child_cmd(fx, fx.plain, "-ignore-manifest"), out), out, "PLAIN"), "the program runs") + t |> equal("compiled (manifests ignored)", descriptor_line(out, UNIT_TEST_FOLDER), "the copied descriptor compiles:\n{out}") + t |> equal("compiled (manifests ignored)", descriptor_line(out, "deferred_probe"), "the pure-das descriptor compiles:\n{out}") + t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads on start:\n{out}") + t |> success(!stat(fx.manifest).is_valid && !stat(fx.utManifest).is_valid, "no manifest was written") + } + t |> run("a deferred row whose artifact is gone brings every deferred module in, and the require fails as on a cold start") @(t : T?) { + var warm : string + t |> success(report_child(t, "rewarm", run_child(child_cmd(fx, fx.plain), warm), warm, "PLAIN"), "a cold start rewrites the manifest") + let text = fread(fx.utManifest) + t |> success(find(text, "\ndm\t") >= 0, "the manifest carries the dm row: {text}") + fwrite(fx.utManifest, replace(text, UNIT_TEST_ARTIFACT, "absent")) + var out : string + let rc = run_child(child_cmd(fx, fx.uses), out) + t |> success(rc != 0 && find(out, "LENGTH") < 0, "the program does not run:\n{out}") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the require asks for the row:\n{out}") + t |> success(find(out, "[module] loading every deferred module (") >= 0, "the failed load brings the rest in:\n{out}") + t |> success(find(out, "error[20605]: missing prerequisite '{UNIT_TEST_MODULE}'") >= 0, "the require fails with the missing-module error:\n{out}") + remove_result(fx.utManifest) + } + t |> run("a module cache written by an eager start serves a lazy start, and -log-compile-time shows the reads and the timeline") @(t : T?) { + let cache = "-module-cache \"{path_join(fx.tmp, "mc.bin")}\"" + var cold : string + t |> success(report_child(t, "cold cache", run_child(child_cmd(fx, fx.plain, "-ignore-manifest -log-compile-time", cache), cold), cold, "PLAIN"), "the eager start writes the cache") + t |> success(find(cold, "startup: main total") >= 0 && find(cold, "\tmodule scan") >= 0 && find(cold, "\tshutdown ") >= 0, "the timeline prints its laps:\n{cold}") + var warm : string + t |> success(report_child(t, "warm cache", run_child(child_cmd(fx, fx.plain, "-log-compile-time", cache), warm), warm, "PLAIN"), "the lazy start, which loads no C++ module, runs") + t |> equal(-1, find(warm, "cumulative hash"), "a builtin module's hash does not depend on the loaded set:\n{warm}") + t |> success(find(warm, "cache read took") >= 0, "the lazy start reads the eager start's records:\n{warm}") + } + t |> run("a dastest stream written where the module had loaded is read where nothing required it yet") @(t : T?) { + let testsDir = path_join(fx.tmp, "tests") + var merr : string + mkdir_rec(testsDir, merr) + fwrite(path_join(testsDir, "test_uses.das"), "options gen2\nrequire dastest/testing_boost public\nrequire {UNIT_TEST_MODULE}\n[test]\ndef test_uses(t : T?) \{\n t |> equal(5l, test_string_arg_length(\"hello\"))\n\}\n") + let stream = path_join(fx.tmp, "tests.ser") + let dastest = path_join(get_das_root(), "dastest/dastest.das") + var wrote : string + t |> success(report_child(t, "ser", run_child("{child_cmd(fx, dastest)} -- --test \"{testsDir}\" --ser \"{stream}\"", wrote), wrote, "Serialized 1 programs"), "the writer serializes the test program") + var read : string + t |> success(report_child(t, "deser", run_child("{child_cmd(fx, dastest)} -- --test \"{testsDir}\" --deser \"{stream}\"", read), read, "1 tests, 0 error(s)"), "the reader runs the test from the stream") + t |> success(find(read, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the reader loaded the module the stream names:\n{read}") + } +} + +def arm_fallback(t : T?; fx : Fixture) { + let modules = path_join(get_das_root(), "modules") + let imgui = path_join(modules, "dasImgui") + if (!has_shared_modules(imgui, ["imguiApp", "dasModuleImgui"]) || !has_shared_modules(path_join(modules, "dasGlfw"), ["dasModuleGlfw"])) { + to_log(LOG_INFO, "test_deferred_modules: no imgui_app, imgui and glfw shared modules in {modules} - the fallback arm has nothing to observe\n") + return + } + t |> run("a module whose C++ dependencies are deferred too brings every deferred module in") @(t : T?) { + var out : string + t |> success(report_child(t, "app", run_child(child_cmd(fx, fx.app), out), out, "APP"), "the program compiles against imgui_app") + t |> success(find(out, "[module] require imgui_app: loading the deferred Module_imgui_app") >= 0, "imgui_app loads at its require:\n{out}") + t |> success(find(out, "[module] loading every deferred module (") >= 0, "its initDependencies asks for glfw and imgui, so the rest loads:\n{out}") + } + t |> run("a half-warm tree - a descriptor compiled cold beside replayed ones - initializes by bringing every deferred module in") @(t : T?) { + let imguiDir = copy_module(fx, "dasImgui", ["dasModuleImgui", "imguiApp", "imguiAppHeadless"]) + copy_module(fx, "dasGlfw", ["dasModuleGlfw"]) + var cold : string + t |> success(report_child(t, "half-warm cold", run_child(child_cmd(fx, fx.plain), cold), cold, "PLAIN"), "the copies record their manifests") + if (!copy_loaded(cold, imguiDir, "imguiApp")) { + to_log(LOG_INFO, "test_deferred_modules: the copied imguiApp does not load from {imguiDir} - its libraries sit outside the copy - so the half-warm arm has nothing to observe\n") + return + } + remove_result(path_join(imguiDir, ".das_module.manifest")) + var out : string + t |> success(report_child(t, "half-warm", run_child(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the program runs with imgui_app loaded on start and glfw waiting") + if (!copy_loaded(out, imguiDir, "imguiApp")) { + to_log(LOG_INFO, "test_deferred_modules: the copied imguiApp did not load on the half-warm start - its libraries sit outside the copy - so initialize had nothing to bring in:\n{out}\n") + return + } + t |> success(find(out, "[module] loading every deferred module (") >= 0, "initialize brings the deferred glfw in:\n{out}") + } +} + +//! the arms share one root and run in this order +[test] +def test_deferred_modules(t : T?) { + if (!das_is_dll_build()) { + to_log(LOG_INFO, "test_deferred_modules: {das_exe()} is a static build - its descriptors register no shared module, so it defers nothing\n") + return + } + let artifact = unit_test_artifact() + if (empty(artifact)) { + to_log(LOG_INFO, "test_deferred_modules: no {UNIT_TEST_ARTIFACT}.shared_module under {get_das_root()}/modules/{UNIT_TEST_FOLDER} - a static build defers nothing\n") + return + } + var terr : string + let tmp = create_temp_directory("das_deferred_modules", terr) + if (empty(tmp)) { + t |> failure("create_temp_directory: {terr}") + return + } + let fx = make_fixture(tmp, artifact) + arms_deferred(t, fx) + arms_guards_and_caches(t, fx) + arm_fallback(t, fx) + var err : string + rmdir_rec(tmp, err) +} diff --git a/tests/module_cache/test_descriptor_manifest.das b/tests/module_cache/test_descriptor_manifest.das index a3ab60d0bc..394b2cad7e 100644 --- a/tests/module_cache/test_descriptor_manifest.das +++ b/tests/module_cache/test_descriptor_manifest.das @@ -6,37 +6,7 @@ require dastest/testing_boost public require strings require daslib/fio require daslib/strings_boost - -//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, -//! so argv[0] is the interpreter, not the test script -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} - -//! the child's environment, as a command prefix: the scan trace on, so the trace names each descriptor's -//! verdict; the prefix also keeps the command from opening with a quote, which cmd.exe would strip -def trace_prefix() : string { - return get_platform_name() == "windows" ? "set \"DAS_TRACE_MODULE_LOAD=1\"&& " : "DAS_TRACE_MODULE_LOAD=1 " -} - -//! stderr joins stdout: the scan trace goes through the logger -def run(cmd : string; var output : string&) : int { - return unsafe(popen_timeout("{cmd} 2>&1", 300.0, $(f) { - if (f != null) { - output = fread(f) - } - })) -} - -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} +require _mc_common let DESCRIPTOR = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n register_native_path(\"manifest_probe\", \"hello\", \"\{project_path\}/hello.das\")\n\}\n" let DESCRIPTOR_OPT_OUT = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n no_manifest()\n register_native_path(\"manifest_probe\", \"hello\", \"\{project_path\}/hello.das\")\n\}\n" @@ -49,7 +19,8 @@ let DESCRIPTOR_DEP = "options gen2\nrequire daslib/fio\nrequire ./helper\n\n[exp let HELPER = "options gen2\nmodule helper\ndef helper_name() : string \{\n return \"hello\"\n\}\n" let DESCRIPTOR_BROKEN = "options gen2\n\n[export]\ndef initialize(project_path : string) \{\n this_is_not_a_function()\n\}\n" -//! the scan trace line for a module's descriptor: `[module] descriptor /.das_module: ` +//! the scan trace line for a module's descriptor: `[module] descriptor /.das_module: `; +//! a replay verdict keeps its deferred count and drops the two timings def probe_line(out : string; mod : string = "manifest_probe") : string { let marker = "{mod}/.das_module: " let at = find(out, marker) @@ -58,7 +29,13 @@ def probe_line(out : string; mod : string = "manifest_probe") : string { } let rest = slice(out, at + length(marker)) let eol = find(rest, "\n") - return eol < 0 ? rest : slice(rest, 0, eol) + let line = eol < 0 ? rest : slice(rest, 0, eol) + let timing = find(line, " in ") + if (timing < 0) { + return line + } + let deferred = find(line, ", deferred ") + return "{slice(line, 0, timing)} ({slice(line, deferred + 2)}" } def count_of(text : string; needle : string) : int { @@ -117,7 +94,7 @@ def with_line_inserted(lines : array; idx : int; text : string) : string def expect_rewrite(t : T?; cmd : string; manifest : string; text : string; why : string) { fwrite(manifest, text) var out : string - t |> success(report_child(t, why, run(cmd, out), out, "PROBE 42"), "'{why}': the require resolves without the manifest") + t |> success(report_child(t, why, run_child(cmd, out), out, "PROBE 42"), "'{why}': the require resolves without the manifest") t |> equal("compiled ({why}), manifest written (1 row(s))", probe_line(out), "'{why}' recompiles and rewrites:\n{out}") } @@ -171,7 +148,7 @@ def arms_record_and_replay(t : T?; fx : Fixture) { let manifest = fx.manifest t |> run("the first start compiles the descriptor and writes its manifest") @(t : T?) { var out : string - t |> success(report_child(t, "cold", run(cmd, out), out, "PROBE 42"), "the require resolves through the compiled descriptor") + t |> success(report_child(t, "cold", run_child(cmd, out), out, "PROBE 42"), "the require resolves through the compiled descriptor") t |> equal("compiled (no manifest), manifest written (1 row(s))", probe_line(out), "the trace says compiled and written:\n{out}") t |> success(stat(manifest).is_valid, "the manifest sits beside the descriptor") let text = fread(manifest) @@ -188,25 +165,25 @@ def arms_record_and_replay(t : T?; fx : Fixture) { } t |> run("a descriptor that does not compile gets no manifest, and the run goes on") @(t : T?) { var out : string - t |> success(report_child(t, "broken", run(cmd, out), out, "PROBE 42"), "the other module's require still resolves") + t |> success(report_child(t, "broken", run_child(cmd, out), out, "PROBE 42"), "the other module's require still resolves") t |> equal("compiled with errors, no manifest", probe_line(out, "broken_probe"), "the trace names the failure:\n{out}") t |> success(!stat(path_join(fx.brokenDir, ".das_module.manifest")).is_valid, "nothing was written for it") } t |> run("the second start replays the manifest and never compiles the descriptor") @(t : T?) { var out : string - t |> success(report_child(t, "warm", run(cmd, out), out, "PROBE 42"), "the require resolves through the replayed rows") - t |> equal("replayed 1 row(s)", probe_line(out), "the trace says replayed:\n{out}") - t |> equal("replayed 1 row(s)", probe_line(out, "dm_probe"), "the dm row replays too:\n{out}") + t |> success(report_child(t, "warm", run_child(cmd, out), out, "PROBE 42"), "the require resolves through the replayed rows") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(out), "the trace says replayed:\n{out}") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(out, "dm_probe"), "the dm row replays too:\n{out}") t |> success(count_of(out, "Module_Absent <- ") >= 1, "the replayed dm row retries its dlopen:\n{out}") } t |> run("an edited descriptor recompiles once and is replayed again after") @(t : T?) { fwrite(fx.descriptor, "{DESCRIPTOR}// edited\n") var out : string - t |> success(report_child(t, "edited", run(cmd, out), out, "PROBE 42"), "the edited descriptor still resolves the require") + t |> success(report_child(t, "edited", run_child(cmd, out), out, "PROBE 42"), "the edited descriptor still resolves the require") t |> equal("compiled (descriptor changed), manifest written (1 row(s))", probe_line(out), "the stamp mismatch recompiles and rewrites:\n{out}") var again : string - t |> success(report_child(t, "edited warm", run(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") - t |> equal("replayed 1 row(s)", probe_line(again), "the rewritten manifest replays:\n{again}") + t |> success(report_child(t, "edited warm", run_child(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(again), "the rewritten manifest replays:\n{again}") } } @@ -218,7 +195,7 @@ def arms_rejections(t : T?; fx : Fixture) { let endAt = find(text, "\nend\t") fwrite(manifest, slice(text, 0, endAt + 1)) //! drop the end line - a write that died before its rename var out : string - t |> success(report_child(t, "damaged", run(cmd, out), out, "PROBE 42"), "the damaged manifest is not replayed") + t |> success(report_child(t, "damaged", run_child(cmd, out), out, "PROBE 42"), "the damaged manifest is not replayed") t |> equal("compiled (no end line), manifest written (1 row(s))", probe_line(out), "the trace names the damage:\n{out}") } t |> run("every other rejection: no row of a manifest the reader cannot vouch for is replayed") @(t : T?) { @@ -257,28 +234,28 @@ def arms_rejections(t : T?; fx : Fixture) { fwrite(helper, HELPER) fwrite(fx.descriptor, DESCRIPTOR_DEP) var out : string - t |> success(report_child(t, "dep", run(cmd, out), out, "PROBE 42"), "the require resolves through the helper's answer") + t |> success(report_child(t, "dep", run_child(cmd, out), out, "PROBE 42"), "the require resolves through the helper's answer") t |> equal("compiled (descriptor changed), manifest written (1 row(s))", probe_line(out), "the dep descriptor compiles once:\n{out}") t |> success(find(fread(manifest), "/helper.das\t") >= 0, "the helper is stamped in the manifest, by the path the require resolved: {fread(manifest)}") var warm : string - t |> success(report_child(t, "dep warm", run(cmd, warm), warm, "PROBE 42"), "the dep descriptor replays") - t |> equal("replayed 1 row(s)", probe_line(warm), "an unchanged helper replays:\n{warm}") + t |> success(report_child(t, "dep warm", run_child(cmd, warm), warm, "PROBE 42"), "the dep descriptor replays") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(warm), "an unchanged helper replays:\n{warm}") fwrite(helper, "{HELPER}// edited\n") var edited : string - t |> success(report_child(t, "dep edited", run(cmd, edited), edited, "PROBE 42"), "the require resolves after the helper edit") + t |> success(report_child(t, "dep edited", run_child(cmd, edited), edited, "PROBE 42"), "the require resolves after the helper edit") t |> equal("compiled (dependency changed), manifest written (1 row(s))", probe_line(edited), "the helper's stamp mismatch recompiles the descriptor:\n{edited}") var again : string - t |> success(report_child(t, "dep edited warm", run(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") - t |> equal("replayed 1 row(s)", probe_line(again), "the rewritten manifest replays:\n{again}") + t |> success(report_child(t, "dep edited warm", run_child(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(again), "the rewritten manifest replays:\n{again}") } t |> run("a cross-compile target is part of the key: a --jit-target run never replays a native manifest") @(t : T?) { //! dasOpenGL's descriptor registers its module only when get_cross_platform_name() says emscripten - //! an argv input, so a native manifest must not serve a --jit-target run, nor the other way round var out : string - t |> success(report_child(t, "target", run("{cmd} -- --jit-target=wasm32-unknown-emscripten", out), out, "PROBE 42"), "the require resolves under the target") + t |> success(report_child(t, "target", run_child("{cmd} -- --jit-target=wasm32-unknown-emscripten", out), out, "PROBE 42"), "the require resolves under the target") t |> equal("compiled (compile target), manifest written (1 row(s))", probe_line(out), "the target keys the manifest apart:\n{out}") var back : string - t |> success(report_child(t, "native again", run(cmd, back), back, "PROBE 42"), "the native run resolves the require") + t |> success(report_child(t, "native again", run_child(cmd, back), back, "PROBE 42"), "the native run resolves the require") t |> equal("compiled (compile target), manifest written (1 row(s))", probe_line(back), "the native run keys apart from the target's manifest:\n{back}") } } @@ -289,19 +266,19 @@ def arms_opt_out_and_write_failures(t : T?; fx : Fixture) { t |> run("no_manifest() inside the descriptor makes it run on every start") @(t : T?) { fwrite(fx.descriptor, DESCRIPTOR_OPT_OUT) var out : string - t |> success(report_child(t, "opt-out", run(cmd, out), out, "PROBE 42"), "the opted-out descriptor resolves the require") + t |> success(report_child(t, "opt-out", run_child(cmd, out), out, "PROBE 42"), "the opted-out descriptor resolves the require") t |> equal("compiled (descriptor changed), no_manifest recorded", probe_line(out), "the first run records the opt-out:\n{out}") let text = fread(manifest) t |> success(find(text, "\nno_manifest\n") >= 0 && find(text, "\nnp\t") < 0, "the manifest carries the flag and no rows: {text}") var again : string - t |> success(report_child(t, "opt-out warm", run(cmd, again), again, "PROBE 42"), "the opted-out descriptor resolves the require again") + t |> success(report_child(t, "opt-out warm", run_child(cmd, again), again, "PROBE 42"), "the opted-out descriptor resolves the require again") t |> equal("compiled (no_manifest)", probe_line(again), "every later start compiles it:\n{again}") } t |> run("a recorded argument the format cannot carry leaves the manifest unwritten") @(t : T?) { fwrite(fx.descriptor, DESCRIPTOR_TABBED) let before = fread(manifest) var out : string - t |> success(report_child(t, "tabbed", run(cmd, out), out, "PROBE 42"), "the descriptor still resolves the require") + t |> success(report_child(t, "tabbed", run_child(cmd, out), out, "PROBE 42"), "the descriptor still resolves the require") t |> equal("compiled (descriptor changed), manifest not written: a recorded argument contains a tab or newline", probe_line(out), "the trace names the field:\n{out}") t |> equal(before, fread(manifest), "the stale manifest is left as it was") } @@ -310,12 +287,12 @@ def arms_opt_out_and_write_failures(t : T?; fx : Fixture) { remove_result(manifest) mkdir_result(fx.manifestTmp) //! a directory where the .tmp goes: the create fails on every platform, root or not var out : string - t |> success(report_child(t, "no tmp", run(cmd, out), out, "PROBE 42"), "the require resolves without a manifest") + t |> success(report_child(t, "no tmp", run_child(cmd, out), out, "PROBE 42"), "the require resolves without a manifest") t |> success(probe_line(out) |> starts_with("compiled (no manifest), manifest not written: cannot create "), "the trace names the create failure:\n{out}") rmdir_result(fx.manifestTmp) mkdir_result(manifest) //! a directory where the manifest goes: the rename fails var again : string - t |> success(report_child(t, "no rename", run(cmd, again), again, "PROBE 42"), "the require resolves without a manifest") + t |> success(report_child(t, "no rename", run_child(cmd, again), again, "PROBE 42"), "the require resolves without a manifest") t |> success(find(probe_line(again), ", manifest not written: cannot rename ") >= 0, "the trace names the rename failure:\n{again}") t |> success(!stat(fx.manifestTmp).is_valid, "the failed rename left no .tmp behind") rmdir_result(manifest) diff --git a/tests/module_cache/test_generic_instance_origin.das b/tests/module_cache/test_generic_instance_origin.das index 5d5aa0be2f..2b8c6eea19 100644 --- a/tests/module_cache/test_generic_instance_origin.das +++ b/tests/module_cache/test_generic_instance_origin.das @@ -5,13 +5,7 @@ require dastest/testing_boost public require strings require daslib/fio - -// argv[0] is the running interpreter; dastest is invoked as -// `daslang(.exe) dastest/dastest.das ...`, so argv[0] is the daslang binary to spawn. -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} +require _mc_common def run_argv(args : array; var output : string&) : int { @@ -23,19 +17,6 @@ def run_argv(args : array; var output : string&) : int { } -// A child that dies takes its diagnosis with it unless the parent prints what it said - the -// exit code alone turns a one-line answer into an exit-code hunt. Echo the child's output on -// any failure, and only then. -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} - - // A generic instance restored from the module cache must keep its origin generic // (Function::fromGeneric). The child host restores its own module graph from the cache and // then compiles a fresh program in the same process - a null origin on any restored diff --git a/tests/module_cache/test_macro_dep_invalidate.das b/tests/module_cache/test_macro_dep_invalidate.das index 9371f961e6..1561bd4d96 100644 --- a/tests/module_cache/test_macro_dep_invalidate.das +++ b/tests/module_cache/test_macro_dep_invalidate.das @@ -5,13 +5,7 @@ require dastest/testing_boost public require strings require daslib/fio - -//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, -//! so argv[0] is the interpreter, not the test script -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} +require _mc_common //! child env rides the command string (the portable set/prefix split popen_argv cannot carry) def env_run(dep, cmd : string; var output : string&) : int { @@ -24,15 +18,6 @@ def env_run(dep, cmd : string; var output : string&) : int { })) } -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} - //! the module cache must gate a macro-registered file input by CONTENT, not mtime: a tune //! sidecar is rewritten byte-identically on every exit, and a record served stale replays //! macro output minted against the old file - stale stamps, empty tune_status, unraced winners. diff --git a/utils/REVIEW.das b/utils/REVIEW.das index f8746f15c5..27a49b649e 100644 --- a/utils/REVIEW.das +++ b/utils/REVIEW.das @@ -13,6 +13,10 @@ require dastest/review_gate // utils/ children that are not tool directories: the shared library hub var private LIBRARY_DIRS <- { "common" } +// tools whose development runs through a python keep-alive supervisor: never built or +// shipped as an exe, since that form would ship without anyone having run it +var private SUPERVISED_TOOLS <- { "mcp", "lsp" } + // Files licensed to name utils/internal outside internal/: var private INTERNAL_REF_EXEMPT <- { "utils/REVIEW.md", // states the rule @@ -54,7 +58,7 @@ def private mentions_path(text : string; p : string) : bool { def private list_dirs(parent : string) : array { var out : array dir(parent) $(name) { - return if (name == "." || name == "..") + return if (name |> starts_with(".")) // `.`, `..`, and a JIT's `.jitted_scripts` cache let st = stat(path_join(parent, name)) if (st.is_valid && st.is_dir) { out |> push(name) @@ -164,6 +168,21 @@ def private check_no_internal_refs { } } +def private check_supervised_tools { + let ucm = fread("utils/CMakeLists.txt") + var inscope exes <- cmake_list_entries(ucm, "DAS_UTILS_SHIPPED_EXES") + var inscope built <- cmake_list_entries(ucm, "DAS_UTILS") + for (s in keys(SUPERVISED_TOOLS)) { + var hit = find_index(exes, s) >= 0 + for (b in built) { + hit ||= base_name(b) == s + } + if (hit) { + gate_finding("utils/CMakeLists.txt", "{s} is in DAS_UTILS or DAS_UTILS_SHIPPED_EXES — its development runs through the python keep-alive supervisor, so an exe form ships without anyone having run it; run it interpreted") + } + } +} + var private BIN_EXE <- %regex~`([A-Za-z0-9_./-]+)\.exe`%% def private check_shipped_exes(externals : array) { @@ -244,5 +263,6 @@ def main() : int { check_registration_triple(tools, internals) check_no_internal_refs() check_shipped_exes(externals) + check_supervised_tools() return gate_verdict("utils") } diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 9d2bf05218..054742f446 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -4,10 +4,10 @@ doc: `CLAUDE.md` (repo root). A tool is a directory that owns the programs it ships - each one's entry point and the files -only those programs use; a program joins a tool through `CMakeLists.txt` (beside this file) -building or shipping it, or through the directory's `.das_package` declaring it with -`release_program` - under `utils/`, or outside `utils/` when `CMakeLists.txt` builds or ships -it. An arm is one `t |> run(...)` case of a `[test]` function. An arm's load-bearing +only those programs use; a program joins a tool through `utils/CMakeLists.txt` (beside this +file) or the repo root's `CMakeLists.txt` building or shipping it, or through the directory's +`.das_package` declaring it with `release_program` - under `utils/`, or outside `utils/` when +one of those two files builds or ships it. An arm is one `t |> run(...)` case of a `[test]` function. An arm's load-bearing assertions are the ones that prove the change, never a skip-path assertion. A CI row is a workflow step whose command reaches the arm. Load-bearing assertions no CI row executes - the arm returns or skips before them, or no suite a CI row runs includes the arm's file - are diff --git a/utils/daslang/main.cpp b/utils/daslang/main.cpp index d39d70d95c..d68bf4d440 100644 --- a/utils/daslang/main.cpp +++ b/utils/daslang/main.cpp @@ -9,6 +9,7 @@ #include "daScript/ast/ast_serializer.h" #include "daScript/misc/crash_handler.h" #include "daScript/misc/job_que.h" +#include "daScript/misc/performance_time.h" #ifdef __APPLE__ #include #endif @@ -73,6 +74,9 @@ static bool gen2MakeSyntax = false; static bool trackAllocations = false; static bool heapReportAtExit = false; static bool logModuleCompileTime = false; +static int64_t startupCompileUsec = 0; +static int64_t startupSimulateUsec = 0; +static int64_t startupRunUsec = 0; static bool buildingDocumentation = false; static vector dllSearchPaths; @@ -235,6 +239,8 @@ int das_aot_main ( int argc, char * argv[] ) { gen2MakeSyntax = true; } else if ( strcmp(argv[ai],"-no-dynamic-modules")==0 ) { noDynamicModules = true; + } else if ( strcmp(argv[ai],"-ignore-manifest")==0 ) { + ignore_dynamic_module_manifests(true); } else if ( strcmp(argv[ai],"-no-lint")==0 ) { noLint = true; } else if ( strcmp(argv[ai],"-log-compile-time")==0 ) { @@ -510,7 +516,9 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP cacheQuiet = true; } moduleCache.install(cacheReadPath, cacheWritePath, cacheQuiet); + auto compile0 = ref_time_ticks(); auto program = compileDaScript(fn,access,tout,dummyGroup,policies); + startupCompileUsec += get_time_usec(compile0); { auto cres = moduleCache.finish(); if ( !cacheQuiet ) { @@ -559,7 +567,9 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP if ( compileOnly ) return 0; + auto simulate0 = ref_time_ticks(); auto pctx = SimulateWithErrReport(program, tout); + startupSimulateUsec += get_time_usec(simulate0); // Check for compiler leaks (TypeDecl nodes left on thread root after compile+simulate) { auto & root = gc_root::gc_get_thread_root(); @@ -604,11 +614,13 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP auto fnTest = fnMVec.back(); pctx->restart(); vec4f res; + auto run0 = ref_time_ticks(); if ( debuggerRequired ) { res = pctx->eval(fnTest, nullptr); } else { res = pctx->evalWithCatch(fnTest, nullptr); } + startupRunUsec += get_time_usec(run0); if ( auto ex = pctx->getException() ) { tout << "EXCEPTION: " << ex << " at " << pctx->exceptionAt.describe() << "\n"; exitCode = 1; @@ -730,10 +742,13 @@ void print_help() { << " --das-profiler-global install profiler as singleton agent (default with --das-profiler-memory)\n" << " --das-profiler-leaks track live heap allocations and dump leaks on context destroy\n" << " -no-dynamic-modules skip loading dynamic modules from dasroot and project root\n" + << " -ignore-manifest compile every .das_module descriptor and load every C++ module on start, reading\n" + << " and writing no .das_module.manifest - by default a manifest's C++ module loads at the first require that names it\n" << " -no-lint skip the lint pass (Program::lint)\n" << " --ast-verify force-include daslib/ast_verify; checks AST structural invariants before each inference pass\n" << " --ast-verify-batch checks the finished tree only (no per-pass walks, no cross-module sweeps): cheap enough to gate many files (CI)\n" - << " -log-compile-time log detailed per-module compile-time breakdown (parse / infer with pass count / optimize / macro (in infer) / macro mods / simulate) + function count\n" + << " -log-compile-time log detailed per-module compile-time breakdown (parse / infer with pass count / optimize / macro (in infer) / macro mods / simulate) + function count,\n" + << " a cached module's read time (and its macro-context simulate), and the process startup timeline (builtin modules / module scan / initialize / compile / simulate / run / teardown / shutdown)\n" << " -- separator for script arguments\n" << "daslang -aot {-q} {-p}\n" << " -project path to project file\n" @@ -773,6 +788,12 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR); _set_error_mode(_OUT_TO_STDERR); #endif + auto startupMain0 = ref_time_ticks(); + int64_t startupScanUsec = 0; + int64_t startupInitUsec = 0; + int64_t startupCompileAndRunUsec = 0; + int64_t startupPreScanUsec = 0; + int64_t startupBuiltinUsec = 0; install_das_crash_handler(); #ifdef __APPLE__ pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0); @@ -1043,6 +1064,8 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { i += 1; } else if ( cmd=="no-dynamic-modules" ) { noDynamicModules = true; + } else if ( cmd=="ignore-manifest" ) { + ignore_dynamic_module_manifests(true); } else if ( cmd=="-dump-leaks" ) { dumpLeaks = true; } else if ( cmd=="-no-dump-leaks" ) { @@ -1072,6 +1095,8 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { printf("-no-module-cache disables the cache; do not combine it with -ser/-deser\n"); return -1; } + startupPreScanUsec = get_time_usec(startupMain0); + auto builtin0 = ref_time_ticks(); // register modules register_builtin_modules(); require_project_specific_modules(); @@ -1079,6 +1104,8 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { // Otherwises search for static modules. #include "modules/external_pull.inc" #endif + startupBuiltinUsec = get_time_usec(builtin0); + auto scan0 = ref_time_ticks(); #ifdef DAS_ENABLE_DYN_INCLUDES if ( !noDynamicModules ) { // Search for external modules and init them. Only if flag is enabled. @@ -1088,7 +1115,10 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { require_dynamic_modules(access, getDasRoot(), project_root, load_modules, disabled_modules, tout); } #endif + startupScanUsec = get_time_usec(scan0); + auto init0 = ref_time_ticks(); Module::Initialize(); + startupInitUsec = get_time_usec(init0); // compile and run int exitCode = 0; @@ -1102,7 +1132,9 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { #endif for ( auto & fn : files ) { replace(fn, "_dasroot_", getDasRoot()); + auto compileAndRun0 = ref_time_ticks(); int rc = compile_and_run(fn, mainName, outputProgramCode, dryRun, compileOnly); + startupCompileAndRunUsec += get_time_usec(compileAndRun0); if ( rc != 0 ) { exitCode = rc; } @@ -1121,10 +1153,26 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { // Handle-leak dump runs inside Module::Shutdown, between module // destruction (drains job threads) and DLL unload (invalidates the // dumpHandleLeaks function pointers registered from shared modules). + auto shutdown0 = ref_time_ticks(); Module::Shutdown(dumpLeaks); if ( dumpLeaks ) { JobStatus::DumpJobQueLeaks(); } + if ( logModuleCompileTime ) { + auto shutdownUsec = get_time_usec(shutdown0); + auto teardownUsec = startupCompileAndRunUsec - startupCompileUsec - startupSimulateUsec - startupRunUsec; + tout << "startup: main total " << (get_time_usec(startupMain0) / 1000000.) << "\n" + << "\targuments " << (startupPreScanUsec / 1000000.) << "\n" + << "\tbuiltin modules " << (startupBuiltinUsec / 1000000.) << "\n" + << "\tmodule scan " << (startupScanUsec / 1000000.) << "\n" + << "\tinitialize " << (startupInitUsec / 1000000.) << "\n" + << "\tcompile " << (startupCompileUsec / 1000000.) << "\n" + << "\tsimulate " << (startupSimulateUsec / 1000000.) << "\n" + << "\trun " << (startupRunUsec / 1000000.) << "\n" + << "\tteardown " << (teardownUsec / 1000000.) << "\n" + << "\tshutdown " << (shutdownUsec / 1000000.) << "\n" + ; + } // das::dump_alloc_leaks registers itself as the FIRST atexit handler, so it // fires after all static destructors — cleaner than dumping here. if ( g_smart_ptr_total!=0 ) { diff --git a/utils/lsp/REVIEW.md b/utils/lsp/REVIEW.md deleted file mode 100644 index 96773db62a..0000000000 --- a/utils/lsp/REVIEW.md +++ /dev/null @@ -1,8 +0,0 @@ -# lsp Code Review Checklist - -**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: -`README.md`. - -**Never add `lsp` to `DAS_UTILS` or `DAS_UTILS_SHIPPED_EXES` in `utils/CMakeLists.txt` - clients -start the server through `lsp_supervisor.py` instead.** An exe form would ship without anyone -having used it. diff --git a/utils/lsp/ROADMAP.md b/utils/lsp/ROADMAP.md index 09ed4bb420..3272db45ca 100644 --- a/utils/lsp/ROADMAP.md +++ b/utils/lsp/ROADMAP.md @@ -296,6 +296,12 @@ PR for the whole branch AFTER wave 4 (single preflight + CI round). needs no LSP wiring - its build already produces the binary `find_compiler` discovers (`build/daslang` et al.); setup's done-message now says so. +## Follow-ups + +- The watchdog does not supervise `lsp_supervisor.py` yet; wire it in. With that, an exe form + of the subtools becomes possible again - the same item as the MCP server's + (`utils/mcp/ROADMAP.md`, Follow-ups). + ## Non-goals - Completion (CC doesn't consume it), formatting-over-LSP (CC has the MCP/CLI diff --git a/utils/mcp/README.md b/utils/mcp/README.md index ad5b8ca63a..20c8db4299 100644 --- a/utils/mcp/README.md +++ b/utils/mcp/README.md @@ -80,11 +80,13 @@ On **Linux/macOS** point each entry at the binary directly (no launcher needed): ```json "mcpServers": { - "daslang": { "command": "./bin/daslang", "args": ["utils/mcp/main.das"] }, - "daslang-cpp": { "command": "./bin/daslang", "args": ["utils/mcp/cpp_main.das"] } + "daslang": { "command": "./bin/daslang", "args": ["-ignore-manifest", "utils/mcp/main.das"] }, + "daslang-cpp": { "command": "./bin/daslang", "args": ["-ignore-manifest", "utils/mcp/cpp_main.das"] } } ``` +An existing `.mcp.json` needs `-ignore-manifest` added by hand (or a rerun of `utils/mcp/setup.das`): without it the server enumerates only the modules a compile loaded, so `list_modules` and the all-modules symbol scans come up short. + Tools are namespaced by server, so the cpp server's tools appear as `mcp__daslang-cpp__cpp_compile_check` etc. `cpp-mcp` - a standalone static AOT build of `cpp_main.das` for C++-only consumers - exists as a gated target (`DAS_BUILD_CPP_MCP`, OFF by default; bundled by `ci/make_cpp_mcp_bundle.sh`, released via `cpp_mcp_release.yml`, setup in `cpp-mcp-setup.md`); the interpreted form above is the same server. It is a separate product: the mcp server itself never ships as a `daslang -exe` binary - development runs it through the python keep-alive supervisor, so that exe form would never be dogfooded. ### Duplicate Detection @@ -123,10 +125,10 @@ No extra build dependencies - the MCP server uses stdio transport. Claude Code m ```bash # Manual test (Windows): -bin/Release/daslang.exe utils/mcp/main.das +bin/Release/daslang.exe -ignore-manifest utils/mcp/main.das # Manual test (Linux): -./bin/daslang utils/mcp/main.das +./bin/daslang -ignore-manifest utils/mcp/main.das ``` Configure in `.mcp.json` (project root): @@ -137,7 +139,7 @@ Configure in `.mcp.json` (project root): "mcpServers": { "daslang": { "command": "bin/Release/daslang.exe", - "args": ["utils/mcp/main.das"] + "args": ["-ignore-manifest", "utils/mcp/main.das"] } } } @@ -147,7 +149,7 @@ Configure in `.mcp.json` (project root): "mcpServers": { "daslang": { "command": "./bin/daslang", - "args": ["utils/mcp/main.das"] + "args": ["-ignore-manifest", "utils/mcp/main.das"] } } } @@ -157,10 +159,10 @@ Or add via CLI: ```bash # Windows -claude mcp add daslang -- bin/Release/daslang.exe utils/mcp/main.das +claude mcp add daslang -- bin/Release/daslang.exe -ignore-manifest utils/mcp/main.das # Linux -claude mcp add daslang -- ./bin/daslang utils/mcp/main.das +claude mcp add daslang -- ./bin/daslang -ignore-manifest utils/mcp/main.das ``` Claude Code starts and stops the server automatically with each session. diff --git a/utils/mcp/REVIEW.md b/utils/mcp/REVIEW.md index 68578cea3e..877cc1839f 100644 --- a/utils/mcp/REVIEW.md +++ b/utils/mcp/REVIEW.md @@ -3,17 +3,12 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `README.md`. Planned work: `ROADMAP.md`. -**Never add `mcp` to `DAS_UTILS` or `DAS_UTILS_SHIPPED_EXES` (`utils/CMakeLists.txt`) - run it -interpreted through `.mcp.json` instead.** -Development runs the server through the python keep-alive supervisor, so an exe form would -never be used in development before it ships. - -**A diff that adds a top-level `.das` under `utils/mcp/` that the shipped SDK runs or loads - -`main.das` reaches it, or it has its own `main` that something in the shipped SDK runs - also -adds it to the `install(FILES ...)` block that lists `utils/mcp/main.das` in `CMakeLists.txt` -(repo root), in the same change.** `tools/` and `subtools/` are globbed; a top-level file left -out of the list dies in the shipped SDK on `error[20605] missing prerequisite` while the -in-tree server keeps working. +**A diff that adds a top-level file under `utils/mcp/` that the shipped SDK runs or loads - +`main.das` reaches it, the supervisor or the `.cmd` launcher runs it, or it has its own `main` +that something in the shipped SDK runs - also adds it to the `install(FILES ...)` block that +lists `utils/mcp/main.das` in `CMakeLists.txt` (repo root), in the same change.** `tools/` and +`subtools/` are globbed; a top-level file left out of the list is absent in the shipped SDK +while the in-tree server keeps working. **Weakening the kept-comment cases in `test_tools.das` is a defect** - they pin the formatter's kept set (the leading header block, `//!` docs, `//fmt:` directives, `nolint:` diff --git a/utils/mcp/ROADMAP.md b/utils/mcp/ROADMAP.md index ebf6eb5803..dd2fbab3ef 100644 --- a/utils/mcp/ROADMAP.md +++ b/utils/mcp/ROADMAP.md @@ -334,3 +334,16 @@ Many complex tools are compositions of simpler ones: - `extract_function` = AST analysis + code generation + `compile_check` Building the foundational tools well creates a platform for everything else. + +## Follow-ups + +- **A custom `modules/` fixture.** A test that starts the server with `-project_root` on a + fixture holding its own `modules//.das_module` - a descriptor that registers a require + path and a C++ module - and checks `list_modules`, `find_symbol` and `compile_check` see + them. The server runs eager (`-ignore-manifest`) while a plain run loads a `.shared_module` + at its first `require`; the fixture is what proves a user's tree behaves under both. +- **Back to an exe form.** Once that fixture passes, the server can build and ship as an exe + again: the reason it runs interpreted - development through the python keep-alive supervisor, + so an exe would ship unrun - is met by the watchdog, which now covers what the supervisor + did. `utils/REVIEW.das` bans the exe today; lifting the ban is part of this item. +- **The watchdog does not supervise this server or `lsp_supervisor.py` yet.** Wire both in. diff --git a/utils/mcp/daslang-mcp-msvc.cmd b/utils/mcp/daslang-mcp-msvc.cmd index 4909879415..23478e12a1 100644 --- a/utils/mcp/daslang-mcp-msvc.cmd +++ b/utils/mcp/daslang-mcp-msvc.cmd @@ -58,9 +58,10 @@ rem First arg selects the server. A *.exe selects a prebuilt server binary in rem bin/ (the AOT cpp-mcp) run directly; otherwise it's an interpreted .das rem script handed to daslang.exe. Defaults to main.das (the full server). rem Any further args are forwarded to the chosen server. +rem -ignore-manifest: the server enumerates modules, so every C++ module loads on start. set "MCPSCRIPT=%~1" if not defined MCPSCRIPT ( - "%DASLANG%" "%MCPDIR%main.das" + "%DASLANG%" -ignore-manifest "%MCPDIR%main.das" exit /b %ERRORLEVEL% ) rem Goto (not a parenthesized block) so MCPBIN's set + use are sequential @@ -68,7 +69,7 @@ rem statements -- this script has no EnableDelayedExpansion, so %MCPBIN% rem read inside an `if ( ... )` block would expand stale (pre-set). if /i "%MCPSCRIPT:~-4%"==".exe" goto runexe shift -"%DASLANG%" "%MCPDIR%%MCPSCRIPT%" %1 %2 %3 %4 %5 %6 %7 %8 +"%DASLANG%" -ignore-manifest "%MCPDIR%%MCPSCRIPT%" %1 %2 %3 %4 %5 %6 %7 %8 exit /b %ERRORLEVEL% :runexe diff --git a/utils/mcp/mcp_supervisor.py b/utils/mcp/mcp_supervisor.py index 14d5a55195..129ef08525 100644 --- a/utils/mcp/mcp_supervisor.py +++ b/utils/mcp/mcp_supervisor.py @@ -294,7 +294,8 @@ def _default_launcher() -> list[str]: return ["cmd", "/c", os.path.join(SCRIPT_DIR, "daslang-mcp-msvc.cmd")] main_das = os.path.join(SCRIPT_DIR, "main.das") picked = _pick_binary(REPO_ROOT) - return [picked or os.path.join(REPO_ROOT, "bin", "daslang"), main_das] + # -ignore-manifest: the server enumerates modules, so every C++ module loads on start + return [picked or os.path.join(REPO_ROOT, "bin", "daslang"), "-ignore-manifest", main_das] def _python_launcher() -> str: @@ -361,7 +362,7 @@ def write_mcp_json(repo_root: str) -> bool: if os.path.exists(os.path.join(repo_root, "utils", "internal", "das-herd", "mcp_main.das")): prev_herd = servers.get("dasherd", {}) herd_entry = {"command": _daslang_binary(repo_root), - "args": ["utils/internal/das-herd/mcp_main.das"]} + "args": ["-ignore-manifest", "utils/internal/das-herd/mcp_main.das"]} if isinstance(prev_herd, dict) and "defer_loading" in prev_herd: herd_entry["defer_loading"] = prev_herd["defer_loading"] servers["dasherd"] = herd_entry