From 871d349d88fe77f335a5d26d74bc239f9a2a6cbd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 7 Sep 2026 19:46:48 -0700 Subject: [PATCH 01/22] -log-compile-time prints the process startup timeline (builtin module registration / module scan / initialize / compile / simulate / run / teardown / shutdown) and, per cached module, its cache read time with the macro-context simulate share, plus a cache read total in the compile summary - the serializer's totMacroTime was accumulated and never read, so a warm -jit hello world (237 ms on the M5) had no way to show that 154 of its 178 ms compile reads 61 cached modules, 92 of that simulating macro-module contexts (llvm_macro alone 49: its context is the whole emitter, simulated to run a 1 ms DLL cache hit) and 12 reparsing llvm_func in place every run; DAS_TRACE_MODULE_LOAD replay lines carry their time and the shared-module load share (dasVulkan's dlopen is 14 of the scan's 27 ms); test_descriptor_manifest's probe_line strips the timing clause --- skills/internal/environment_variables.md | 2 +- src/ast/ARCHITECTURE.md | 7 ++- src/ast/ast_parse.cpp | 19 ++++++++ src/ast/dyn_modules.cpp | 8 +++- .../module_cache/test_descriptor_manifest.das | 4 +- utils/daslang/main.cpp | 44 ++++++++++++++++++- 6 files changed, 78 insertions(+), 6 deletions(-) diff --git a/skills/internal/environment_variables.md b/skills/internal/environment_variables.md index 518a1b9f78..87ccc264fa 100644 --- a/skills/internal/environment_variables.md +++ b/skills/internal/environment_variables.md @@ -72,7 +72,7 @@ and unknown codes are harmless. The `-no-lint` command-line flag skips the lint |---|---|---| | `DAS_GC_STAGE_REPORT` | flag | Report gc_node deltas per compilation stage - the first thing to reach for on a `GC APP LEAK` at exit. | | `DAS_GC_BREAK_ON_ID` | number | Break when the gc_node with this id is allocated. Pair it with the id from a leak report. | -| `DAS_TRACE_MODULE_LOAD` | flag | Log every module as it loads, with its resolved path - the fastest way to see which of two same-named modules actually won - and one line per `.das_module` descriptor saying whether the scan replayed its manifest or compiled it, and why (`src/ast/ARCHITECTURE.md` sec.2). | +| `DAS_TRACE_MODULE_LOAD` | flag | Log every module as it loads, with its resolved path - the fastest way to see which of two same-named modules actually won - and one line per `.das_module` descriptor saying whether the scan replayed its manifest or compiled it, and why; a replayed line carries its time and the share its shared-module loads took (`src/ast/ARCHITECTURE.md` sec.2). | ## Ambient variables daslang reads but does not own diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 94e87b99a0..298eaa9073 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -83,5 +83,8 @@ recorded rows in recorded order, so the Quiet deferral and the post-scan retry o `DT_NEEDED` dlopen behave as on a compiled start. `no_manifest()` inside `initialize` marks the descriptor as one that runs on every start: its manifest carries the stamp and the flag and no rows, and is not rewritten. With `DAS_TRACE_MODULE_LOAD=1` the scan prints one line per -descriptor - `replayed N row(s)`, `compiled (), manifest written (N row(s))`, -`compiled (no_manifest)`, or why a manifest was not written. +descriptor - `replayed N row(s) in (shared module load )`, `compiled (), +manifest written (N row(s))`, `compiled (no_manifest)`, or why a manifest was not written. A +replayed descriptor's time is its manifest read plus its rows, and the second number is the +share the `.shared_module` dlopen and module constructor took - on a warm start that is nearly +all of it. diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index f978c52c31..6742a64c2e 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -580,6 +580,9 @@ namespace das { static DAS_THREAD_LOCAL(int64_t) totInfer; static DAS_THREAD_LOCAL(int64_t) totOpt; static DAS_THREAD_LOCAL(int64_t) totM; + static DAS_THREAD_LOCAL(int64_t) totCacheRead; + static DAS_THREAD_LOCAL(int64_t) cntCacheRead; + static DAS_THREAD_LOCAL(int64_t) totCacheMacroSim; // deserialization may have left the active gc root pointing at (or the old program // owning) a module root that dies with the old program — repoint around the swap so @@ -1071,7 +1074,19 @@ namespace das { program->inferPassesUsed = 0; // reset once per module; inferTypesDirty accumulates across all inferTypes legs (incl. restartInfer) program->policies = policies; // before the cache read: the reader compares the record's policies against this compile's + auto & serializer_read = daScriptEnvironment::getBound()->serializer_read; + uint64_t macroSim0 = serializer_read ? serializer_read->totMacroTime : 0; if ( trySerializeProgramModule(program, access, fileName, libGroup, logs) ) { + auto readT = get_time_usec(time0); + auto macroSimT = int64_t(serializer_read->totMacroTime - macroSim0); + *totCacheRead += readT; + *cntCacheRead += 1; + *totCacheMacroSim += macroSimT; + if ( policies.log_module_compile_time ) { + logs << "cache read took " << (readT / 1000000.) << ", " << program->thisModule->name << " (" << fileName << ")"; + if ( macroSimT ) logs << " -- macro simulate " << (macroSimT / 1000000.); + logs << "\n"; + } return program; } else { // Serialization failed and the program changed, so set it for proper GC collection on exit. @@ -1800,6 +1815,9 @@ namespace das { *totInfer = 0; *totOpt = 0; *totM = 0; + *totCacheRead = 0; + *cntCacheRead = 0; + *totCacheMacroSim = 0; daScriptEnvironment::getBound()->macroTimeTicks = 0; vector req; vector missing; @@ -1933,6 +1951,7 @@ namespace das { auto totT = get_time_usec(time0); logs << "total compile took " << (totT / 1000000.) << ", " << fileName << " -- " << res->totalFunctions << " functions\n" << "\trequire " << (preqT / 1000000.) << "\n" + << "\tcache read " << (*totCacheRead / 1000000.) << " (" << *cntCacheRead << " modules, macro simulate " << (*totCacheMacroSim / 1000000.) << ")\n" << "\tparse " << (*totParse / 1000000.) << "\n" << "\tinfer " << (*totInfer / 1000000.) << "\n" << "\toptimize " << (*totOpt / 1000000.) << "\n" diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 5ef03c5cd9..2062b5a045 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -8,6 +8,7 @@ #include // TextWriter, LOG (the env-gated scan trace) #include // hash_block64 #include // get_dasenv_trace_module_load +#include #include // tolower (case-insensitive basename normalize) #include // fprintf(stderr) for the shadow-shadows-global diagnostic @@ -343,17 +344,22 @@ static Result init_dyn_modules(smart_ptr fa, string path, TextWriter const uint64_t stamp = src ? hash_block64((const uint8_t *) src, len) : 0; const string manifest = path + "/" + MANIFEST_SUFFIX; const ManifestKey key = manifest_key(path); + auto time0 = ref_time_ticks(); auto mr = src ? read_manifest(manifest, len, stamp, key, fa) : ManifestRead(); if ( mr.verdict == ManifestVerdict::Replay ) { + int64_t dllUsec = 0; for ( auto & row : mr.rows ) { if ( row.dynamic ) { + auto dll0 = ref_time_ticks(); replay_dynamic_module(row.a.c_str(), row.b.c_str(), row.on_error); + dllUsec += get_time_usec(dll0); } else { replay_native_path(row.a.c_str(), row.b.c_str(), row.c.c_str()); } } if ( trace_scan() ) { - LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": replayed " << mr.rows.size() << " row(s)\n"; + LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": replayed " << mr.rows.size() << " row(s) in " + << (get_time_usec(time0) / 1000000.) << " (shared module load " << (dllUsec / 1000000.) << ")\n"; } return Result::OK; } diff --git a/tests/module_cache/test_descriptor_manifest.das b/tests/module_cache/test_descriptor_manifest.das index a3ab60d0bc..9d9f89f7ab 100644 --- a/tests/module_cache/test_descriptor_manifest.das +++ b/tests/module_cache/test_descriptor_manifest.das @@ -58,7 +58,9 @@ def probe_line(out : string; mod : string = "manifest_probe") : string { } let rest = slice(out, at + length(marker)) let eol = find(rest, "\n") - return eol < 0 ? rest : slice(rest, 0, eol) + let line = eol < 0 ? rest : slice(rest, 0, eol) + let timing = find(line, " in ") + return timing < 0 ? line : slice(line, 0, timing) } def count_of(text : string; needle : string) : int { diff --git a/utils/daslang/main.cpp b/utils/daslang/main.cpp index d39d70d95c..cef658681b 100644 --- a/utils/daslang/main.cpp +++ b/utils/daslang/main.cpp @@ -9,6 +9,7 @@ #include "daScript/ast/ast_serializer.h" #include "daScript/misc/crash_handler.h" #include "daScript/misc/job_que.h" +#include "daScript/misc/performance_time.h" #ifdef __APPLE__ #include #endif @@ -73,6 +74,9 @@ static bool gen2MakeSyntax = false; static bool trackAllocations = false; static bool heapReportAtExit = false; static bool logModuleCompileTime = false; +static int64_t startupCompileUsec = 0; +static int64_t startupSimulateUsec = 0; +static int64_t startupRunUsec = 0; static bool buildingDocumentation = false; static vector dllSearchPaths; @@ -510,7 +514,9 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP cacheQuiet = true; } moduleCache.install(cacheReadPath, cacheWritePath, cacheQuiet); + auto compile0 = ref_time_ticks(); auto program = compileDaScript(fn,access,tout,dummyGroup,policies); + startupCompileUsec = get_time_usec(compile0); { auto cres = moduleCache.finish(); if ( !cacheQuiet ) { @@ -559,7 +565,9 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP if ( compileOnly ) return 0; + auto simulate0 = ref_time_ticks(); auto pctx = SimulateWithErrReport(program, tout); + startupSimulateUsec = get_time_usec(simulate0); // Check for compiler leaks (TypeDecl nodes left on thread root after compile+simulate) { auto & root = gc_root::gc_get_thread_root(); @@ -604,11 +612,13 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP auto fnTest = fnMVec.back(); pctx->restart(); vec4f res; + auto run0 = ref_time_ticks(); if ( debuggerRequired ) { res = pctx->eval(fnTest, nullptr); } else { res = pctx->evalWithCatch(fnTest, nullptr); } + startupRunUsec = get_time_usec(run0); if ( auto ex = pctx->getException() ) { tout << "EXCEPTION: " << ex << " at " << pctx->exceptionAt.describe() << "\n"; exitCode = 1; @@ -733,7 +743,8 @@ void print_help() { << " -no-lint skip the lint pass (Program::lint)\n" << " --ast-verify force-include daslib/ast_verify; checks AST structural invariants before each inference pass\n" << " --ast-verify-batch checks the finished tree only (no per-pass walks, no cross-module sweeps): cheap enough to gate many files (CI)\n" - << " -log-compile-time log detailed per-module compile-time breakdown (parse / infer with pass count / optimize / macro (in infer) / macro mods / simulate) + function count\n" + << " -log-compile-time log detailed per-module compile-time breakdown (parse / infer with pass count / optimize / macro (in infer) / macro mods / simulate) + function count,\n" + << " a cached module's read time (and its macro-context simulate), and the process startup timeline (builtin modules / module scan / initialize / compile / simulate / run / teardown / shutdown)\n" << " -- separator for script arguments\n" << "daslang -aot {-q} {-p}\n" << " -project path to project file\n" @@ -773,6 +784,12 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR); _set_error_mode(_OUT_TO_STDERR); #endif + auto startupMain0 = ref_time_ticks(); + int64_t startupScanUsec = 0; + int64_t startupInitUsec = 0; + int64_t startupCompileAndRunUsec = 0; + int64_t startupPreScanUsec = 0; + int64_t startupBuiltinUsec = 0; install_das_crash_handler(); #ifdef __APPLE__ pthread_set_qos_class_self_np(QOS_CLASS_USER_INTERACTIVE, 0); @@ -1072,6 +1089,8 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { printf("-no-module-cache disables the cache; do not combine it with -ser/-deser\n"); return -1; } + startupPreScanUsec = get_time_usec(startupMain0); + auto builtin0 = ref_time_ticks(); // register modules register_builtin_modules(); require_project_specific_modules(); @@ -1079,6 +1098,8 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { // Otherwises search for static modules. #include "modules/external_pull.inc" #endif + startupBuiltinUsec = get_time_usec(builtin0); + auto scan0 = ref_time_ticks(); #ifdef DAS_ENABLE_DYN_INCLUDES if ( !noDynamicModules ) { // Search for external modules and init them. Only if flag is enabled. @@ -1088,7 +1109,10 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { require_dynamic_modules(access, getDasRoot(), project_root, load_modules, disabled_modules, tout); } #endif + startupScanUsec = get_time_usec(scan0); + auto init0 = ref_time_ticks(); Module::Initialize(); + startupInitUsec = get_time_usec(init0); // compile and run int exitCode = 0; @@ -1102,7 +1126,9 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { #endif for ( auto & fn : files ) { replace(fn, "_dasroot_", getDasRoot()); + auto compileAndRun0 = ref_time_ticks(); int rc = compile_and_run(fn, mainName, outputProgramCode, dryRun, compileOnly); + startupCompileAndRunUsec += get_time_usec(compileAndRun0); if ( rc != 0 ) { exitCode = rc; } @@ -1121,10 +1147,26 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { // Handle-leak dump runs inside Module::Shutdown, between module // destruction (drains job threads) and DLL unload (invalidates the // dumpHandleLeaks function pointers registered from shared modules). + auto shutdown0 = ref_time_ticks(); Module::Shutdown(dumpLeaks); if ( dumpLeaks ) { JobStatus::DumpJobQueLeaks(); } + if ( logModuleCompileTime ) { + auto shutdownUsec = get_time_usec(shutdown0); + auto teardownUsec = startupCompileAndRunUsec - startupCompileUsec - startupSimulateUsec - startupRunUsec; + tout << "startup: main total " << (get_time_usec(startupMain0) / 1000000.) << "\n" + << "\targuments " << (startupPreScanUsec / 1000000.) << "\n" + << "\tbuiltin modules " << (startupBuiltinUsec / 1000000.) << "\n" + << "\tmodule scan " << (startupScanUsec / 1000000.) << "\n" + << "\tinitialize " << (startupInitUsec / 1000000.) << "\n" + << "\tcompile " << (startupCompileUsec / 1000000.) << "\n" + << "\tsimulate " << (startupSimulateUsec / 1000000.) << "\n" + << "\trun " << (startupRunUsec / 1000000.) << "\n" + << "\tteardown " << (teardownUsec / 1000000.) << "\n" + << "\tshutdown " << (shutdownUsec / 1000000.) << "\n" + ; + } // das::dump_alloc_leaks registers itself as the FIRST atexit handler, so it // fires after all static destructors — cleaner than dumping here. if ( g_smart_ptr_total!=0 ) { From ac56703403e93f7de2dfde0b928f585af0b8c8e8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 7 Sep 2026 22:49:08 -0700 Subject: [PATCH 02/22] a C++ module the scan replayed from a manifest loads at the first require that names it, not on start - a replayed dm row carrying the das-visible name the recording start learned waits (defer_dynamic_module), and the prerequisite walk's miss asks the loader the scan installed (setDeferredModuleLoader), which dlopens, runs the factored initDependencies fixed point (Module::InitializeDependencies) and, when a module reports it needs another deferred one, brings the whole deferred set in and runs it again; the load runs under one gc root with the thread root's own nodes parked, since a constructor's builtin das module dumps its leftovers on the thread root and TypeDecl::gc_collect stops at a node owned elsewhere, and the collect walks every module, since a constructor registers into existing ones too; a `require ?mod` and builtin_module_exists mean "a require of this compile named mod" (a scan-loaded module is unrequired until a require resolves to it, so a cold start answers as a warm one), a plain-name guard rides the RequireRecord and is tested at its line, compileDaScript walks again while a skipped guard's module got loaded so order never matters, and the parser reads the walk's verdict (walkedGuardVerdict) instead of re-testing; the daslang host names the llvm witness under -jit and -exe, since no das file requires it unguarded and a static host runs the JIT without it; -ignore-manifest keeps the eager start for the MCP server and the LSP subtools, which enumerate; has_module answers loaded-or-deferred so the sweep gates keep their answer; a vector of a module's own handled type registers into that module (vectorHomeModule), since library.front() is always `$` and a load must not move the builtin module's cumulative hash under the module cache, and LLVM_JIT_CODEGEN_VERSION goes 0x76 because the externs a DLL binds by name moved; manifest read and write are guarded by DAS_NO_FILEIO like the fio builtins; hello world on the M5: interpreter 44 -> 14 ms, -jit 240 -> 211; tests/module_cache/test_deferred_modules.das covers cold, warm, every guard order, -ignore-manifest and the fallback on a copied dasUnitTest that shadows the tree's --- daslib/ARCHITECTURE.md | 4 +- ...ion-rtti-has_module-0x2f9e9a6e19be1ef0.rst | 2 +- include/daScript/ast/ast.h | 9 + include/daScript/ast/ast_handle.h | 12 +- include/daScript/ast/dyn_modules.h | 11 + include/daScript/simulate/debug_info.h | 1 + modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- .../daslang/references/modules-and-stdlib.md | 5 +- skills/dynamic_modules.md | 19 +- skills/internal/environment_variables.md | 2 +- skills/mcp_tools.md | 2 +- src/ast/ARCHITECTURE.md | 54 ++++- src/ast/ast_infer_type.cpp | 3 +- src/ast/ast_module.cpp | 69 +++--- src/ast/ast_parse.cpp | 87 +++++-- src/ast/dyn_modules.cpp | 94 +++++++- src/builtin/module_builtin_fio.cpp | 68 +++++- src/builtin/module_builtin_rtti.cpp | 4 +- src/parser/parser_impl.cpp | 6 +- tests/module_cache/ARCHITECTURE.md | 12 + tests/module_cache/test_deferred_modules.das | 221 ++++++++++++++++++ utils/daslang/main.cpp | 9 + utils/lsp/lsp_supervisor.py | 3 +- utils/mcp/README.md | 16 +- utils/mcp/daslang-mcp-msvc.cmd | 5 +- utils/mcp/mcp_supervisor.py | 5 +- 26 files changed, 639 insertions(+), 86 deletions(-) create mode 100644 tests/module_cache/test_deferred_modules.das diff --git a/daslib/ARCHITECTURE.md b/daslib/ARCHITECTURE.md index 9dba5c40eb..00c5d35ac3 100644 --- a/daslib/ARCHITECTURE.md +++ b/daslib/ARCHITECTURE.md @@ -243,4 +243,6 @@ Three companions carry a concern each; a section number is unique across all fou dasLLVM compiles with the shells inert and no `_variants()` registry - a program that reads one is framework-only and says so with that direct require. `daslib/just_in_time` keeps its direct require for the opposite reason: a static host that never registered the witness - still runs the JIT through the LLVM library, and the guard would switch it off. + still runs the JIT through the LLVM library, and the guard would switch it off. A host that + loads C++ modules at their first require (`daslang`) names the witness itself for a `-jit` or + `-exe` run, since no das file requires it unguarded (`src/ast/ARCHITECTURE.md` sec.2). diff --git a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst index 845b1660ab..9c3fbb972f 100644 --- a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst +++ b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst @@ -1 +1 @@ -Returns ``true`` if a module with the given name is registered, ``false`` otherwise. +Returns ``true`` if a module with the given name is registered, or waits in a ``.das_module`` manifest for the first ``require`` that names it, ``false`` otherwise. It answers what the tree has; ``typeinfo builtin_module_exists(name)`` answers what an earlier require loaded. diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 53b664c3fb..795adf7cc6 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1106,6 +1106,13 @@ namespace das DAS_API bool isValidBuiltinName ( const string & name, bool canPunkt = false ); + // the scan's loader for a deferred .shared_module, asked by name at a require miss (src/ast/ARCHITECTURE.md sec.2) + typedef bool (*DeferredModuleLoader) ( const string & name ); + DAS_API void setDeferredModuleLoader ( DeferredModuleLoader loader ); + DAS_API DeferredModuleLoader getDeferredModuleLoader (); + // the require walk's verdict on a `require ?guard x` line: 1 taken, 0 skipped, -1 the walk never saw it + DAS_API int walkedGuardVerdict ( const string & fileName, int32_t line ); + class DAS_API Module { public: Module ( const string & n = "" ); @@ -1154,6 +1161,8 @@ namespace das static Module * require ( const string & name ); static Module * requireEx ( const string & name, bool allowPromoted, const string & requireName = string(), const string & expectedFileName = string() ); static void Initialize(); + // the initDependencies fixed point; false = the named modules never initialized (src/ast/ARCHITECTURE.md sec.2) + static bool InitializeDependencies ( string & notInitialized ); static void CollectFileInfo(das::vector &accesses); static void Shutdown( bool dumpHandleLeaks = true ); // Runtime-only shutdown — for standalone exes built with `daslang -exe`, diff --git a/include/daScript/ast/ast_handle.h b/include/daScript/ast/ast_handle.h index 3032233a2f..d9b45ca985 100644 --- a/include/daScript/ast/ast_handle.h +++ b/include/daScript/ast/ast_handle.h @@ -748,6 +748,16 @@ namespace das } }; + // a vector lives with its element's type: one of a module's own handled type registers into + // that module, so loading the module never changes the builtin module's hash; one of a + // builtin element stays in `$` (library.front(): a module's dependencies sit before it) + __forceinline Module * vectorHomeModule ( const TypeDeclPtr & elem, const ModuleLibrary & library ) { + auto t = elem; + while ( t && t->isPointer() && t->firstType ) t = t->firstType; + if ( t && t->isHandle() && t->annotation && t->annotation->module ) return t->annotation->module; + return library.front(); + } + template struct typeFactory> { using VT = vector; @@ -759,7 +769,7 @@ namespace das ann->cppName = "das::vector<" + describeCppType(declT, CpptSubstitureRef::no, CpptSkipRef::no, CpptSkipConst::no, CpptRedundantConst::yes, ChooseSmartPtr::yes) + ">"; - auto mod = library.front(); + auto mod = vectorHomeModule(declT, library); mod->addAnnotation(ann); registerVectorFunctions>::init(mod,library, declT->canCopy(), diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index 53787740b3..2a546d32a0 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -40,4 +40,15 @@ DAS_API void begin_dynamic_module_recording(); DAS_API void end_dynamic_module_recording(vector & rows, bool & optOut); DAS_API void replay_native_path(const char * mod_name, const char * src, const char * dst); DAS_API void replay_dynamic_module(const char * path, const char * cpp_class, int on_error); +// a replayed dm row with a das name waits under it for the first require (ARCHITECTURE.md sec.2) +DAS_API void defer_dynamic_module(const char * path, const char * cpp_class, int on_error, const char * das_name); +DAS_API bool load_deferred_dynamic_module(const char * das_name); // true = the module registered +DAS_API size_t load_all_deferred_dynamic_modules(); // the count it attempted +DAS_API bool has_deferred_dynamic_modules(); +DAS_API bool is_dynamic_module_deferred(const char * das_name); +// a module the recording scan loaded is not "required" until a require names it: a guard reads it +// absent on a cold start as on a warm one, where the row waits (ARCHITECTURE.md sec.2) +DAS_API bool is_dynamic_module_unrequired(const char * das_name); +DAS_API void mark_dynamic_module_required(const char * das_name); +DAS_CC_API void ignore_dynamic_module_manifests(bool ignore); // -ignore-manifest: no read, no write, every module loads on start } diff --git a/include/daScript/simulate/debug_info.h b/include/daScript/simulate/debug_info.h index 6b48979d92..1267b1eec6 100644 --- a/include/daScript/simulate/debug_info.h +++ b/include/daScript/simulate/debug_info.h @@ -238,6 +238,7 @@ namespace das struct RequireRecord : BaseRequireRecord { bool isPublic = false; bool cantBeRequired = false; + string guard; // `require ?guard target`, a plain module name: tested when the walk reaches the line, not when it collects the file }; enum class MissingHint { diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 7806e05c47..21f2784167 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -38,7 +38,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x76ul // 0x76: a statement after a terminator in the same block list lands in its own dead block instead of after the ret (0x75: the global-offset lookup is memory(none) and emitted at its use site - LLVM dedups and hoists it, an untaken branch never pays it; a solid-context global resolves once per function at entry (0x74: a runtime-only exe emits no register_native_path rows, and a whole-lib exe emits them once (0x73: computed goto lowers to one switch with the trap as its default, not an icmp chain (0x72: policies.fast_math defaults to the host's float flags, so a fast-math host now JITs fast-math (0x71: a CPU class row's cpu is the arch's bare baseline, so a DAS_JIT_BASELINE build enables the row's set and nothing a level implies (0x70: the wasm feature string drops +relaxed-simd and the idot family keeps only the exact extmul + extadd_pairwise lowering on wasm SIMD128 (0x6f: the first wasm idot lowering; 0x6e: the aarch64 SDOT / SMMLA tables gate on DotProd / i8mm, not the arch alone, and the force env reaches the generic exe machine (0x6d: the inline polynomial rail carries NaN: tanh selects the operand back over its ordered clamp, and the sincos quadrant / tan octant convert through llvm.fptosi.sat instead of poisoning on NaN and out-of-range (0x6c: aarch64 vector tan/exp2/log2/log/pow join the inline polynomial rail bit-exactly with the interpreter, sinh/cosh/tanh ride the exp one; 0x6b: aarch64 vector sin/cos ride the inline polynomial; 0x6a: srem/urem for 32-bit %; 0x69: every string argument of an extern is substituted, not just the ones which asked) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x77ul // 0x77: a vector of a handled element type registers into the element's module, so the externs a DLL binds by mangled name moved out of `$` (0x76: a statement after a terminator in the same block list lands in its own dead block instead of after the ret (0x75: the global-offset lookup is memory(none) and emitted at its use site - LLVM dedups and hoists it, an untaken branch never pays it; a solid-context global resolves once per function at entry (0x74: a runtime-only exe emits no register_native_path rows, and a whole-lib exe emits them once (0x73: computed goto lowers to one switch with the trap as its default, not an icmp chain (0x72: policies.fast_math defaults to the host's float flags, so a fast-math host now JITs fast-math (0x71: a CPU class row's cpu is the arch's bare baseline, so a DAS_JIT_BASELINE build enables the row's set and nothing a level implies (0x70: the wasm feature string drops +relaxed-simd and the idot family keeps only the exact extmul + extadd_pairwise lowering on wasm SIMD128 (0x6f: the first wasm idot lowering; 0x6e: the aarch64 SDOT / SMMLA tables gate on DotProd / i8mm, not the arch alone, and the force env reaches the generic exe machine (0x6d: the inline polynomial rail carries NaN: tanh selects the operand back over its ordered clamp, and the sincos quadrant / tan octant convert through llvm.fptosi.sat instead of poisoning on NaN and out-of-range (0x6c: aarch64 vector tan/exp2/log2/log/pow join the inline polynomial rail bit-exactly with the interpreter, sinh/cosh/tanh ride the exp one; 0x6b: aarch64 vector sin/cos ride the inline polynomial; 0x6a: srem/urem for 32-bit %; 0x69: every string argument of an extern is substituted, not just the ones which asked)) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/skills/daslang/references/modules-and-stdlib.md b/skills/daslang/references/modules-and-stdlib.md index 3d1562f449..fc850d9e18 100644 --- a/skills/daslang/references/modules-and-stdlib.md +++ b/skills/daslang/references/modules-and-stdlib.md @@ -75,7 +75,7 @@ require geom // a bare name also finds geom.das next to require ./helpers.das // file-relative path require %/daslib/random.das as rng // `%` is the daslang root; `as` binds a local qualifier require dastest/testing_boost public // re-export to whoever requires me -require ?pugixml pugixml/PUGIXML_boost // load only if module `pugixml` is available +require ?pugixml pugixml/PUGIXML_boost // only if module `pugixml` is linked or required in this compile ``` - **Path form needs the `.das`:** a require is a literal path only when it starts with `./`, `../`, @@ -183,7 +183,8 @@ else is a require. | `fio_core`, `rtti_core`, `ast_core`, `network_core` | Low-level C++ layers; require the wrapper instead - `daslib/fio`, `daslib/rtti`, `daslib/ast`, `daslib/network`. Bare `require rtti` / `require ast` do **not** resolve. | Which built-in modules exist depends on how the host embedded daslang - guard anything non-core -with `require ?mod ...`. +with `require ?mod ...`. The guard passes when `mod` is linked into the host or required +somewhere in this compile; a guard loads nothing itself. ## Container operations diff --git a/skills/dynamic_modules.md b/skills/dynamic_modules.md index 4bbd5fcf15..88aa10f169 100644 --- a/skills/dynamic_modules.md +++ b/skills/dynamic_modules.md @@ -100,8 +100,23 @@ of the machine, a variant picked by hardware - opts out by calling `no_manifest( `initialize`; it then runs on every start. Everything a descriptor registers is replayed, so the opt-out is only for a descriptor whose answer changes between starts. -`DAS_TRACE_MODULE_LOAD=1` prints one line per descriptor saying whether it was replayed or -compiled, and why. +A replayed C++ module loads at the first `require` naming it, not in the scan: the require +walk loads the row and runs the `initDependencies` fixed point; a module needing another +deferred one pulls the whole deferred set in. A program pays for the C++ modules it requires. +Two consequences: + +- `require ?mod x` and `typeinfo builtin_module_exists(mod)` say whether some require in the + compile named `mod`, not what `modules/` holds, and a cold start answers as a warm one. + The `daslang` host names the `llvm` witness under `-jit` and `-exe`, so `?llvm` is true + there; an interpreter run that never requires `llvm` reads it false. Require order does not + matter. +- A tool that enumerates the process's modules (the MCP server, the LSP subtools) runs with + `-ignore-manifest`: no manifest read or written, every descriptor compiles, every C++ module + loads on start. `has_module(name)` (`daslib/rtti`) answers loaded-or-deferred, so a sweep + gate asking what the tree has keeps its answer. + +`DAS_TRACE_MODULE_LOAD=1` prints one line per descriptor (replayed or compiled, why, the +deferred count) and one per deferred module as a require loads it. ## Adding a `.das` file to an existing module needs the same edit diff --git a/skills/internal/environment_variables.md b/skills/internal/environment_variables.md index 87ccc264fa..8eda8d5971 100644 --- a/skills/internal/environment_variables.md +++ b/skills/internal/environment_variables.md @@ -72,7 +72,7 @@ and unknown codes are harmless. The `-no-lint` command-line flag skips the lint |---|---|---| | `DAS_GC_STAGE_REPORT` | flag | Report gc_node deltas per compilation stage - the first thing to reach for on a `GC APP LEAK` at exit. | | `DAS_GC_BREAK_ON_ID` | number | Break when the gc_node with this id is allocated. Pair it with the id from a leak report. | -| `DAS_TRACE_MODULE_LOAD` | flag | Log every module as it loads, with its resolved path - the fastest way to see which of two same-named modules actually won - and one line per `.das_module` descriptor saying whether the scan replayed its manifest or compiled it, and why; a replayed line carries its time and the share its shared-module loads took (`src/ast/ARCHITECTURE.md` sec.2). | +| `DAS_TRACE_MODULE_LOAD` | flag | Log every module as it loads, with its resolved path - the fastest way to see which of two same-named modules actually won - and one line per `.das_module` descriptor: replayed (with its time, shared-module load share and deferred count) or compiled, and why; a deferred module's load prints `require : loading the deferred ` (`src/ast/ARCHITECTURE.md` sec.2). | ## Ambient variables daslang reads but does not own diff --git a/skills/mcp_tools.md b/skills/mcp_tools.md index 032e92e07a..66bf1f52e3 100644 --- a/skills/mcp_tools.md +++ b/skills/mcp_tools.md @@ -72,7 +72,7 @@ The daslang MCP server (`utils/mcp/main.das`) exposes compiler diagnostics, prog **`shutdown` tool.** Shuts down the MCP server process. Claude Code auto-restarts it, picking up code changes to `.das` tool files. Tool registration changes (adding/removing tools) still require a manual MCP restart. -**Configuration.** Configure `.mcp.json` with `"command"` pointing at the daslang binary (`bin/daslang` on Windows MSVC, `build/daslang` on Linux/macOS, `bin/daslang` for the installed SDK), `"args": ["utils/mcp/main.das"]`. See `utils/mcp/README.md` for details and Claude Code permissions. +**Configuration.** Configure `.mcp.json` with `"command"` pointing at the daslang binary (`bin/daslang` on Windows MSVC, `build/daslang` on Linux/macOS, `bin/daslang` for the installed SDK), `"args": ["-ignore-manifest", "utils/mcp/main.das"]` (the flag loads every C++ module on start; the server enumerates them). See `utils/mcp/README.md` for details and Claude Code permissions. **Fresh checkouts / worktrees.** `.mcp.json`, `sgconfig.yml`, `bin/`, and the tree-sitter grammar lib are all gitignored, so a new `git worktree add` (or clone) has no daslang MCP at all. Bootstrap it with `daslang utils/mcp/setup.das -- --root ` - it configures `build/` on the cmake generator of the tree running the setup (platform default when that tree has no `build/CMakeCache.txt`), builds a worktree-local binary (+ grammar), copies the platform `sgconfig.yml`, and merges a `daslang` entry into `.mcp.json` (adds no new secrets; existing servers, including any secret env blocks, are preserved as-is). `--no-build` skips the build. Restart the session to pick it up. diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 298eaa9073..4567145d0f 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -78,13 +78,47 @@ than an escaped form the reader would have to decode, and goes through a `.tmp` Recording is armed around one descriptor run: each builtin appends the arguments it actually received, in order, and `register_dynamic_module` records its call whatever the outcome and adds -the das-visible module name once the load succeeded. Replay calls the same two builtins with the -recorded rows in recorded order, so the Quiet deferral and the post-scan retry of a sibling -`DT_NEEDED` dlopen behave as on a compiled start. `no_manifest()` inside `initialize` marks the -descriptor as one that runs on every start: its manifest carries the stamp and the flag and no -rows, and is not rewritten. With `DAS_TRACE_MODULE_LOAD=1` the scan prints one line per -descriptor - `replayed N row(s) in (shared module load )`, `compiled (), -manifest written (N row(s))`, `compiled (no_manifest)`, or why a manifest was not written. A -replayed descriptor's time is its manifest read plus its rows, and the second number is the -share the `.shared_module` dlopen and module constructor took - on a warm start that is nearly -all of it. +the das-visible module name once the load succeeded. Replay registers every `np` row as +recorded. A `dm` row carrying that name is not loaded by the scan: the row waits under the name +(`defer_dynamic_module`), and the load happens at the first require that names it. The +prerequisite walk (`getPrerequisits`) finds no module under the name and asks the loader the +scan installed (`setDeferredModuleLoader`); the loader dlopens and registers the module, runs the +`initDependencies` fixed point `Module::Initialize` runs (`Module::InitializeDependencies`) over +the grown list, and when a module reports it cannot initialize - what it needs is deferred too - +brings every deferred module in and runs the fixed point again, which is the set an eager start +has; the new modules' TypeDecls, made on the active root, move to their module roots. A `dm` row +with no name - the recording start's load failed - replays as recorded, so the Quiet deferral +and the post-scan retry of a sibling `DT_NEEDED` dlopen behave as on a compiled start. A +require guard (`require ?mod`) and `builtin_module_exists` answer whether a require of the +process named the module, not whether the tree holds it: the `daslang` host names the `llvm` +witness itself when `-jit` or `-exe` is on (the module carries no symbol, so no das file +requires it unguarded, and a static host runs the JIT without it), so `?llvm` is taken there; +an interpreter run of a program that never requires `llvm` reads it absent whatever `modules/` +holds. The recording start loads every module +eagerly, so the module a `dm` row names counts as unrequired (`is_dynamic_module_unrequired`) +until a require resolves to it (`mark_dynamic_module_required`), and a cold start answers a +guard as a warm one does. Order inside a compile does not matter: the collector +(`getAllRequireReq`) decides a path guard (its file resolves) as it reads the file, but carries +a plain-name guard on the record (`RequireRecord::guard`) for `getPrerequisits` to test when +the walk reaches the line, and `compileDaScript` walks again whenever a skipped guard's module +was loaded later in the walk, so the guarded target lands in dependency order; the parser +(`ast_requireModule`) reads the walk's verdict for its file and line (`walkedGuardVerdict`) +rather than testing the guard itself, so a module loaded between the walk and the parse +cannot make the two disagree. A load must not change another module's content either: a +module-cache record carries each builtin module's cumulative hash of mangled names, and a +process that loaded a different set of C++ modules would otherwise fail every record on `$`, +so a `vector` of a module's own handled type registers into that module +(`vectorHomeModule`, `ast_handle.h`), and only a vector of a builtin element lands in `$`, which +every library lists first because `ModuleLibrary::addModule` puts a module's dependencies +before it. `-ignore-manifest` +reads and writes no manifest: every descriptor compiles and every C++ module loads on start, +the form a tool that enumerates modules - the MCP server, the LSP subtools - runs under. +`no_manifest()` inside `initialize` marks the descriptor as one that runs on every start: its +manifest carries the stamp and the flag and no rows, and is not rewritten. With +`DAS_TRACE_MODULE_LOAD=1` the scan prints one line per descriptor - `replayed N row(s) in +(shared module load , deferred K)`, `compiled (), manifest written (N row(s))`, +`compiled (no_manifest)`, `compiled (manifests ignored)`, or why a manifest was not written - +and a deferred load prints `[module] require : loading the deferred `, the +fallback `[module] loading every deferred module (K)`. A replayed descriptor's time is its +manifest read plus its rows, and the second number is the share the `.shared_module` dlopen and +module constructor took. diff --git a/src/ast/ast_infer_type.cpp b/src/ast/ast_infer_type.cpp index 13eabd93ae..e9c59abd7c 100644 --- a/src/ast/ast_infer_type.cpp +++ b/src/ast/ast_infer_type.cpp @@ -4,6 +4,7 @@ #include "daScript/ast/ast_generate.h" #include "daScript/ast/ast_infer_type.h" #include "daScript/ast/ast_pass_macros.h" +#include "daScript/ast/dyn_modules.h" #include "daScript/ast/ast_visitor.h" #define DAS_XSTR(s) #s @@ -2846,7 +2847,7 @@ namespace das { // also accepts shared das modules compiled earlier in the process auto mod = Module::requireEx(evar->name, true); reportAstChanged(); - return new ExprConstBool(expr->at, mod != nullptr); + return new ExprConstBool(expr->at, mod != nullptr && !is_dynamic_module_unrequired(evar->name.c_str())); } else { error("unsupported module name subexpression ", expr->subexpr->__rtti, "", expr->at, CompilationError::invalid_typeinfo_module_subexpression); diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index 3678db73cf..238ae790d5 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -135,16 +135,7 @@ namespace das { } } - void Module::Initialize() { - daScriptEnvironment::ensure(); - static bool atexit_registered = (atexit(daslang_atexit_audit), true); - (void)atexit_registered; - g_envTotal ++; - - if (daScriptEnvironment::getBound()->modules == nullptr) { - DAS_FATAL_ERROR("No modules founds. You should add modules before call that function."); - } - + bool Module::InitializeDependencies ( string & notInitialized ) { // InitDependencies do not add new modules. vector mod_state; bool any = true; @@ -167,24 +158,48 @@ namespace das { } } } - if (!any) { - // Some modules was not initialized! - size_t i = 0; - string error = ""; - for ( auto m = daScriptEnvironment::getBound()->modules; m ; m = m->next, i++ ) { - DAS_ASSERT(mod_state.size() == i); - if (!mod_state.at(i)) { - error += " " + m->name; - } + if ( all ) return true; + // Some modules was not initialized! + size_t i = 0; + for ( auto m = daScriptEnvironment::getBound()->modules; m ; m = m->next, i++ ) { + DAS_ASSERT(mod_state.size() > i); + if (!mod_state.at(i)) { + notInitialized += " " + m->name; } - // A module that never initializes usually failed Module::require on a dependency - // whose .shared_module dlopen failed QUIETLY during the startup scan. Name those - // load failures (with their dlerror) so this doesn't read as a missing C++ module. - auto pendingNote = describe_pending_dynamic_modules(); - if ( !pendingNote.empty() ) { - error += "\nnote: these dynamic modules failed to load - an unresolved dependency may live in one of them:\n" + pendingNote; - } - DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", error.c_str()); + } + // A module that never initializes usually failed Module::require on a dependency + // whose .shared_module dlopen failed QUIETLY during the startup scan. Name those + // load failures (with their dlerror) so this doesn't read as a missing C++ module. + auto pendingNote = describe_pending_dynamic_modules(); + if ( !pendingNote.empty() ) { + notInitialized += "\nnote: these dynamic modules failed to load - an unresolved dependency may live in one of them:\n" + pendingNote; + } + return false; + } + + static DeferredModuleLoader g_deferredModuleLoader = nullptr; + + void setDeferredModuleLoader ( DeferredModuleLoader loader ) { + g_deferredModuleLoader = loader; + } + + DeferredModuleLoader getDeferredModuleLoader () { + return g_deferredModuleLoader; + } + + void Module::Initialize() { + daScriptEnvironment::ensure(); + static bool atexit_registered = (atexit(daslang_atexit_audit), true); + (void)atexit_registered; + g_envTotal ++; + + if (daScriptEnvironment::getBound()->modules == nullptr) { + DAS_FATAL_ERROR("No modules founds. You should add modules before call that function."); + } + + string notInitialized; + if ( !InitializeDependencies(notInitialized) ) { + DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); } // Collect reachable TypeDecl from thread root into module roots, sweep the rest. auto & threadRoot = gc_root::gc_get_thread_root(); diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index 6742a64c2e..aaa7d1a82a 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -1,6 +1,7 @@ #include "daScript/misc/platform.h" #include "daScript/ast/ast.h" +#include "daScript/ast/dyn_modules.h" #include "daScript/ast/ast_infer_type.h" #include "daScript/ast/ast_serializer.h" #include "daScript/ast/ast_expressions.h" @@ -61,6 +62,20 @@ DAS_CC_API das::smart_ptr get_file_access( char * pak ) { namespace das { + // the walk in flight's `require ?guard x` verdicts by file and line, and the skipped guards' module + // names - a later require may load one, and then the walk runs again (ARCHITECTURE.md sec.2) + static thread_local das_hash_map g_guardVerdicts; + static thread_local das_hash_set g_skippedGuards; + + static string guardVerdictKey ( const string & fileName, int32_t line ) { + return fileName + "\t" + to_string(line); + } + + int walkedGuardVerdict ( const string & fileName, int32_t line ) { + auto it = g_guardVerdicts.find(guardVerdictKey(fileName, line)); + return it == g_guardVerdicts.end() ? -1 : it->second; + } + void applyPostRewriteMacros ( Program * program ) { program->library.foreach([&](Module * mod) -> bool { for ( const auto & pm : mod->postRewriteMacros ) { @@ -260,13 +275,15 @@ namespace das { // linked C++ module); no target-resolvability fallback (module source // dirs exist in every checkout regardless of build config). Otherwise — // skip silently. Must match ast_requireModule (parser_impl.cpp). + // a plain-name guard is tested by getPrerequisits at its line (ARCHITECTURE.md sec.2) + string plainGuard; if ( hasReqGuard && reqGuard.find('/')!=string::npos ) { auto ginfo = access->getModuleInfo(reqGuard, fi->name); if ( ginfo.fileName.empty() || !access->getFileInfo(ginfo.fileName) ) { continue; } - } else if ( hasReqGuard && Module::requireEx(reqGuard, false)==nullptr ) { - continue; + } else if ( hasReqGuard ) { + plainGuard = reqGuard; } bool isPublic = false; while ( src < src_end && src[0] == ' ' ) { @@ -276,6 +293,7 @@ namespace das { isPublic = true; } req.push_back({mod, line, chain, isPublic}); + req.back().guard = plainGuard; } else if ( isInc ) { string incFileName = access->getIncludeFileName(fi->name,mod); auto info = access->getFileInfo(incFileName); @@ -409,6 +427,17 @@ namespace das { vector ownReq = getAllRequire(fi, modName, chain, access); for ( auto & modRec : ownReq ) { string mod = modRec.name; + if ( !modRec.guard.empty() ) { + bool taken = Module::requireEx(modRec.guard, false)!=nullptr && !is_dynamic_module_unrequired(modRec.guard.c_str()); + g_guardVerdicts[guardVerdictKey(fi->name, modRec.line)] = taken ? 1 : 0; + if ( !taken ) { + g_skippedGuards.insert(modRec.guard); + if ( log ) { + *log << string(tab,'\t') << "require ?" << modRec.guard << " " << mod << " - guard module not required, skipped\n"; + } + continue; + } + } if ( log ) { *log << string(tab,'\t') << "require " << mod << "\n"; } @@ -433,6 +462,15 @@ namespace das { } } module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); // try native with that name AGAIN (promoted?) + if ( !module ) { + // a C++ module the scan deferred loads at the require that names it (ARCHITECTURE.md sec.2) + if ( auto loader = getDeferredModuleLoader(); loader && loader(mod) ) { + module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); + if ( log && module ) { + *log << string(tab,'\t') << " loaded deferred shared module " << mod << "\n"; + } + } + } if ( !module ) { auto it_r = find_if(req.begin(), req.end(), [&] ( const ModuleInfo & reqM ) { return reqM.moduleName == mod; @@ -556,6 +594,7 @@ namespace das { return false; } else { libGroup.addModule(module); + mark_dynamic_module_required(module->name.c_str()); } } } @@ -1830,20 +1869,38 @@ namespace das { string modName; [[maybe_unused]] auto builtinModule = Module::require("$"); DAS_ASSERTF(builtinModule, "Somehow `builtin` module is missing."); - bool allGood = addExtraDependency("builtin", get_builtin_path(), missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, &logs); - if ( !allGood ) { - auto res = make_smart(); - res->error("internal error: failed to build builtin.das", logs.str(), "", LineInfo(), CompilationError::internal_module); - return res; - } - for ( const auto & em : access->getExtraModules() ) { - allGood = addExtraDependency(em.first, em.second, missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, nullptr) && allGood; - } - if ( !allGood ) { - return reportPrerequisitesErrors(fileName, missing, circular, notAllowed, namelessMismatches, libGroup, policies); + bool walked = false; + for ( ;; ) { + req.clear(); missing.clear(); circular.clear(); notAllowed.clear(); chain.clear(); + dependencies.clear(); namelessReq.clear(); namelessMismatches.clear(); modName.clear(); + g_guardVerdicts.clear(); g_skippedGuards.clear(); + bool allGood = addExtraDependency("builtin", get_builtin_path(), missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, &logs); + if ( !allGood ) { + auto res = make_smart(); + res->error("internal error: failed to build builtin.das", logs.str(), "", LineInfo(), CompilationError::internal_module); + return res; + } + for ( const auto & em : access->getExtraModules() ) { + allGood = addExtraDependency(em.first, em.second, missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, nullptr) && allGood; + } + if ( !allGood ) { + return reportPrerequisitesErrors(fileName, missing, circular, notAllowed, namelessMismatches, libGroup, policies); + } + walked = getPrerequisits(fileName, access, modName, req, missing, circular, notAllowed, chain, + dependencies, namelessReq, namelessMismatches, libGroup, nullptr, 1, !policies.ignore_shared_modules); + if ( !walked ) break; + // a guard skipped before a later require loaded its module: the walk runs again, and + // the guard's target lands in dependency order (ARCHITECTURE.md sec.2) + bool flipped = false; + for ( auto & guard : g_skippedGuards ) { + if ( Module::requireEx(guard, false)!=nullptr && !is_dynamic_module_unrequired(guard.c_str()) ) { + flipped = true; + break; + } + } + if ( !flipped ) break; } - if ( getPrerequisits(fileName, access, modName, req, missing, circular, notAllowed, chain, - dependencies, namelessReq, namelessMismatches, libGroup, nullptr, 1, !policies.ignore_shared_modules) ) { + if ( walked ) { preqT = get_time_usec(time0); disableSerializationOnDebugger(req); if ( !verifyModuleNamesUnique(req, logs) ) { diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 2062b5a045..85b957903b 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -10,6 +10,7 @@ #include // get_dasenv_trace_module_load #include #include // tolower (case-insensitive basename normalize) +#include // the deferred-load lock #include // fprintf(stderr) for the shadow-shadows-global diagnostic das::FileAccessPtr get_file_access( char * pak ); @@ -91,6 +92,12 @@ static Result run_descriptor(smart_ptr fa, const string & mod_filena static constexpr const char *MANIFEST_SUFFIX = ".das_module.manifest"; // ARCHITECTURE.md sec.2 static constexpr const char *MANIFEST_HEADER = "das_module_manifest\t2"; +static bool g_ignore_manifests = false; + +void ignore_dynamic_module_manifests(bool ignore) { + g_ignore_manifests = ignore; +} + static bool trace_scan() { static const bool on = []{ const char * e = get_dasenv_trace_module_load(); @@ -183,6 +190,11 @@ static ManifestKey manifest_key(const string & path) { static ManifestRead read_manifest(const string & file, uint32_t descSize, uint64_t descHash, const ManifestKey & key, const smart_ptr & fa) { ManifestRead res; +#if DAS_NO_FILEIO + // the guard the fio builtins carry: a build without file IO reads no manifest + (void)file; (void)descSize; (void)descHash; (void)key; (void)fa; + return res; +#else FILE * f = fopen(file.c_str(), "rb"); if ( !f ) { return res; @@ -273,6 +285,7 @@ static ManifestRead read_manifest(const string & file, uint32_t descSize, uint64 if ( optOut && !res.rows.empty() ) return damaged("no_manifest with rows"); res.verdict = optOut ? ManifestVerdict::OptOut : ManifestVerdict::Replay; return res; +#endif } static bool field_ok(const string & s) { @@ -310,6 +323,11 @@ static bool write_manifest(const string & file, uint32_t descSize, uint64_t desc } text += "end\t" + to_string(rows.size()) + "\n"; } +#if DAS_NO_FILEIO + (void)file; + why = "no file io"; + return false; +#else const string tmp = file + ".tmp"; FILE * f = fopen(tmp.c_str(), "wb"); if ( !f ) { why = "cannot create " + tmp; return false; } @@ -321,6 +339,7 @@ static bool write_manifest(const string & file, uint32_t descSize, uint64_t desc #endif if ( rename(tmp.c_str(), file.c_str()) != 0 ) { remove(tmp.c_str()); why = "cannot rename " + tmp; return false; } return true; +#endif } static Result init_dyn_modules(smart_ptr fa, string path, TextWriter &tout, bool debug = false) { @@ -345,31 +364,44 @@ static Result init_dyn_modules(smart_ptr fa, string path, TextWriter const string manifest = path + "/" + MANIFEST_SUFFIX; const ManifestKey key = manifest_key(path); auto time0 = ref_time_ticks(); - auto mr = src ? read_manifest(manifest, len, stamp, key, fa) : ManifestRead(); +#if DAS_NO_FILEIO + const bool useManifest = false; // the guard the fio builtins carry: no manifest read or written + const char * noManifestWhy = "no file io"; +#else + const bool useManifest = src && !g_ignore_manifests; + const char * noManifestWhy = !src ? "no source" : g_ignore_manifests ? "manifests ignored" : "no_manifest"; +#endif + auto mr = useManifest ? read_manifest(manifest, len, stamp, key, fa) : ManifestRead(); if ( mr.verdict == ManifestVerdict::Replay ) { int64_t dllUsec = 0; + size_t deferred = 0; for ( auto & row : mr.rows ) { - if ( row.dynamic ) { + if ( !row.dynamic ) { + replay_native_path(row.a.c_str(), row.b.c_str(), row.c.c_str()); + } else if ( !row.c.empty() ) { + // a row without a das name failed to load on the recording start, and replays as it did + defer_dynamic_module(row.a.c_str(), row.b.c_str(), row.on_error, row.c.c_str()); + deferred ++; + } else { auto dll0 = ref_time_ticks(); replay_dynamic_module(row.a.c_str(), row.b.c_str(), row.on_error); dllUsec += get_time_usec(dll0); - } else { - replay_native_path(row.a.c_str(), row.b.c_str(), row.c.c_str()); } } if ( trace_scan() ) { LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": replayed " << mr.rows.size() << " row(s) in " - << (get_time_usec(time0) / 1000000.) << " (shared module load " << (dllUsec / 1000000.) << ")\n"; + << (get_time_usec(time0) / 1000000.) << " (shared module load " << (dllUsec / 1000000.) + << ", deferred " << deferred << ")\n"; } return Result::OK; } - const bool record = src && mr.verdict != ManifestVerdict::OptOut; + const bool record = useManifest && mr.verdict != ManifestVerdict::OptOut; if ( record ) begin_dynamic_module_recording(); das::vector depFiles; auto res = run_descriptor(fa, mod_filename, path, tout, record ? &depFiles : nullptr); if ( !record ) { if ( trace_scan() ) { - LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": compiled (" << (src ? "no_manifest" : "no source") << ")\n"; + LOG(LogLevel::info) << "[module] descriptor " << mod_filename << ": compiled (" << noManifestWhy << ")\n"; } return res; } @@ -540,12 +572,60 @@ static das::string path_basename(const das::string &path) { return path.substr(slash + 1, end - slash - 1); } +static void move_all_nodes(gc_root & from, gc_root & to) { + while ( from.gc_first ) { + auto node = from.gc_first; + from.gc_unlink(node); + to.gc_link(node); + } +} + +// ARCHITECTURE.md sec.2: the prerequisite walk found no module under `name` +static bool load_deferred_module_for_require(const string & name) { + static recursive_mutex loadMutex; + lock_guard guard(loadMutex); + // one root for all the load makes (a collect walks one root): the thread root's own nodes sit + // aside while a compiled builtin das module dumps its leftovers there, then join loadRoot + auto & threadRoot = gc_root::gc_get_thread_root(); + gc_root parked, loadRoot; + move_all_nodes(threadRoot, parked); + bool loaded = false; + { + gc_active_scope scope(&loadRoot); + loaded = load_deferred_dynamic_module(name.c_str()); + if ( loaded ) { + string notInitialized; + if ( !Module::InitializeDependencies(notInitialized) ) { + // what it needs is deferred too: the whole deferred set, the eager start's, then the fixed point again + notInitialized.clear(); + load_all_deferred_dynamic_modules(); + if ( !Module::InitializeDependencies(notInitialized) ) { + DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); + } + } + } + } + move_all_nodes(threadRoot, loadRoot); + if ( loaded ) { + // every module: a constructor registers into existing ones too (a vector type's functions) + Module::foreach([&](Module * m) { + m->gc_collect(&loadRoot); + return true; + }); + } + loadRoot.gc_sweep(); + move_all_nodes(parked, threadRoot); + return loaded; +} + bool require_dynamic_modules(FileAccessPtr file_access, const das::string &das_root, const das::string &project_root, const das::vector &load_modules, const das::vector &disabled_modules, das::TextWriter &tout) { + // before the walk: a descriptor compiled mid-scan may require a module an earlier replay deferred + setDeferredModuleLoader(&load_deferred_module_for_require); // Explicitly-disabled modules (case-insensitive on every platform) are never // loaded/registered — keeps a native-only module out of a wasm cross-compile. das_hash_set disabled_set; diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 9d9208e13c..a4abf3f797 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -312,6 +312,13 @@ namespace das { DAS_API void end_dynamic_module_recording ( vector &, bool & ) GENERATE_IO_STUB DAS_API void replay_native_path ( const char *, const char *, const char * ) GENERATE_IO_STUB DAS_API void replay_dynamic_module ( const char *, const char *, int ) GENERATE_IO_STUB + DAS_API void defer_dynamic_module ( const char *, const char *, int, const char * ) GENERATE_IO_STUB + DAS_API bool load_deferred_dynamic_module ( const char * ) GENERATE_IO_STUB_RET + DAS_API size_t load_all_deferred_dynamic_modules () GENERATE_IO_STUB_RET + DAS_API bool has_deferred_dynamic_modules () GENERATE_IO_STUB_RET + DAS_API bool is_dynamic_module_deferred ( const char * ) GENERATE_IO_STUB_RET + DAS_API bool is_dynamic_module_unrequired ( const char * ) GENERATE_IO_STUB_RET + DAS_API void mark_dynamic_module_required ( const char * ) GENERATE_IO_STUB #undef GENERATE_IO_STUB #undef GENERATE_IO_STUB_RET @@ -2282,6 +2289,16 @@ namespace das { // after the folder scan, so module enumeration order stops mattering. static vector> g_pending_dynamic_modules; // path, cpp_class_name, last dlopen error + static das_hash_set g_unrequired_dynamic_modules; // loaded by the recording scan, no require yet (dyn_modules.h) + + DAS_API bool is_dynamic_module_unrequired ( const char * das_name ) { + return das_name && g_unrequired_dynamic_modules.count(das_name) != 0; + } + + DAS_API void mark_dynamic_module_required ( const char * das_name ) { + if ( das_name ) g_unrequired_dynamic_modules.erase(das_name); + } + // the descriptor manifest recorder (dyn_modules.h, src/ast/ARCHITECTURE.md sec.2) static thread_local bool g_manifest_recording = false; static thread_local bool g_manifest_opt_out = false; @@ -2404,7 +2421,10 @@ namespace das { } *ModuleKarma += unsigned(intptr_t(mod)); g_registered_dynamic_modules.emplace_back(path, mod_name, mod->name); - if ( recordedRowIndex != size_t(-1) ) g_manifest_rows[recordedRowIndex].c = mod->name; + if ( recordedRowIndex != size_t(-1) ) { + g_manifest_rows[recordedRowIndex].c = mod->name; + g_unrequired_dynamic_modules.insert(mod->name); + } return lib; } void *register_dynamic_module_silent(const char *path, const char *mod_name, Context * context, LineInfoArg * at ) { @@ -2415,6 +2435,52 @@ namespace das { register_dynamic_module(path, cpp_class, on_error, nullptr, nullptr); } + DAS_API void retry_pending_dynamic_modules(); + + struct DeferredDynamicModule { + string path, cpp_class, das_name; + int on_error = 0; + }; + static vector g_deferred_dynamic_modules; // manifest rows waiting for a require (ARCHITECTURE.md sec.2) + + DAS_API void defer_dynamic_module ( const char * path, const char * cpp_class, int on_error, const char * das_name ) { + g_deferred_dynamic_modules.push_back({path ? path : "", cpp_class ? cpp_class : "", das_name ? das_name : "", on_error}); + } + + DAS_API bool load_deferred_dynamic_module ( const char * das_name ) { + auto it = find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), + [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }); + if ( it == g_deferred_dynamic_modules.end() ) return false; + auto dm = das::move(*it); + g_deferred_dynamic_modules.erase(it); + if ( trace_module_load() ) { + LOG(LogLevel::info) << "[module] require " << dm.das_name << ": loading the deferred " << dm.cpp_class << "\n"; + } + return register_dynamic_module(dm.path.c_str(), dm.cpp_class.c_str(), dm.on_error, nullptr, nullptr) != nullptr; + } + + DAS_API size_t load_all_deferred_dynamic_modules () { + vector all; + all.swap(g_deferred_dynamic_modules); + if ( trace_module_load() && !all.empty() ) { + LOG(LogLevel::info) << "[module] loading every deferred module (" << all.size() << ")\n"; + } + for ( auto & dm : all ) { + register_dynamic_module(dm.path.c_str(), dm.cpp_class.c_str(), dm.on_error, nullptr, nullptr); + } + retry_pending_dynamic_modules(); + return all.size(); + } + + DAS_API bool has_deferred_dynamic_modules () { + return !g_deferred_dynamic_modules.empty(); + } + + DAS_API bool is_dynamic_module_deferred ( const char * das_name ) { + return das_name && find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), + [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }) != g_deferred_dynamic_modules.end(); + } + // Re-attempt modules whose dlopen was deferred (Quiet failure during the // module-folder scan — usually a sibling-module DT_NEEDED dep not yet loaded // because directory enumeration visited the dependent before its dependency). diff --git a/src/builtin/module_builtin_rtti.cpp b/src/builtin/module_builtin_rtti.cpp index 39e1db22df..f6fc7ba25e 100644 --- a/src/builtin/module_builtin_rtti.cpp +++ b/src/builtin/module_builtin_rtti.cpp @@ -11,6 +11,7 @@ #include "daScript/misc/performance_time.h" #include "daScript/ast/ast_serializer.h" +#include "daScript/ast/dyn_modules.h" #include "daScript/misc/gc_node.h" using namespace das; @@ -1187,8 +1188,9 @@ namespace das { return Module::require(name); } + // loaded or deferred: what the tree has, where `require ?name` asks what an earlier require loaded bool rtti_has_module ( const char * name ) { - return Module::require(name) != nullptr; + return Module::require(name) != nullptr || is_dynamic_module_deferred(name); } void rtti_builtin_program_for_each_module ( smart_ptr_raw program, const TBlock & block, Context * context, LineInfoArg * at ) { diff --git a/src/parser/parser_impl.cpp b/src/parser/parser_impl.cpp index 91dc287566..1c0df9a890 100644 --- a/src/parser/parser_impl.cpp +++ b/src/parser/parser_impl.cpp @@ -4,6 +4,7 @@ #include "parser_state.h" #include "daScript/ast/ast_generate.h" +#include "daScript/ast/dyn_modules.h" #include "daScript/ast/ast_handle.h" #undef yyextra @@ -1240,7 +1241,10 @@ namespace das { auto ginfo = yyextra->g_Access->getModuleInfo(*guard, yyextra->g_FileAccessStack.back()->name); guardAvailable = !ginfo.fileName.empty() && yyextra->g_Access->getFileInfo(ginfo.fileName) != nullptr; } else { - guardAvailable = Module::requireEx(*guard, false) != nullptr; + // the walk decided this line already (a later require may have loaded the guard module since) + auto verdict = walkedGuardVerdict(yyextra->g_FileAccessStack.back()->name, int32_t(atName.line)); + guardAvailable = verdict >= 0 ? verdict != 0 + : Module::requireEx(*guard, false) != nullptr && !is_dynamic_module_unrequired(guard->c_str()); } delete guard; if ( !guardAvailable ) { diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 234f725a75..d73b437d35 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -31,6 +31,18 @@ this document states what the folder is and why its tests take the shape they do tab leaves the manifest unwritten, and a directory sitting where the `.tmp` or the manifest goes fails the create or the rename so the start just compiles; each verdict is read from the `DAS_TRACE_MODULE_LOAD=1` line the child prints. +- `test_deferred_modules.das` - a replayed `dm` row is not loaded by the scan. The project root + holds a copy of the tree's `dasUnitTest` (descriptor and artifact), which shadows the tree's, + so the copy's manifest is the test's to make cold or warm: a cold start compiles the + descriptor, loads the module to record its name, and still reads a `require ?UnitTest x` + as skipped; a warm start defers the row and loads nothing for a program that requires + nothing; the first `require UnitTest` loads it and the program calls into it; the guard is + taken whether the `require UnitTest` sits above it, below it, or in the entry while the + guard sits in a module walked earlier, and skipped when no require names the module; + `-ignore-manifest` compiles every descriptor, loads every C++ module on start and writes no + manifest; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every + deferred module in because its `initDependencies` asks for two more. A static build, whose + tree holds no `.shared_module`, has nothing to observe and the test says so and returns. - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, `mc_generic_origin_*`); a case needing a macro-bearing module graph puts it here instead of writing the script inline. diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das new file mode 100644 index 0000000000..916787698c --- /dev/null +++ b/tests/module_cache/test_deferred_modules.das @@ -0,0 +1,221 @@ +options gen2 +options indenting = 4 + +require dastest/testing_boost public + +require strings +require daslib/fio +require daslib/strings_boost + +//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, +//! so argv[0] is the interpreter, not the test script +def das_exe() : string { + let args <- get_command_line_arguments() + return empty(args) ? "" : args[0] +} + +//! the child's environment, as a command prefix: the scan trace on, so the trace names each +//! descriptor's verdict and each shared-module load +def trace_prefix() : string { + return get_platform_name() == "windows" ? "set \"DAS_TRACE_MODULE_LOAD=1\"&& " : "DAS_TRACE_MODULE_LOAD=1 " +} + +//! stderr joins stdout: the scan trace goes through the logger +def run(cmd : string; var output : string&) : int { + return unsafe(popen_timeout("{cmd} 2>&1", 300.0, $(f) { + if (f != null) { + output = fread(f) + } + })) +} + +def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { + let ok = rc == 0 && find(out, marker) >= 0 + if (!ok) { + t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") + t |> failure("{phase} child output follows:\n{out}") + } + return ok +} + +//! the whole scan trace line for a module's descriptor, timing clause included +def descriptor_line(out : string; mod : string) : string { + let marker = "{mod}/.das_module: " + let at = find(out, marker) + if (at < 0) { + return "" + } + let rest = slice(out, at + length(marker)) + let eol = find(rest, "\n") + return eol < 0 ? rest : slice(rest, 0, eol) +} + +let UNIT_TEST_MODULE = "UnitTest" +let UNIT_TEST_CLASS = "Module_UnitTest" +let UNIT_TEST_FOLDER = "dasUnitTest" +let UNIT_TEST_ARTIFACT = "dasModuleUnitTest" + +//! the tree's UnitTest shared module, in whichever configuration this binary was built - absent +//! in a static build, where nothing is deferred and this test has nothing to observe +def unit_test_artifact() : string { + let dir = path_join(path_join(get_das_root(), "modules"), UNIT_TEST_FOLDER) + for (name in ["{UNIT_TEST_ARTIFACT}.shared_module", "{UNIT_TEST_ARTIFACT}_debug.shared_module"]) { + let candidate = path_join(dir, name) + if (stat(candidate).is_valid) { + return candidate + } + } + return "" +} + +let DESCRIPTOR = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n register_native_path(\"deferred_probe\", \"hello\", \"\{project_path\}/hello.das\")\n register_native_path(\"deferred_probe\", \"guarded_dep\", \"\{project_path\}/guarded_dep.das\")\n\}\n" + +//! a project root holding a copy of the tree's dasUnitTest - descriptor and artifact - so the +//! copy's manifest is the test's to make cold or warm (the copy shadows the tree's), a pure-das +//! module, and the driver scripts the arms spawn +struct Fixture { + tmp : string + manifest : string + utManifest : string + plain : string + uses : string + guarded : string + guardedAfter : string + guardedBefore : string + guardedDep : string + app : string +} + +def make_fixture(tmp : string; artifact : string) : Fixture { + let modules = path_join(tmp, "modules") + let modDir = path_join(modules, "deferred_probe") + let utDir = path_join(modules, UNIT_TEST_FOLDER) + var merr : string + mkdir_rec(modDir, merr) + mkdir_rec(utDir, merr) + fwrite(path_join(modDir, ".das_module"), DESCRIPTOR) + fwrite(path_join(modDir, "hello.das"), "options gen2\nmodule hello\ndef hello_value() : int \{\n return 42\n\}\n") + fwrite(path_join(modDir, "guarded_dep.das"), "options gen2\nmodule guarded_dep\nrequire ?{UNIT_TEST_MODULE} deferred_probe/hello public\n") + let treeUt = path_join(path_join(get_das_root(), "modules"), UNIT_TEST_FOLDER) + var cerr : string + copy_file(path_join(treeUt, ".das_module"), path_join(utDir, ".das_module"), true, cerr) + copy_file(artifact, path_join(utDir, base_name(artifact)), true, cerr) + let fx = Fixture( + tmp = tmp, + manifest = path_join(modDir, ".das_module.manifest"), + utManifest = path_join(utDir, ".das_module.manifest"), + plain = path_join(tmp, "plain.das"), + uses = path_join(tmp, "uses.das"), + guarded = path_join(tmp, "guarded.das"), + guardedAfter = path_join(tmp, "guarded_after.das"), + guardedBefore = path_join(tmp, "guarded_before.das"), + guardedDep = path_join(tmp, "guarded_dep_driver.das"), + app = path_join(tmp, "app.das") + ) + let guardLine = "require ?{UNIT_TEST_MODULE} deferred_probe/hello\n" + let guardMain = "[export]\ndef main \{\n print(\"GUARD \{typeinfo module_exists(hello)\}\\n\")\n\}\n" + fwrite(fx.plain, "options gen2\n[export]\ndef main \{\n print(\"PLAIN\\n\")\n\}\n") + fwrite(fx.uses, "options gen2\nrequire {UNIT_TEST_MODULE}\n[export]\ndef main \{\n print(\"LENGTH \{test_string_arg_length(\"hello\")\}\\n\")\n\}\n") + fwrite(fx.guarded, "options gen2\n{guardLine}{guardMain}") + fwrite(fx.guardedAfter, "options gen2\nrequire {UNIT_TEST_MODULE}\n{guardLine}{guardMain}") + fwrite(fx.guardedBefore, "options gen2\n{guardLine}require {UNIT_TEST_MODULE}\n{guardMain}") + fwrite(fx.guardedDep, "options gen2\nrequire deferred_probe/guarded_dep\nrequire {UNIT_TEST_MODULE}\n{guardMain}") + fwrite(fx.app, "options gen2\nrequire imgui_app\n[export]\ndef main \{\n print(\"APP\\n\")\n\}\n") + return fx +} + +def child_cmd(fx : Fixture; script : string; flags : string = "") : string { + return "{trace_prefix()}\"{das_exe()}\" -dasroot \"{get_das_root()}\" -no-module-cache -project_root \"{fx.tmp}\" {flags} \"{script}\"" +} + +def has_shared_modules(dir : string; names : array) : bool { + for (name in names) { + var found = false + for (suffix in [".shared_module", "_debug.shared_module"]) { + found ||= stat(path_join(dir, "{name}{suffix}")).is_valid + } + if (!found) { + return false + } + } + return true +} + +def arms_deferred(t : T?; fx : Fixture) { + t |> run("a cold start loads the module to record its name, and a guard still reads it unrequired") @(t : T?) { + var out : string + t |> success(report_child(t, "cold", run(child_cmd(fx, fx.guarded), out), out, "GUARD false"), "the cold start runs, and the guard alone takes nothing") + t |> equal("compiled (no manifest), manifest written (1 row(s))", descriptor_line(out, UNIT_TEST_FOLDER), "the copied descriptor compiles and records:\n{out}") + t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the recording start loads the module:\n{out}") + } + t |> run("a warm start defers the module and loads nothing for a program that requires nothing") @(t : T?) { + var out : string + t |> success(report_child(t, "warm", run(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the warm start runs the program") + let line = descriptor_line(out, UNIT_TEST_FOLDER) + t |> success(line |> starts_with("replayed 1 row(s) in ") && find(line, ", deferred 1)") >= 0, "the descriptor replays its one row deferred:\n{out}") + t |> equal(-1, find(out, "{UNIT_TEST_CLASS} <- "), "nothing loads the shared module:\n{out}") + } + t |> run("the first require naming a deferred module loads it, and the program calls into it") @(t : T?) { + var out : string + t |> success(report_child(t, "uses", run(child_cmd(fx, fx.uses), out), out, "LENGTH 5"), "the program calls the module's function") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the trace names the require that loaded it:\n{out}") + t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads:\n{out}") + } + t |> run("a require guard reads whether some require of the compile named the module, in any order") @(t : T?) { + var alone : string + t |> success(report_child(t, "guarded", run(child_cmd(fx, fx.guarded), alone), alone, "GUARD false"), "no require names the module, so the guarded require is skipped") + t |> equal(-1, find(alone, "{UNIT_TEST_CLASS} <- "), "the guard alone loads nothing:\n{alone}") + var after : string + t |> success(report_child(t, "guarded after", run(child_cmd(fx, fx.guardedAfter), after), after, "GUARD true"), "a require above the guard") + var before : string + t |> success(report_child(t, "guarded before", run(child_cmd(fx, fx.guardedBefore), before), before, "GUARD true"), "a require below the guard") + var dep : string + t |> success(report_child(t, "guarded dep", run(child_cmd(fx, fx.guardedDep), dep), dep, "GUARD true"), "a guard inside a module walked before the require that loads its module") + } + t |> run("-ignore-manifest compiles every descriptor, loads every C++ module on start, and writes no manifest") @(t : T?) { + remove_result(fx.manifest) + remove_result(fx.utManifest) + var out : string + t |> success(report_child(t, "ignore", run(child_cmd(fx, fx.plain, "-ignore-manifest"), out), out, "PLAIN"), "the program runs") + t |> equal("compiled (manifests ignored)", descriptor_line(out, UNIT_TEST_FOLDER), "the copied descriptor compiles:\n{out}") + t |> equal("compiled (manifests ignored)", descriptor_line(out, "deferred_probe"), "the pure-das descriptor compiles:\n{out}") + t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads on start:\n{out}") + t |> success(!stat(fx.manifest).is_valid && !stat(fx.utManifest).is_valid, "no manifest was written") + } +} + +def arm_fallback(t : T?; fx : Fixture) { + let modules = path_join(get_das_root(), "modules") + let imgui = path_join(modules, "dasImgui") + if (!has_shared_modules(imgui, ["imguiApp", "dasModuleImgui"]) || !has_shared_modules(path_join(modules, "dasGlfw"), ["dasModuleGlfw"])) { + to_log(LOG_INFO, "test_deferred_modules: no imgui_app, imgui and glfw shared modules in {modules} - the fallback arm has nothing to observe\n") + return + } + t |> run("a module whose C++ dependencies are deferred too brings every deferred module in") @(t : T?) { + var out : string + t |> success(report_child(t, "app", run(child_cmd(fx, fx.app), out), out, "APP"), "the program compiles against imgui_app") + t |> success(find(out, "[module] require imgui_app: loading the deferred Module_imgui_app") >= 0, "imgui_app loads at its require:\n{out}") + t |> success(find(out, "[module] loading every deferred module (") >= 0, "its initDependencies asks for glfw and imgui, so the rest loads:\n{out}") + } +} + +//! the arms share one root and run in this order +[test] +def test_deferred_modules(t : T?) { + let artifact = unit_test_artifact() + if (empty(artifact)) { + to_log(LOG_INFO, "test_deferred_modules: no {UNIT_TEST_ARTIFACT}.shared_module under {get_das_root()}/modules/{UNIT_TEST_FOLDER} - a static build defers nothing\n") + return + } + var terr : string + let tmp = create_temp_directory("das_deferred_modules", terr) + if (empty(tmp)) { + t |> failure("create_temp_directory: {terr}") + return + } + let fx = make_fixture(tmp, artifact) + arms_deferred(t, fx) + arm_fallback(t, fx) + var err : string + rmdir_rec(tmp, err) +} diff --git a/utils/daslang/main.cpp b/utils/daslang/main.cpp index cef658681b..fc759822c2 100644 --- a/utils/daslang/main.cpp +++ b/utils/daslang/main.cpp @@ -239,6 +239,8 @@ int das_aot_main ( int argc, char * argv[] ) { gen2MakeSyntax = true; } else if ( strcmp(argv[ai],"-no-dynamic-modules")==0 ) { noDynamicModules = true; + } else if ( strcmp(argv[ai],"-ignore-manifest")==0 ) { + ignore_dynamic_module_manifests(true); } else if ( strcmp(argv[ai],"-no-lint")==0 ) { noLint = true; } else if ( strcmp(argv[ai],"-log-compile-time")==0 ) { @@ -464,6 +466,9 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP if ( jitNoCache ) policies.jit_dll_mode = false; policies.jit_emit_prologue = jitStack; access->addExtraModule("just_in_time", getDasRoot() + "/daslib/just_in_time.das"); + // the witness every `require ?llvm` guards on: a JIT run names it (src/ast/ARCHITECTURE.md sec.2) + if ( auto loader = getDeferredModuleLoader() ) loader("llvm"); + mark_dynamic_module_required("llvm"); policies.jit_output_path = jitOutPath; policies.dll_search_paths.emplace_back(getDasRoot() + "/lib"); } @@ -740,6 +745,8 @@ void print_help() { << " --das-profiler-global install profiler as singleton agent (default with --das-profiler-memory)\n" << " --das-profiler-leaks track live heap allocations and dump leaks on context destroy\n" << " -no-dynamic-modules skip loading dynamic modules from dasroot and project root\n" + << " -ignore-manifest compile every .das_module descriptor and load every C++ module on start, reading\n" + << " and writing no .das_module.manifest - by default a manifest's C++ module loads at the first require that names it\n" << " -no-lint skip the lint pass (Program::lint)\n" << " --ast-verify force-include daslib/ast_verify; checks AST structural invariants before each inference pass\n" << " --ast-verify-batch checks the finished tree only (no per-pass walks, no cross-module sweeps): cheap enough to gate many files (CI)\n" @@ -1060,6 +1067,8 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { i += 1; } else if ( cmd=="no-dynamic-modules" ) { noDynamicModules = true; + } else if ( cmd=="ignore-manifest" ) { + ignore_dynamic_module_manifests(true); } else if ( cmd=="-dump-leaks" ) { dumpLeaks = true; } else if ( cmd=="-no-dump-leaks" ) { diff --git a/utils/lsp/lsp_supervisor.py b/utils/lsp/lsp_supervisor.py index 646a3f2cc9..04fc38441a 100644 --- a/utils/lsp/lsp_supervisor.py +++ b/utils/lsp/lsp_supervisor.py @@ -137,7 +137,8 @@ def subtool_argv(self, subtool: str, args: list[str]) -> list[str] | None: self.compiler = find_compiler(self.init_options) if self.compiler is None: return None - argv = [self.compiler] + # -ignore-manifest: a subtool enumerates modules, so every C++ module loads on start + argv = [self.compiler, "-ignore-manifest"] if self.init_options.get("project_root"): argv += ["-project_root", self.init_options["project_root"]] for lm in self.init_options.get("load_module") or []: diff --git a/utils/mcp/README.md b/utils/mcp/README.md index ad5b8ca63a..0a07d22536 100644 --- a/utils/mcp/README.md +++ b/utils/mcp/README.md @@ -80,8 +80,8 @@ On **Linux/macOS** point each entry at the binary directly (no launcher needed): ```json "mcpServers": { - "daslang": { "command": "./bin/daslang", "args": ["utils/mcp/main.das"] }, - "daslang-cpp": { "command": "./bin/daslang", "args": ["utils/mcp/cpp_main.das"] } + "daslang": { "command": "./bin/daslang", "args": ["-ignore-manifest", "utils/mcp/main.das"] }, + "daslang-cpp": { "command": "./bin/daslang", "args": ["-ignore-manifest", "utils/mcp/cpp_main.das"] } } ``` @@ -123,10 +123,10 @@ No extra build dependencies - the MCP server uses stdio transport. Claude Code m ```bash # Manual test (Windows): -bin/Release/daslang.exe utils/mcp/main.das +bin/Release/daslang.exe -ignore-manifest utils/mcp/main.das # Manual test (Linux): -./bin/daslang utils/mcp/main.das +./bin/daslang -ignore-manifest utils/mcp/main.das ``` Configure in `.mcp.json` (project root): @@ -137,7 +137,7 @@ Configure in `.mcp.json` (project root): "mcpServers": { "daslang": { "command": "bin/Release/daslang.exe", - "args": ["utils/mcp/main.das"] + "args": ["-ignore-manifest", "utils/mcp/main.das"] } } } @@ -147,7 +147,7 @@ Configure in `.mcp.json` (project root): "mcpServers": { "daslang": { "command": "./bin/daslang", - "args": ["utils/mcp/main.das"] + "args": ["-ignore-manifest", "utils/mcp/main.das"] } } } @@ -157,10 +157,10 @@ Or add via CLI: ```bash # Windows -claude mcp add daslang -- bin/Release/daslang.exe utils/mcp/main.das +claude mcp add daslang -- bin/Release/daslang.exe -ignore-manifest utils/mcp/main.das # Linux -claude mcp add daslang -- ./bin/daslang utils/mcp/main.das +claude mcp add daslang -- ./bin/daslang -ignore-manifest utils/mcp/main.das ``` Claude Code starts and stops the server automatically with each session. diff --git a/utils/mcp/daslang-mcp-msvc.cmd b/utils/mcp/daslang-mcp-msvc.cmd index 4909879415..23478e12a1 100644 --- a/utils/mcp/daslang-mcp-msvc.cmd +++ b/utils/mcp/daslang-mcp-msvc.cmd @@ -58,9 +58,10 @@ rem First arg selects the server. A *.exe selects a prebuilt server binary in rem bin/ (the AOT cpp-mcp) run directly; otherwise it's an interpreted .das rem script handed to daslang.exe. Defaults to main.das (the full server). rem Any further args are forwarded to the chosen server. +rem -ignore-manifest: the server enumerates modules, so every C++ module loads on start. set "MCPSCRIPT=%~1" if not defined MCPSCRIPT ( - "%DASLANG%" "%MCPDIR%main.das" + "%DASLANG%" -ignore-manifest "%MCPDIR%main.das" exit /b %ERRORLEVEL% ) rem Goto (not a parenthesized block) so MCPBIN's set + use are sequential @@ -68,7 +69,7 @@ rem statements -- this script has no EnableDelayedExpansion, so %MCPBIN% rem read inside an `if ( ... )` block would expand stale (pre-set). if /i "%MCPSCRIPT:~-4%"==".exe" goto runexe shift -"%DASLANG%" "%MCPDIR%%MCPSCRIPT%" %1 %2 %3 %4 %5 %6 %7 %8 +"%DASLANG%" -ignore-manifest "%MCPDIR%%MCPSCRIPT%" %1 %2 %3 %4 %5 %6 %7 %8 exit /b %ERRORLEVEL% :runexe diff --git a/utils/mcp/mcp_supervisor.py b/utils/mcp/mcp_supervisor.py index 14d5a55195..129ef08525 100644 --- a/utils/mcp/mcp_supervisor.py +++ b/utils/mcp/mcp_supervisor.py @@ -294,7 +294,8 @@ def _default_launcher() -> list[str]: return ["cmd", "/c", os.path.join(SCRIPT_DIR, "daslang-mcp-msvc.cmd")] main_das = os.path.join(SCRIPT_DIR, "main.das") picked = _pick_binary(REPO_ROOT) - return [picked or os.path.join(REPO_ROOT, "bin", "daslang"), main_das] + # -ignore-manifest: the server enumerates modules, so every C++ module loads on start + return [picked or os.path.join(REPO_ROOT, "bin", "daslang"), "-ignore-manifest", main_das] def _python_launcher() -> str: @@ -361,7 +362,7 @@ def write_mcp_json(repo_root: str) -> bool: if os.path.exists(os.path.join(repo_root, "utils", "internal", "das-herd", "mcp_main.das")): prev_herd = servers.get("dasherd", {}) herd_entry = {"command": _daslang_binary(repo_root), - "args": ["utils/internal/das-herd/mcp_main.das"]} + "args": ["-ignore-manifest", "utils/internal/das-herd/mcp_main.das"]} if isinstance(prev_herd, dict) and "defer_loading" in prev_herd: herd_entry["defer_loading"] = prev_herd["defer_loading"] servers["dasherd"] = herd_entry From 612fb56384ee543af9e0fb8a80e9f86e8160d7aa Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 7 Sep 2026 23:20:55 -0700 Subject: [PATCH 03/22] a require guard loads a waiting module - `require ?X x` and builtin_module_exists ask whether the build has X (guardModuleAvailable: linked in, or a manifest row the loader brings in now), so `require ?das_metal metal/das_metal_boost` keeps meaning "on a build with Metal" and dasLLAMA's GPU tier, tests/metal and daslib/tune's llvm arms reach their modules as they did on an eager start, which the sweeps could not show because every such body sits inside a static_if; the "required earlier" reading needed an unguarded require somewhere for each witness, which one source that must also compile without the module cannot carry, and it leaves with the unrequired set, the RequireRecord guard field, the walk's fixed point, the parser's verdict map and the host naming llvm under -jit --- daslib/ARCHITECTURE.md | 4 +- ...ion-rtti-has_module-0x2f9e9a6e19be1ef0.rst | 2 +- include/daScript/ast/ast.h | 4 +- include/daScript/ast/dyn_modules.h | 4 - include/daScript/simulate/debug_info.h | 1 - .../daslang/references/modules-and-stdlib.md | 5 +- skills/dynamic_modules.md | 7 +- src/ast/ARCHITECTURE.md | 35 +++------ src/ast/ast_infer_type.cpp | 3 +- src/ast/ast_module.cpp | 5 ++ src/ast/ast_parse.cpp | 77 ++++--------------- src/builtin/module_builtin_fio.cpp | 17 +--- src/parser/parser_impl.cpp | 6 +- tests/module_cache/ARCHITECTURE.md | 10 +-- tests/module_cache/test_deferred_modules.das | 12 +-- utils/daslang/main.cpp | 3 - utils/mcp/README.md | 2 + 17 files changed, 56 insertions(+), 141 deletions(-) diff --git a/daslib/ARCHITECTURE.md b/daslib/ARCHITECTURE.md index 00c5d35ac3..9dba5c40eb 100644 --- a/daslib/ARCHITECTURE.md +++ b/daslib/ARCHITECTURE.md @@ -243,6 +243,4 @@ Three companions carry a concern each; a section number is unique across all fou dasLLVM compiles with the shells inert and no `_variants()` registry - a program that reads one is framework-only and says so with that direct require. `daslib/just_in_time` keeps its direct require for the opposite reason: a static host that never registered the witness - still runs the JIT through the LLVM library, and the guard would switch it off. A host that - loads C++ modules at their first require (`daslang`) names the witness itself for a `-jit` or - `-exe` run, since no das file requires it unguarded (`src/ast/ARCHITECTURE.md` sec.2). + still runs the JIT through the LLVM library, and the guard would switch it off. diff --git a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst index 9c3fbb972f..484a1bb373 100644 --- a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst +++ b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst @@ -1 +1 @@ -Returns ``true`` if a module with the given name is registered, or waits in a ``.das_module`` manifest for the first ``require`` that names it, ``false`` otherwise. It answers what the tree has; ``typeinfo builtin_module_exists(name)`` answers what an earlier require loaded. +Returns ``true`` if a module with the given name is registered, or waits in a ``.das_module`` manifest for the first ``require`` that names it, ``false`` otherwise. Unlike ``typeinfo builtin_module_exists(name)`` it loads nothing. diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 795adf7cc6..0312ffe18f 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1110,8 +1110,8 @@ namespace das typedef bool (*DeferredModuleLoader) ( const string & name ); DAS_API void setDeferredModuleLoader ( DeferredModuleLoader loader ); DAS_API DeferredModuleLoader getDeferredModuleLoader (); - // the require walk's verdict on a `require ?guard x` line: 1 taken, 0 skipped, -1 the walk never saw it - DAS_API int walkedGuardVerdict ( const string & fileName, int32_t line ); + // what `require ?guard x` and builtin_module_exists ask: the module is linked in, or the loader brings it in now + DAS_API bool guardModuleAvailable ( const string & name ); class DAS_API Module { public: diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index 2a546d32a0..f59f6b0ea4 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -46,9 +46,5 @@ DAS_API bool load_deferred_dynamic_module(const char * das_name); // true = th DAS_API size_t load_all_deferred_dynamic_modules(); // the count it attempted DAS_API bool has_deferred_dynamic_modules(); DAS_API bool is_dynamic_module_deferred(const char * das_name); -// a module the recording scan loaded is not "required" until a require names it: a guard reads it -// absent on a cold start as on a warm one, where the row waits (ARCHITECTURE.md sec.2) -DAS_API bool is_dynamic_module_unrequired(const char * das_name); -DAS_API void mark_dynamic_module_required(const char * das_name); DAS_CC_API void ignore_dynamic_module_manifests(bool ignore); // -ignore-manifest: no read, no write, every module loads on start } diff --git a/include/daScript/simulate/debug_info.h b/include/daScript/simulate/debug_info.h index 1267b1eec6..6b48979d92 100644 --- a/include/daScript/simulate/debug_info.h +++ b/include/daScript/simulate/debug_info.h @@ -238,7 +238,6 @@ namespace das struct RequireRecord : BaseRequireRecord { bool isPublic = false; bool cantBeRequired = false; - string guard; // `require ?guard target`, a plain module name: tested when the walk reaches the line, not when it collects the file }; enum class MissingHint { diff --git a/skills/daslang/references/modules-and-stdlib.md b/skills/daslang/references/modules-and-stdlib.md index fc850d9e18..3d1562f449 100644 --- a/skills/daslang/references/modules-and-stdlib.md +++ b/skills/daslang/references/modules-and-stdlib.md @@ -75,7 +75,7 @@ require geom // a bare name also finds geom.das next to require ./helpers.das // file-relative path require %/daslib/random.das as rng // `%` is the daslang root; `as` binds a local qualifier require dastest/testing_boost public // re-export to whoever requires me -require ?pugixml pugixml/PUGIXML_boost // only if module `pugixml` is linked or required in this compile +require ?pugixml pugixml/PUGIXML_boost // load only if module `pugixml` is available ``` - **Path form needs the `.das`:** a require is a literal path only when it starts with `./`, `../`, @@ -183,8 +183,7 @@ else is a require. | `fio_core`, `rtti_core`, `ast_core`, `network_core` | Low-level C++ layers; require the wrapper instead - `daslib/fio`, `daslib/rtti`, `daslib/ast`, `daslib/network`. Bare `require rtti` / `require ast` do **not** resolve. | Which built-in modules exist depends on how the host embedded daslang - guard anything non-core -with `require ?mod ...`. The guard passes when `mod` is linked into the host or required -somewhere in this compile; a guard loads nothing itself. +with `require ?mod ...`. ## Container operations diff --git a/skills/dynamic_modules.md b/skills/dynamic_modules.md index 88aa10f169..cf2ff7afae 100644 --- a/skills/dynamic_modules.md +++ b/skills/dynamic_modules.md @@ -105,11 +105,8 @@ walk loads the row and runs the `initDependencies` fixed point; a module needing deferred one pulls the whole deferred set in. A program pays for the C++ modules it requires. Two consequences: -- `require ?mod x` and `typeinfo builtin_module_exists(mod)` say whether some require in the - compile named `mod`, not what `modules/` holds, and a cold start answers as a warm one. - The `daslang` host names the `llvm` witness under `-jit` and `-exe`, so `?llvm` is true - there; an interpreter run that never requires `llvm` reads it false. Require order does not - matter. +- `require ?mod x` and `typeinfo builtin_module_exists(mod)` still ask whether the build has + `mod`: a guard loads a waiting module, so a cold start and a warm one answer alike. - A tool that enumerates the process's modules (the MCP server, the LSP subtools) runs with `-ignore-manifest`: no manifest read or written, every descriptor compiles, every C++ module loads on start. `has_module(name)` (`daslib/rtti`) answers loaded-or-deferred, so a sweep diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 4567145d0f..3c2c013444 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -83,34 +83,23 @@ recorded. A `dm` row carrying that name is not loaded by the scan: the row waits (`defer_dynamic_module`), and the load happens at the first require that names it. The prerequisite walk (`getPrerequisits`) finds no module under the name and asks the loader the scan installed (`setDeferredModuleLoader`); the loader dlopens and registers the module, runs the -`initDependencies` fixed point `Module::Initialize` runs (`Module::InitializeDependencies`) over +`initDependencies` fixed point that `Module::Initialize` runs (`Module::InitializeDependencies`) over the grown list, and when a module reports it cannot initialize - what it needs is deferred too - brings every deferred module in and runs the fixed point again, which is the set an eager start has; the new modules' TypeDecls, made on the active root, move to their module roots. A `dm` row with no name - the recording start's load failed - replays as recorded, so the Quiet deferral and the post-scan retry of a sibling `DT_NEEDED` dlopen behave as on a compiled start. A -require guard (`require ?mod`) and `builtin_module_exists` answer whether a require of the -process named the module, not whether the tree holds it: the `daslang` host names the `llvm` -witness itself when `-jit` or `-exe` is on (the module carries no symbol, so no das file -requires it unguarded, and a static host runs the JIT without it), so `?llvm` is taken there; -an interpreter run of a program that never requires `llvm` reads it absent whatever `modules/` -holds. The recording start loads every module -eagerly, so the module a `dm` row names counts as unrequired (`is_dynamic_module_unrequired`) -until a require resolves to it (`mark_dynamic_module_required`), and a cold start answers a -guard as a warm one does. Order inside a compile does not matter: the collector -(`getAllRequireReq`) decides a path guard (its file resolves) as it reads the file, but carries -a plain-name guard on the record (`RequireRecord::guard`) for `getPrerequisits` to test when -the walk reaches the line, and `compileDaScript` walks again whenever a skipped guard's module -was loaded later in the walk, so the guarded target lands in dependency order; the parser -(`ast_requireModule`) reads the walk's verdict for its file and line (`walkedGuardVerdict`) -rather than testing the guard itself, so a module loaded between the walk and the parse -cannot make the two disagree. A load must not change another module's content either: a -module-cache record carries each builtin module's cumulative hash of mangled names, and a -process that loaded a different set of C++ modules would otherwise fail every record on `$`, -so a `vector` of a module's own handled type registers into that module -(`vectorHomeModule`, `ast_handle.h`), and only a vector of a builtin element lands in `$`, which -every library lists first because `ModuleLibrary::addModule` puts a module's dependencies -before it. `-ignore-manifest` +require guard (`require ?mod`) and `builtin_module_exists` ask whether the build has the +module (`guardModuleAvailable`): linked in, or waiting in a manifest row, which the guard +loads then - so `require ?das_metal metal/das_metal_boost` still means "on a build with +Metal", a cold start and a warm start answer alike, and `llvm`, a witness module no das file +requires unguarded, comes in through the guards `daslib/tune` places on it. A load changes no +other module's content: a module-cache record carries each builtin module's cumulative hash +of mangled names, and a process that loaded a different set of C++ modules would otherwise +fail every record on `$`, so a `vector` of a handled element registers into the element's +module (`vectorHomeModule`, `ast_handle.h`) whichever module builds it, and only a vector of a +builtin element lands in `$`, which every library lists first because +`ModuleLibrary::addModule` puts a module's dependencies before it. `-ignore-manifest` reads and writes no manifest: every descriptor compiles and every C++ module loads on start, the form a tool that enumerates modules - the MCP server, the LSP subtools - runs under. `no_manifest()` inside `initialize` marks the descriptor as one that runs on every start: its diff --git a/src/ast/ast_infer_type.cpp b/src/ast/ast_infer_type.cpp index e9c59abd7c..80f5b6c633 100644 --- a/src/ast/ast_infer_type.cpp +++ b/src/ast/ast_infer_type.cpp @@ -4,7 +4,6 @@ #include "daScript/ast/ast_generate.h" #include "daScript/ast/ast_infer_type.h" #include "daScript/ast/ast_pass_macros.h" -#include "daScript/ast/dyn_modules.h" #include "daScript/ast/ast_visitor.h" #define DAS_XSTR(s) #s @@ -2847,7 +2846,7 @@ namespace das { // also accepts shared das modules compiled earlier in the process auto mod = Module::requireEx(evar->name, true); reportAstChanged(); - return new ExprConstBool(expr->at, mod != nullptr && !is_dynamic_module_unrequired(evar->name.c_str())); + return new ExprConstBool(expr->at, mod != nullptr || guardModuleAvailable(evar->name)); } else { error("unsupported module name subexpression ", expr->subexpr->__rtti, "", expr->at, CompilationError::invalid_typeinfo_module_subexpression); diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index 238ae790d5..33215b1bfa 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -187,6 +187,11 @@ namespace das { return g_deferredModuleLoader; } + bool guardModuleAvailable ( const string & name ) { + if ( Module::requireEx(name, false) ) return true; + return g_deferredModuleLoader && g_deferredModuleLoader(name) && Module::requireEx(name, false); + } + void Module::Initialize() { daScriptEnvironment::ensure(); static bool atexit_registered = (atexit(daslang_atexit_audit), true); diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index aaa7d1a82a..bd12aa4fc4 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -62,20 +62,6 @@ DAS_CC_API das::smart_ptr get_file_access( char * pak ) { namespace das { - // the walk in flight's `require ?guard x` verdicts by file and line, and the skipped guards' module - // names - a later require may load one, and then the walk runs again (ARCHITECTURE.md sec.2) - static thread_local das_hash_map g_guardVerdicts; - static thread_local das_hash_set g_skippedGuards; - - static string guardVerdictKey ( const string & fileName, int32_t line ) { - return fileName + "\t" + to_string(line); - } - - int walkedGuardVerdict ( const string & fileName, int32_t line ) { - auto it = g_guardVerdicts.find(guardVerdictKey(fileName, line)); - return it == g_guardVerdicts.end() ? -1 : it->second; - } - void applyPostRewriteMacros ( Program * program ) { program->library.foreach([&](Module * mod) -> bool { for ( const auto & pm : mod->postRewriteMacros ) { @@ -275,15 +261,13 @@ namespace das { // linked C++ module); no target-resolvability fallback (module source // dirs exist in every checkout regardless of build config). Otherwise — // skip silently. Must match ast_requireModule (parser_impl.cpp). - // a plain-name guard is tested by getPrerequisits at its line (ARCHITECTURE.md sec.2) - string plainGuard; if ( hasReqGuard && reqGuard.find('/')!=string::npos ) { auto ginfo = access->getModuleInfo(reqGuard, fi->name); if ( ginfo.fileName.empty() || !access->getFileInfo(ginfo.fileName) ) { continue; } - } else if ( hasReqGuard ) { - plainGuard = reqGuard; + } else if ( hasReqGuard && !guardModuleAvailable(reqGuard) ) { + continue; } bool isPublic = false; while ( src < src_end && src[0] == ' ' ) { @@ -293,7 +277,6 @@ namespace das { isPublic = true; } req.push_back({mod, line, chain, isPublic}); - req.back().guard = plainGuard; } else if ( isInc ) { string incFileName = access->getIncludeFileName(fi->name,mod); auto info = access->getFileInfo(incFileName); @@ -427,17 +410,6 @@ namespace das { vector ownReq = getAllRequire(fi, modName, chain, access); for ( auto & modRec : ownReq ) { string mod = modRec.name; - if ( !modRec.guard.empty() ) { - bool taken = Module::requireEx(modRec.guard, false)!=nullptr && !is_dynamic_module_unrequired(modRec.guard.c_str()); - g_guardVerdicts[guardVerdictKey(fi->name, modRec.line)] = taken ? 1 : 0; - if ( !taken ) { - g_skippedGuards.insert(modRec.guard); - if ( log ) { - *log << string(tab,'\t') << "require ?" << modRec.guard << " " << mod << " - guard module not required, skipped\n"; - } - continue; - } - } if ( log ) { *log << string(tab,'\t') << "require " << mod << "\n"; } @@ -594,7 +566,6 @@ namespace das { return false; } else { libGroup.addModule(module); - mark_dynamic_module_required(module->name.c_str()); } } } @@ -1869,38 +1840,20 @@ namespace das { string modName; [[maybe_unused]] auto builtinModule = Module::require("$"); DAS_ASSERTF(builtinModule, "Somehow `builtin` module is missing."); - bool walked = false; - for ( ;; ) { - req.clear(); missing.clear(); circular.clear(); notAllowed.clear(); chain.clear(); - dependencies.clear(); namelessReq.clear(); namelessMismatches.clear(); modName.clear(); - g_guardVerdicts.clear(); g_skippedGuards.clear(); - bool allGood = addExtraDependency("builtin", get_builtin_path(), missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, &logs); - if ( !allGood ) { - auto res = make_smart(); - res->error("internal error: failed to build builtin.das", logs.str(), "", LineInfo(), CompilationError::internal_module); - return res; - } - for ( const auto & em : access->getExtraModules() ) { - allGood = addExtraDependency(em.first, em.second, missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, nullptr) && allGood; - } - if ( !allGood ) { - return reportPrerequisitesErrors(fileName, missing, circular, notAllowed, namelessMismatches, libGroup, policies); - } - walked = getPrerequisits(fileName, access, modName, req, missing, circular, notAllowed, chain, - dependencies, namelessReq, namelessMismatches, libGroup, nullptr, 1, !policies.ignore_shared_modules); - if ( !walked ) break; - // a guard skipped before a later require loaded its module: the walk runs again, and - // the guard's target lands in dependency order (ARCHITECTURE.md sec.2) - bool flipped = false; - for ( auto & guard : g_skippedGuards ) { - if ( Module::requireEx(guard, false)!=nullptr && !is_dynamic_module_unrequired(guard.c_str()) ) { - flipped = true; - break; - } - } - if ( !flipped ) break; + bool allGood = addExtraDependency("builtin", get_builtin_path(), missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, &logs); + if ( !allGood ) { + auto res = make_smart(); + res->error("internal error: failed to build builtin.das", logs.str(), "", LineInfo(), CompilationError::internal_module); + return res; } - if ( walked ) { + for ( const auto & em : access->getExtraModules() ) { + allGood = addExtraDependency(em.first, em.second, missing, circular, notAllowed, req, dependencies, namelessReq, namelessMismatches, access, libGroup, policies, nullptr) && allGood; + } + if ( !allGood ) { + return reportPrerequisitesErrors(fileName, missing, circular, notAllowed, namelessMismatches, libGroup, policies); + } + if ( getPrerequisits(fileName, access, modName, req, missing, circular, notAllowed, chain, + dependencies, namelessReq, namelessMismatches, libGroup, nullptr, 1, !policies.ignore_shared_modules) ) { preqT = get_time_usec(time0); disableSerializationOnDebugger(req); if ( !verifyModuleNamesUnique(req, logs) ) { diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index a4abf3f797..8471ef54f3 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -317,8 +317,6 @@ namespace das { DAS_API size_t load_all_deferred_dynamic_modules () GENERATE_IO_STUB_RET DAS_API bool has_deferred_dynamic_modules () GENERATE_IO_STUB_RET DAS_API bool is_dynamic_module_deferred ( const char * ) GENERATE_IO_STUB_RET - DAS_API bool is_dynamic_module_unrequired ( const char * ) GENERATE_IO_STUB_RET - DAS_API void mark_dynamic_module_required ( const char * ) GENERATE_IO_STUB #undef GENERATE_IO_STUB #undef GENERATE_IO_STUB_RET @@ -2289,16 +2287,6 @@ namespace das { // after the folder scan, so module enumeration order stops mattering. static vector> g_pending_dynamic_modules; // path, cpp_class_name, last dlopen error - static das_hash_set g_unrequired_dynamic_modules; // loaded by the recording scan, no require yet (dyn_modules.h) - - DAS_API bool is_dynamic_module_unrequired ( const char * das_name ) { - return das_name && g_unrequired_dynamic_modules.count(das_name) != 0; - } - - DAS_API void mark_dynamic_module_required ( const char * das_name ) { - if ( das_name ) g_unrequired_dynamic_modules.erase(das_name); - } - // the descriptor manifest recorder (dyn_modules.h, src/ast/ARCHITECTURE.md sec.2) static thread_local bool g_manifest_recording = false; static thread_local bool g_manifest_opt_out = false; @@ -2421,10 +2409,7 @@ namespace das { } *ModuleKarma += unsigned(intptr_t(mod)); g_registered_dynamic_modules.emplace_back(path, mod_name, mod->name); - if ( recordedRowIndex != size_t(-1) ) { - g_manifest_rows[recordedRowIndex].c = mod->name; - g_unrequired_dynamic_modules.insert(mod->name); - } + if ( recordedRowIndex != size_t(-1) ) g_manifest_rows[recordedRowIndex].c = mod->name; return lib; } void *register_dynamic_module_silent(const char *path, const char *mod_name, Context * context, LineInfoArg * at ) { diff --git a/src/parser/parser_impl.cpp b/src/parser/parser_impl.cpp index 1c0df9a890..fe46851b48 100644 --- a/src/parser/parser_impl.cpp +++ b/src/parser/parser_impl.cpp @@ -4,7 +4,6 @@ #include "parser_state.h" #include "daScript/ast/ast_generate.h" -#include "daScript/ast/dyn_modules.h" #include "daScript/ast/ast_handle.h" #undef yyextra @@ -1241,10 +1240,7 @@ namespace das { auto ginfo = yyextra->g_Access->getModuleInfo(*guard, yyextra->g_FileAccessStack.back()->name); guardAvailable = !ginfo.fileName.empty() && yyextra->g_Access->getFileInfo(ginfo.fileName) != nullptr; } else { - // the walk decided this line already (a later require may have loaded the guard module since) - auto verdict = walkedGuardVerdict(yyextra->g_FileAccessStack.back()->name, int32_t(atName.line)); - guardAvailable = verdict >= 0 ? verdict != 0 - : Module::requireEx(*guard, false) != nullptr && !is_dynamic_module_unrequired(guard->c_str()); + guardAvailable = guardModuleAvailable(*guard); } delete guard; if ( !guardAvailable ) { diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index d73b437d35..0e113b17de 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -34,11 +34,11 @@ this document states what the folder is and why its tests take the shape they do - `test_deferred_modules.das` - a replayed `dm` row is not loaded by the scan. The project root holds a copy of the tree's `dasUnitTest` (descriptor and artifact), which shadows the tree's, so the copy's manifest is the test's to make cold or warm: a cold start compiles the - descriptor, loads the module to record its name, and still reads a `require ?UnitTest x` - as skipped; a warm start defers the row and loads nothing for a program that requires - nothing; the first `require UnitTest` loads it and the program calls into it; the guard is - taken whether the `require UnitTest` sits above it, below it, or in the entry while the - guard sits in a module walked earlier, and skipped when no require names the module; + descriptor, loads the module to record its name, and a `require ?UnitTest x` is taken; a + warm start defers the row and loads nothing for a program that requires nothing; the first + `require UnitTest` loads it and the program calls into it; a guard alone loads the module + and is taken, as it is with a `require UnitTest` above it, below it, or in the entry while + the guard sits in a module walked earlier; `-ignore-manifest` compiles every descriptor, loads every C++ module on start and writes no manifest; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every deferred module in because its `initDependencies` asks for two more. A static build, whose diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index 916787698c..06d410db32 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -142,9 +142,9 @@ def has_shared_modules(dir : string; names : array) : bool { } def arms_deferred(t : T?; fx : Fixture) { - t |> run("a cold start loads the module to record its name, and a guard still reads it unrequired") @(t : T?) { + t |> run("a cold start loads the module to record its name, and a guard reads it available") @(t : T?) { var out : string - t |> success(report_child(t, "cold", run(child_cmd(fx, fx.guarded), out), out, "GUARD false"), "the cold start runs, and the guard alone takes nothing") + t |> success(report_child(t, "cold", run(child_cmd(fx, fx.guarded), out), out, "GUARD true"), "the cold start runs, and the guard takes its target") t |> equal("compiled (no manifest), manifest written (1 row(s))", descriptor_line(out, UNIT_TEST_FOLDER), "the copied descriptor compiles and records:\n{out}") t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the recording start loads the module:\n{out}") } @@ -161,16 +161,16 @@ def arms_deferred(t : T?; fx : Fixture) { t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the trace names the require that loaded it:\n{out}") t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads:\n{out}") } - t |> run("a require guard reads whether some require of the compile named the module, in any order") @(t : T?) { + t |> run("a require guard loads a waiting module, whatever else the compile requires") @(t : T?) { var alone : string - t |> success(report_child(t, "guarded", run(child_cmd(fx, fx.guarded), alone), alone, "GUARD false"), "no require names the module, so the guarded require is skipped") - t |> equal(-1, find(alone, "{UNIT_TEST_CLASS} <- "), "the guard alone loads nothing:\n{alone}") + t |> success(report_child(t, "guarded", run(child_cmd(fx, fx.guarded), alone), alone, "GUARD true"), "the guard alone takes its target") + t |> success(find(alone, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the guard loaded the module:\n{alone}") var after : string t |> success(report_child(t, "guarded after", run(child_cmd(fx, fx.guardedAfter), after), after, "GUARD true"), "a require above the guard") var before : string t |> success(report_child(t, "guarded before", run(child_cmd(fx, fx.guardedBefore), before), before, "GUARD true"), "a require below the guard") var dep : string - t |> success(report_child(t, "guarded dep", run(child_cmd(fx, fx.guardedDep), dep), dep, "GUARD true"), "a guard inside a module walked before the require that loads its module") + t |> success(report_child(t, "guarded dep", run(child_cmd(fx, fx.guardedDep), dep), dep, "GUARD true"), "a guard inside a module walked before the entry's own require") } t |> run("-ignore-manifest compiles every descriptor, loads every C++ module on start, and writes no manifest") @(t : T?) { remove_result(fx.manifest) diff --git a/utils/daslang/main.cpp b/utils/daslang/main.cpp index fc759822c2..18ac9a5a71 100644 --- a/utils/daslang/main.cpp +++ b/utils/daslang/main.cpp @@ -466,9 +466,6 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP if ( jitNoCache ) policies.jit_dll_mode = false; policies.jit_emit_prologue = jitStack; access->addExtraModule("just_in_time", getDasRoot() + "/daslib/just_in_time.das"); - // the witness every `require ?llvm` guards on: a JIT run names it (src/ast/ARCHITECTURE.md sec.2) - if ( auto loader = getDeferredModuleLoader() ) loader("llvm"); - mark_dynamic_module_required("llvm"); policies.jit_output_path = jitOutPath; policies.dll_search_paths.emplace_back(getDasRoot() + "/lib"); } diff --git a/utils/mcp/README.md b/utils/mcp/README.md index 0a07d22536..20c8db4299 100644 --- a/utils/mcp/README.md +++ b/utils/mcp/README.md @@ -85,6 +85,8 @@ On **Linux/macOS** point each entry at the binary directly (no launcher needed): } ``` +An existing `.mcp.json` needs `-ignore-manifest` added by hand (or a rerun of `utils/mcp/setup.das`): without it the server enumerates only the modules a compile loaded, so `list_modules` and the all-modules symbol scans come up short. + Tools are namespaced by server, so the cpp server's tools appear as `mcp__daslang-cpp__cpp_compile_check` etc. `cpp-mcp` - a standalone static AOT build of `cpp_main.das` for C++-only consumers - exists as a gated target (`DAS_BUILD_CPP_MCP`, OFF by default; bundled by `ci/make_cpp_mcp_bundle.sh`, released via `cpp_mcp_release.yml`, setup in `cpp-mcp-setup.md`); the interpreted form above is the same server. It is a separate product: the mcp server itself never ships as a `daslang -exe` binary - development runs it through the python keep-alive supervisor, so that exe form would never be dogfooded. ### Duplicate Detection From 055bbf518a072c97d1dd11357f35ff552bde659b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 7 Sep 2026 23:58:08 -0700 Subject: [PATCH 04/22] the diff's added C++ comments settle into the documents: the loader-before-the-walk ordering is a duty in src/ast/REVIEW.md, the load's single-root gc mechanism is a sentence of src/ast/ARCHITECTURE.md sec.2, the rest are one-line pointers to that section or gone --- include/daScript/ast/ast.h | 4 +--- include/daScript/ast/ast_handle.h | 4 +--- include/daScript/ast/dyn_modules.h | 5 ++--- src/ast/ARCHITECTURE.md | 6 +++++- src/ast/REVIEW.md | 5 +++++ src/ast/ast_module.cpp | 1 - src/ast/ast_parse.cpp | 2 +- src/ast/dyn_modules.cpp | 13 ++++--------- src/builtin/module_builtin_fio.cpp | 2 +- src/builtin/module_builtin_rtti.cpp | 2 +- 10 files changed, 21 insertions(+), 23 deletions(-) diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 0312ffe18f..b2bf8374af 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1106,11 +1106,10 @@ namespace das DAS_API bool isValidBuiltinName ( const string & name, bool canPunkt = false ); - // the scan's loader for a deferred .shared_module, asked by name at a require miss (src/ast/ARCHITECTURE.md sec.2) + // src/ast/ARCHITECTURE.md sec.2 typedef bool (*DeferredModuleLoader) ( const string & name ); DAS_API void setDeferredModuleLoader ( DeferredModuleLoader loader ); DAS_API DeferredModuleLoader getDeferredModuleLoader (); - // what `require ?guard x` and builtin_module_exists ask: the module is linked in, or the loader brings it in now DAS_API bool guardModuleAvailable ( const string & name ); class DAS_API Module { @@ -1161,7 +1160,6 @@ namespace das static Module * require ( const string & name ); static Module * requireEx ( const string & name, bool allowPromoted, const string & requireName = string(), const string & expectedFileName = string() ); static void Initialize(); - // the initDependencies fixed point; false = the named modules never initialized (src/ast/ARCHITECTURE.md sec.2) static bool InitializeDependencies ( string & notInitialized ); static void CollectFileInfo(das::vector &accesses); static void Shutdown( bool dumpHandleLeaks = true ); diff --git a/include/daScript/ast/ast_handle.h b/include/daScript/ast/ast_handle.h index d9b45ca985..6e7356b58b 100644 --- a/include/daScript/ast/ast_handle.h +++ b/include/daScript/ast/ast_handle.h @@ -748,9 +748,7 @@ namespace das } }; - // a vector lives with its element's type: one of a module's own handled type registers into - // that module, so loading the module never changes the builtin module's hash; one of a - // builtin element stays in `$` (library.front(): a module's dependencies sit before it) + // src/ast/ARCHITECTURE.md sec.2 __forceinline Module * vectorHomeModule ( const TypeDeclPtr & elem, const ModuleLibrary & library ) { auto t = elem; while ( t && t->isPointer() && t->firstType ) t = t->firstType; diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index f59f6b0ea4..3eb33b09ae 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -40,11 +40,10 @@ DAS_API void begin_dynamic_module_recording(); DAS_API void end_dynamic_module_recording(vector & rows, bool & optOut); DAS_API void replay_native_path(const char * mod_name, const char * src, const char * dst); DAS_API void replay_dynamic_module(const char * path, const char * cpp_class, int on_error); -// a replayed dm row with a das name waits under it for the first require (ARCHITECTURE.md sec.2) DAS_API void defer_dynamic_module(const char * path, const char * cpp_class, int on_error, const char * das_name); -DAS_API bool load_deferred_dynamic_module(const char * das_name); // true = the module registered +DAS_API bool load_deferred_dynamic_module(const char * das_name); DAS_API size_t load_all_deferred_dynamic_modules(); // the count it attempted DAS_API bool has_deferred_dynamic_modules(); DAS_API bool is_dynamic_module_deferred(const char * das_name); -DAS_CC_API void ignore_dynamic_module_manifests(bool ignore); // -ignore-manifest: no read, no write, every module loads on start +DAS_CC_API void ignore_dynamic_module_manifests(bool ignore); } diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 3c2c013444..9561098966 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -86,7 +86,11 @@ scan installed (`setDeferredModuleLoader`); the loader dlopens and registers the `initDependencies` fixed point that `Module::Initialize` runs (`Module::InitializeDependencies`) over the grown list, and when a module reports it cannot initialize - what it needs is deferred too - brings every deferred module in and runs the fixed point again, which is the set an eager start -has; the new modules' TypeDecls, made on the active root, move to their module roots. A `dm` row +has. The load runs under one gc root of its own with the thread root's nodes parked meanwhile, +because a constructor's nodes go to the active root while a builtin das module it compiles +dumps its leftovers on the thread root, and a collect stops at a node owned by another root; +after the load every module, not only the new ones, collects from that root, since a +constructor registers into modules that exist already, and the rest is swept. A `dm` row with no name - the recording start's load failed - replays as recorded, so the Quiet deferral and the post-scan retry of a sibling `DT_NEEDED` dlopen behave as on a compiled start. A require guard (`require ?mod`) and `builtin_module_exists` ask whether the build has the diff --git a/src/ast/REVIEW.md b/src/ast/REVIEW.md index cffe38d4b6..2dd8e2373e 100644 --- a/src/ast/REVIEW.md +++ b/src/ast/REVIEW.md @@ -24,3 +24,8 @@ replay loop in `init_dyn_modules` (`dyn_modules.cpp`) a branch for it in the same change.** The loop dispatches on one flag with `replay_native_path` as the other arm, so a kind it does not know replays as a native path. + +- **A diff that moves or removes the `setDeferredModuleLoader` call in `require_dynamic_modules` + (`dyn_modules.cpp`) keeps it above the descriptor walk, in the same change.** A descriptor + compiled during the walk can require a module an earlier replay deferred, and with no loader + installed that require fails. diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index 33215b1bfa..ff5b984d5e 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -159,7 +159,6 @@ namespace das { } } if ( all ) return true; - // Some modules was not initialized! size_t i = 0; for ( auto m = daScriptEnvironment::getBound()->modules; m ; m = m->next, i++ ) { DAS_ASSERT(mod_state.size() > i); diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index bd12aa4fc4..8f3b17840a 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -435,7 +435,7 @@ namespace das { } module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); // try native with that name AGAIN (promoted?) if ( !module ) { - // a C++ module the scan deferred loads at the require that names it (ARCHITECTURE.md sec.2) + // ARCHITECTURE.md sec.2 if ( auto loader = getDeferredModuleLoader(); loader && loader(mod) ) { module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); if ( log && module ) { diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 85b957903b..e2fc5d54b8 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -10,7 +10,7 @@ #include // get_dasenv_trace_module_load #include #include // tolower (case-insensitive basename normalize) -#include // the deferred-load lock +#include #include // fprintf(stderr) for the shadow-shadows-global diagnostic das::FileAccessPtr get_file_access( char * pak ); @@ -191,7 +191,6 @@ static ManifestRead read_manifest(const string & file, uint32_t descSize, uint64 const smart_ptr & fa) { ManifestRead res; #if DAS_NO_FILEIO - // the guard the fio builtins carry: a build without file IO reads no manifest (void)file; (void)descSize; (void)descHash; (void)key; (void)fa; return res; #else @@ -365,7 +364,7 @@ static Result init_dyn_modules(smart_ptr fa, string path, TextWriter const ManifestKey key = manifest_key(path); auto time0 = ref_time_ticks(); #if DAS_NO_FILEIO - const bool useManifest = false; // the guard the fio builtins carry: no manifest read or written + const bool useManifest = false; const char * noManifestWhy = "no file io"; #else const bool useManifest = src && !g_ignore_manifests; @@ -379,7 +378,6 @@ static Result init_dyn_modules(smart_ptr fa, string path, TextWriter if ( !row.dynamic ) { replay_native_path(row.a.c_str(), row.b.c_str(), row.c.c_str()); } else if ( !row.c.empty() ) { - // a row without a das name failed to load on the recording start, and replays as it did defer_dynamic_module(row.a.c_str(), row.b.c_str(), row.on_error, row.c.c_str()); deferred ++; } else { @@ -580,12 +578,11 @@ static void move_all_nodes(gc_root & from, gc_root & to) { } } -// ARCHITECTURE.md sec.2: the prerequisite walk found no module under `name` +// ARCHITECTURE.md sec.2 static bool load_deferred_module_for_require(const string & name) { static recursive_mutex loadMutex; lock_guard guard(loadMutex); - // one root for all the load makes (a collect walks one root): the thread root's own nodes sit - // aside while a compiled builtin das module dumps its leftovers there, then join loadRoot + // a collect walks one root: the thread root's own nodes park while the load's leftovers gather on loadRoot auto & threadRoot = gc_root::gc_get_thread_root(); gc_root parked, loadRoot; move_all_nodes(threadRoot, parked); @@ -596,7 +593,6 @@ static bool load_deferred_module_for_require(const string & name) { if ( loaded ) { string notInitialized; if ( !Module::InitializeDependencies(notInitialized) ) { - // what it needs is deferred too: the whole deferred set, the eager start's, then the fixed point again notInitialized.clear(); load_all_deferred_dynamic_modules(); if ( !Module::InitializeDependencies(notInitialized) ) { @@ -624,7 +620,6 @@ bool require_dynamic_modules(FileAccessPtr file_access, const das::vector &load_modules, const das::vector &disabled_modules, das::TextWriter &tout) { - // before the walk: a descriptor compiled mid-scan may require a module an earlier replay deferred setDeferredModuleLoader(&load_deferred_module_for_require); // Explicitly-disabled modules (case-insensitive on every platform) are never // loaded/registered — keeps a native-only module out of a wasm cross-compile. diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 8471ef54f3..106c5e91eb 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -2426,7 +2426,7 @@ namespace das { string path, cpp_class, das_name; int on_error = 0; }; - static vector g_deferred_dynamic_modules; // manifest rows waiting for a require (ARCHITECTURE.md sec.2) + static vector g_deferred_dynamic_modules; // src/ast/ARCHITECTURE.md sec.2 DAS_API void defer_dynamic_module ( const char * path, const char * cpp_class, int on_error, const char * das_name ) { g_deferred_dynamic_modules.push_back({path ? path : "", cpp_class ? cpp_class : "", das_name ? das_name : "", on_error}); diff --git a/src/builtin/module_builtin_rtti.cpp b/src/builtin/module_builtin_rtti.cpp index f6fc7ba25e..3cb2c5521f 100644 --- a/src/builtin/module_builtin_rtti.cpp +++ b/src/builtin/module_builtin_rtti.cpp @@ -1188,7 +1188,7 @@ namespace das { return Module::require(name); } - // loaded or deferred: what the tree has, where `require ?name` asks what an earlier require loaded + // true for a module the tree has, loaded or waiting in a manifest row - this asks, it never loads bool rtti_has_module ( const char * name ) { return Module::require(name) != nullptr || is_dynamic_module_deferred(name); } From 0147b5d821d70703d320be80f3c042dcc30d64da Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 00:15:30 -0700 Subject: [PATCH 05/22] a deferred module whose own dlopen fails brings every deferred module in and retries the pending ones before its require fails, and a module set that grew runs the initDependencies fixed point and the collect whether or not the named module came in - the fallback skipped both, so the sweep freed nodes the constructors had just registered (SIGBUS on a manifest row naming an absent artifact); the serializer version is 205 because a vector of a handled element streams under the element's module; the src/ast gate scans the cache-read timing line outside the reader for its log_module_compile_time guard; test_descriptor_manifest's probe_line keeps the deferred count; checklist self-review fixes in src/ast, src/builtin, src/parser, modules/dasLLVM and tests/module_cache --- include/daScript/ast/ast_serializer.h | 2 +- modules/dasLLVM/REVIEW.md | 12 ++-- src/ast/ARCHITECTURE.md | 4 +- src/ast/REVIEW.das | 61 +++++++++++-------- src/ast/REVIEW.md | 44 +++++++------ src/ast/ast_parse.cpp | 8 +-- src/ast/dyn_modules.cpp | 13 +++- src/builtin/REVIEW.md | 18 +++--- src/parser/REVIEW.md | 5 ++ src/parser/parser_impl.cpp | 9 ++- tests/module_cache/REVIEW.md | 11 ++-- tests/module_cache/test_deferred_modules.das | 14 +++++ .../module_cache/test_descriptor_manifest.das | 19 +++--- 13 files changed, 137 insertions(+), 83 deletions(-) diff --git a/include/daScript/ast/ast_serializer.h b/include/daScript/ast/ast_serializer.h index ce0466c0c5..76d94e9b56 100644 --- a/include/daScript/ast/ast_serializer.h +++ b/include/daScript/ast/ast_serializer.h @@ -258,7 +258,7 @@ namespace das { AstSerializer & serializeModule ( Module & module, bool already_exists ); static constexpr uint32_t getVersion () { - return 204; // 204: the record header stamps the source by content hash, not mtime; the policy stream carries every CodeOfPolicies field + return 205; // 205: a vector of a handled element streams under the element's module (204: the record header stamps the source by content hash; the policy stream carries every CodeOfPolicies field) } void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept; diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 6dd7d6a318..74bafe1240 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -35,11 +35,13 @@ it, while that phase's line still prints** (phase inventory: `ARCHITECTURE.md` sec.1). Option resolution before the first timer, and log lines, are not work. -- **A change that can alter the machine code the JIT's DLL or split-obj cache serves back for - identical inputs bumps `LLVM_JIT_CODEGEN_VERSION`** (`daslib/llvm_jit_run.das`; what counts - as emitting: `ARCHITECTURE.md` sec.1.2). The constant folds only into those two keys; - selecting among existing generators' `[llvm_code]` arguments - the `[tune]` stamping - is not - such a change, because stamped arguments fold into the cache keys per function. +- **A change to what emits the JIT's machine code - IR generation, target-machine setup, a + `[llvm_code]` generator body, or the call ABI the generated code binds: function signatures, + the name scheme, the prologue, the externs the install phase binds - bumps + `LLVM_JIT_CODEGEN_VERSION`** (`daslib/llvm_jit_run.das`); selecting among existing generators' + `[llvm_code]` arguments, the `[tune]` stamping, is not such a change. The DLL and split-obj + caches are addressed by the AST hashes and this constant, so an emitter change without the bump + serves the old machine code back (`ARCHITECTURE.md` sec.1.2). - **A diff that adds an environment or config input to a JIT cache key folds it inside `jit_env_salt` (`daslib/llvm_jit_run.das`), never directly into either JIT key - the DLL diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 9561098966..573d36eeda 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -86,7 +86,9 @@ scan installed (`setDeferredModuleLoader`); the loader dlopens and registers the `initDependencies` fixed point that `Module::Initialize` runs (`Module::InitializeDependencies`) over the grown list, and when a module reports it cannot initialize - what it needs is deferred too - brings every deferred module in and runs the fixed point again, which is the set an eager start -has. The load runs under one gc root of its own with the thread root's nodes parked meanwhile, +has. A row whose own dlopen fails takes the same road - every deferred module comes in, the +pending retry runs - and the require then finds the module or fails as a cold start would. The +load runs under one gc root of its own with the thread root's nodes parked meanwhile, because a constructor's nodes go to the active root while a builtin das module it compiles dumps its leftovers on the thread root, and a collect stops at a node owned by another root; after the load every module, not only the new ones, collects from that root, since a diff --git a/src/ast/REVIEW.das b/src/ast/REVIEW.das index 84ea19098d..ffefe338d1 100644 --- a/src/ast/REVIEW.das +++ b/src/ast/REVIEW.das @@ -12,6 +12,8 @@ let private SOURCE = "src/ast/ast_parse.cpp" let private READER = "trySerializeProgramModule" let private GUARD = "quietCache" let private PRINT = "logs <<" +let private CACHE_READ_PRINT = "logs << \"cache read" +let private TIMING_GUARD = "log_module_compile_time" def private count_char(line : string; ch : int) : int { var n = 0 @@ -23,35 +25,22 @@ def private count_char(line : string; ch : int) : int { return n } -// Every print inside the reader sits under a quietCache guard - on the guard's own line, or in -// the block a guard line opens. Brace depth is tracked from the reader's opening line to its -// closing brace; a guard line that opens a block guards every line until the depth drops back. -def private check_reader_prints { - let text = fread(SOURCE) - if (empty(text)) { - gate_finding(SOURCE, "cannot read the file the gate scans") - return - } - var inscope lines <- split(text, "\n") - var start = -1 - for (ln, i in lines, count()) { - if (find(ln, READER) >= 0 && find(ln, "(") >= 0 && find(ln, ";") < 0 && find(ln, "return") < 0 && find(ln, "if") < 0) { - start = i - break - } - } - if (start < 0) { - gate_finding(SOURCE, "no definition of {READER} found - the gate scans its body for module-cache diagnostics that print without a {GUARD} guard") - return - } +//! Every print matching `marker` from `start` on sits under one of `guards` - on the guard's own +//! line, or in the block a guard line opens. Brace depth is tracked from `start`; a guard line +//! that opens a block guards every line until the depth drops back. With `oneFunction` the walk +//! stops at the closing brace of the function `start` opens. +def private check_guarded_prints(lines : array; start : int; oneFunction : bool; marker : string; guards : array; fix : string) { var depth = 0 var opened = false var guardDepth = -1 for (i in range(start, length(lines))) { let ln = lines[i] - let guardLine = find(ln, GUARD) >= 0 && find(ln, "if") >= 0 - if (find(ln, PRINT) >= 0 && !guardLine && guardDepth < 0) { - gate_finding("{SOURCE}:{i + 1}", "a module-cache read diagnostic prints without a {GUARD} guard - the default cache is on unasked for an ordinary run, so this line is output every user sees; wrap it in `if ( !serializer_read->{GUARD} )`") + var guardLine = false + for (g in guards) { + guardLine ||= find(ln, g) >= 0 && find(ln, "if") >= 0 + } + if (find(ln, marker) >= 0 && !guardLine && guardDepth < 0) { + gate_finding("{SOURCE}:{i + 1}", "a module-cache read diagnostic prints without a guard - the default cache is on unasked for an ordinary run, so this line is output every user sees; {fix}") } let opens = count_char(ln, '{') let closes = count_char(ln, '}') @@ -65,12 +54,34 @@ def private check_reader_prints { if (guardDepth >= 0 && depth < guardDepth) { guardDepth = -1 } - if (opened && depth <= 0) { + if (oneFunction && opened && depth <= 0) { break } } } +def private check_reader_prints { + let text = fread(SOURCE) + if (empty(text)) { + gate_finding(SOURCE, "cannot read the file the gate scans") + return + } + var inscope lines <- split(text, "\n") + var start = -1 + for (ln, i in lines, count()) { + if (find(ln, READER) >= 0 && find(ln, "(") >= 0 && find(ln, ";") < 0 && find(ln, "return") < 0 && find(ln, "if") < 0) { + start = i + break + } + } + if (start < 0) { + gate_finding(SOURCE, "no definition of {READER} found - the gate scans its body for module-cache diagnostics that print without a {GUARD} guard") + return + } + check_guarded_prints(lines, start, true, PRINT, [GUARD], "wrap it in `if ( !serializer_read->{GUARD} )`") + check_guarded_prints(lines, 0, false, CACHE_READ_PRINT, [GUARD, TIMING_GUARD], "wrap it in `if ( !serializer_read->{GUARD} )` or `if ( policies.{TIMING_GUARD} )`") +} + [export] def main { check_reader_prints() diff --git a/src/ast/REVIEW.md b/src/ast/REVIEW.md index 2dd8e2373e..bbd6ca2851 100644 --- a/src/ast/REVIEW.md +++ b/src/ast/REVIEW.md @@ -4,28 +4,32 @@ `ARCHITECTURE.md`. - **Weakening `REVIEW.das` (beside this file) is a defect:** dropping its scan of the prints in - `trySerializeProgramModule` (`ast_parse.cpp`, `ARCHITECTURE.md` sec.1), or a finding text that - no longer names what failed. What the gate enforces is read from the gate itself. + `trySerializeProgramModule` (`ast_parse.cpp`, `ARCHITECTURE.md` sec.1) or of the cache-read + lines elsewhere in that file, or a finding text that no longer names what failed. What the gate + enforces is read from the gate itself. -- **A diff that moves a module-cache read diagnostic out of `trySerializeProgramModule` - into - a helper it calls, or another function - extends `REVIEW.das`'s scan to the new home in the - same change.** The gate scans that one function's body, so a print moved out of it is a print - the gate no longer checks, and an ungated line there is output every user of the default cache - sees. +- **A diff that adds a module-cache read diagnostic outside `trySerializeProgramModule`, or moves + one out of it - into a helper it calls, or another function - extends `REVIEW.das`'s scan to the + new home in the same change, and the line prints only when the user asked: the serializer's + `quietCache` off, or `log_module_compile_time` set.** A print the gate does not scan is a print + nobody checks, and an ungated line there is output every user of the default cache sees. -- **A diff that changes the manifest format - a field added, removed, reordered or re-typed in - `read_manifest` or `write_manifest` (`dyn_modules.cpp`), or a line added beside the key lines - `stamp`, `dll`, `root`, `dasroot`, `target` and `dep` - bumps the version in `MANIFEST_HEADER` - in the same change.** A reader accepts a manifest whose first line equals `MANIFEST_HEADER`, - so without the bump an older manifest decodes the changed bytes as a wrong registration with - no diagnostic. +- **A diff that changes what a manifest written by an earlier binary replays to - a field added, + removed, reordered or re-typed in `read_manifest` or `write_manifest` (`dyn_modules.cpp`), a + line added beside the key lines `stamp`, `dll`, `root`, `dasroot`, `target` and `dep`, or a + change to what an existing row registers - bumps the version in `MANIFEST_HEADER` in the same + change.** A reader accepts a manifest whose first line equals `MANIFEST_HEADER`, so without the + bump an older manifest decodes the changed bytes as a wrong registration with no diagnostic. - **A diff that gives `read_manifest` a new kind of row - one it pushes into `rows` - gives the - replay loop in `init_dyn_modules` (`dyn_modules.cpp`) a branch for it in the same change.** The loop - dispatches on one flag with `replay_native_path` as the other arm, so a kind it does not know - replays as a native path. + replay loop in `init_dyn_modules` (`dyn_modules.cpp`) a branch for it in the same change.** The + loop dispatches on `row.dynamic` and then on whether the row carries a das-visible name, so a + kind it does not know replays as a native path or waits under a name nothing requires. -- **A diff that moves or removes the `setDeferredModuleLoader` call in `require_dynamic_modules` - (`dyn_modules.cpp`) keeps it above the descriptor walk, in the same change.** A descriptor - compiled during the walk can require a module an earlier replay deferred, and with no loader - installed that require fails. +- **Removing the `setDeferredModuleLoader` call from `require_dynamic_modules` + (`dyn_modules.cpp`) is a defect.** A descriptor compiled during the scan can require a module + an earlier replay deferred, and with no loader installed that require fails. + +- **A diff that moves the `setDeferredModuleLoader` call in `require_dynamic_modules` keeps it + above the first `init_modules_for_folder` call, in the same change.** That call compiles the + descriptors, and a descriptor can require a module an earlier replay deferred. diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index 8f3b17840a..331e1df35f 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -257,10 +257,10 @@ namespace das { // guarded optional require. Path guard (contains '/'): proceed only when // the guard's OWN file resolves — the rail for pure-das packages (nothing // C++ to guard on) and cross-package dependency witnesses. Plain-name - // guard: STRICT — proceed only when the guard module is registered (a - // linked C++ module); no target-resolvability fallback (module source - // dirs exist in every checkout regardless of build config). Otherwise — - // skip silently. Must match ast_requireModule (parser_impl.cpp). + // guard: proceed only when the build has the module (guardModuleAvailable, + // src/ast/ARCHITECTURE.md sec.2); no target-resolvability fallback (module + // source dirs exist in every checkout regardless of build config). + // Otherwise skip silently. Must match ast_requireModule (parser_impl.cpp). if ( hasReqGuard && reqGuard.find('/')!=string::npos ) { auto ginfo = access->getModuleInfo(reqGuard, fi->name); if ( ginfo.fileName.empty() || !access->getFileInfo(ginfo.fileName) ) { diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index e2fc5d54b8..7389396b72 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -587,10 +587,19 @@ static bool load_deferred_module_for_require(const string & name) { gc_root parked, loadRoot; move_all_nodes(threadRoot, parked); bool loaded = false; + bool grown = false; // the module set changed - the fixed point and the collect are owed whether or not `name` came in { gc_active_scope scope(&loadRoot); + bool wasDeferred = is_dynamic_module_deferred(name.c_str()); loaded = load_deferred_dynamic_module(name.c_str()); - if ( loaded ) { + grown = loaded; + if ( !loaded && wasDeferred ) { + // its dlopen failed - a sibling it links may be deferred too; bring every row in (retries the pending ones) + load_all_deferred_dynamic_modules(); + grown = true; + loaded = Module::require(name) != nullptr; + } + if ( grown ) { string notInitialized; if ( !Module::InitializeDependencies(notInitialized) ) { notInitialized.clear(); @@ -602,7 +611,7 @@ static bool load_deferred_module_for_require(const string & name) { } } move_all_nodes(threadRoot, loadRoot); - if ( loaded ) { + if ( grown ) { // every module: a constructor registers into existing ones too (a vector type's functions) Module::foreach([&](Module * m) { m->gc_collect(&loadRoot); diff --git a/src/builtin/REVIEW.md b/src/builtin/REVIEW.md index b446a7928b..9a787c1310 100644 --- a/src/builtin/REVIEW.md +++ b/src/builtin/REVIEW.md @@ -5,10 +5,8 @@ - **Weakening `review_nttp.das`'s bind-flavor scan, which `REVIEW.das` runs, is a defect - fix a bind the scan reports by switching the bind.** The Inline modules are `$` (builtin), `math`, - `strings` and `jit`. In those, a plain-value bind - one returning nothing, or a value that is - neither a reference nor written into the caller's result slot - registers through - `addExternInline` or `addExternInlineEx`; in every other module it registers through - `addExtern`. + `strings` and `jit`. What the scan enforces, and which shared generic helpers it exempts, is + read from the scan itself. - **A diff that adds or changes a bind in a module the scan covers rebuilds the binary from that diff before the folder's gate runs** - the scan reads the binds compiled into the running @@ -21,11 +19,13 @@ - **Never drop a module from `review_nttp.das`'s `require` list.** The list is what sets the modules the scan covers. -- **A diff that changes what `module_builtin_ast_serialize.cpp` streams - a field added, - removed, reordered, re-typed, or given a new meaning - bumps the version `getVersion()` - returns in `include/daScript/ast/ast_serializer.h`, in the same change** - a reader accepts a - stream only when its stored version equals `getVersion()`, so without the bump an older cache - passes that check and decodes the changed bytes as something else. +- **A diff that changes the bytes a module-cache record carries or what they resolve to - a + field added, removed, reordered, re-typed or given a new meaning in + `module_builtin_ast_serialize.cpp`, or a change anywhere to which module owns a streamed + annotation, function or type - bumps the version `getVersion()` returns in + `include/daScript/ast/ast_serializer.h`, in the same change** - a reader accepts a stream only + when its stored version equals `getVersion()`, so without the bump an older cache passes that + check and decodes the changed bytes as something else. - **A diff that streams or compares a `CodeOfPolicies` field in `module_builtin_ast_serialize.cpp` outside `DAS_MODULE_CACHE_POLICY_FIELDS` is a defect - put the field on the list instead** - the diff --git a/src/parser/REVIEW.md b/src/parser/REVIEW.md index cfdf6910b6..4c9e5e2fb9 100644 --- a/src/parser/REVIEW.md +++ b/src/parser/REVIEW.md @@ -7,3 +7,8 @@ rule to `tree-sitter-daslang/grammar.js` (repo root), in the same change.** The editor and the MCP search tools parse with the tree-sitter grammar, not with bison, so syntax missing from `grammar.js` does not appear in code folding, outline or `grep_usage`. + +**A diff that lets `ds2_parser.ypp` or `ds2_lexer.lpp` accept new syntax also adds a section +exercising it to `modules/dasImgui/tests/test_grammar_canary.das` (repo root), in the same +change.** The canary reds only for syntax it already carries, so syntax it lacks can drift from +the tree-sitter grammar with no test to say so. diff --git a/src/parser/parser_impl.cpp b/src/parser/parser_impl.cpp index fe46851b48..c71b5ede37 100644 --- a/src/parser/parser_impl.cpp +++ b/src/parser/parser_impl.cpp @@ -1230,11 +1230,10 @@ namespace das { if ( guard ) { // Path guard (contains '/'): availability = the guard's OWN file resolves — the rail for // pure-das packages (nothing C++ to guard on) and cross-package dependencies the target's - // resolvability can't express. Plain-name guard: STRICT — the guard module is registered - // (a linked C++ module). No target-resolvability fallback: `require ?sqlite ...`-style - // guards mean "loaded only when is linked", and module source dirs are present in - // every checkout regardless of build config. Must match the require collector's rule - // (ast_parse.cpp getAllRequireReq). + // resolvability can't express. Plain-name guard: the build has the module + // (guardModuleAvailable, src/ast/ARCHITECTURE.md sec.2). No target-resolvability + // fallback: module source dirs are present in every checkout regardless of build config. + // Must match the require collector's rule (ast_parse.cpp getAllRequireReq). bool guardAvailable; if ( guard->find('/') != string::npos ) { auto ginfo = yyextra->g_Access->getModuleInfo(*guard, yyextra->g_FileAccessStack.back()->name); diff --git a/tests/module_cache/REVIEW.md b/tests/module_cache/REVIEW.md index 8b84493c83..6ef5cc28c1 100644 --- a/tests/module_cache/REVIEW.md +++ b/tests/module_cache/REVIEW.md @@ -3,10 +3,13 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE.md`. -- **Weakening what a test in this folder checks a spawned child's output against is a defect; - an edited assertion weakens when it accepts an output the old one rejected, and re-pinning a - count or a verdict form to the child's new true output does not.** A child's output is the - only instrument a human has for what the cache and the scan served. +- **Weakening the text a test in this folder compares a spawned child's output against - an + assertion literal, or the helper that produces the compared text - is a defect; an edit + weakens when it accepts an output the old one rejected, and re-pinning a count or the fixed + words of a scan or cache line to the child's new true output does not. A field the child newly + prints that no run can pin - a timing - leaves the compared text only while every deterministic + field beside it stays compared.** A child's output is the only instrument a human has for what + the cache and the scan served. - **A test in this folder writes only under a directory it created for this process - its own files and its children's - and removes that directory.** diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index 06d410db32..93dccbae37 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -182,6 +182,20 @@ def arms_deferred(t : T?; fx : Fixture) { t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads on start:\n{out}") t |> success(!stat(fx.manifest).is_valid && !stat(fx.utManifest).is_valid, "no manifest was written") } + t |> run("a deferred row whose artifact is gone brings every deferred module in, and the require fails as on a cold start") @(t : T?) { + var warm : string + t |> success(report_child(t, "rewarm", run(child_cmd(fx, fx.plain), warm), warm, "PLAIN"), "a cold start rewrites the manifest") + let text = fread(fx.utManifest) + t |> success(find(text, "\ndm\t") >= 0, "the manifest carries the dm row: {text}") + fwrite(fx.utManifest, replace(text, UNIT_TEST_ARTIFACT, "absent")) + var out : string + let rc = run(child_cmd(fx, fx.uses), out) + t |> success(rc != 0 && find(out, "LENGTH") < 0, "the program does not run:\n{out}") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the require asks for the row:\n{out}") + t |> success(find(out, "[module] loading every deferred module (") >= 0, "the failed load brings the rest in:\n{out}") + t |> success(find(out, "error[20605]: missing prerequisite '{UNIT_TEST_MODULE}'") >= 0, "the require fails with the missing-module error:\n{out}") + remove_result(fx.utManifest) + } } def arm_fallback(t : T?; fx : Fixture) { diff --git a/tests/module_cache/test_descriptor_manifest.das b/tests/module_cache/test_descriptor_manifest.das index 9d9f89f7ab..909c3507a2 100644 --- a/tests/module_cache/test_descriptor_manifest.das +++ b/tests/module_cache/test_descriptor_manifest.das @@ -49,7 +49,8 @@ let DESCRIPTOR_DEP = "options gen2\nrequire daslib/fio\nrequire ./helper\n\n[exp let HELPER = "options gen2\nmodule helper\ndef helper_name() : string \{\n return \"hello\"\n\}\n" let DESCRIPTOR_BROKEN = "options gen2\n\n[export]\ndef initialize(project_path : string) \{\n this_is_not_a_function()\n\}\n" -//! the scan trace line for a module's descriptor: `[module] descriptor /.das_module: ` +//! the scan trace line for a module's descriptor: `[module] descriptor /.das_module: `; +//! a replay verdict keeps its deferred count and drops the two timings def probe_line(out : string; mod : string = "manifest_probe") : string { let marker = "{mod}/.das_module: " let at = find(out, marker) @@ -60,7 +61,11 @@ def probe_line(out : string; mod : string = "manifest_probe") : string { let eol = find(rest, "\n") let line = eol < 0 ? rest : slice(rest, 0, eol) let timing = find(line, " in ") - return timing < 0 ? line : slice(line, 0, timing) + if (timing < 0) { + return line + } + let deferred = find(line, ", deferred ") + return "{slice(line, 0, timing)} ({slice(line, deferred + 2)}" } def count_of(text : string; needle : string) : int { @@ -197,8 +202,8 @@ def arms_record_and_replay(t : T?; fx : Fixture) { t |> run("the second start replays the manifest and never compiles the descriptor") @(t : T?) { var out : string t |> success(report_child(t, "warm", run(cmd, out), out, "PROBE 42"), "the require resolves through the replayed rows") - t |> equal("replayed 1 row(s)", probe_line(out), "the trace says replayed:\n{out}") - t |> equal("replayed 1 row(s)", probe_line(out, "dm_probe"), "the dm row replays too:\n{out}") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(out), "the trace says replayed:\n{out}") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(out, "dm_probe"), "the dm row replays too:\n{out}") t |> success(count_of(out, "Module_Absent <- ") >= 1, "the replayed dm row retries its dlopen:\n{out}") } t |> run("an edited descriptor recompiles once and is replayed again after") @(t : T?) { @@ -208,7 +213,7 @@ def arms_record_and_replay(t : T?; fx : Fixture) { t |> equal("compiled (descriptor changed), manifest written (1 row(s))", probe_line(out), "the stamp mismatch recompiles and rewrites:\n{out}") var again : string t |> success(report_child(t, "edited warm", run(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") - t |> equal("replayed 1 row(s)", probe_line(again), "the rewritten manifest replays:\n{again}") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(again), "the rewritten manifest replays:\n{again}") } } @@ -264,14 +269,14 @@ def arms_rejections(t : T?; fx : Fixture) { t |> success(find(fread(manifest), "/helper.das\t") >= 0, "the helper is stamped in the manifest, by the path the require resolved: {fread(manifest)}") var warm : string t |> success(report_child(t, "dep warm", run(cmd, warm), warm, "PROBE 42"), "the dep descriptor replays") - t |> equal("replayed 1 row(s)", probe_line(warm), "an unchanged helper replays:\n{warm}") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(warm), "an unchanged helper replays:\n{warm}") fwrite(helper, "{HELPER}// edited\n") var edited : string t |> success(report_child(t, "dep edited", run(cmd, edited), edited, "PROBE 42"), "the require resolves after the helper edit") t |> equal("compiled (dependency changed), manifest written (1 row(s))", probe_line(edited), "the helper's stamp mismatch recompiles the descriptor:\n{edited}") var again : string t |> success(report_child(t, "dep edited warm", run(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") - t |> equal("replayed 1 row(s)", probe_line(again), "the rewritten manifest replays:\n{again}") + t |> equal("replayed 1 row(s) (deferred 0)", probe_line(again), "the rewritten manifest replays:\n{again}") } t |> run("a cross-compile target is part of the key: a --jit-target run never replays a native manifest") @(t : T?) { //! dasOpenGL's descriptor registers its module only when get_cross_platform_name() says emscripten - From 0e7d395af23b1aacf1b508b9d181454de33b74f8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 00:22:53 -0700 Subject: [PATCH 06/22] tests/module_cache shares one _common module for the spawn helpers the five tests carried verbatim - das_exe, trace_prefix, run_child and report_child - a test whose child needs another spawn shape keeps that helper local; utils/mcp's install-block rule binds every top-level file the shipped SDK runs, not only .das; utils/lsp's README says the subtools spawn with -ignore-manifest --- tests/module_cache/ARCHITECTURE.md | 5 ++ tests/module_cache/_common.das | 41 +++++++++++ .../module_cache/test_default_cache_path.das | 17 +---- tests/module_cache/test_deferred_modules.das | 54 ++++----------- .../module_cache/test_descriptor_manifest.das | 68 ++++++------------- .../test_generic_instance_origin.das | 21 +----- .../test_macro_dep_invalidate.das | 17 +---- utils/lsp/README.md | 5 +- utils/mcp/REVIEW.md | 12 ++-- 9 files changed, 90 insertions(+), 150 deletions(-) create mode 100644 tests/module_cache/_common.das diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 0e113b17de..0424551f48 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -46,6 +46,11 @@ this document states what the folder is and why its tests take the shape they do - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, `mc_generic_origin_*`); a case needing a macro-bearing module graph puts it here instead of writing the script inline. +- `_common.das` - the spawn helpers every test here shares: the binary to spawn (`das_exe`), + the scan-trace command prefix (`trace_prefix`), the stderr-joining child run (`run_child`) + and the failure report that echoes the child's output (`report_child`). A test whose child + needs a different spawn shape - an argv spawn, an environment variable - keeps that one + helper local. ## 2. Why every case is a spawned process diff --git a/tests/module_cache/_common.das b/tests/module_cache/_common.das new file mode 100644 index 0000000000..12bdad224b --- /dev/null +++ b/tests/module_cache/_common.das @@ -0,0 +1,41 @@ +options gen2 +options indenting = 4 + +module _common shared public + +require dastest/testing_boost public + +require strings +require daslib/fio + +//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, +//! so argv[0] is the interpreter, not the test script +def das_exe() : string { + let args <- get_command_line_arguments() + return empty(args) ? "" : args[0] +} + +//! the child's environment, as a command prefix: the scan trace on, so the trace names each +//! descriptor's verdict and each shared-module load; the prefix also keeps the command from +//! opening with a quote, which cmd.exe would strip +def trace_prefix() : string { + return get_platform_name() == "windows" ? "set \"DAS_TRACE_MODULE_LOAD=1\"&& " : "DAS_TRACE_MODULE_LOAD=1 " +} + +//! stderr joins stdout: the scan trace goes through the logger +def run_child(cmd : string; var output : string&) : int { + return unsafe(popen_timeout("{cmd} 2>&1", 300.0, $(f) { + if (f != null) { + output = fread(f) + } + })) +} + +def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { + let ok = rc == 0 && find(out, marker) >= 0 + if (!ok) { + t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") + t |> failure("{phase} child output follows:\n{out}") + } + return ok +} diff --git a/tests/module_cache/test_default_cache_path.das b/tests/module_cache/test_default_cache_path.das index 71f4e4f265..52e90058b3 100644 --- a/tests/module_cache/test_default_cache_path.das +++ b/tests/module_cache/test_default_cache_path.das @@ -7,13 +7,7 @@ require strings require math require daslib/fio require daslib/strings_boost - -//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, -//! so argv[0] is the interpreter, not the test script -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} +require _common //! the child shares dastest's cwd (argv[0] is a cwd-relative interpreter path), so the default //! cache lands in this tree's own .jitted_scripts/module_cache - the test removes what it wrote. @@ -28,15 +22,6 @@ def run(cmd : string; var output : string&) : int { let CACHE_DIR = ".jitted_scripts/module_cache" -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} - //! the cache files written for one root: `/-.dascache` def cache_files(stem : string) : int { var n = 0 diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index 93dccbae37..61c83cc42d 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -6,37 +6,7 @@ require dastest/testing_boost public require strings require daslib/fio require daslib/strings_boost - -//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, -//! so argv[0] is the interpreter, not the test script -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} - -//! the child's environment, as a command prefix: the scan trace on, so the trace names each -//! descriptor's verdict and each shared-module load -def trace_prefix() : string { - return get_platform_name() == "windows" ? "set \"DAS_TRACE_MODULE_LOAD=1\"&& " : "DAS_TRACE_MODULE_LOAD=1 " -} - -//! stderr joins stdout: the scan trace goes through the logger -def run(cmd : string; var output : string&) : int { - return unsafe(popen_timeout("{cmd} 2>&1", 300.0, $(f) { - if (f != null) { - output = fread(f) - } - })) -} - -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} +require _common //! the whole scan trace line for a module's descriptor, timing clause included def descriptor_line(out : string; mod : string) : string { @@ -144,39 +114,39 @@ def has_shared_modules(dir : string; names : array) : bool { def arms_deferred(t : T?; fx : Fixture) { t |> run("a cold start loads the module to record its name, and a guard reads it available") @(t : T?) { var out : string - t |> success(report_child(t, "cold", run(child_cmd(fx, fx.guarded), out), out, "GUARD true"), "the cold start runs, and the guard takes its target") + t |> success(report_child(t, "cold", run_child(child_cmd(fx, fx.guarded), out), out, "GUARD true"), "the cold start runs, and the guard takes its target") t |> equal("compiled (no manifest), manifest written (1 row(s))", descriptor_line(out, UNIT_TEST_FOLDER), "the copied descriptor compiles and records:\n{out}") t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the recording start loads the module:\n{out}") } t |> run("a warm start defers the module and loads nothing for a program that requires nothing") @(t : T?) { var out : string - t |> success(report_child(t, "warm", run(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the warm start runs the program") + t |> success(report_child(t, "warm", run_child(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the warm start runs the program") let line = descriptor_line(out, UNIT_TEST_FOLDER) t |> success(line |> starts_with("replayed 1 row(s) in ") && find(line, ", deferred 1)") >= 0, "the descriptor replays its one row deferred:\n{out}") t |> equal(-1, find(out, "{UNIT_TEST_CLASS} <- "), "nothing loads the shared module:\n{out}") } t |> run("the first require naming a deferred module loads it, and the program calls into it") @(t : T?) { var out : string - t |> success(report_child(t, "uses", run(child_cmd(fx, fx.uses), out), out, "LENGTH 5"), "the program calls the module's function") + t |> success(report_child(t, "uses", run_child(child_cmd(fx, fx.uses), out), out, "LENGTH 5"), "the program calls the module's function") t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the trace names the require that loaded it:\n{out}") t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads:\n{out}") } t |> run("a require guard loads a waiting module, whatever else the compile requires") @(t : T?) { var alone : string - t |> success(report_child(t, "guarded", run(child_cmd(fx, fx.guarded), alone), alone, "GUARD true"), "the guard alone takes its target") + t |> success(report_child(t, "guarded", run_child(child_cmd(fx, fx.guarded), alone), alone, "GUARD true"), "the guard alone takes its target") t |> success(find(alone, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the guard loaded the module:\n{alone}") var after : string - t |> success(report_child(t, "guarded after", run(child_cmd(fx, fx.guardedAfter), after), after, "GUARD true"), "a require above the guard") + t |> success(report_child(t, "guarded after", run_child(child_cmd(fx, fx.guardedAfter), after), after, "GUARD true"), "a require above the guard") var before : string - t |> success(report_child(t, "guarded before", run(child_cmd(fx, fx.guardedBefore), before), before, "GUARD true"), "a require below the guard") + t |> success(report_child(t, "guarded before", run_child(child_cmd(fx, fx.guardedBefore), before), before, "GUARD true"), "a require below the guard") var dep : string - t |> success(report_child(t, "guarded dep", run(child_cmd(fx, fx.guardedDep), dep), dep, "GUARD true"), "a guard inside a module walked before the entry's own require") + t |> success(report_child(t, "guarded dep", run_child(child_cmd(fx, fx.guardedDep), dep), dep, "GUARD true"), "a guard inside a module walked before the entry's own require") } t |> run("-ignore-manifest compiles every descriptor, loads every C++ module on start, and writes no manifest") @(t : T?) { remove_result(fx.manifest) remove_result(fx.utManifest) var out : string - t |> success(report_child(t, "ignore", run(child_cmd(fx, fx.plain, "-ignore-manifest"), out), out, "PLAIN"), "the program runs") + t |> success(report_child(t, "ignore", run_child(child_cmd(fx, fx.plain, "-ignore-manifest"), out), out, "PLAIN"), "the program runs") t |> equal("compiled (manifests ignored)", descriptor_line(out, UNIT_TEST_FOLDER), "the copied descriptor compiles:\n{out}") t |> equal("compiled (manifests ignored)", descriptor_line(out, "deferred_probe"), "the pure-das descriptor compiles:\n{out}") t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads on start:\n{out}") @@ -184,12 +154,12 @@ def arms_deferred(t : T?; fx : Fixture) { } t |> run("a deferred row whose artifact is gone brings every deferred module in, and the require fails as on a cold start") @(t : T?) { var warm : string - t |> success(report_child(t, "rewarm", run(child_cmd(fx, fx.plain), warm), warm, "PLAIN"), "a cold start rewrites the manifest") + t |> success(report_child(t, "rewarm", run_child(child_cmd(fx, fx.plain), warm), warm, "PLAIN"), "a cold start rewrites the manifest") let text = fread(fx.utManifest) t |> success(find(text, "\ndm\t") >= 0, "the manifest carries the dm row: {text}") fwrite(fx.utManifest, replace(text, UNIT_TEST_ARTIFACT, "absent")) var out : string - let rc = run(child_cmd(fx, fx.uses), out) + let rc = run_child(child_cmd(fx, fx.uses), out) t |> success(rc != 0 && find(out, "LENGTH") < 0, "the program does not run:\n{out}") t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the require asks for the row:\n{out}") t |> success(find(out, "[module] loading every deferred module (") >= 0, "the failed load brings the rest in:\n{out}") @@ -207,7 +177,7 @@ def arm_fallback(t : T?; fx : Fixture) { } t |> run("a module whose C++ dependencies are deferred too brings every deferred module in") @(t : T?) { var out : string - t |> success(report_child(t, "app", run(child_cmd(fx, fx.app), out), out, "APP"), "the program compiles against imgui_app") + t |> success(report_child(t, "app", run_child(child_cmd(fx, fx.app), out), out, "APP"), "the program compiles against imgui_app") t |> success(find(out, "[module] require imgui_app: loading the deferred Module_imgui_app") >= 0, "imgui_app loads at its require:\n{out}") t |> success(find(out, "[module] loading every deferred module (") >= 0, "its initDependencies asks for glfw and imgui, so the rest loads:\n{out}") } diff --git a/tests/module_cache/test_descriptor_manifest.das b/tests/module_cache/test_descriptor_manifest.das index 909c3507a2..4c6d73834b 100644 --- a/tests/module_cache/test_descriptor_manifest.das +++ b/tests/module_cache/test_descriptor_manifest.das @@ -6,37 +6,7 @@ require dastest/testing_boost public require strings require daslib/fio require daslib/strings_boost - -//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, -//! so argv[0] is the interpreter, not the test script -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} - -//! the child's environment, as a command prefix: the scan trace on, so the trace names each descriptor's -//! verdict; the prefix also keeps the command from opening with a quote, which cmd.exe would strip -def trace_prefix() : string { - return get_platform_name() == "windows" ? "set \"DAS_TRACE_MODULE_LOAD=1\"&& " : "DAS_TRACE_MODULE_LOAD=1 " -} - -//! stderr joins stdout: the scan trace goes through the logger -def run(cmd : string; var output : string&) : int { - return unsafe(popen_timeout("{cmd} 2>&1", 300.0, $(f) { - if (f != null) { - output = fread(f) - } - })) -} - -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} +require _common let DESCRIPTOR = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n register_native_path(\"manifest_probe\", \"hello\", \"\{project_path\}/hello.das\")\n\}\n" let DESCRIPTOR_OPT_OUT = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n no_manifest()\n register_native_path(\"manifest_probe\", \"hello\", \"\{project_path\}/hello.das\")\n\}\n" @@ -124,7 +94,7 @@ def with_line_inserted(lines : array; idx : int; text : string) : string def expect_rewrite(t : T?; cmd : string; manifest : string; text : string; why : string) { fwrite(manifest, text) var out : string - t |> success(report_child(t, why, run(cmd, out), out, "PROBE 42"), "'{why}': the require resolves without the manifest") + t |> success(report_child(t, why, run_child(cmd, out), out, "PROBE 42"), "'{why}': the require resolves without the manifest") t |> equal("compiled ({why}), manifest written (1 row(s))", probe_line(out), "'{why}' recompiles and rewrites:\n{out}") } @@ -178,7 +148,7 @@ def arms_record_and_replay(t : T?; fx : Fixture) { let manifest = fx.manifest t |> run("the first start compiles the descriptor and writes its manifest") @(t : T?) { var out : string - t |> success(report_child(t, "cold", run(cmd, out), out, "PROBE 42"), "the require resolves through the compiled descriptor") + t |> success(report_child(t, "cold", run_child(cmd, out), out, "PROBE 42"), "the require resolves through the compiled descriptor") t |> equal("compiled (no manifest), manifest written (1 row(s))", probe_line(out), "the trace says compiled and written:\n{out}") t |> success(stat(manifest).is_valid, "the manifest sits beside the descriptor") let text = fread(manifest) @@ -195,13 +165,13 @@ def arms_record_and_replay(t : T?; fx : Fixture) { } t |> run("a descriptor that does not compile gets no manifest, and the run goes on") @(t : T?) { var out : string - t |> success(report_child(t, "broken", run(cmd, out), out, "PROBE 42"), "the other module's require still resolves") + t |> success(report_child(t, "broken", run_child(cmd, out), out, "PROBE 42"), "the other module's require still resolves") t |> equal("compiled with errors, no manifest", probe_line(out, "broken_probe"), "the trace names the failure:\n{out}") t |> success(!stat(path_join(fx.brokenDir, ".das_module.manifest")).is_valid, "nothing was written for it") } t |> run("the second start replays the manifest and never compiles the descriptor") @(t : T?) { var out : string - t |> success(report_child(t, "warm", run(cmd, out), out, "PROBE 42"), "the require resolves through the replayed rows") + t |> success(report_child(t, "warm", run_child(cmd, out), out, "PROBE 42"), "the require resolves through the replayed rows") t |> equal("replayed 1 row(s) (deferred 0)", probe_line(out), "the trace says replayed:\n{out}") t |> equal("replayed 1 row(s) (deferred 0)", probe_line(out, "dm_probe"), "the dm row replays too:\n{out}") t |> success(count_of(out, "Module_Absent <- ") >= 1, "the replayed dm row retries its dlopen:\n{out}") @@ -209,10 +179,10 @@ def arms_record_and_replay(t : T?; fx : Fixture) { t |> run("an edited descriptor recompiles once and is replayed again after") @(t : T?) { fwrite(fx.descriptor, "{DESCRIPTOR}// edited\n") var out : string - t |> success(report_child(t, "edited", run(cmd, out), out, "PROBE 42"), "the edited descriptor still resolves the require") + t |> success(report_child(t, "edited", run_child(cmd, out), out, "PROBE 42"), "the edited descriptor still resolves the require") t |> equal("compiled (descriptor changed), manifest written (1 row(s))", probe_line(out), "the stamp mismatch recompiles and rewrites:\n{out}") var again : string - t |> success(report_child(t, "edited warm", run(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") + t |> success(report_child(t, "edited warm", run_child(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") t |> equal("replayed 1 row(s) (deferred 0)", probe_line(again), "the rewritten manifest replays:\n{again}") } } @@ -225,7 +195,7 @@ def arms_rejections(t : T?; fx : Fixture) { let endAt = find(text, "\nend\t") fwrite(manifest, slice(text, 0, endAt + 1)) //! drop the end line - a write that died before its rename var out : string - t |> success(report_child(t, "damaged", run(cmd, out), out, "PROBE 42"), "the damaged manifest is not replayed") + t |> success(report_child(t, "damaged", run_child(cmd, out), out, "PROBE 42"), "the damaged manifest is not replayed") t |> equal("compiled (no end line), manifest written (1 row(s))", probe_line(out), "the trace names the damage:\n{out}") } t |> run("every other rejection: no row of a manifest the reader cannot vouch for is replayed") @(t : T?) { @@ -264,28 +234,28 @@ def arms_rejections(t : T?; fx : Fixture) { fwrite(helper, HELPER) fwrite(fx.descriptor, DESCRIPTOR_DEP) var out : string - t |> success(report_child(t, "dep", run(cmd, out), out, "PROBE 42"), "the require resolves through the helper's answer") + t |> success(report_child(t, "dep", run_child(cmd, out), out, "PROBE 42"), "the require resolves through the helper's answer") t |> equal("compiled (descriptor changed), manifest written (1 row(s))", probe_line(out), "the dep descriptor compiles once:\n{out}") t |> success(find(fread(manifest), "/helper.das\t") >= 0, "the helper is stamped in the manifest, by the path the require resolved: {fread(manifest)}") var warm : string - t |> success(report_child(t, "dep warm", run(cmd, warm), warm, "PROBE 42"), "the dep descriptor replays") + t |> success(report_child(t, "dep warm", run_child(cmd, warm), warm, "PROBE 42"), "the dep descriptor replays") t |> equal("replayed 1 row(s) (deferred 0)", probe_line(warm), "an unchanged helper replays:\n{warm}") fwrite(helper, "{HELPER}// edited\n") var edited : string - t |> success(report_child(t, "dep edited", run(cmd, edited), edited, "PROBE 42"), "the require resolves after the helper edit") + t |> success(report_child(t, "dep edited", run_child(cmd, edited), edited, "PROBE 42"), "the require resolves after the helper edit") t |> equal("compiled (dependency changed), manifest written (1 row(s))", probe_line(edited), "the helper's stamp mismatch recompiles the descriptor:\n{edited}") var again : string - t |> success(report_child(t, "dep edited warm", run(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") + t |> success(report_child(t, "dep edited warm", run_child(cmd, again), again, "PROBE 42"), "the rewritten manifest resolves the require") t |> equal("replayed 1 row(s) (deferred 0)", probe_line(again), "the rewritten manifest replays:\n{again}") } t |> run("a cross-compile target is part of the key: a --jit-target run never replays a native manifest") @(t : T?) { //! dasOpenGL's descriptor registers its module only when get_cross_platform_name() says emscripten - //! an argv input, so a native manifest must not serve a --jit-target run, nor the other way round var out : string - t |> success(report_child(t, "target", run("{cmd} -- --jit-target=wasm32-unknown-emscripten", out), out, "PROBE 42"), "the require resolves under the target") + t |> success(report_child(t, "target", run_child("{cmd} -- --jit-target=wasm32-unknown-emscripten", out), out, "PROBE 42"), "the require resolves under the target") t |> equal("compiled (compile target), manifest written (1 row(s))", probe_line(out), "the target keys the manifest apart:\n{out}") var back : string - t |> success(report_child(t, "native again", run(cmd, back), back, "PROBE 42"), "the native run resolves the require") + t |> success(report_child(t, "native again", run_child(cmd, back), back, "PROBE 42"), "the native run resolves the require") t |> equal("compiled (compile target), manifest written (1 row(s))", probe_line(back), "the native run keys apart from the target's manifest:\n{back}") } } @@ -296,19 +266,19 @@ def arms_opt_out_and_write_failures(t : T?; fx : Fixture) { t |> run("no_manifest() inside the descriptor makes it run on every start") @(t : T?) { fwrite(fx.descriptor, DESCRIPTOR_OPT_OUT) var out : string - t |> success(report_child(t, "opt-out", run(cmd, out), out, "PROBE 42"), "the opted-out descriptor resolves the require") + t |> success(report_child(t, "opt-out", run_child(cmd, out), out, "PROBE 42"), "the opted-out descriptor resolves the require") t |> equal("compiled (descriptor changed), no_manifest recorded", probe_line(out), "the first run records the opt-out:\n{out}") let text = fread(manifest) t |> success(find(text, "\nno_manifest\n") >= 0 && find(text, "\nnp\t") < 0, "the manifest carries the flag and no rows: {text}") var again : string - t |> success(report_child(t, "opt-out warm", run(cmd, again), again, "PROBE 42"), "the opted-out descriptor resolves the require again") + t |> success(report_child(t, "opt-out warm", run_child(cmd, again), again, "PROBE 42"), "the opted-out descriptor resolves the require again") t |> equal("compiled (no_manifest)", probe_line(again), "every later start compiles it:\n{again}") } t |> run("a recorded argument the format cannot carry leaves the manifest unwritten") @(t : T?) { fwrite(fx.descriptor, DESCRIPTOR_TABBED) let before = fread(manifest) var out : string - t |> success(report_child(t, "tabbed", run(cmd, out), out, "PROBE 42"), "the descriptor still resolves the require") + t |> success(report_child(t, "tabbed", run_child(cmd, out), out, "PROBE 42"), "the descriptor still resolves the require") t |> equal("compiled (descriptor changed), manifest not written: a recorded argument contains a tab or newline", probe_line(out), "the trace names the field:\n{out}") t |> equal(before, fread(manifest), "the stale manifest is left as it was") } @@ -317,12 +287,12 @@ def arms_opt_out_and_write_failures(t : T?; fx : Fixture) { remove_result(manifest) mkdir_result(fx.manifestTmp) //! a directory where the .tmp goes: the create fails on every platform, root or not var out : string - t |> success(report_child(t, "no tmp", run(cmd, out), out, "PROBE 42"), "the require resolves without a manifest") + t |> success(report_child(t, "no tmp", run_child(cmd, out), out, "PROBE 42"), "the require resolves without a manifest") t |> success(probe_line(out) |> starts_with("compiled (no manifest), manifest not written: cannot create "), "the trace names the create failure:\n{out}") rmdir_result(fx.manifestTmp) mkdir_result(manifest) //! a directory where the manifest goes: the rename fails var again : string - t |> success(report_child(t, "no rename", run(cmd, again), again, "PROBE 42"), "the require resolves without a manifest") + t |> success(report_child(t, "no rename", run_child(cmd, again), again, "PROBE 42"), "the require resolves without a manifest") t |> success(find(probe_line(again), ", manifest not written: cannot rename ") >= 0, "the trace names the rename failure:\n{again}") t |> success(!stat(fx.manifestTmp).is_valid, "the failed rename left no .tmp behind") rmdir_result(manifest) diff --git a/tests/module_cache/test_generic_instance_origin.das b/tests/module_cache/test_generic_instance_origin.das index 5d5aa0be2f..9b562a9f54 100644 --- a/tests/module_cache/test_generic_instance_origin.das +++ b/tests/module_cache/test_generic_instance_origin.das @@ -5,13 +5,7 @@ require dastest/testing_boost public require strings require daslib/fio - -// argv[0] is the running interpreter; dastest is invoked as -// `daslang(.exe) dastest/dastest.das ...`, so argv[0] is the daslang binary to spawn. -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} +require _common def run_argv(args : array; var output : string&) : int { @@ -23,19 +17,6 @@ def run_argv(args : array; var output : string&) : int { } -// A child that dies takes its diagnosis with it unless the parent prints what it said - the -// exit code alone turns a one-line answer into an exit-code hunt. Echo the child's output on -// any failure, and only then. -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} - - // A generic instance restored from the module cache must keep its origin generic // (Function::fromGeneric). The child host restores its own module graph from the cache and // then compiles a fresh program in the same process - a null origin on any restored diff --git a/tests/module_cache/test_macro_dep_invalidate.das b/tests/module_cache/test_macro_dep_invalidate.das index 9371f961e6..23e1c1d09a 100644 --- a/tests/module_cache/test_macro_dep_invalidate.das +++ b/tests/module_cache/test_macro_dep_invalidate.das @@ -5,13 +5,7 @@ require dastest/testing_boost public require strings require daslib/fio - -//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, -//! so argv[0] is the interpreter, not the test script -def das_exe() : string { - let args <- get_command_line_arguments() - return empty(args) ? "" : args[0] -} +require _common //! child env rides the command string (the portable set/prefix split popen_argv cannot carry) def env_run(dep, cmd : string; var output : string&) : int { @@ -24,15 +18,6 @@ def env_run(dep, cmd : string; var output : string&) : int { })) } -def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { - let ok = rc == 0 && find(out, marker) >= 0 - if (!ok) { - t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") - t |> failure("{phase} child output follows:\n{out}") - } - return ok -} - //! the module cache must gate a macro-registered file input by CONTENT, not mtime: a tune //! sidecar is rewritten byte-identically on every exit, and a record served stale replays //! macro output minted against the old file - stale stamps, empty tune_status, unraced winners. diff --git a/utils/lsp/README.md b/utils/lsp/README.md index 3e143573fe..c09b5768df 100644 --- a/utils/lsp/README.md +++ b/utils/lsp/README.md @@ -79,7 +79,10 @@ Two processes, hard split - full rationale and wave history in - **`subtools/*.das`** - stateless batch tools (`validate.das`, `nav.das`). One fresh `daslang` process per request; argv in, LSP-shaped JSON out, exit. The document shadow rides along as a `--overlay` temp file, so compiles see - the client's buffer even when unsaved. + the client's buffer even when unsaved. Every subtool spawns with + `-ignore-manifest`: the module scan compiles every descriptor and loads every + C++ module on start, so a symbol scan sees the whole tree rather than the + modules a plain run would load at their first `require`. No resident daslang, by design: no macro-state leaks across compiles, no binary/DLL locks while builds run, per-request crash isolation. diff --git a/utils/mcp/REVIEW.md b/utils/mcp/REVIEW.md index 68578cea3e..388bbd8d0c 100644 --- a/utils/mcp/REVIEW.md +++ b/utils/mcp/REVIEW.md @@ -8,12 +8,12 @@ interpreted through `.mcp.json` instead.** Development runs the server through the python keep-alive supervisor, so an exe form would never be used in development before it ships. -**A diff that adds a top-level `.das` under `utils/mcp/` that the shipped SDK runs or loads - -`main.das` reaches it, or it has its own `main` that something in the shipped SDK runs - also -adds it to the `install(FILES ...)` block that lists `utils/mcp/main.das` in `CMakeLists.txt` -(repo root), in the same change.** `tools/` and `subtools/` are globbed; a top-level file left -out of the list dies in the shipped SDK on `error[20605] missing prerequisite` while the -in-tree server keeps working. +**A diff that adds a top-level file under `utils/mcp/` that the shipped SDK runs or loads - +`main.das` reaches it, the supervisor or the `.cmd` launcher runs it, or it has its own `main` +that something in the shipped SDK runs - also adds it to the `install(FILES ...)` block that +lists `utils/mcp/main.das` in `CMakeLists.txt` (repo root), in the same change.** `tools/` and +`subtools/` are globbed; a top-level file left out of the list is absent in the shipped SDK +while the in-tree server keeps working. **Weakening the kept-comment cases in `test_tools.das` is a defect** - they pin the formatter's kept set (the leading header block, `//!` docs, `//fmt:` directives, `nolint:` From ca26aa86527c15f4414e49f66d57a9cb62461022 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 00:30:00 -0700 Subject: [PATCH 07/22] the deferred-module rows and the loader belong to the scan: require_dynamic_modules clears the rows before its walk and Module::Shutdown clears both, so a host that runs several Initialize/Shutdown cycles neither replays the last environment's rows twice nor answers a guard from a tree it no longer has; -log-compile-time's compile, simulate and run laps accumulate across the input files the way the total does, so the teardown residual no longer swallows every file but the last; has_module's page names the manifest file and the bare-name form of builtin_module_exists --- doc/REVIEW.md | 4 ++-- .../function-rtti-has_module-0x2f9e9a6e19be1ef0.rst | 2 +- include/daScript/ast/dyn_modules.h | 1 + src/ast/ARCHITECTURE.md | 4 +++- src/ast/ast_module.cpp | 3 +++ src/ast/dyn_modules.cpp | 1 + src/builtin/module_builtin_fio.cpp | 5 +++++ utils/REVIEW.md | 8 ++++---- utils/daslang/main.cpp | 6 +++--- 9 files changed, 23 insertions(+), 11 deletions(-) diff --git a/doc/REVIEW.md b/doc/REVIEW.md index b5d3d53170..fb09af2fd3 100644 --- a/doc/REVIEW.md +++ b/doc/REVIEW.md @@ -12,7 +12,7 @@ reader sends that host and whether the host sets cookies.** **A diff that adds, to any text that reaches a built page - an authored `.rst` under `source`, a stub under `source/stdlib/handmade`, or page text a `reflections` generator writes - anything -a reader is told to fetch, run, or type - an address, a file, a command, a flag, an API a -`.das_package` manifest calls - states, in the PR body, that each exists and works at merge, and +a reader is told to fetch, run, or type - an address, a file, a command, a flag, an API, a name +a `.das_package` manifest resolves - states, in the PR body, that each exists and works at merge, and where that was checked** - the build proves the page renders, never that what it points a reader at is there. diff --git a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst index 484a1bb373..6aa693a3ca 100644 --- a/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst +++ b/doc/source/stdlib/handmade/function-rtti-has_module-0x2f9e9a6e19be1ef0.rst @@ -1 +1 @@ -Returns ``true`` if a module with the given name is registered, or waits in a ``.das_module`` manifest for the first ``require`` that names it, ``false`` otherwise. Unlike ``typeinfo builtin_module_exists(name)`` it loads nothing. +Returns ``true`` if a module with the given name is registered, or waits in a ``.das_module.manifest`` row for the first ``require`` that names it, ``false`` otherwise. Unlike ``typeinfo builtin_module_exists(mod)``, which takes a bare module name and loads a waiting module, it loads nothing. diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index 3eb33b09ae..17d8812c6c 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -45,5 +45,6 @@ DAS_API bool load_deferred_dynamic_module(const char * das_name); DAS_API size_t load_all_deferred_dynamic_modules(); // the count it attempted DAS_API bool has_deferred_dynamic_modules(); DAS_API bool is_dynamic_module_deferred(const char * das_name); +DAS_API void clear_deferred_dynamic_modules(); // the rows are the scan's: cleared at scan start and at shutdown DAS_CC_API void ignore_dynamic_module_manifests(bool ignore); } diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 573d36eeda..a9d358615a 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -88,7 +88,9 @@ the grown list, and when a module reports it cannot initialize - what it needs i brings every deferred module in and runs the fixed point again, which is the set an eager start has. A row whose own dlopen fails takes the same road - every deferred module comes in, the pending retry runs - and the require then finds the module or fails as a cold start would. The -load runs under one gc root of its own with the thread root's nodes parked meanwhile, +rows and the loader are the scan's: `require_dynamic_modules` clears the rows before its +walk and installs the loader, and `Module::Shutdown` clears both, so an environment that +follows sees neither the last one's rows nor its loader. The load runs under one gc root of its own with the thread root's nodes parked meanwhile, because a constructor's nodes go to the active root while a builtin das module it compiles dumps its leftovers on the thread root, and a collect stops at a node owned by another root; after the load every module, not only the new ones, collects from that root, since a diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index ff5b984d5e..025d0b7bc3 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -7,6 +7,7 @@ #include "daScript/daScriptModule.h" #include "daScript/misc/handle_registry.h" #include "daScript/simulate/simulate_fusion.h" +#include "daScript/ast/dyn_modules.h" #include @@ -248,6 +249,8 @@ namespace das { if ( dumpHandleLeaks ) handleRegistry_dumpAll(); // Free allocated structures for dynamic modules (unloads DLLs). delete daScriptEnvironment::getBound()->g_dyn_modules_resolve; + clear_deferred_dynamic_modules(); + setDeferredModuleLoader(nullptr); clearGlobalAotLibrary(); if ( resetFusion ) { diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 7389396b72..8b03c163b1 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -629,6 +629,7 @@ bool require_dynamic_modules(FileAccessPtr file_access, const das::vector &load_modules, const das::vector &disabled_modules, das::TextWriter &tout) { + clear_deferred_dynamic_modules(); setDeferredModuleLoader(&load_deferred_module_for_require); // Explicitly-disabled modules (case-insensitive on every platform) are never // loaded/registered — keeps a native-only module out of a wasm cross-compile. diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 106c5e91eb..5d3d897c94 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -317,6 +317,7 @@ namespace das { DAS_API size_t load_all_deferred_dynamic_modules () GENERATE_IO_STUB_RET DAS_API bool has_deferred_dynamic_modules () GENERATE_IO_STUB_RET DAS_API bool is_dynamic_module_deferred ( const char * ) GENERATE_IO_STUB_RET + DAS_API void clear_deferred_dynamic_modules () GENERATE_IO_STUB #undef GENERATE_IO_STUB #undef GENERATE_IO_STUB_RET @@ -2466,6 +2467,10 @@ namespace das { [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }) != g_deferred_dynamic_modules.end(); } + DAS_API void clear_deferred_dynamic_modules () { + g_deferred_dynamic_modules.clear(); + } + // Re-attempt modules whose dlopen was deferred (Quiet failure during the // module-folder scan — usually a sibling-module DT_NEEDED dep not yet loaded // because directory enumeration visited the dependent before its dependency). diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 9d2bf05218..1c58f20b10 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -4,10 +4,10 @@ doc: `CLAUDE.md` (repo root). A tool is a directory that owns the programs it ships - each one's entry point and the files -only those programs use; a program joins a tool through `CMakeLists.txt` (beside this file) -building or shipping it, or through the directory's `.das_package` declaring it with -`release_program` - under `utils/`, or outside `utils/` when `CMakeLists.txt` builds or ships -it. An arm is one `t |> run(...)` case of a `[test]` function. An arm's load-bearing +only those programs use; a program joins a tool through a `CMakeLists.txt` anywhere in the +tree - the one beside this file or the repo root's - building or shipping it, or through the +directory's `.das_package` declaring it with `release_program` - under `utils/`, or outside +`utils/` when a `CMakeLists.txt` builds or ships it. An arm is one `t |> run(...)` case of a `[test]` function. An arm's load-bearing assertions are the ones that prove the change, never a skip-path assertion. A CI row is a workflow step whose command reaches the arm. Load-bearing assertions no CI row executes - the arm returns or skips before them, or no suite a CI row runs includes the arm's file - are diff --git a/utils/daslang/main.cpp b/utils/daslang/main.cpp index 18ac9a5a71..d68bf4d440 100644 --- a/utils/daslang/main.cpp +++ b/utils/daslang/main.cpp @@ -518,7 +518,7 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP moduleCache.install(cacheReadPath, cacheWritePath, cacheQuiet); auto compile0 = ref_time_ticks(); auto program = compileDaScript(fn,access,tout,dummyGroup,policies); - startupCompileUsec = get_time_usec(compile0); + startupCompileUsec += get_time_usec(compile0); { auto cres = moduleCache.finish(); if ( !cacheQuiet ) { @@ -569,7 +569,7 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP auto simulate0 = ref_time_ticks(); auto pctx = SimulateWithErrReport(program, tout); - startupSimulateUsec = get_time_usec(simulate0); + startupSimulateUsec += get_time_usec(simulate0); // Check for compiler leaks (TypeDecl nodes left on thread root after compile+simulate) { auto & root = gc_root::gc_get_thread_root(); @@ -620,7 +620,7 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP } else { res = pctx->evalWithCatch(fnTest, nullptr); } - startupRunUsec = get_time_usec(run0); + startupRunUsec += get_time_usec(run0); if ( auto ex = pctx->getException() ) { tout << "EXCEPTION: " << ex << " at " << pctx->exceptionAt.describe() << "\n"; exitCode = 1; From e46a9e0e5ef80db6ed063b740b5e861caaf4b9ab Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 00:50:23 -0700 Subject: [PATCH 08/22] test_deferred_modules pins what the audit found unpinned: builtin_module_exists loads a waiting module with no require naming it, rtti has_module answers true for it and loads nothing, a lazy start and an eager start count the same functions in $ (the vector-home invariant, red at 2485 vs 2501 with it reverted), and a module cache an eager start wrote serves a lazy start with -log-compile-time printing the reads and the timeline; has_deferred_dynamic_modules had no caller and is gone --- include/daScript/ast/dyn_modules.h | 1 - src/builtin/module_builtin_fio.cpp | 5 -- tests/module_cache/ARCHITECTURE.md | 10 +++- tests/module_cache/test_deferred_modules.das | 57 +++++++++++++++++++- 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index 17d8812c6c..2eb753c387 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -43,7 +43,6 @@ DAS_API void replay_dynamic_module(const char * path, const char * cpp_class, in DAS_API void defer_dynamic_module(const char * path, const char * cpp_class, int on_error, const char * das_name); DAS_API bool load_deferred_dynamic_module(const char * das_name); DAS_API size_t load_all_deferred_dynamic_modules(); // the count it attempted -DAS_API bool has_deferred_dynamic_modules(); DAS_API bool is_dynamic_module_deferred(const char * das_name); DAS_API void clear_deferred_dynamic_modules(); // the rows are the scan's: cleared at scan start and at shutdown DAS_CC_API void ignore_dynamic_module_manifests(bool ignore); diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 5d3d897c94..8a4dc0de09 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -315,7 +315,6 @@ namespace das { DAS_API void defer_dynamic_module ( const char *, const char *, int, const char * ) GENERATE_IO_STUB DAS_API bool load_deferred_dynamic_module ( const char * ) GENERATE_IO_STUB_RET DAS_API size_t load_all_deferred_dynamic_modules () GENERATE_IO_STUB_RET - DAS_API bool has_deferred_dynamic_modules () GENERATE_IO_STUB_RET DAS_API bool is_dynamic_module_deferred ( const char * ) GENERATE_IO_STUB_RET DAS_API void clear_deferred_dynamic_modules () GENERATE_IO_STUB @@ -2458,10 +2457,6 @@ namespace das { return all.size(); } - DAS_API bool has_deferred_dynamic_modules () { - return !g_deferred_dynamic_modules.empty(); - } - DAS_API bool is_dynamic_module_deferred ( const char * das_name ) { return das_name && find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }) != g_deferred_dynamic_modules.end(); diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 0424551f48..1344246d99 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -36,11 +36,17 @@ this document states what the folder is and why its tests take the shape they do so the copy's manifest is the test's to make cold or warm: a cold start compiles the descriptor, loads the module to record its name, and a `require ?UnitTest x` is taken; a warm start defers the row and loads nothing for a program that requires nothing; the first - `require UnitTest` loads it and the program calls into it; a guard alone loads the module + `require UnitTest` loads it and the program calls into it; `typeinfo builtin_module_exists` + loads it with no require naming it, while rtti `has_module` answers true and loads nothing; + a lazy start and an eager start count the same functions in `$`, so a load registers + nothing into another module; a guard alone loads the module and is taken, as it is with a `require UnitTest` above it, below it, or in the entry while the guard sits in a module walked earlier; `-ignore-manifest` compiles every descriptor, loads every C++ module on start and writes no - manifest; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every + manifest; a manifest row hand-edited to an absent artifact makes the require bring every + deferred module in and then fail on the missing prerequisite; a module cache an eager start + wrote serves a lazy start, with `-log-compile-time` printing the reads and the startup + timeline; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every deferred module in because its `initDependencies` asks for two more. A static build, whose tree holds no `.shared_module`, has nothing to observe and the test says so and returns. - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index 61c83cc42d..4aa9f58ccb 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -20,6 +20,18 @@ def descriptor_line(out : string; mod : string) : string { return eol < 0 ? rest : slice(rest, 0, eol) } +//! the `DOLLAR N` line the dollar.das child prints: the builtin module's function count +def dollar_count(out : string) : int { + let marker = "DOLLAR " + let at = find(out, marker) + if (at < 0) { + return -1 + } + let rest = slice(out, at + length(marker)) + let eol = find(rest, "\n") + return to_int(eol < 0 ? rest : slice(rest, 0, eol)) +} + let UNIT_TEST_MODULE = "UnitTest" let UNIT_TEST_CLASS = "Module_UnitTest" let UNIT_TEST_FOLDER = "dasUnitTest" @@ -49,6 +61,9 @@ struct Fixture { utManifest : string plain : string uses : string + exists : string + hasModule : string + dollar : string guarded : string guardedAfter : string guardedBefore : string @@ -76,6 +91,9 @@ def make_fixture(tmp : string; artifact : string) : Fixture { utManifest = path_join(utDir, ".das_module.manifest"), plain = path_join(tmp, "plain.das"), uses = path_join(tmp, "uses.das"), + exists = path_join(tmp, "exists.das"), + hasModule = path_join(tmp, "has_module.das"), + dollar = path_join(tmp, "dollar.das"), guarded = path_join(tmp, "guarded.das"), guardedAfter = path_join(tmp, "guarded_after.das"), guardedBefore = path_join(tmp, "guarded_before.das"), @@ -86,6 +104,9 @@ def make_fixture(tmp : string; artifact : string) : Fixture { let guardMain = "[export]\ndef main \{\n print(\"GUARD \{typeinfo module_exists(hello)\}\\n\")\n\}\n" fwrite(fx.plain, "options gen2\n[export]\ndef main \{\n print(\"PLAIN\\n\")\n\}\n") fwrite(fx.uses, "options gen2\nrequire {UNIT_TEST_MODULE}\n[export]\ndef main \{\n print(\"LENGTH \{test_string_arg_length(\"hello\")\}\\n\")\n\}\n") + fwrite(fx.exists, "options gen2\n[export]\ndef main \{\n print(\"EXISTS \{typeinfo builtin_module_exists({UNIT_TEST_MODULE})\}\\n\")\n\}\n") + fwrite(fx.hasModule, "options gen2\nrequire daslib/rtti\n[export]\ndef main \{\n print(\"HAS \{has_module(\"{UNIT_TEST_MODULE}\")\}\\n\")\n\}\n") + fwrite(fx.dollar, "options gen2\nrequire daslib/rtti\n[export]\ndef main \{\n program_for_each_registered_module() $(mod) \{\n if (mod.name == \"$\") \{\n var n = 0\n module_for_each_function(mod) $(fn) \{\n n++\n \}\n print(\"DOLLAR \{n\}\\n\")\n \}\n \}\n\}\n") fwrite(fx.guarded, "options gen2\n{guardLine}{guardMain}") fwrite(fx.guardedAfter, "options gen2\nrequire {UNIT_TEST_MODULE}\n{guardLine}{guardMain}") fwrite(fx.guardedBefore, "options gen2\n{guardLine}require {UNIT_TEST_MODULE}\n{guardMain}") @@ -94,8 +115,8 @@ def make_fixture(tmp : string; artifact : string) : Fixture { return fx } -def child_cmd(fx : Fixture; script : string; flags : string = "") : string { - return "{trace_prefix()}\"{das_exe()}\" -dasroot \"{get_das_root()}\" -no-module-cache -project_root \"{fx.tmp}\" {flags} \"{script}\"" +def child_cmd(fx : Fixture; script : string; flags : string = ""; cache : string = "-no-module-cache") : string { + return "{trace_prefix()}\"{das_exe()}\" -dasroot \"{get_das_root()}\" {cache} -project_root \"{fx.tmp}\" {flags} \"{script}\"" } def has_shared_modules(dir : string; names : array) : bool { @@ -131,6 +152,27 @@ def arms_deferred(t : T?; fx : Fixture) { t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the trace names the require that loaded it:\n{out}") t |> success(find(out, "{UNIT_TEST_CLASS} <- ") >= 0 && find(out, ": loaded") >= 0, "the shared module loads:\n{out}") } + t |> run("builtin_module_exists loads a waiting module, with no require naming it") @(t : T?) { + var out : string + t |> success(report_child(t, "exists", run_child(child_cmd(fx, fx.exists), out), out, "EXISTS true"), "the build has the module") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the question loaded it:\n{out}") + } + t |> run("has_module answers true for a waiting module and loads nothing") @(t : T?) { + var out : string + t |> success(report_child(t, "has_module", run_child(child_cmd(fx, fx.hasModule), out), out, "HAS true"), "the build has the module") + t |> equal(-1, find(out, "{UNIT_TEST_CLASS} <- "), "nothing loads the shared module:\n{out}") + } + t |> run("a lazy start and an eager start build the same builtin module") @(t : T?) { + var lazy : string + t |> success(report_child(t, "dollar lazy", run_child(child_cmd(fx, fx.dollar), lazy), lazy, "DOLLAR "), "the lazy start counts") + var eager : string + t |> success(report_child(t, "dollar eager", run_child(child_cmd(fx, fx.dollar, "-ignore-manifest"), eager), eager, "DOLLAR "), "the eager start counts") + t |> success(dollar_count(lazy) > 0, "the count is a count:\n{lazy}") + t |> equal(dollar_count(lazy), dollar_count(eager), "a C++ module's load registers nothing into $:\nlazy:\n{lazy}\neager:\n{eager}") + } +} + +def arms_guards_and_caches(t : T?; fx : Fixture) { t |> run("a require guard loads a waiting module, whatever else the compile requires") @(t : T?) { var alone : string t |> success(report_child(t, "guarded", run_child(child_cmd(fx, fx.guarded), alone), alone, "GUARD true"), "the guard alone takes its target") @@ -166,6 +208,16 @@ def arms_deferred(t : T?; fx : Fixture) { t |> success(find(out, "error[20605]: missing prerequisite '{UNIT_TEST_MODULE}'") >= 0, "the require fails with the missing-module error:\n{out}") remove_result(fx.utManifest) } + t |> run("a module cache written by an eager start serves a lazy start, and -log-compile-time shows the reads and the timeline") @(t : T?) { + let cache = "-module-cache \"{path_join(fx.tmp, "mc.bin")}\"" + var cold : string + t |> success(report_child(t, "cold cache", run_child(child_cmd(fx, fx.plain, "-ignore-manifest -log-compile-time", cache), cold), cold, "PLAIN"), "the eager start writes the cache") + t |> success(find(cold, "startup: main total") >= 0 && find(cold, "\tmodule scan") >= 0 && find(cold, "\tshutdown ") >= 0, "the timeline prints its laps:\n{cold}") + var warm : string + t |> success(report_child(t, "warm cache", run_child(child_cmd(fx, fx.plain, "-log-compile-time", cache), warm), warm, "PLAIN"), "the lazy start, which loads no C++ module, runs") + t |> equal(-1, find(warm, "cumulative hash"), "a builtin module's hash does not depend on the loaded set:\n{warm}") + t |> success(find(warm, "cache read took") >= 0, "the lazy start reads the eager start's records:\n{warm}") + } } def arm_fallback(t : T?; fx : Fixture) { @@ -199,6 +251,7 @@ def test_deferred_modules(t : T?) { } let fx = make_fixture(tmp, artifact) arms_deferred(t, fx) + arms_guards_and_caches(t, fx) arm_fallback(t, fx) var err : string rmdir_rec(tmp, err) From 8e8552abd4eef3467dd20064a6671379bf979135 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 01:14:47 -0700 Subject: [PATCH 09/22] a parse no prerequisite walk precedes - the compile of a string - meets a deferred module at the parser's own require and loads it there, and the loader puts the parse's program back as the bound one, since a module's builtin das part parses under a program of its own (tests/language/reflection's sample requires UnitTest through compile()); the module-cache spawn helpers live in _mc_common - a sweep worker keeps every shared module it met under one name and tests/linq owns _common, so the five tests resolved to the wrong module there --- src/ast/ARCHITECTURE.md | 5 ++++- src/ast/dyn_modules.cpp | 2 ++ src/parser/parser_impl.cpp | 9 ++++++++- tests/module_cache/ARCHITECTURE.md | 4 +++- tests/module_cache/{_common.das => _mc_common.das} | 2 +- tests/module_cache/test_default_cache_path.das | 2 +- tests/module_cache/test_deferred_modules.das | 10 +++++++++- tests/module_cache/test_descriptor_manifest.das | 2 +- tests/module_cache/test_generic_instance_origin.das | 2 +- tests/module_cache/test_macro_dep_invalidate.das | 2 +- 10 files changed, 31 insertions(+), 9 deletions(-) rename tests/module_cache/{_common.das => _mc_common.das} (97%) diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index a9d358615a..dd4877113c 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -90,7 +90,10 @@ has. A row whose own dlopen fails takes the same road - every deferred module co pending retry runs - and the require then finds the module or fails as a cold start would. The rows and the loader are the scan's: `require_dynamic_modules` clears the rows before its walk and installs the loader, and `Module::Shutdown` clears both, so an environment that -follows sees neither the last one's rows nor its loader. The load runs under one gc root of its own with the thread root's nodes parked meanwhile, +follows sees neither the last one's rows nor its loader. A parse that no prerequisite walk +precedes - the `compile` of a string - meets a deferred module at the parser's own require +(`ast_requireModule`) and loads it there; the loader puts the parse's program back as the +bound one, since a module's builtin das part parses under a program of its own. The load runs under one gc root of its own with the thread root's nodes parked meanwhile, because a constructor's nodes go to the active root while a builtin das module it compiles dumps its leftovers on the thread root, and a collect stops at a node owned by another root; after the load every module, not only the new ones, collects from that root, since a diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 8b03c163b1..a27ce54462 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -586,6 +586,7 @@ static bool load_deferred_module_for_require(const string & name) { auto & threadRoot = gc_root::gc_get_thread_root(); gc_root parked, loadRoot; move_all_nodes(threadRoot, parked); + auto boundProgram = daScriptEnvironment::getBound()->g_Program; // a module's builtin das part parses under its own program; the caller may be mid-parse bool loaded = false; bool grown = false; // the module set changed - the fixed point and the collect are owed whether or not `name` came in { @@ -620,6 +621,7 @@ static bool load_deferred_module_for_require(const string & name) { } loadRoot.gc_sweep(); move_all_nodes(parked, threadRoot); + daScriptEnvironment::getBound()->g_Program = boundProgram; return loaded; } diff --git a/src/parser/parser_impl.cpp b/src/parser/parser_impl.cpp index c71b5ede37..7fcf78bc92 100644 --- a/src/parser/parser_impl.cpp +++ b/src/parser/parser_impl.cpp @@ -1249,7 +1249,14 @@ namespace das { } } auto info = yyextra->g_Access->getModuleInfo(*name, yyextra->g_FileAccessStack.back()->name); - if ( auto mod = yyextra->g_Program->addModule(info.moduleName) ) { + auto mod = yyextra->g_Program->addModule(info.moduleName); + if ( !mod ) { + // a parse with no prerequisite walk (compile of a string) meets a deferred module here (src/ast/ARCHITECTURE.md sec.2) + if ( auto loader = getDeferredModuleLoader(); loader && loader(info.moduleName) ) { + mod = yyextra->g_Program->addModule(info.moduleName); + } + } + if ( mod ) { yyextra->g_Program->allRequireDecl.push_back(make_tuple(mod,*name,info.fileName,pub,atName)); yyextra->g_Program->thisModule->addDependency(mod, pub); das_collect_all_keywords(mod,scanner); diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 1344246d99..8322912d52 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -52,7 +52,9 @@ this document states what the folder is and why its tests take the shape they do - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, `mc_generic_origin_*`); a case needing a macro-bearing module graph puts it here instead of writing the script inline. -- `_common.das` - the spawn helpers every test here shares: the binary to spawn (`das_exe`), +- `_mc_common.das` - the spawn helpers every test here shares (the name is the folder's, since + a sweep worker holds every shared module it met under one name and `tests/linq` has a + `_common` already): the binary to spawn (`das_exe`), the scan-trace command prefix (`trace_prefix`), the stderr-joining child run (`run_child`) and the failure report that echoes the child's output (`report_child`). A test whose child needs a different spawn shape - an argv spawn, an environment variable - keeps that one diff --git a/tests/module_cache/_common.das b/tests/module_cache/_mc_common.das similarity index 97% rename from tests/module_cache/_common.das rename to tests/module_cache/_mc_common.das index 12bdad224b..b605278509 100644 --- a/tests/module_cache/_common.das +++ b/tests/module_cache/_mc_common.das @@ -1,7 +1,7 @@ options gen2 options indenting = 4 -module _common shared public +module _mc_common shared public require dastest/testing_boost public diff --git a/tests/module_cache/test_default_cache_path.das b/tests/module_cache/test_default_cache_path.das index 52e90058b3..5b9e9f808d 100644 --- a/tests/module_cache/test_default_cache_path.das +++ b/tests/module_cache/test_default_cache_path.das @@ -7,7 +7,7 @@ require strings require math require daslib/fio require daslib/strings_boost -require _common +require _mc_common //! the child shares dastest's cwd (argv[0] is a cwd-relative interpreter path), so the default //! cache lands in this tree's own .jitted_scripts/module_cache - the test removes what it wrote. diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index 4aa9f58ccb..f443dd987c 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -6,7 +6,7 @@ require dastest/testing_boost public require strings require daslib/fio require daslib/strings_boost -require _common +require _mc_common //! the whole scan trace line for a module's descriptor, timing clause included def descriptor_line(out : string; mod : string) : string { @@ -64,6 +64,7 @@ struct Fixture { exists : string hasModule : string dollar : string + nested : string guarded : string guardedAfter : string guardedBefore : string @@ -94,6 +95,7 @@ def make_fixture(tmp : string; artifact : string) : Fixture { exists = path_join(tmp, "exists.das"), hasModule = path_join(tmp, "has_module.das"), dollar = path_join(tmp, "dollar.das"), + nested = path_join(tmp, "nested.das"), guarded = path_join(tmp, "guarded.das"), guardedAfter = path_join(tmp, "guarded_after.das"), guardedBefore = path_join(tmp, "guarded_before.das"), @@ -107,6 +109,7 @@ def make_fixture(tmp : string; artifact : string) : Fixture { fwrite(fx.exists, "options gen2\n[export]\ndef main \{\n print(\"EXISTS \{typeinfo builtin_module_exists({UNIT_TEST_MODULE})\}\\n\")\n\}\n") fwrite(fx.hasModule, "options gen2\nrequire daslib/rtti\n[export]\ndef main \{\n print(\"HAS \{has_module(\"{UNIT_TEST_MODULE}\")\}\\n\")\n\}\n") fwrite(fx.dollar, "options gen2\nrequire daslib/rtti\n[export]\ndef main \{\n program_for_each_registered_module() $(mod) \{\n if (mod.name == \"$\") \{\n var n = 0\n module_for_each_function(mod) $(fn) \{\n n++\n \}\n print(\"DOLLAR \{n\}\\n\")\n \}\n \}\n\}\n") + fwrite(fx.nested, "options gen2\nrequire daslib/ast\n[export]\ndef main \{\n compile(\"nested\", \"require {UNIT_TEST_MODULE}\\n[export]\\ndef main \\\{\\n\\\}\\n\", CodeOfPolicies()) $(ok; _prog; errors) \{\n print(\"NESTED \{ok\}\\n\{errors\}\\n\")\n \}\n\}\n") fwrite(fx.guarded, "options gen2\n{guardLine}{guardMain}") fwrite(fx.guardedAfter, "options gen2\nrequire {UNIT_TEST_MODULE}\n{guardLine}{guardMain}") fwrite(fx.guardedBefore, "options gen2\n{guardLine}require {UNIT_TEST_MODULE}\n{guardMain}") @@ -170,6 +173,11 @@ def arms_deferred(t : T?; fx : Fixture) { t |> success(dollar_count(lazy) > 0, "the count is a count:\n{lazy}") t |> equal(dollar_count(lazy), dollar_count(eager), "a C++ module's load registers nothing into $:\nlazy:\n{lazy}\neager:\n{eager}") } + t |> run("a compile of a string, which walks no prerequisites, meets the waiting module at its require") @(t : T?) { + var out : string + t |> success(report_child(t, "nested", run_child(child_cmd(fx, fx.nested), out), out, "NESTED true"), "the nested program compiles") + t |> success(find(out, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the parser's require loaded it:\n{out}") + } } def arms_guards_and_caches(t : T?; fx : Fixture) { diff --git a/tests/module_cache/test_descriptor_manifest.das b/tests/module_cache/test_descriptor_manifest.das index 4c6d73834b..394b2cad7e 100644 --- a/tests/module_cache/test_descriptor_manifest.das +++ b/tests/module_cache/test_descriptor_manifest.das @@ -6,7 +6,7 @@ require dastest/testing_boost public require strings require daslib/fio require daslib/strings_boost -require _common +require _mc_common let DESCRIPTOR = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n register_native_path(\"manifest_probe\", \"hello\", \"\{project_path\}/hello.das\")\n\}\n" let DESCRIPTOR_OPT_OUT = "options gen2\nrequire daslib/fio\n\n[export]\ndef initialize(project_path : string) \{\n no_manifest()\n register_native_path(\"manifest_probe\", \"hello\", \"\{project_path\}/hello.das\")\n\}\n" diff --git a/tests/module_cache/test_generic_instance_origin.das b/tests/module_cache/test_generic_instance_origin.das index 9b562a9f54..2b8c6eea19 100644 --- a/tests/module_cache/test_generic_instance_origin.das +++ b/tests/module_cache/test_generic_instance_origin.das @@ -5,7 +5,7 @@ require dastest/testing_boost public require strings require daslib/fio -require _common +require _mc_common def run_argv(args : array; var output : string&) : int { diff --git a/tests/module_cache/test_macro_dep_invalidate.das b/tests/module_cache/test_macro_dep_invalidate.das index 23e1c1d09a..1561bd4d96 100644 --- a/tests/module_cache/test_macro_dep_invalidate.das +++ b/tests/module_cache/test_macro_dep_invalidate.das @@ -5,7 +5,7 @@ require dastest/testing_boost public require strings require daslib/fio -require _common +require _mc_common //! child env rides the command string (the portable set/prefix split popen_argv cannot carry) def env_run(dep, cmd : string; var output : string&) : int { From 5a19cc211164abf719c019a22964edeb0a74778f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 01:21:27 -0700 Subject: [PATCH 10/22] ast_boost's printer names a block's expression and variable lists by the part after '::' - a vector of a handled element now describes under the element's module (ast::dasvector`ptr`Expression), so the $:: comparison printed every list empty and the MCP ast_dump test lost its ExprReturn --- daslib/ast_boost.das | 4 ++-- src/ast/ARCHITECTURE.md | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/daslib/ast_boost.das b/daslib/ast_boost.das index 8991ef8604..a300be9e42 100644 --- a/daslib/ast_boost.das +++ b/daslib/ast_boost.das @@ -1302,7 +1302,7 @@ def private debug_expression_impl(var writer : StringBuilderWriter; expr : Expre unsafe { p8 = (reinterpret(expr)) + int(offset) } - if (tstr == "$::dasvector`ptr`Expression") { + if (tstr |> ends_with("::dasvector`ptr`Expression")) { let pv = unsafe(reinterpret(p8)) if (!empty(*pv)) { let ts = repeat(" ", tabs + 2) @@ -1316,7 +1316,7 @@ def private debug_expression_impl(var writer : StringBuilderWriter; expr : Expre } writer |> write("]") } - } elif (tstr == "$::dasvector`ptr`Variable") { + } elif (tstr |> ends_with("::dasvector`ptr`Variable")) { let pv = unsafe(reinterpret(p8)) if (!empty(*pv)) { let ts = repeat(" ", tabs + 2) diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index dd4877113c..045bb05651 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -110,7 +110,9 @@ of mangled names, and a process that loaded a different set of C++ modules would fail every record on `$`, so a `vector` of a handled element registers into the element's module (`vectorHomeModule`, `ast_handle.h`) whichever module builds it, and only a vector of a builtin element lands in `$`, which every library lists first because -`ModuleLibrary::addModule` puts a module's dependencies before it. `-ignore-manifest` +`ModuleLibrary::addModule` puts a module's dependencies before it. The described name of such +a vector carries that module - ``ast::dasvector`ptr`Expression``, not ``$::...`` - so code +that names one compares the part after `::` (`daslib/ast_boost`'s printer). `-ignore-manifest` reads and writes no manifest: every descriptor compiles and every C++ module loads on start, the form a tool that enumerates modules - the MCP server, the LSP subtools - runs under. `no_manifest()` inside `initialize` marks the descriptor as one that runs on every start: its From f9cdc933ceea9fb6816bfab1c67b5724fd2d2570 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 01:31:01 -0700 Subject: [PATCH 11/22] test_deferred_modules returns on a static host - the AOT test binary's descriptors register no shared module, so nothing is deferred there and the arms had nothing to observe --- tests/module_cache/ARCHITECTURE.md | 5 +++-- tests/module_cache/test_deferred_modules.das | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 8322912d52..cbed01d744 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -47,8 +47,9 @@ this document states what the folder is and why its tests take the shape they do deferred module in and then fail on the missing prerequisite; a module cache an eager start wrote serves a lazy start, with `-log-compile-time` printing the reads and the startup timeline; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every - deferred module in because its `initDependencies` asks for two more. A static build, whose - tree holds no `.shared_module`, has nothing to observe and the test says so and returns. + deferred module in because its `initDependencies` asks for two more. A static host - the AOT + test binary, whose descriptors register no shared module, or a tree holding no + `.shared_module` - has nothing to observe and the test says so and returns. - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, `mc_generic_origin_*`); a case needing a macro-bearing module graph puts it here instead of writing the script inline. diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index f443dd987c..cf9c5aad00 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -246,6 +246,10 @@ def arm_fallback(t : T?; fx : Fixture) { //! the arms share one root and run in this order [test] def test_deferred_modules(t : T?) { + if (!das_is_dll_build()) { + to_log(LOG_INFO, "test_deferred_modules: {das_exe()} is a static build - its descriptors register no shared module, so it defers nothing\n") + return + } let artifact = unit_test_artifact() if (empty(artifact)) { to_log(LOG_INFO, "test_deferred_modules: no {UNIT_TEST_ARTIFACT}.shared_module under {get_das_root()}/modules/{UNIT_TEST_FOLDER} - a static build defers nothing\n") From 5cf70ba345fd0d82dab413883069b1b58a55544b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 07:22:50 -0700 Subject: [PATCH 12/22] the LSP subtools spawn plain - neither validate nor nav enumerates the process's modules, so -ignore-manifest cost every keystroke a quarter second for nothing (the MCP server keeps it); the vector-home sentence claims what holds and is tested - a load adds nothing to $, and a vector's home may be a module that exists already; the dasLLVM codegen-bump rule's criterion is the outcome again with the surfaces as its content; utils/REVIEW.md names the two CMakeLists.txt its gate reads; the module-cache weakening rule names the timing as the field that leaves the compared text --- modules/dasLLVM/REVIEW.md | 15 ++++++++------- skills/dynamic_modules.md | 2 +- src/ast/ARCHITECTURE.md | 15 ++++++++------- tests/module_cache/ARCHITECTURE.md | 4 ++-- tests/module_cache/REVIEW.md | 7 +++---- utils/REVIEW.md | 8 ++++---- utils/lsp/README.md | 5 +---- utils/lsp/lsp_supervisor.py | 3 +-- 8 files changed, 28 insertions(+), 31 deletions(-) diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 74bafe1240..2dffc9aad9 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -35,13 +35,14 @@ it, while that phase's line still prints** (phase inventory: `ARCHITECTURE.md` sec.1). Option resolution before the first timer, and log lines, are not work. -- **A change to what emits the JIT's machine code - IR generation, target-machine setup, a - `[llvm_code]` generator body, or the call ABI the generated code binds: function signatures, - the name scheme, the prologue, the externs the install phase binds - bumps - `LLVM_JIT_CODEGEN_VERSION`** (`daslib/llvm_jit_run.das`); selecting among existing generators' - `[llvm_code]` arguments, the `[tune]` stamping, is not such a change. The DLL and split-obj - caches are addressed by the AST hashes and this constant, so an emitter change without the bump - serves the old machine code back (`ARCHITECTURE.md` sec.1.2). +- **A change that can alter the machine code the JIT's DLL or split-obj cache serves back for + identical inputs - IR generation, target-machine setup, a `[llvm_code]` generator body, or the + call ABI the generated code binds: function signatures, the name scheme, the prologue, the + externs the install phase binds - bumps `LLVM_JIT_CODEGEN_VERSION`** + (`daslib/llvm_jit_run.das`); selecting among existing generators' `[llvm_code]` arguments, the + `[tune]` stamping, is not such a change. The caches are addressed by the AST hashes and this + constant, so such a change without the bump serves the old machine code back + (`ARCHITECTURE.md` sec.1.2). - **A diff that adds an environment or config input to a JIT cache key folds it inside `jit_env_salt` (`daslib/llvm_jit_run.das`), never directly into either JIT key - the DLL diff --git a/skills/dynamic_modules.md b/skills/dynamic_modules.md index cf2ff7afae..bb5f1d0f38 100644 --- a/skills/dynamic_modules.md +++ b/skills/dynamic_modules.md @@ -107,7 +107,7 @@ Two consequences: - `require ?mod x` and `typeinfo builtin_module_exists(mod)` still ask whether the build has `mod`: a guard loads a waiting module, so a cold start and a warm one answer alike. -- A tool that enumerates the process's modules (the MCP server, the LSP subtools) runs with +- A tool that enumerates the process's modules (the MCP server) runs with `-ignore-manifest`: no manifest read or written, every descriptor compiles, every C++ module loads on start. `has_module(name)` (`daslib/rtti`) answers loaded-or-deferred, so a sweep gate asking what the tree has keeps its answer. diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 045bb05651..6ef42b97d9 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -104,17 +104,18 @@ require guard (`require ?mod`) and `builtin_module_exists` ask whether the build module (`guardModuleAvailable`): linked in, or waiting in a manifest row, which the guard loads then - so `require ?das_metal metal/das_metal_boost` still means "on a build with Metal", a cold start and a warm start answer alike, and `llvm`, a witness module no das file -requires unguarded, comes in through the guards `daslib/tune` places on it. A load changes no -other module's content: a module-cache record carries each builtin module's cumulative hash -of mangled names, and a process that loaded a different set of C++ modules would otherwise -fail every record on `$`, so a `vector` of a handled element registers into the element's -module (`vectorHomeModule`, `ast_handle.h`) whichever module builds it, and only a vector of a -builtin element lands in `$`, which every library lists first because +requires unguarded, comes in through the guards `daslib/tune` places on it. A load adds +nothing to `$`: a module-cache record carries each builtin module's cumulative hash of +mangled names, and a process that loaded a different set of C++ modules would otherwise fail +every record on `$`, so a `vector` of a handled element registers into the element's +module (`vectorHomeModule`, `ast_handle.h`) whichever module builds it - a module that exists +already, when the element is another module's - and only a vector of a builtin element lands +in `$`, which every library lists first because `ModuleLibrary::addModule` puts a module's dependencies before it. The described name of such a vector carries that module - ``ast::dasvector`ptr`Expression``, not ``$::...`` - so code that names one compares the part after `::` (`daslib/ast_boost`'s printer). `-ignore-manifest` reads and writes no manifest: every descriptor compiles and every C++ module loads on start, -the form a tool that enumerates modules - the MCP server, the LSP subtools - runs under. +the form a tool that enumerates modules - the MCP server - runs under. `no_manifest()` inside `initialize` marks the descriptor as one that runs on every start: its manifest carries the stamp and the flag and no rows, and is not rewritten. With `DAS_TRACE_MODULE_LOAD=1` the scan prints one line per descriptor - `replayed N row(s) in diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index cbed01d744..0fd27b7b40 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -38,8 +38,8 @@ this document states what the folder is and why its tests take the shape they do warm start defers the row and loads nothing for a program that requires nothing; the first `require UnitTest` loads it and the program calls into it; `typeinfo builtin_module_exists` loads it with no require naming it, while rtti `has_module` answers true and loads nothing; - a lazy start and an eager start count the same functions in `$`, so a load registers - nothing into another module; a guard alone loads the module + a lazy start and an eager start count the same functions in `$`, so a load adds nothing + to `$`; a guard alone loads the module and is taken, as it is with a `require UnitTest` above it, below it, or in the entry while the guard sits in a module walked earlier; `-ignore-manifest` compiles every descriptor, loads every C++ module on start and writes no diff --git a/tests/module_cache/REVIEW.md b/tests/module_cache/REVIEW.md index 6ef5cc28c1..843e9776ac 100644 --- a/tests/module_cache/REVIEW.md +++ b/tests/module_cache/REVIEW.md @@ -6,10 +6,9 @@ - **Weakening the text a test in this folder compares a spawned child's output against - an assertion literal, or the helper that produces the compared text - is a defect; an edit weakens when it accepts an output the old one rejected, and re-pinning a count or the fixed - words of a scan or cache line to the child's new true output does not. A field the child newly - prints that no run can pin - a timing - leaves the compared text only while every deterministic - field beside it stays compared.** A child's output is the only instrument a human has for what - the cache and the scan served. + words of a scan or cache line to the child's new true output does not. A timing the child + newly prints leaves the compared text; every other field stays compared.** A child's output is + the only instrument a human has for what the cache and the scan served. - **A test in this folder writes only under a directory it created for this process - its own files and its children's - and removes that directory.** diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 1c58f20b10..054742f446 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -4,10 +4,10 @@ doc: `CLAUDE.md` (repo root). A tool is a directory that owns the programs it ships - each one's entry point and the files -only those programs use; a program joins a tool through a `CMakeLists.txt` anywhere in the -tree - the one beside this file or the repo root's - building or shipping it, or through the -directory's `.das_package` declaring it with `release_program` - under `utils/`, or outside -`utils/` when a `CMakeLists.txt` builds or ships it. An arm is one `t |> run(...)` case of a `[test]` function. An arm's load-bearing +only those programs use; a program joins a tool through `utils/CMakeLists.txt` (beside this +file) or the repo root's `CMakeLists.txt` building or shipping it, or through the directory's +`.das_package` declaring it with `release_program` - under `utils/`, or outside `utils/` when +one of those two files builds or ships it. An arm is one `t |> run(...)` case of a `[test]` function. An arm's load-bearing assertions are the ones that prove the change, never a skip-path assertion. A CI row is a workflow step whose command reaches the arm. Load-bearing assertions no CI row executes - the arm returns or skips before them, or no suite a CI row runs includes the arm's file - are diff --git a/utils/lsp/README.md b/utils/lsp/README.md index c09b5768df..3e143573fe 100644 --- a/utils/lsp/README.md +++ b/utils/lsp/README.md @@ -79,10 +79,7 @@ Two processes, hard split - full rationale and wave history in - **`subtools/*.das`** - stateless batch tools (`validate.das`, `nav.das`). One fresh `daslang` process per request; argv in, LSP-shaped JSON out, exit. The document shadow rides along as a `--overlay` temp file, so compiles see - the client's buffer even when unsaved. Every subtool spawns with - `-ignore-manifest`: the module scan compiles every descriptor and loads every - C++ module on start, so a symbol scan sees the whole tree rather than the - modules a plain run would load at their first `require`. + the client's buffer even when unsaved. No resident daslang, by design: no macro-state leaks across compiles, no binary/DLL locks while builds run, per-request crash isolation. diff --git a/utils/lsp/lsp_supervisor.py b/utils/lsp/lsp_supervisor.py index 04fc38441a..646a3f2cc9 100644 --- a/utils/lsp/lsp_supervisor.py +++ b/utils/lsp/lsp_supervisor.py @@ -137,8 +137,7 @@ def subtool_argv(self, subtool: str, args: list[str]) -> list[str] | None: self.compiler = find_compiler(self.init_options) if self.compiler is None: return None - # -ignore-manifest: a subtool enumerates modules, so every C++ module loads on start - argv = [self.compiler, "-ignore-manifest"] + argv = [self.compiler] if self.init_options.get("project_root"): argv += ["-project_root", self.init_options["project_root"]] for lm in self.init_options.get("load_module") or []: From ef12ffac6457f9914132967b3f4e051827194c6b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 07:28:04 -0700 Subject: [PATCH 13/22] the mcp/lsp exe ban is a utils/REVIEW.das check - the two folder checklists carried it where it could never fire, since its trigger is utils/CMakeLists.txt; utils/lsp/REVIEW.md had nothing else and is gone; the gate skips dot-directories, so a JIT's .jitted_scripts cache under utils/ is not read as a tool; the MCP and LSP roadmaps ledger the custom-modules fixture, the return to an exe form, and the watchdog supervising both --- utils/REVIEW.das | 22 +++++++++++++++++++++- utils/lsp/REVIEW.md | 8 -------- utils/lsp/ROADMAP.md | 6 ++++++ utils/mcp/REVIEW.md | 5 ----- utils/mcp/ROADMAP.md | 13 +++++++++++++ 5 files changed, 40 insertions(+), 14 deletions(-) delete mode 100644 utils/lsp/REVIEW.md diff --git a/utils/REVIEW.das b/utils/REVIEW.das index f8746f15c5..27a49b649e 100644 --- a/utils/REVIEW.das +++ b/utils/REVIEW.das @@ -13,6 +13,10 @@ require dastest/review_gate // utils/ children that are not tool directories: the shared library hub var private LIBRARY_DIRS <- { "common" } +// tools whose development runs through a python keep-alive supervisor: never built or +// shipped as an exe, since that form would ship without anyone having run it +var private SUPERVISED_TOOLS <- { "mcp", "lsp" } + // Files licensed to name utils/internal outside internal/: var private INTERNAL_REF_EXEMPT <- { "utils/REVIEW.md", // states the rule @@ -54,7 +58,7 @@ def private mentions_path(text : string; p : string) : bool { def private list_dirs(parent : string) : array { var out : array dir(parent) $(name) { - return if (name == "." || name == "..") + return if (name |> starts_with(".")) // `.`, `..`, and a JIT's `.jitted_scripts` cache let st = stat(path_join(parent, name)) if (st.is_valid && st.is_dir) { out |> push(name) @@ -164,6 +168,21 @@ def private check_no_internal_refs { } } +def private check_supervised_tools { + let ucm = fread("utils/CMakeLists.txt") + var inscope exes <- cmake_list_entries(ucm, "DAS_UTILS_SHIPPED_EXES") + var inscope built <- cmake_list_entries(ucm, "DAS_UTILS") + for (s in keys(SUPERVISED_TOOLS)) { + var hit = find_index(exes, s) >= 0 + for (b in built) { + hit ||= base_name(b) == s + } + if (hit) { + gate_finding("utils/CMakeLists.txt", "{s} is in DAS_UTILS or DAS_UTILS_SHIPPED_EXES — its development runs through the python keep-alive supervisor, so an exe form ships without anyone having run it; run it interpreted") + } + } +} + var private BIN_EXE <- %regex~`([A-Za-z0-9_./-]+)\.exe`%% def private check_shipped_exes(externals : array) { @@ -244,5 +263,6 @@ def main() : int { check_registration_triple(tools, internals) check_no_internal_refs() check_shipped_exes(externals) + check_supervised_tools() return gate_verdict("utils") } diff --git a/utils/lsp/REVIEW.md b/utils/lsp/REVIEW.md deleted file mode 100644 index 96773db62a..0000000000 --- a/utils/lsp/REVIEW.md +++ /dev/null @@ -1,8 +0,0 @@ -# lsp Code Review Checklist - -**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: -`README.md`. - -**Never add `lsp` to `DAS_UTILS` or `DAS_UTILS_SHIPPED_EXES` in `utils/CMakeLists.txt` - clients -start the server through `lsp_supervisor.py` instead.** An exe form would ship without anyone -having used it. diff --git a/utils/lsp/ROADMAP.md b/utils/lsp/ROADMAP.md index 09ed4bb420..3272db45ca 100644 --- a/utils/lsp/ROADMAP.md +++ b/utils/lsp/ROADMAP.md @@ -296,6 +296,12 @@ PR for the whole branch AFTER wave 4 (single preflight + CI round). needs no LSP wiring - its build already produces the binary `find_compiler` discovers (`build/daslang` et al.); setup's done-message now says so. +## Follow-ups + +- The watchdog does not supervise `lsp_supervisor.py` yet; wire it in. With that, an exe form + of the subtools becomes possible again - the same item as the MCP server's + (`utils/mcp/ROADMAP.md`, Follow-ups). + ## Non-goals - Completion (CC doesn't consume it), formatting-over-LSP (CC has the MCP/CLI diff --git a/utils/mcp/REVIEW.md b/utils/mcp/REVIEW.md index 388bbd8d0c..877cc1839f 100644 --- a/utils/mcp/REVIEW.md +++ b/utils/mcp/REVIEW.md @@ -3,11 +3,6 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `README.md`. Planned work: `ROADMAP.md`. -**Never add `mcp` to `DAS_UTILS` or `DAS_UTILS_SHIPPED_EXES` (`utils/CMakeLists.txt`) - run it -interpreted through `.mcp.json` instead.** -Development runs the server through the python keep-alive supervisor, so an exe form would -never be used in development before it ships. - **A diff that adds a top-level file under `utils/mcp/` that the shipped SDK runs or loads - `main.das` reaches it, the supervisor or the `.cmd` launcher runs it, or it has its own `main` that something in the shipped SDK runs - also adds it to the `install(FILES ...)` block that diff --git a/utils/mcp/ROADMAP.md b/utils/mcp/ROADMAP.md index ebf6eb5803..dd2fbab3ef 100644 --- a/utils/mcp/ROADMAP.md +++ b/utils/mcp/ROADMAP.md @@ -334,3 +334,16 @@ Many complex tools are compositions of simpler ones: - `extract_function` = AST analysis + code generation + `compile_check` Building the foundational tools well creates a platform for everything else. + +## Follow-ups + +- **A custom `modules/` fixture.** A test that starts the server with `-project_root` on a + fixture holding its own `modules//.das_module` - a descriptor that registers a require + path and a C++ module - and checks `list_modules`, `find_symbol` and `compile_check` see + them. The server runs eager (`-ignore-manifest`) while a plain run loads a `.shared_module` + at its first `require`; the fixture is what proves a user's tree behaves under both. +- **Back to an exe form.** Once that fixture passes, the server can build and ship as an exe + again: the reason it runs interpreted - development through the python keep-alive supervisor, + so an exe would ship unrun - is met by the watchdog, which now covers what the supervisor + did. `utils/REVIEW.das` bans the exe today; lifting the ban is part of this item. +- **The watchdog does not supervise this server or `lsp_supervisor.py` yet.** Wire both in. From fc733d5403c6272a15f6b6ccbedc82de4f45fd8f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 07:37:03 -0700 Subject: [PATCH 14/22] the codegen-version bump has a duty where its out-of-folder triggers live: include/daScript/ast (a bind's home module or name - vectorHomeModule, the vector functions, an annotation's name), src/ast (how a mangled name forms, which module addFunction files a builtin under) and src/builtin (a bind moved between modules) - the JIT's DLL cache key folds the codegen version and each function's AST hash, never the module an extern lives in, so without the bump a cached DLL binds the old name and crashes on the hit --- include/daScript/ast/REVIEW.md | 11 +++++++++++ src/ast/REVIEW.md | 6 ++++++ src/builtin/REVIEW.md | 7 +++++++ 3 files changed, 24 insertions(+) create mode 100644 include/daScript/ast/REVIEW.md diff --git a/include/daScript/ast/REVIEW.md b/include/daScript/ast/REVIEW.md new file mode 100644 index 0000000000..471a307ff7 --- /dev/null +++ b/include/daScript/ast/REVIEW.md @@ -0,0 +1,11 @@ +# AST Headers Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: +`src/ast/ARCHITECTURE.md` (repo root). + +- **A diff that changes which module a bind registers into, or the name it registers under - + `vectorHomeModule`, `typeFactory>::make`, `registerVectorFunctions`, the name a + `ManagedVectorAnnotation` or `ManagedStructureAnnotation` takes (`ast_handle.h`) - bumps + `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` (repo root), in the + same change.** The JIT's DLL cache key folds the codegen version and each function's AST hash, + never the module an extern lives in, so a cached DLL binds the old name and crashes on the hit. diff --git a/src/ast/REVIEW.md b/src/ast/REVIEW.md index bbd6ca2851..66ce2fb022 100644 --- a/src/ast/REVIEW.md +++ b/src/ast/REVIEW.md @@ -33,3 +33,9 @@ - **A diff that moves the `setDeferredModuleLoader` call in `require_dynamic_modules` keeps it above the first `init_modules_for_folder` call, in the same change.** That call compiles the descriptors, and a descriptor can require a module an earlier replay deferred. + +- **A diff that changes how `Function::getMangledName` (`ast.cpp`) forms a name, or which module + `Module::addFunction` (`ast_module.cpp`) files a builtin function under, bumps + `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` (repo root), in the + same change.** The JIT's DLL cache key folds the codegen version and each function's AST hash, + never the name an extern binds under, so a cached DLL binds the old name and crashes on the hit. diff --git a/src/builtin/REVIEW.md b/src/builtin/REVIEW.md index 9a787c1310..2ee28e6a25 100644 --- a/src/builtin/REVIEW.md +++ b/src/builtin/REVIEW.md @@ -46,6 +46,13 @@ A replayed start never runs the descriptor, so an effect the recorder does not see is an effect every warm start silently lacks. +- **A diff that moves a bind between modules - an `addExtern` or `addExternInline` call whose + module changes, or a builtin whose `vector` functions follow a type to another module - + bumps `LLVM_JIT_CODEGEN_VERSION` in `modules/dasLLVM/daslib/llvm_jit_run.das` (repo root), in + the same change.** The JIT's DLL cache key folds the codegen version and each function's AST + hash, never the module an extern lives in, so a cached DLL binds the old name and crashes on + the hit. + - **A diff that changes what `ModuleFileCache::defaultPath` folds into the module-cache key - the binary, the command line, the environment names, or which script arguments count - updates the cache-key paragraph of `ARCHITECTURE.md` in the same change.** The key is what stops a From f9c0fe3dfbca66348b25e33123a085096bf6282b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 07:40:43 -0700 Subject: [PATCH 15/22] plans/review_md_splits.md ledgers the four checklist splits the audits proposed - utils, doc, dasLLVM, handmade - for a docs-only PR at the end of the startup chain --- plans/review_md_splits.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 plans/review_md_splits.md diff --git a/plans/review_md_splits.md b/plans/review_md_splits.md new file mode 100644 index 0000000000..03443d9cce --- /dev/null +++ b/plans/review_md_splits.md @@ -0,0 +1,28 @@ +# Checklist splits owed - a docs-only PR at the end of the startup chain + +Four rules the per-PR audits found working but hard to apply: each fuses two checks, or takes +the wrong subject. Splitting is a restructuring, so it rides its own docs-only PR (those merge +gate-free) rather than a code PR. Meaning-preserving; the dragon reads the result. + +- `utils/REVIEW.md` - the three CI-row rules take "an arm" as subject while the definition + above them says one arm may hold both assertions a CI row can run and assertions none can, + so a mixed arm fits neither rule. Make the assertion set the subject: "the load-bearing + assertions a CI row can run ship with a row that executes them on every pull request; the + ones no CI row can run ship with a compile-check row and a recorded run". Then split each + 60-word rule into a trigger sentence and a duty sentence, and drop "one arm may hold both", + which follows. +- `doc/REVIEW.md` - the fetch/run/type rule's subject and verb sit 45 words apart in two nested + dash lists. Split into the duty ("A diff that adds anything a reader is told to fetch, run, + or type to text that reaches a built page states, in the PR body, that each exists and works + at merge, and where that was checked"), then the scope sentence naming the three page kinds + and the `.. include::` arm (a diff touching only an included `.md` never surfaces this + checklist in the folder walk, so that arm needs a routing line in the opening), then the WHY. +- `modules/dasLLVM/REVIEW.md` - the module-cache flag rule fuses two checks (a child asserting + a compile-time macro line spawns with `-no-module-cache`; a test whose subject is the cache + pins its own file with `-module-cache `) behind a five-clause qualifier and a five-line + mechanism tail. One rule per check; the replay mechanism moves to `ARCHITECTURE.md`. Same + shape, weaker: the intrinsic emitter reference-cell rule fuses three obligations. +- `doc/source/stdlib/handmade/REVIEW.md` - the type-file format rule (one type-description + line, one line per member, declaration order) carries a placement duty in a subordinate + clause (a property's description is its own `function-` file). Two rules; drop "handmade + file in this folder" from the head, which the folder walk already scopes. From abbc1093c42cc4fba43c1d6697714542586c5620 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 07:53:36 -0700 Subject: [PATCH 16/22] the manifest reader's two helpers - split_tabs and parse_on_error - sit under the same file-io guard as the reader, so a build with no file io does not carry functions nothing calls (the no_fileio lane builds with -Werror=unused-function) --- src/ast/dyn_modules.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index a27ce54462..1996334c48 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -114,6 +114,7 @@ struct ManifestRead { string why; }; +#if !DAS_NO_FILEIO // the reader's helpers: a build with no file io reads no manifest static das::vector split_tabs(const string & line) { das::vector fields; size_t start = 0; @@ -127,6 +128,7 @@ static das::vector split_tabs(const string & line) { start = tab + 1; } } +#endif static string hex64(uint64_t v) { char buf[32]; @@ -167,11 +169,13 @@ static bool dep_stamp(const smart_ptr & fa, const string & path, uin return true; } +#if !DAS_NO_FILEIO static bool parse_on_error(const string & s, int & value) { // RegisterOnError: Quiet, ErrorMsg, Fail if ( s.size() != 1 || s[0] < '0' || s[0] > '2' ) return false; value = s[0] - '0'; return true; } +#endif // the inputs a descriptor's rows can depend on beyond its own bytes: its folder, the das root // (`get_das_root()`), and the cross-compile target (`get_cross_platform_name()`, read from argv) From 48a06aeeb452098a86edca1f8e0ab7b3354154d0 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 07:55:39 -0700 Subject: [PATCH 17/22] the deferred-module rows sit behind one recursive mutex across defer, load, load-all, the deferred query and the clear - a run-time has_module on one thread reads the list while another thread's compile loads from it --- src/builtin/module_builtin_fio.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 8a4dc0de09..7a7be2fd06 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -327,6 +327,7 @@ namespace das { #include #include +#include #include #if _WIN32 #include @@ -2427,12 +2428,15 @@ namespace das { int on_error = 0; }; static vector g_deferred_dynamic_modules; // src/ast/ARCHITECTURE.md sec.2 + static std::recursive_mutex g_deferred_dynamic_modules_mutex; // a run-time has_module reads while another thread's compile loads DAS_API void defer_dynamic_module ( const char * path, const char * cpp_class, int on_error, const char * das_name ) { + lock_guard guard(g_deferred_dynamic_modules_mutex); g_deferred_dynamic_modules.push_back({path ? path : "", cpp_class ? cpp_class : "", das_name ? das_name : "", on_error}); } DAS_API bool load_deferred_dynamic_module ( const char * das_name ) { + lock_guard guard(g_deferred_dynamic_modules_mutex); auto it = find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }); if ( it == g_deferred_dynamic_modules.end() ) return false; @@ -2445,6 +2449,7 @@ namespace das { } DAS_API size_t load_all_deferred_dynamic_modules () { + lock_guard guard(g_deferred_dynamic_modules_mutex); vector all; all.swap(g_deferred_dynamic_modules); if ( trace_module_load() && !all.empty() ) { @@ -2458,11 +2463,14 @@ namespace das { } DAS_API bool is_dynamic_module_deferred ( const char * das_name ) { - return das_name && find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), + if ( !das_name ) return false; + lock_guard guard(g_deferred_dynamic_modules_mutex); + return find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }) != g_deferred_dynamic_modules.end(); } DAS_API void clear_deferred_dynamic_modules () { + lock_guard guard(g_deferred_dynamic_modules_mutex); g_deferred_dynamic_modules.clear(); } From e06a43fe5a271560be19412c692f0ed1f8ada5cf Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 08:16:14 -0700 Subject: [PATCH 18/22] Module::Initialize on a half-warm tree - one descriptor compiled cold, so its module loaded on start, beside replayed ones whose rows wait - brings every deferred module in when the eager fixed point fails and runs it once more; two processes warming one tree make it half-warm, which the parallel AOT batches of a cold CI checkout did (Linux Debug: 'Unable to initialize some modules: imgui_app'); the test copies dasImgui and dasGlfw under its fixture and removes the imgui copy's manifest --- src/ast/ARCHITECTURE.md | 6 +++- src/ast/ast_module.cpp | 7 ++++- tests/module_cache/ARCHITECTURE.md | 4 ++- tests/module_cache/test_deferred_modules.das | 30 ++++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 6ef42b97d9..8a59ef405d 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -87,7 +87,11 @@ scan installed (`setDeferredModuleLoader`); the loader dlopens and registers the the grown list, and when a module reports it cannot initialize - what it needs is deferred too - brings every deferred module in and runs the fixed point again, which is the set an eager start has. A row whose own dlopen fails takes the same road - every deferred module comes in, the -pending retry runs - and the require then finds the module or fails as a cold start would. The +pending retry runs - and the require then finds the module or fails as a cold start would. +`Module::Initialize` takes it too: on a half-warm tree - one descriptor compiled cold, so its +module loaded on start, beside replayed ones whose rows wait - the eager fixed point fails on +the first pass, brings every deferred module in and runs once more. A tree is half-warm +whenever two processes warm it at once, which parallel AOT batches do. The rows and the loader are the scan's: `require_dynamic_modules` clears the rows before its walk and installs the loader, and `Module::Shutdown` clears both, so an environment that follows sees neither the last one's rows nor its loader. A parse that no prerequisite walk diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index 025d0b7bc3..aa0321269f 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -204,7 +204,12 @@ namespace das { string notInitialized; if ( !InitializeDependencies(notInitialized) ) { - DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); + // a half-warm tree: a descriptor compiled cold loaded its module, its dependency's replayed row waits (ARCHITECTURE.md sec.2) + notInitialized.clear(); + load_all_deferred_dynamic_modules(); + if ( !InitializeDependencies(notInitialized) ) { + DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); + } } // Collect reachable TypeDecl from thread root into module roots, sweep the rest. auto & threadRoot = gc_root::gc_get_thread_root(); diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 0fd27b7b40..473b7e5f23 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -47,7 +47,9 @@ this document states what the folder is and why its tests take the shape they do deferred module in and then fail on the missing prerequisite; a module cache an eager start wrote serves a lazy start, with `-log-compile-time` printing the reads and the startup timeline; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every - deferred module in because its `initDependencies` asks for two more. A static host - the AOT + deferred module in because its `initDependencies` asks for two more, and a half-warm tree - + copies of dasImgui and dasGlfw under the fixture, the imgui copy's manifest removed so its + modules load on start while glfw's row waits - initializes by bringing the rest in. A static host - the AOT test binary, whose descriptors register no shared module, or a tree holding no `.shared_module` - has nothing to observe and the test says so and returns. - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index cf9c5aad00..c8bd12f080 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -118,6 +118,26 @@ def make_fixture(tmp : string; artifact : string) : Fixture { return fx } +//! a copy of one of the tree's module folders - descriptor and the artifacts that exist, in +//! either configuration's name - under the fixture's modules, where it shadows the tree's +def copy_module(fx : Fixture; folder : string; artifacts : array) : string { + let src = path_join(path_join(get_das_root(), "modules"), folder) + let dst = path_join(path_join(fx.tmp, "modules"), folder) + var merr : string + mkdir_rec(dst, merr) + var cerr : string + copy_file(path_join(src, ".das_module"), path_join(dst, ".das_module"), true, cerr) + for (name in artifacts) { + for (suffix in [".shared_module", "_debug.shared_module"]) { + let f = path_join(src, "{name}{suffix}") + if (stat(f).is_valid) { + copy_file(f, path_join(dst, "{name}{suffix}"), true, cerr) + } + } + } + return dst +} + def child_cmd(fx : Fixture; script : string; flags : string = ""; cache : string = "-no-module-cache") : string { return "{trace_prefix()}\"{das_exe()}\" -dasroot \"{get_das_root()}\" {cache} -project_root \"{fx.tmp}\" {flags} \"{script}\"" } @@ -241,6 +261,16 @@ def arm_fallback(t : T?; fx : Fixture) { t |> success(find(out, "[module] require imgui_app: loading the deferred Module_imgui_app") >= 0, "imgui_app loads at its require:\n{out}") t |> success(find(out, "[module] loading every deferred module (") >= 0, "its initDependencies asks for glfw and imgui, so the rest loads:\n{out}") } + t |> run("a half-warm tree - a descriptor compiled cold beside replayed ones - initializes by bringing every deferred module in") @(t : T?) { + let imguiDir = copy_module(fx, "dasImgui", ["dasModuleImgui", "imguiApp", "imguiAppHeadless"]) + copy_module(fx, "dasGlfw", ["dasModuleGlfw"]) + var cold : string + t |> success(report_child(t, "half-warm cold", run_child(child_cmd(fx, fx.plain), cold), cold, "PLAIN"), "the copies record their manifests") + remove_result(path_join(imguiDir, ".das_module.manifest")) + var out : string + t |> success(report_child(t, "half-warm", run_child(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the program runs with imgui_app loaded on start and glfw waiting") + t |> success(find(out, "[module] loading every deferred module (") >= 0, "initialize brings the deferred glfw in:\n{out}") + } } //! the arms share one root and run in this order From 7445b330456bce4d7ea324c80edb7e5760b72d7b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 08:25:02 -0700 Subject: [PATCH 19/22] load_deferred_dynamic_module answers false for a null name, as the deferred query does; InitializeDependencies clears the names it reports before it runs, so a caller reusing the string never reads a stale one - the two callers' own clears go --- src/ast/ast_module.cpp | 2 +- src/ast/dyn_modules.cpp | 1 - src/builtin/module_builtin_fio.cpp | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index aa0321269f..38b14624b7 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -137,6 +137,7 @@ namespace das { } bool Module::InitializeDependencies ( string & notInitialized ) { + notInitialized.clear(); // InitDependencies do not add new modules. vector mod_state; bool any = true; @@ -205,7 +206,6 @@ namespace das { string notInitialized; if ( !InitializeDependencies(notInitialized) ) { // a half-warm tree: a descriptor compiled cold loaded its module, its dependency's replayed row waits (ARCHITECTURE.md sec.2) - notInitialized.clear(); load_all_deferred_dynamic_modules(); if ( !InitializeDependencies(notInitialized) ) { DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 1996334c48..68c9858d32 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -607,7 +607,6 @@ static bool load_deferred_module_for_require(const string & name) { if ( grown ) { string notInitialized; if ( !Module::InitializeDependencies(notInitialized) ) { - notInitialized.clear(); load_all_deferred_dynamic_modules(); if ( !Module::InitializeDependencies(notInitialized) ) { DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 7a7be2fd06..3a7bf26a29 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -2436,6 +2436,7 @@ namespace das { } DAS_API bool load_deferred_dynamic_module ( const char * das_name ) { + if ( !das_name ) return false; lock_guard guard(g_deferred_dynamic_modules_mutex); auto it = find_if(g_deferred_dynamic_modules.begin(), g_deferred_dynamic_modules.end(), [&](const DeferredDynamicModule & dm) { return dm.das_name == das_name; }); From 7aa624ad0e7f92e728f78afb76b6bb0e58b05a7a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 09:12:42 -0700 Subject: [PATCH 20/22] the deserializer fetches a builtin module the way the parser's require does - a stream written where a C++ module had loaded lazily is read where nothing required it yet, so Program::serialize's builtin list and the module-cache record header ask the deferred loader when Module::require answers null (the darwin modules lane's ser/deser sweep: program 123 referenced sqlite, loaded by a require ?sqlite guard in an earlier test, and the reader silently skipped a null module); the test writes a dastest --ser stream of a test requiring UnitTest and reads it with --deser in a child that nothing made require it --- src/ast/ARCHITECTURE.md | 4 +++- src/builtin/module_builtin_ast_serialize.cpp | 13 +++++++++++-- tests/module_cache/ARCHITECTURE.md | 3 ++- tests/module_cache/test_deferred_modules.das | 13 +++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 8a59ef405d..d9ef4373ee 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -88,7 +88,9 @@ the grown list, and when a module reports it cannot initialize - what it needs i brings every deferred module in and runs the fixed point again, which is the set an eager start has. A row whose own dlopen fails takes the same road - every deferred module comes in, the pending retry runs - and the require then finds the module or fails as a cold start would. -`Module::Initialize` takes it too: on a half-warm tree - one descriptor compiled cold, so its +The deserializer (`Program::serialize` reading, and the module-cache record header) fetches a +builtin module by name the same way: a stream written where a module had loaded lazily is read +where nothing required it yet. `Module::Initialize` takes it too: on a half-warm tree - one descriptor compiled cold, so its module loaded on start, beside replayed ones whose rows wait - the eager fixed point fails on the first pass, brings every deferred module in and runs once more. A tree is half-warm whenever two processes warm it at once, which parallel AOT batches do. The diff --git a/src/builtin/module_builtin_ast_serialize.cpp b/src/builtin/module_builtin_ast_serialize.cpp index 536056a672..d359a8edf4 100644 --- a/src/builtin/module_builtin_ast_serialize.cpp +++ b/src/builtin/module_builtin_ast_serialize.cpp @@ -1092,6 +1092,15 @@ namespace das { return *this; } + // a read no prerequisite walk precedes meets a deferred C++ module here (src/ast/ARCHITECTURE.md sec.2) + static Module * requireBuiltinModule ( const string & name ) { + auto m = Module::require(name); + if ( !m ) { + if ( auto loader = getDeferredModuleLoader(); loader && loader(name) ) m = Module::require(name); + } + return m; + } + AstSerializer & AstSerializer::operator << ( Module * & module ) { bool is_null = module == nullptr; *this << is_null; @@ -3009,7 +3018,7 @@ namespace das { string name; ser << name; if ( builtin && !promoted ) { - auto m = Module::require(name); + auto m = requireBuiltinModule(name); // a corrupted record can hand this arm a garbage name - require() // answers null, and the deref was a SIGSEGV (recoverable throw now; // the resume reparses the record in place) @@ -3221,7 +3230,7 @@ namespace das { uint64_t size_builtin = 0; ser << size_builtin; for ( uint64_t i = 0; i < size_builtin; i++ ) { string name; ser << name; - Module * m = Module::require(name); + Module * m = requireBuiltinModule(name); library.addModule(m); } diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 473b7e5f23..9bba2c3acc 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -46,7 +46,8 @@ this document states what the folder is and why its tests take the shape they do manifest; a manifest row hand-edited to an absent artifact makes the require bring every deferred module in and then fail on the missing prerequisite; a module cache an eager start wrote serves a lazy start, with `-log-compile-time` printing the reads and the startup - timeline; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every + timeline; a dastest `--ser` stream of a test requiring the module is read by a `--deser` + child that nothing made require it, and the reader loads it; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every deferred module in because its `initDependencies` asks for two more, and a half-warm tree - copies of dasImgui and dasGlfw under the fixture, the imgui copy's manifest removed so its modules load on start while glfw's row waits - initializes by bringing the rest in. A static host - the AOT diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index c8bd12f080..1defb9d6a2 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -246,6 +246,19 @@ def arms_guards_and_caches(t : T?; fx : Fixture) { t |> equal(-1, find(warm, "cumulative hash"), "a builtin module's hash does not depend on the loaded set:\n{warm}") t |> success(find(warm, "cache read took") >= 0, "the lazy start reads the eager start's records:\n{warm}") } + t |> run("a dastest stream written where the module had loaded is read where nothing required it yet") @(t : T?) { + let testsDir = path_join(fx.tmp, "tests") + var merr : string + mkdir_rec(testsDir, merr) + fwrite(path_join(testsDir, "test_uses.das"), "options gen2\nrequire dastest/testing_boost public\nrequire {UNIT_TEST_MODULE}\n[test]\ndef test_uses(t : T?) \{\n t |> equal(5l, test_string_arg_length(\"hello\"))\n\}\n") + let stream = path_join(fx.tmp, "tests.ser") + let dastest = path_join(get_das_root(), "dastest/dastest.das") + var wrote : string + t |> success(report_child(t, "ser", run_child("{child_cmd(fx, dastest)} -- --test \"{testsDir}\" --ser \"{stream}\"", wrote), wrote, "Serialized 1 programs"), "the writer serializes the test program") + var read : string + t |> success(report_child(t, "deser", run_child("{child_cmd(fx, dastest)} -- --test \"{testsDir}\" --deser \"{stream}\"", read), read, "1 tests, 0 error(s)"), "the reader runs the test from the stream") + t |> success(find(read, "[module] require {UNIT_TEST_MODULE}: loading the deferred {UNIT_TEST_CLASS}") >= 0, "the reader loaded the module the stream names:\n{read}") + } } def arm_fallback(t : T?; fx : Fixture) { From 8a2a4b92cdb69d692354be7047a7da56c04dfbc3 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 09:44:45 -0700 Subject: [PATCH 21/22] the half-warm arm's copies bring the libraries beside the artifacts along, and where the copied imguiApp still cannot load - its libraries sit relative to the tree, as on the darwin CI box - the arm says so and returns instead of asserting on a state it never reached --- tests/module_cache/ARCHITECTURE.md | 4 +++- tests/module_cache/test_deferred_modules.das | 21 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 9bba2c3acc..6d16b91b8b 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -50,7 +50,9 @@ this document states what the folder is and why its tests take the shape they do child that nothing made require it, and the reader loads it; and, where the tree holds dasImgui and dasGlfw, `require imgui_app` brings every deferred module in because its `initDependencies` asks for two more, and a half-warm tree - copies of dasImgui and dasGlfw under the fixture, the imgui copy's manifest removed so its - modules load on start while glfw's row waits - initializes by bringing the rest in. A static host - the AOT + modules load on start while glfw's row waits - initializes by bringing the rest in; a build + whose copied artifact cannot find its libraries from the copy has nothing to observe there + and the arm says so. A static host - the AOT test binary, whose descriptors register no shared module, or a tree holding no `.shared_module` - has nothing to observe and the test says so and returns. - `_fixtures/` - the driver and module scripts the spawned children compile (`mc_dep_*`, diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index 1defb9d6a2..b94bd40d90 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -135,9 +135,26 @@ def copy_module(fx : Fixture; folder : string; artifacts : array) : stri } } } + dir(src) $(name) { // a library an artifact loads from beside itself + let ext = extension(name) + if (ext == ".dylib" || ext == ".so" || ext == ".dll") { + copy_file(path_join(src, name), path_join(dst, name), true, cerr) + } + } return dst } +//! whether the cold run loaded the copied artifact - a build whose artifacts find their +//! libraries relative to the tree cannot load a copy, and the arms on copies say so and return +def copy_loaded(out : string; dir : string; artifact : string) : bool { + for (suffix in [".shared_module", "_debug.shared_module"]) { + if (find(out, "{path_join(dir, artifact)}{suffix} : loaded") >= 0) { + return true + } + } + return false +} + def child_cmd(fx : Fixture; script : string; flags : string = ""; cache : string = "-no-module-cache") : string { return "{trace_prefix()}\"{das_exe()}\" -dasroot \"{get_das_root()}\" {cache} -project_root \"{fx.tmp}\" {flags} \"{script}\"" } @@ -279,6 +296,10 @@ def arm_fallback(t : T?; fx : Fixture) { copy_module(fx, "dasGlfw", ["dasModuleGlfw"]) var cold : string t |> success(report_child(t, "half-warm cold", run_child(child_cmd(fx, fx.plain), cold), cold, "PLAIN"), "the copies record their manifests") + if (!copy_loaded(cold, imguiDir, "imguiApp")) { + to_log(LOG_INFO, "test_deferred_modules: the copied imguiApp does not load from {imguiDir} - its libraries sit outside the copy - so the half-warm arm has nothing to observe\n") + return + } remove_result(path_join(imguiDir, ".das_module.manifest")) var out : string t |> success(report_child(t, "half-warm", run_child(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the program runs with imgui_app loaded on start and glfw waiting") From de5d19c978ccf9b03291a12589409dd9ebdd059c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 10:13:25 -0700 Subject: [PATCH 22/22] the half-warm arm reads its premise off the half-warm start too - where the copied imguiApp fails to dlopen there, initialize had nothing to bring in and the arm says so with the child's output, instead of asserting on the line that start could never print --- tests/module_cache/test_deferred_modules.das | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/module_cache/test_deferred_modules.das b/tests/module_cache/test_deferred_modules.das index b94bd40d90..f89928d9ff 100644 --- a/tests/module_cache/test_deferred_modules.das +++ b/tests/module_cache/test_deferred_modules.das @@ -303,6 +303,10 @@ def arm_fallback(t : T?; fx : Fixture) { remove_result(path_join(imguiDir, ".das_module.manifest")) var out : string t |> success(report_child(t, "half-warm", run_child(child_cmd(fx, fx.plain), out), out, "PLAIN"), "the program runs with imgui_app loaded on start and glfw waiting") + if (!copy_loaded(out, imguiDir, "imguiApp")) { + to_log(LOG_INFO, "test_deferred_modules: the copied imguiApp did not load on the half-warm start - its libraries sit outside the copy - so initialize had nothing to bring in:\n{out}\n") + return + } t |> success(find(out, "[module] loading every deferred module (") >= 0, "initialize brings the deferred glfw in:\n{out}") } }