diff --git a/CHANGELIST.md b/CHANGELIST.md index eb1c6740d2..1c7c53b93f 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -231,6 +231,7 @@ Z3 SMT solver bindings as a dynamic module, dasLLVM-style. - **Unwind tables on emitted functions** when the host uses C++ exceptions (#3347) - **JIT debug tooling** (#3511); **`llvm_tune` per-box UX** (#3403) - scope / policy / `--tune`, self-tuning servers, `-exe` fix - **AOT batch-composition hash fix** (`g_isInAot` leak) + `aot_cpp` made AOT-linkable (#3409); **AOT fuzzer-failure hardening** (#3303) +- **Standalone contexts link C++ modules** (#3947) - a `-ctx` context that reaches dasHV, fio or any handled type registers the modules it links (the builtin set in the C++ registrar's order, then dependencies-first) on first construction through a process-wide list (`include/daScript/simulate/standalone_modules.h`), so several generated contexts in one binary share one registry lifetime that ends with `Module::ShutdownStandalone` when the last context is destroyed; a builtin module the program calls at run time keeps its AOT header, the function table is dense (a slot per function the context still reaches), and `examples/standalone/06_full_runtime` (dasHV + fio, static, compiles nothing at run time, loads no shared module) is the worked example and a small-lane test - **LLVM-AOT in a large embedding host** (#3715) - scalar call ABI matched at bool/reference seams, target triple + data layout pinned on emitted objects, `-dll-path`/`DAS_DLL_PATH` dasbind search, per-object glob-init deferred to first link #### Runtime, Tooling, and Hosting diff --git a/CLAUDE.md b/CLAUDE.md index 4fb73f4968..6f42336857 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ Task-specific instructions are split into skill files under `skills/`. You MUST | `skills/internal/documentation_rst.md` | Editing RST in `doc/source/`, `//!` doc-comments in `daslib/*.das`, tutorial RST pages | | `skills/internal/tutorials.md` | Anything that looks like a tutorial - they live under `/tutorials//`, NEVER `modules//tutorial/` | | `skills/internal/tutorial_prose.md` | WRITING or revising general-reader doc/tutorial prose (`documentation_rst.md` is mechanics, this is the words) | -| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`) | +| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`, or a standalone context on the full runtime) | | `skills/internal/cpp_codebase_notes.md` | Working on daslang's own C++ - where inference/builtins/errors/parser live, AST function flags | | `skills/internal/clang_bind_build.md` | Enabling `dasClangBind` / bumping the libclang SDK / running any `bind_*.das` self-binder | | `skills/daslib_modules.md` | Working with `daslib/` modules or extending the stdlib | diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b7d7218c9..06a9a9b819 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1771,6 +1771,7 @@ SET(DAS_SIMULATE_INCLUDES include/daScript/simulate/simulate_visit_op.h include/daScript/simulate/simulate_visit_op_undef.h include/daScript/simulate/standalone_ctx_utils.h + include/daScript/simulate/standalone_modules.h ) install(FILES diff --git a/daslib/ARCHITECTURE_EMIT.md b/daslib/ARCHITECTURE_EMIT.md index ee073e9330..df9cd8849c 100644 --- a/daslib/ARCHITECTURE_EMIT.md +++ b/daslib/ARCHITECTURE_EMIT.md @@ -37,6 +37,20 @@ Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across `set_aot_main_module_name` writes the daslib global AND forwards to `set_aot_main_module_name_cpp`; a spelling rule changed on one side only produces a TU where the definition and its debug-info references disagree. +- **`fromExtraDependency` describes the host process, not the program being emitted.** The + flag is set when the host first loaded a shared module as an extra dependency (a `-jit` + host loads `strings`, `fio_core` and `math` that way) and stays on the shared module for + every later compile in the process. `getRequiredModulesFor` skips such modules only on the + regular AOT path; a standalone context decides by its linked set alone, otherwise a + generator running under a `-jit` host (the JIT test lane) prunes the modules the program + calls. +- **`DEFAULT_MODULE_ORDER` mirrors `register_builtin_modules_impl`** in + `src/builtin/modules.cpp` - the same builtin C++ modules in the same order, a + daslib-to-C++ pair `tests/aot/test_standalone_emit.das` reads back from the C++ source and + pins. A standalone context registers these first because + module constructors `Module::require` earlier ones by name (`fio_core` takes `strings`, + dasHV takes `rtti_core`). A module missing from the daslib list still registers, only in + the dependencies-first pass that follows; a module added to the C++ side joins the list. ## 6. aot_standalone @@ -79,6 +93,46 @@ Companion to `ARCHITECTURE.md` in this folder; section numbers are unique across `fnByMangledName` call that would crash at runtime. `prepareProgramForEmission` runs `NoAotMarker` first (the regular AOT paths run it too; standalone must match) and then `checkAllUsedFunctionsCanAot` walks used, non-builtin functions. +- **The function table is dense** - `addFunctionInfo` numbers the emitted rows in emission + order and the ctor sizes `context.functions` by that count, not by the program's + `totalFunctions`. The program count includes every function the interpreter simulated + (externs among them), while the context emits only what it still reaches; a table sized + by the larger count leaves rows of uninitialized memory that the destructor's shutdown + walk (`runShutdownScript`) reads. Nothing reads a function by program index: AOT calls + are direct, function pointers resolve through `fnByMangledName`, and `FillFunction` + matches rows by AOT hash. +- **A context links the modules the program reaches, and registers the C++ ones itself.** + `aot_cpp.das`'s `collectLinkedModules` marks the module of every used function, of every + function a used function or global calls (`UseTypeMarker` reads the callee of each call, + operator and `@@` address - externs carry no `used` flag of their own), and of every + struct, enum and handled type; a program that reaches no C++ module beyond the builtin one + links exactly that set. A nano program reaches none by contract (every builtin module is + absent there), so its output stays registry-free. + Once it reaches one, the default C++ modules the compiler loaded and every linked C++ + module's `module_for_each_dependency` closure join the set, because module constructors + `Module::require` those by name (dasHV takes `rtti_core` this way). The pruned modules + (`compile time only, not linked`) get no `aotRequire` include and no registration. + `standaloneModuleRegistration` orders the C++ subset and ranks it: `DEFAULT_MODULE_ORDER` + first (the pair it mirrors is recorded in sec. 5), then the rest dependencies-first, ranked + by dependency depth. Each generated TU emits its list as a `StandaloneModule` table added to + the process-wide list before main, and the context class takes `StandaloneModuleScope` + (`include/daScript/simulate/standalone_modules.h`, inline so every TU shares one copy) as + its FIRST base, ahead of `Context`: constructed before it, destroyed after it. The first + scope constructed registers the union, rank order, and initializes once; the last scope + destroyed shuts down - inside main, after the last context's own teardown and while the + runtime's statics still stand. A static destructor cannot do this: it runs after main, + when statics constructed later than it (libhv's, the runtime's own) are already gone, and + on Windows that shutdown corrupts the heap - the exit-time race the `-exe` path avoids by + shutting down before main returns. Two contexts with different module sets in one binary + therefore share one lifetime. A host that registered the builtin module owns the registry + instead: the context registers nothing and stops the program, by name, on a linked module + the host did not register - the runtime cannot add to an initialized registry and keep + `Initialize`/`Shutdown` balanced, and a second shutdown would walk a registry the first + one deleted. The shutdown is `Module::ShutdownStandalone`, never `Module::Shutdown`: the + latter resets the fusion engine through a function pointer only the interpreter's + `simulate_fusion.cpp` installs, and a standalone binary links no interpreter. Handled + types need the registry because `TypeInfo::resolveAnnotation` walks the bound + environment's module list; a context with none dereferences a null environment. - **Type definitions live in the header, once** - struct/enum definitions (the dependency dump plus the entry module's own `declarations` capture) are emitted into the `.das.h`, which the `.das.cpp` includes; the source never redefines them. They sit diff --git a/daslib/REVIEW.md b/daslib/REVIEW.md index bfac46d858..173f4ad68a 100644 --- a/daslib/REVIEW.md +++ b/daslib/REVIEW.md @@ -38,8 +38,8 @@ survives into the sibling loop's exit path and unbalances its counter. **A diff that adds or changes a daslib fact - code or a `//!` contract - whose truth is decided by a C++-side definition, with no test, lint, or compile error failing when the two -sides no longer match, records the pair in the architecture doc, in its module's section, -naming both sides.** +sides no longer match, records the pair in the architecture doc, in the section of the file +the daslib side lives in, naming both sides.** **When a diff changes one side of a recorded daslib/C++ pair so the two no longer match, it changes the other side and updates the pair's architecture-doc entry in the same diff.** @@ -117,8 +117,8 @@ set without the cap is a silently missed finding; raising a cap without the over suggestion that does not compile. **A diff that adds or changes an emit entry point - a function that runs the emit visitor -(`CppAot` or any subclass of it) and then returns or writes the generated C++ - keeps the -error check ahead of that return or write.** The error check is the program's +(`CppAot` or any subclass of it) and then writes the generated C++ to a file - keeps the +error check ahead of that file write.** The error check is the program's `macroException`/`failToCompile` state, read directly or through `log_aot_emit_errors`; a codegen exception mid-visit leaves partial C++. diff --git a/daslib/aot_cpp.das b/daslib/aot_cpp.das index ac1815b48e..33223965d8 100644 --- a/daslib/aot_cpp.das +++ b/daslib/aot_cpp.das @@ -619,9 +619,13 @@ class public PrologueMarker : AstVisitor { }; class UseTypeMarker : AstVisitor { - //! AST visitor that collects structs and enums used in AOT-eligible functions. + //! AST visitor that collects structs, enums and handled types used in AOT-eligible functions, + //! and the modules whose functions the used functions and globals call (externs included). useStructs : table; useEnums : table; + useHandles : table; + useModules : table; + inUsedCode : bool = false; def override preVisitTypeDecl(typeDecl : TypeDeclPtr) { } @@ -630,9 +634,31 @@ class UseTypeMarker : AstVisitor { return !fn.flags.noAot; } + def override preVisitFunction(fun : FunctionPtr) { + inUsedCode = fun.flags.used; + } + def override visitFunction(var fun : FunctionPtr) : FunctionPtr { + inUsedCode = false; + return <- fun; + } def override preVisitExpression(expr : ExpressionPtr) { mark(expr._type); } + def override preVisitExprCall(expr : ExprCall?) { + markCallee(expr.func); + } + def override preVisitExprOp1(expr : ExprOp1?) { + markCallee(expr.func); + } + def override preVisitExprOp2(expr : ExprOp2?) { + markCallee(expr.func); + } + def override preVisitExprOp3(expr : ExprOp3?) { + markCallee(expr.func); + } + def override preVisitExprAddr(expr : ExprAddr?) { + markCallee(expr.func); + } def override preVisitFunctionArgument(fn : FunctionPtr; variable : VariablePtr; lastArg : bool) { mark(variable._type); } @@ -640,8 +666,18 @@ class UseTypeMarker : AstVisitor { mark(variable._type); } def override preVisitGlobalLetVariable(variable : Variable?; lastArg : bool) { + inUsedCode = variable.flags.used; mark(variable._type); } + def override visitGlobalLetVariable(var variable : VariablePtr; lastArg : bool) : VariablePtr { + inUsedCode = false; + return <- variable; + } + def markCallee(fn : Function?) { + if (inUsedCode && fn != null && fn._module != null) { + useModules.insert(fn._module); + } + } def mark(decl : TypeDeclPtr) { if (decl == null) return ; if (decl.baseType == Type.tStructure) { @@ -658,6 +694,12 @@ class UseTypeMarker : AstVisitor { decl.baseType == Type.tEnumeration64) { assert(decl.enumType != null); useEnums.insert(decl.enumType); + } elif (decl.baseType == Type.tHandle) { + //! a handled type decides whether a C++ module is linked, so only code the runtime + //! program reaches counts - a dead function naming FILE? must not pull fio_core in + if (inUsedCode && decl.annotation != null) { + useHandles.insert(decl.annotation); + } } else { if (decl.firstType != null) mark(decl.firstType); if (decl.secondType != null) mark(decl.secondType); @@ -4372,12 +4414,33 @@ def public dumpRegisterAot(var tw : StringBuilderWriter?; program : ProgramPtr; write(*tw, "static AotListBase impl(registerAotFunctions);\n"); } +//! The builtin C++ modules in the order `register_builtin_modules_impl` (`src/builtin/modules.cpp`) +//! registers them. A module constructor requires `strings` / `rtti_core` by name, so this order is +//! the one every host already relies on; it goes first in a standalone context's registration. +let public DEFAULT_MODULE_ORDER = [ + "Module_BuiltIn", "Module_Math", "Module_Strings", "Module_Rtti", "Module_Ast", "Module_Jit", + "Module_Debugger", "Module_Network", "Module_UriParser", "Module_JobQue", "Module_FIO", "Module_DASBIND" +] + +def private isCppModule(mod : Module?) : bool { + return !empty(mod.cppClassName) +} + +def private canAotModule(mod : Module?) : bool { + var ok = false + build_string() $(wr) { + ok = mod |> aot_require(unsafe(addr(wr))) + } + return ok +} + def private collectUsedModules(program : ProgramPtr) : table { - //! Modules owning a function, struct or enum the runtime program still reaches. + //! Modules owning a function (externs included), struct, enum or handled type the runtime + //! program still reaches. var used : table program.get_ptr() |> for_each_module_no_order($(pm) { pm |> for_each_module_function($(pfun) { - if (pfun.index >= 0 && pfun.flags.used && !pfun.flags.builtIn && pfun._module != null) { + if (pfun.index >= 0 && pfun.flags.used && pfun._module != null) { used |> insert(pfun._module) } }) @@ -4396,23 +4459,148 @@ def private collectUsedModules(program : ProgramPtr) : table { used |> insert(en._module) } } + for (ann in keys(utm.useHandles)) { + if (ann._module != null) { + used |> insert(ann._module) + } + } + for (mod in keys(utm.useModules)) { + used |> insert(mod) + } unsafe { delete utm } return <- used } +//! a module of a standalone context's registration list, with its cross-TU registration rank +struct public StandaloneModuleEntry { + mod : Module? + rank : int +} + +def private insertWithDependencies(var linked, visited : table; var mod : Module?) { + //! `visited` tracks the walk, not membership: a module already linked as used still has + //! its dependencies to bring in + if (visited |> key_exists(mod)) return + visited |> insert(mod) + linked |> insert(mod) + module_for_each_dependency(mod) $(dep; _pub) { + if (isCppModule(dep) && canAotModule(dep)) { + insertWithDependencies(linked, visited, dep) + } + } +} + +def private cppDependencyDepth(linked : table; var visiting : table; var mod : Module?) : int { + //! 1 + the deepest linked C++ dependency - a rank that puts dependencies first across TUs; + //! `visiting` cuts dependency cycles (modules require each other), which count as depth 0 + if (visiting |> key_exists(mod)) return 0 + visiting |> insert(mod) + var depth = 0 + module_for_each_dependency(mod) $(dep; _pub) { + if (isCppModule(dep) && (linked |> key_exists(dep))) { + let depDepth = cppDependencyDepth(linked, visiting, dep) + if (depDepth > depth) { + depth = depDepth + } + } + } + visiting |> erase(mod) + return depth + 1 +} + +def public collectLinkedModules(program : ProgramPtr) : table { + //! Modules a standalone context links: what the runtime program uses, and - once that reaches a + //! C++ module beyond the builtin one - the default C++ modules the compiler loaded plus every + //! linked C++ module's dependencies, because module constructors require those by name. + var linked <- collectUsedModules(program) + var reachesCpp = false + for (mod in keys(linked)) { + if (isCppModule(mod) && mod.cppClassName != "Module_BuiltIn") { + reachesCpp = true + } + } + if (!reachesCpp) { + return <- linked + } + var cppModules : array + program.get_ptr() |> for_each_module($(mod) { + if (mod.name != "" && isCppModule(mod)) { + cppModules |> push(mod) + } + }) + var visited : table + for (mod in cppModules) { + let isDefault = DEFAULT_MODULE_ORDER |> find_index(string(mod.cppClassName)) >= 0 + if ((isDefault || (linked |> key_exists(mod))) && canAotModule(mod)) { + insertWithDependencies(linked, visited, mod) + } + } + return <- linked +} + +def public standaloneModuleRegistration(program : ProgramPtr) : array { + //! The C++ modules a standalone context registers: the builtin set first in + //! `DEFAULT_MODULE_ORDER`, then the rest dependencies-first; the rank orders the same way + //! across TUs. Empty when the program reaches no C++ module beyond the builtin one. + let linked <- collectLinkedModules(program) + var ordered : array + var visited : table + var cppModules : array + program.get_ptr() |> for_each_module($(mod) { + if (mod.name != "" && isCppModule(mod) && (linked |> key_exists(mod))) { + cppModules |> push(mod) + } + }) + if (length(cppModules) <= 1) { + return <- ordered + } + ordered |> reserve(length(cppModules)) + var rank = 0 + for (className in DEFAULT_MODULE_ORDER) { + for (mod in cppModules) { + if (mod.cppClassName == className && !(visited |> key_exists(mod))) { + visited |> insert(mod) + ordered |> push(StandaloneModuleEntry(mod = mod, rank = rank)) + } + } + rank++ + } + var rest : array + for (mod in cppModules) { + appendDependenciesFirst(rest, visited, linked, mod) + } + for (mod in rest) { + var visiting : table + ordered |> push(StandaloneModuleEntry(mod = mod, rank = length(DEFAULT_MODULE_ORDER) + cppDependencyDepth(linked, visiting, mod))) + } + return <- ordered +} + +def private appendDependenciesFirst(var ordered : array; var visited : table; linked : table; var mod : Module?) { + if (visited |> key_exists(mod)) return + visited |> insert(mod) + module_for_each_dependency(mod) $(dep; _pub) { + if (linked |> key_exists(dep) && isCppModule(dep)) { + appendDependenciesFirst(ordered, visited, linked, dep) + } + } + ordered |> push(mod) +} + def public getRequiredModulesFor(program : ProgramPtr; prune_to_used : bool = false) { //! Collects required module declarations and checks for no-AOT modules in the program. - //! `prune_to_used` drops modules the runtime program never reaches - standalone only. + //! `prune_to_used` drops modules the runtime program never reaches - standalone only - and then + //! the linked set alone decides; `fromExtraDependency` is a host-process fact (ARCHITECTURE_EMIT sec. 5). var modules_str : array var noAotModule = false - let used <- prune_to_used ? collectUsedModules(program) : table() + let linked <- prune_to_used ? collectLinkedModules(program) : table() program.get_ptr() |> for_each_module($(mod) { if (mod.name == "") { - } elif (mod.moduleFlags.fromExtraDependency && mod.name != "builtin") { - } elif (prune_to_used && mod.name != "$" && mod.name != "builtin" && !(used |> key_exists(mod))) { + } elif (prune_to_used && mod.name != "$" && mod.name != "builtin" && !(linked |> key_exists(mod))) { modules_str |> push(" // require {mod.name} - compile time only, not linked\n") + } elif (!prune_to_used && mod.moduleFlags.fromExtraDependency && mod.name != "builtin") { } else { modules_str |> push(build_string() $(wr) { write(wr, " // require {mod.name}\n") diff --git a/daslib/aot_standalone.das b/daslib/aot_standalone.das index eea71bbb78..00cbd4fe97 100644 --- a/daslib/aot_standalone.das +++ b/daslib/aot_standalone.das @@ -24,7 +24,9 @@ struct StandaloneContextCfg { context_name : string; class_name : string; cpp_output_dir : string; - cross_platform : bool + cross_platform : bool; + //! the context links C++ modules beyond the builtin one, so its constructor registers them + registers_modules : bool }; def aotFunctionName(str : string) { @@ -127,6 +129,7 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var let requested_stack = stack_arg ?as tInt ?? int(program.policies.stack) let stack_base = requested_stack > 0 && stack_arg is tInt ? requested_stack : max(requested_stack, min_init_stack) let stack_size = stack_base + int(program.globalInitStackSize) + let usedFunctionCount = length(collectProgramUsedFunctions(program, false, false)) write(tw, "{cfg.class_name}::{cfg.class_name}() : Context({stack_size}/*stack*/) \{\n"); write(tw, " auto & context = *this;\n"); write(tw, " CodeOfPolicies policies;"); @@ -148,8 +151,8 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var write(tw, " context.allocateGlobalsAndShared();\n"); write(tw, " if ( context.globals ) memset(context.globals, 0, context.getGlobalSize());\n"); write(tw, " context.totalVariables = {program.totalVariables}/*totalVariables*/;\n"); - write(tw, " context.functions = (SimFunction *) context.code->allocate( {program.totalFunctions}/*totalFunctions*/*sizeof(SimFunction) );\n"); - write(tw, " context.totalFunctions = {program.totalFunctions}/*totalFunctions*/;\n"); + write(tw, " context.functions = (SimFunction *) context.code->allocate( {usedFunctionCount}/*totalFunctions*/*sizeof(SimFunction) );\n"); + write(tw, " context.totalFunctions = {usedFunctionCount}/*totalFunctions*/;\n"); write(tw, " bool anyPInvoke = false;\n"); write(tw, " if ( anyPInvoke || {program.policies.threadlock_context || program.policies.debugger}"); @@ -202,7 +205,10 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var def writeStandaloneContext(var program : ProgramPtr, initFunctions : string, var header : StringBuilderWriter, var source : StringBuilderWriter; cfg : StandaloneContextCfg; var context : Context) { write(header, "\n\n"); - write(header, "class {cfg.class_name} : public Context \{\n"); + //! the module scope is the FIRST base: constructed before Context, destroyed after it, so the + //! registry outlives the context's own teardown + let bases = cfg.registers_modules ? "public StandaloneModuleScope, public Context" : "public Context" + write(header, "class {cfg.class_name} : {bases} \{\n"); write(header, "public:\n"); write(header, " {cfg.class_name}();\n"); writeStandaloneContextMethods(program, header, "", true, cfg); @@ -268,22 +274,48 @@ class StandaloneContextGen : CppAot { }; +def writeModuleDeclarations(var header, source : StringBuilderWriter; registrations : array) { + //! the header carries the registry include (the class derives from its scope base); the + //! source carries the module declarations its registration table takes addresses of + if (empty(registrations)) return + write(header, "#include \"daScript/simulate/standalone_modules.h\"\n"); + write(source, "#include \"daScript/simulate/standalone_modules.h\"\n"); + for (entry in registrations) { + write(source, "DECLARE_MODULE({entry.mod.cppClassName});\n"); + } +} + +def writeModuleRegistration(var source : StringBuilderWriter; registrations : array) { + //! This TU's linked C++ modules join the process-wide list before main; the first context's + //! `StandaloneModuleScope` base (standalone_modules.h) registers the union once, and the last + //! context's destruction shuts down, so several contexts in one binary share one lifetime. + if (empty(registrations)) return + write(source, "static const StandaloneModule das_standalone_modules[] = \{\n"); + for (entry in registrations) { + write(source, " \{ \"{entry.mod.name}\", &::register_{entry.mod.cppClassName}, {entry.rank} \},\n"); + } + write(source, "\};\n"); + write(source, "static const bool das_standalone_modules_added = standaloneAddModules(das_standalone_modules, {length(registrations)});\n\n"); +} + def writeRegistration(var header : StringBuilderWriter; var source : StringBuilderWriter; initFunctions : string; var program : ProgramPtr; cfg : StandaloneContextCfg; + registrations : array; var context : Context) { write(source, "using namespace {program.thisNamespace};\n"); write(header, "namespace {cfg.context_name} \{\n"); write(source, "namespace {cfg.context_name} \{\n"); dumpRegisterAot(unsafe(addr(source)), program, context, false, cfg.cross_platform); + writeModuleRegistration(source, registrations); writeStandaloneContext(program, initFunctions, header, source, cfg, context); write(header, "\} // namespace {cfg.context_name}\n"); write(source, "\} // namespace {cfg.context_name}\n"); } -def GetFunctionInfo(pfun : Function?, info : string) { +def GetFunctionInfo(pfun : Function?; index : int; info : string) { return build_string() $(tw) { let args = ["\"{pfun.name}\"", "\"{pfun |> get_mangled_name()}\"", @@ -296,7 +328,7 @@ def GetFunctionInfo(pfun : Function?, info : string) { "{pfun._module.moduleFlags.promoted}", "{pfun.result.isRefType && !pfun.result.flags.ref}", "{pfun.moreFlags.pinvoke}"] - write(tw, " \{{pfun.index}, FunctionInfo({args|>join(", ")}), &{info}\},\n"); + write(tw, " \{{index}, FunctionInfo({args|>join(", ")}), &{info}\},\n"); } } @@ -308,9 +340,13 @@ def addFunctionInfo(fnn : array; var helper : AotDebugInfoHelper?) { lookupFunctionTable.push((pfun, info)); } + //! the table is dense: a slot per function the context still reaches, in emission order, + //! so a standalone context carries no row for what the interpreter would have removed return build_string() $(tw) { + var index = 0 for ((fun_info, info) in lookupFunctionTable) { - write(tw, GetFunctionInfo(fun_info, funcInfoName(info))); + write(tw, GetFunctionInfo(fun_info, index, funcInfoName(info))); + index++ } } } @@ -414,8 +450,9 @@ def private prepareProgramForEmission(var program : ProgramPtr; context : Contex return coll } -def public runStandaloneVisitor(var program : ProgramPtr, modules : array, var pctx : smart_ptr; cfg : StandaloneContextCfg) : bool { +def public runStandaloneVisitor(var program : ProgramPtr, modules : array; registrations : array; var pctx : smart_ptr; cfg : StandaloneContextCfg) : bool { //! Runs the standalone AOT visitor on the program to generate C++ source and header files. + //! `registrations` are the C++ modules the generated constructor registers, in order. //! Returns false (writing nothing) when emission collected errors. assume context = *pctx; @@ -440,13 +477,14 @@ def public runStandaloneVisitor(var program : ProgramPtr, modules : array split_by_chars("/\\") |> back() let ctx_name = (file_name |> split("."))[0] - let cfg = StandaloneContextCfg(context_name = ctx_name, + var cfg = StandaloneContextCfg(context_name = ctx_name, class_name = "Standalone", cpp_output_dir = output_dir, cross_platform = cross_platform) @@ -507,7 +545,9 @@ def public standalone_aot(input : string; output_dir : string; cross_platform : panic("Standalone context called on non aot module {input}") } else { before_emit |> invoke(program) - ok = runStandaloneVisitor(program, modules_str, pctx, cfg) + let registrations <- standaloneModuleRegistration(program) + cfg.registers_modules = !empty(registrations) + ok = runStandaloneVisitor(program, modules_str, registrations, pctx, cfg) } } } diff --git a/examples/standalone/06_full_runtime/README.md b/examples/standalone/06_full_runtime/README.md new file mode 100644 index 0000000000..7ee557dcd3 --- /dev/null +++ b/examples/standalone/06_full_runtime/README.md @@ -0,0 +1,35 @@ +# 06_full_runtime - a standalone context that links C++ modules + +The same `-ctx` generation as the nano examples beside it, linked against the full +`libDaScript` and the static archive of every C++ module the script reaches, instead +of nano. This one reaches two: dasHV (an HTTP client) and fio (a child process). + +What the binary still does without: the compiler, the interpreter, and every shared +module. It parses nothing at run time and holds no lock on any file a deploy +replaces - the shape a supervisor needs. + +What it costs over nano is the module registry. Handled types resolve their +annotations through it, so the first context constructed registers the C++ modules +it links (the builtin set in the C++ registrar's order, then the rest +dependencies-first), and the last context destroyed shuts them down with +`Module::ShutdownStandalone` - no interpreter is linked to own a fusion reset. That +happens inside `main`, where it belongs: keep contexts there. `main.cpp` constructs +the context and nothing else. A host that registered the +builtin module owns the registry instead: it registers every module the context +links before constructing it, and shuts down itself; a module it missed stops the +program at construction, by name. + +The generated code names only what the program reaches: a module the script used at +compile time alone is neither included nor registered, and the function table has a +slot per function the context still calls - nothing for what the interpreter would +have removed. + +## Running it + +``` +standalone_06_full_runtime [url] +``` + +Probes `url` (default: a port nobody listens on, so the status is -1), then runs a +copy of itself with `--child` and prints what the child wrote. It builds in-tree with +dasHV enabled and runs as a small-lane test, exit 0 when the child round trip held. diff --git a/examples/standalone/06_full_runtime/main.cpp b/examples/standalone/06_full_runtime/main.cpp new file mode 100644 index 0000000000..8ba4d0d6d2 --- /dev/null +++ b/examples/standalone/06_full_runtime/main.cpp @@ -0,0 +1,34 @@ +// A standalone context on the full runtime, linking C++ modules. +// +// dasHV (an HTTP client) and fio (a child process) come in as static archives, +// so the binary compiles nothing at run time and loads no shared module - the +// shape of a supervisor that must never hold a lock on the files it deploys. The +// generated constructor registers the modules it links; main constructs the +// context and calls what the script exported. +// +// service_probe [url] probe `url` (default: a port nobody listens on), +// then run a copy of this program as the child +// service_probe --child the child: print one line, exit 7 + +#include "daScript/daScript.h" +#include "service_probe.das.h" + +#include +#include + +using namespace das; + +int main ( int argc, char * argv[] ) { + if ( argc > 1 && strcmp(argv[1], "--child") == 0 ) { + printf("hello from the child\n"); + return 7; + } + const char * url = argc > 1 ? argv[1] : "http://127.0.0.1:1/"; + service_probe::Standalone ctx; + const int status = ctx.http_status((char *)url); + printf("GET %s -> %d%s\n", url, status, status < 0 ? " (nobody answered)" : ""); + const bool status_is_sane = status == -1 || (status >= 100 && status <= 599); + const int code = ctx.run_child(argv[0], (char *)"--child"); + printf("child exit code %d\n", code); + return (status_is_sane && code == 7) ? 0 : 1; +} diff --git a/examples/standalone/06_full_runtime/service_probe.das b/examples/standalone/06_full_runtime/service_probe.das new file mode 100644 index 0000000000..7f12a37250 --- /dev/null +++ b/examples/standalone/06_full_runtime/service_probe.das @@ -0,0 +1,42 @@ +options gen2 +options stack = 65536 + +// A standalone context on the full runtime: dasHV and fio are C++ modules, linked as static +// archives. The generated context registers them (their handled types resolve through the +// module registry) and the last context destroyed shuts them down - main.cpp constructs the +// context and nothing else. + +require dashv/dashv_boost +require daslib/fio + +//! HTTP status of `url`, or -1 when nothing answers within two seconds. +[export] +def http_status(url : string) : int { + var status = -1 + with_http_request() $(var req) { + req.method = http_method.GET + req.url := url + req.timeout = 2 + req.connect_timeout = 2 + request(req) $(resp) { + if (resp != null) { + status = int(resp.status_code) + } + } + } + return status +} + +//! Runs `exe arg`, prints every line the child writes, and returns its exit code. +[export] +def run_child(exe, arg : string) : int { + let code = unsafe(popen_argv([exe, arg], 10.0, $(f) { + while (!feof(f)) { + let line = fgets(f) + if (!empty(line)) { + print("child: {line}") + } + } + })) + return code +} diff --git a/examples/standalone/CMakeLists.txt b/examples/standalone/CMakeLists.txt index 9c5be20ad1..256ad82b7d 100644 --- a/examples/standalone/CMakeLists.txt +++ b/examples/standalone/CMakeLists.txt @@ -5,7 +5,7 @@ # links the generated code and nano - no daslang binary at run time, and no # compiler in the linked program at all. # -# SDK users build the same four targets from CMakeLists.standalone.cmake. +# SDK users build 01-04 from CMakeLists.standalone.cmake; 05 and the full-runtime 06 are in-tree. ########################################################### # nano decides the header search order for everything that links it, and a @@ -51,3 +51,46 @@ das_nano_example(standalone_04_c_binding 04_c_binding blinker.das) das_nano_example(standalone_05_table 05_compile_time_table thermometer.das "${CMAKE_CURRENT_SOURCE_DIR}/05_compile_time_table/thermistor.csv" "${CMAKE_CURRENT_SOURCE_DIR}/05_compile_time_table/csv_bake.das") + +# Not a nano example: a standalone context on the full runtime, linking C++ modules (dasHV, +# fio) as static archives - it compiles nothing at run time and loads no shared module. It +# compiles against the full runtime's headers, so it takes them back explicitly after the nano +# clear above. It also runs as a small test: the generated constructor registering its modules, +# and the context tearing down cleanly, is what no nano example exercises. +if(NOT DAS_HV_DISABLED) + set(_full_runtime_gen_cpp "${NANO_EXAMPLE_GEN}/service_probe.das.cpp") + add_custom_command( + OUTPUT "${_full_runtime_gen_cpp}" "${NANO_EXAMPLE_GEN}/service_probe.das.h" + COMMAND $ + "${PROJECT_SOURCE_DIR}/utils/aot/main.das" + -- -ctx "${CMAKE_CURRENT_SOURCE_DIR}/06_full_runtime/service_probe.das" + "${NANO_EXAMPLE_GEN}/" + DEPENDS daslang + "${CMAKE_CURRENT_SOURCE_DIR}/06_full_runtime/service_probe.das" + "${PROJECT_SOURCE_DIR}/utils/aot/main.das" + "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" + "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + COMMENT "Standalone AOT (full runtime): service_probe.das" + VERBATIM + ) + add_executable(standalone_06_full_runtime + "${CMAKE_CURRENT_SOURCE_DIR}/06_full_runtime/main.cpp" "${_full_runtime_gen_cpp}") + target_include_directories(standalone_06_full_runtime PRIVATE + "${NANO_EXAMPLE_GEN}" + "${PROJECT_SOURCE_DIR}/include" + "${PROJECT_SOURCE_DIR}/3rdparty/fmt/include" + ${NEED_MODULES_PATH} + "${PROJECT_SOURCE_DIR}/modules/dasHV/hv/$/include") + if(DEFINED DAS_CONFIG_INCLUDE_DIR) + target_include_directories(standalone_06_full_runtime PRIVATE ${DAS_CONFIG_INCLUDE_DIR}) + endif() + target_link_libraries(standalone_06_full_runtime PRIVATE + libDaScript ${SRC_LIBRARIES} libDasModuleHV) + SETUP_CPP11(standalone_06_full_runtime) + set_target_properties(standalone_06_full_runtime PROPERTIES FOLDER "examples/standalone") + add_test(NAME standalone_full_runtime COMMAND standalone_06_full_runtime + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) + set_tests_properties(standalone_full_runtime PROPERTIES LABELS "small") + add_dependencies(test-small standalone_06_full_runtime) +endif() diff --git a/include/daScript/simulate/standalone_modules.h b/include/daScript/simulate/standalone_modules.h new file mode 100644 index 0000000000..4b764e866b --- /dev/null +++ b/include/daScript/simulate/standalone_modules.h @@ -0,0 +1,87 @@ +#pragma once + +#include + +#include "daScript/ast/ast.h" +#include "daScript/daScriptModule.h" + +namespace das { + + // The module registry of a standalone context that links C++ modules. Every generated TU + // adds its modules before main; the first context constructed registers the union, lowest + // rank first, and initializes once; the destruction of the last live context shuts the + // registry down - inside main, never from a static destructor, which runs after the + // runtime's own statics are gone - with ShutdownStandalone, since no interpreter is linked + // to own a fusion reset. A host that registered the builtin module owns the registry + // instead: it must have registered every module the contexts link (a missing one stops + // the program, by name), and it initializes and shuts down itself - the runtime has no way + // to add to an initialized registry and keep Initialize/Shutdown balanced. + struct StandaloneModule { + const char * name; + Module * (*pull)(); + int rank; + }; + + inline vector & standaloneModules () { + static vector modules; + return modules; + } + + inline bool standaloneAddModules ( const StandaloneModule * first, size_t count ) { + auto & modules = standaloneModules(); + modules.insert(modules.end(), first, first + count); + return true; + } + + struct StandaloneModuleState { + int live = 0; // contexts alive + bool owner = false; // this process's contexts registered the modules and own the shutdown + }; + + inline StandaloneModuleState & standaloneModuleState () { + static StandaloneModuleState state; + return state; + } + + inline void standaloneAcquireModules () { + auto & state = standaloneModuleState(); + if ( state.live++ != 0 ) return; + auto & modules = standaloneModules(); + for ( size_t i = 1; i < modules.size(); ++i ) { // stable insertion sort by rank + auto entry = modules[i]; + size_t j = i; + for ( ; j > 0 && modules[j-1].rank > entry.rank; --j ) modules[j] = modules[j-1]; + modules[j] = entry; + } + const bool hostOwns = Module::require("$") != nullptr; + for ( auto & m : modules ) { + if ( Module::require(m.name) ) continue; + if ( hostOwns ) { + // a configuration error, not an invariant: the message names the fix, so no + // stack trace and no debugger break - the same clean exit in every build config + DAS_FATAL_LOG("standalone context: module '%s' is not registered, and the host owns " + "the module registry - register it before constructing the context\n", m.name); + exit(-1); + } + *ModuleKarma += unsigned(intptr_t(m.pull())); + state.owner = true; + } + if ( state.owner ) Module::Initialize(); + } + + inline void standaloneReleaseModules () { + auto & state = standaloneModuleState(); + if ( --state.live != 0 || !state.owner ) return; + state.owner = false; + Module::ShutdownStandalone(false); + } + + // The first base of every generated context that links C++ modules: constructed before the + // Context base, destroyed after it, so the registry outlives every context's teardown. + struct StandaloneModuleScope { + StandaloneModuleScope () { standaloneAcquireModules(); } + ~StandaloneModuleScope () { standaloneReleaseModules(); } + StandaloneModuleScope ( const StandaloneModuleScope & ) = delete; + StandaloneModuleScope & operator = ( const StandaloneModuleScope & ) = delete; + }; +} diff --git a/install/CLAUDE.md b/install/CLAUDE.md index 12efafe746..d741df0ea9 100644 --- a/install/CLAUDE.md +++ b/install/CLAUDE.md @@ -53,7 +53,7 @@ Task-specific instructions are in skill files under `skills/`. Read the relevant | `skills/mcp_tools.md` | Full MCP tool table + live-API reference | | `skills/das_formatting.md` | Creating or modifying any `.das` file | | `skills/comment_style_hygiene.md` | Writing or reviewing comments, names, or local code shape in ANY language | -| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`) | +| `skills/cpp_integration.md` | Embedding daslang in C++; binding types/functions/enums; shipping without the compiler (`libDaScriptNano`, or a standalone context on the full runtime) | | `skills/daslib_modules.md` | Using `daslib/` modules (linq, json, regex, etc.) | | `skills/das_macros.md` | Compile-time macros, AST manipulation, qmacro/quote, gc_node patterns | | `skills/daspkg.md` | Creating `.das_package` manifests, daspkg commands | diff --git a/nano/README.md b/nano/README.md index 93d2a58607..3688116850 100644 --- a/nano/README.md +++ b/nano/README.md @@ -41,6 +41,10 @@ cortex-m4 with `-Os --gc-sections` and newlib. | `05_compile_time_table` | a CSV baked into a table by the compiler | 144 KB | 78,028 | | full-runtime standalone context, for scale | | 472 KB | - | +`06_full_runtime` beside those five is not a nano program: the same `-ctx` output +linked against the full runtime, because its script reaches C++ modules (dasHV, fio). +Its README says what that keeps and what it costs; it is not in the table above. + The x64 column includes the platform's C runtime, so what it measures is the difference nano makes on a host, not an embedded footprint. The cortex-m4 column is the real one: of `01_pure`'s 75,540 bytes, roughly 29 KB is the nano runtime, diff --git a/skills/cpp_integration.md b/skills/cpp_integration.md index 2a53276368..da8c8f7712 100644 --- a/skills/cpp_integration.md +++ b/skills/cpp_integration.md @@ -315,6 +315,34 @@ compiler, and cross-compiling for bare metal still needs portability work in the `nano/README.md` is the build recipe and `nano/ARCHITECTURE.md` lists what nano trades away. +## A standalone context on the full runtime + +**The same `-ctx` generation (`daslang utils/aot/main.das -- -ctx script.das out/`) links +against the full runtime when the script reaches a C++ module beyond the `builtin` module +itself** - dasHV, fio, or any module the script calls a function from or names a type from. Link +`libDaScript`, which already carries `builtin`, `math`, `strings`, `fio` and the rest of the +builtin set, plus the static archive of each module outside it (dasHV is `libDasModuleHV`), and +add the include directory its AOT header pulls in (dasHV's pulls libhv's headers). The binary +compiles nothing at run time - it parses no source and loads no shared module - but unlike nano +it links the full `libDaScript`. + +**The generated context owns module registration.** Handled types resolve through the module +registry, so the first context constructed registers the modules it links, and the last +context destroyed shuts them down with `Module::ShutdownStandalone`, not `Module::Shutdown` - +the latter also resets the interpreter's fused-node tables, and no interpreter is linked. Keep +contexts inside `main`: a context that outlives it would shut the registry down from a static +destructor, after the runtime's own statics are gone. The embedder constructs the context and +makes no `Module::Initialize()` / `Module::Shutdown()` call. A host that registered the builtin module owns the registry instead: it registers every +module the context links and calls `Module::Initialize()` before constructing it, and calls +`Module::Shutdown()` itself; a module it did not register stops the program at construction, +by name - the runtime cannot add to an initialized registry and keep `Initialize`/`Shutdown` +balanced. + +**Only what the program reaches is linked** - a module the script used at compile time alone is +neither included nor registered. Worked example: `examples/standalone/06_full_runtime/` - read +it for the shape; building it needs the daslang repository, since the bundle carries no dasHV +headers or archive. The recipe above works from a bundle for any C++ module you build yourself. + ## Diagnostics - `TextPrinter`, never `fprintf(stderr, ...)` ```cpp diff --git a/skills/internal/writing_cpp_tests.md b/skills/internal/writing_cpp_tests.md index df1f7bd599..608485d7cb 100644 --- a/skills/internal/writing_cpp_tests.md +++ b/skills/internal/writing_cpp_tests.md @@ -64,7 +64,7 @@ add_dependencies(test-big my_stress_test) Big tests that are C++ executables **don't include doctest** - they keep their own `int main()` returning 0/1 to ctest. -**A big test's `int main()` owns its module lifetime** - nothing runs `doctest_main.cpp` for it. An exe that resolves modules through the registry calls `Module::Initialize()` / `Module::Shutdown()` itself. An exe whose only context comes from a generated standalone AOT constructor calls neither: that constructor is self-contained and consults no module registry (example: `big/standalone_ctx/`). +**A big test's `int main()` owns its module lifetime** - nothing runs `doctest_main.cpp` for it. An exe that resolves modules through the registry calls `Module::Initialize()` / `Module::Shutdown()` itself. An exe whose only context comes from a generated standalone AOT constructor calls neither: a context that reaches no C++ module beyond the builtin one consults no registry (example: `big/standalone_ctx/test_standalone_ctx.cpp`), and one that does registers what it links itself and shuts it down when the last context is destroyed (example: `examples/standalone/06_full_runtime/`, which doubles as a small-lane test; `big/standalone_ctx/test_standalone_modules.cpp` puts two such contexts in one binary). A test that registered the builtin module - `NEED_ALL_DEFAULT_MODULES` or `NEED_MODULE(Module_BuiltIn)` - owns the registry: it registers every module the context links and calls `Module::Initialize()` before constructing it; a module it missed stops the program at construction, by name (`standalone_modules_host_partial` pins that as a `WILL_FAIL` test). A big-labelled test is not gated by CI - only the small suite runs there. Run `ninja test-big` locally before pushing one. diff --git a/tests-cpp/big/standalone_ctx/CMakeLists.txt b/tests-cpp/big/standalone_ctx/CMakeLists.txt index 561f4bd78b..4879c2f353 100644 --- a/tests-cpp/big/standalone_ctx/CMakeLists.txt +++ b/tests-cpp/big/standalone_ctx/CMakeLists.txt @@ -35,3 +35,52 @@ add_test(NAME standalone_ctx COMMAND test_standalone_ctx WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) set_tests_properties(standalone_ctx PROPERTIES LABELS "big") add_dependencies(test-big test_standalone_ctx) + +# Two contexts with different C++ module sets in one binary - the dasHV + fio example and a +# fio-only fixture - share one module registry lifetime. Small-labelled: a second owner of the +# shutdown, or an unbalanced Initialize, fails at exit and nothing else would notice. +if(NOT DAS_HV_DISABLED) + set(STANDALONE_MODULES_GENERATED) + foreach(_das "${PROJECT_SOURCE_DIR}/examples/standalone/06_full_runtime/service_probe.das" + "${CMAKE_CURRENT_SOURCE_DIR}/standalone_modules_fixture.das") + get_filename_component(_das_name "${_das}" NAME) + add_custom_command( + OUTPUT "${STANDALONE_CTX_GEN}/${_das_name}.cpp" "${STANDALONE_CTX_GEN}/${_das_name}.h" + COMMAND $ + "${PROJECT_SOURCE_DIR}/utils/aot/main.das" + -- -ctx "${_das}" "${STANDALONE_CTX_GEN}/" + DEPENDS daslang + "${_das}" + "${PROJECT_SOURCE_DIR}/utils/aot/main.das" + "${PROJECT_SOURCE_DIR}/daslib/aot_standalone.das" + "${PROJECT_SOURCE_DIR}/daslib/aot_cpp.das" + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + COMMENT "Standalone AOT: ${_das_name}" + VERBATIM + ) + list(APPEND STANDALONE_MODULES_GENERATED "${STANDALONE_CTX_GEN}/${_das_name}.cpp") + endforeach() + + add_executable(test_standalone_modules test_standalone_modules.cpp ${STANDALONE_MODULES_GENERATED}) + target_link_libraries(test_standalone_modules PRIVATE + libDaScript ${SRC_LIBRARIES} libDasModuleHV) + target_include_directories(test_standalone_modules PRIVATE "${STANDALONE_CTX_GEN}" ${NEED_MODULES_PATH} + "${PROJECT_SOURCE_DIR}/modules/dasHV/hv/$/include") + SETUP_CPP11(test_standalone_modules) + set_target_properties(test_standalone_modules PROPERTIES FOLDER "tests-cpp/big") + + add_test(NAME standalone_modules COMMAND test_standalone_modules + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) + # the host registered every module first: the generated code registers nothing and owns + # no shutdown, so the host's own Initialize/Shutdown pair stays balanced + add_test(NAME standalone_modules_host COMMAND test_standalone_modules --host-registers + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) + # the host registered the builtin set but not dasHV: the generated code must stop the + # program by name rather than initialize a second time and shut down a registry twice + add_test(NAME standalone_modules_host_partial COMMAND test_standalone_modules --host-partial + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) + set_tests_properties(standalone_modules_host_partial PROPERTIES WILL_FAIL TRUE) + set_tests_properties(standalone_modules standalone_modules_host standalone_modules_host_partial + PROPERTIES LABELS "small") + add_dependencies(test-small test_standalone_modules) +endif() diff --git a/tests-cpp/big/standalone_ctx/standalone_modules_fixture.das b/tests-cpp/big/standalone_ctx/standalone_modules_fixture.das new file mode 100644 index 0000000000..742b7d18b2 --- /dev/null +++ b/tests-cpp/big/standalone_ctx/standalone_modules_fixture.das @@ -0,0 +1,12 @@ +options gen2 + +// A second context in the same binary as the dasHV example, reaching a different C++ module +// set (fio only): the two generated TUs add their module tables to one process-wide list, the +// first constructor registers the union, and one shutdown runs at exit. + +require daslib/fio + +[export] +def has_path_variable : bool { + return has_env_variable("PATH") +} diff --git a/tests-cpp/big/standalone_ctx/test_standalone_modules.cpp b/tests-cpp/big/standalone_ctx/test_standalone_modules.cpp new file mode 100644 index 0000000000..a55cc64b3b --- /dev/null +++ b/tests-cpp/big/standalone_ctx/test_standalone_modules.cpp @@ -0,0 +1,70 @@ +// Two generated standalone contexts with different C++ module sets in one binary: the dasHV +// + fio example and a fio-only fixture. Each TU adds its modules before main; the first context +// constructed registers the union and initializes once; the last one destroyed shuts down. A +// second owner would shut the registry down twice, and an unbalanced Initialize trips the exit +// audit - both fail this test's exit code. +// +// test_standalone_modules the generated code owns the registry lifetime +// test_standalone_modules --host-registers main registers every module first and shuts +// down itself; the generated code registers nothing +// test_standalone_modules --host-partial main registers the builtin set but not dasHV: +// the generated code stops the program by name +// instead of taking a second registry lifetime +// test_standalone_modules --child the child process run_child spawns + +#include "daScript/daScript.h" +#include "daScript/daScriptModule.h" +#include "service_probe.das.h" +#include "standalone_modules_fixture.das.h" + +#include +#include + +using namespace das; + +DECLARE_MODULE(Module_HV); + +static int run_contexts ( char * self ) { + int failures = 0; + standalone_modules_fixture::Standalone fio_only; + if ( !fio_only.has_path_variable() ) { + printf("has_path_variable() = false, expected true\n"); + failures ++; + } + service_probe::Standalone probe; + const int status = probe.http_status((char *)"http://127.0.0.1:1/"); + if ( status != -1 ) { + printf("http_status(dead port) = %d, expected -1\n", status); + failures ++; + } + const int code = probe.run_child(self, (char *)"--child"); + if ( code != 7 ) { + printf("run_child() = %d, expected 7\n", code); + failures ++; + } + return failures; +} + +int main ( int argc, char * argv[] ) { + if ( argc > 1 && strcmp(argv[1], "--child") == 0 ) { + printf("child\n"); + return 7; + } + const bool host_registers = argc > 1 && strcmp(argv[1], "--host-registers") == 0; + const bool host_partial = argc > 1 && strcmp(argv[1], "--host-partial") == 0; + if ( host_registers || host_partial ) { + NEED_ALL_DEFAULT_MODULES; + NEED_MODULE(Module_UriParser); + NEED_MODULE(Module_JobQue); + if ( host_registers ) { + NEED_MODULE(Module_HV); + } + Module::Initialize(); + } + const int failures = run_contexts(argv[0]); + if ( host_registers || host_partial ) { + Module::Shutdown(); + } + printf(failures ? "standalone_modules: %d failure(s)\n" : "standalone_modules: ok\n", failures); + return failures ? 1 : 0; +} diff --git a/tests/aot/_standalone_dead_fio_fixture.das b/tests/aot/_standalone_dead_fio_fixture.das new file mode 100644 index 0000000000..9bc132243f --- /dev/null +++ b/tests/aot/_standalone_dead_fio_fixture.das @@ -0,0 +1,18 @@ +options gen2 +// the dead function must survive into the AST the emitter walks: with symbol removal on it is +// gone before emission and the case proves nothing +options remove_unused_symbols = false + +// only a DEAD function reaches fio_core - through its FILE? parameter and an extern call in its +// body - so the module stays compile-time-only and nothing registers + +require daslib/fio + +def dead_helper(f : FILE?) : bool { + return f == null && has_env_variable("PATH") +} + +[export] +def answer : int { + return 42 +} diff --git a/tests/aot/_standalone_fio_fixture.das b/tests/aot/_standalone_fio_fixture.das new file mode 100644 index 0000000000..52f2445f45 --- /dev/null +++ b/tests/aot/_standalone_fio_fixture.das @@ -0,0 +1,11 @@ +options gen2 + +// reaches fio_core through an extern call and nothing else - the module must be linked and +// registered, and its AOT header included + +require daslib/fio + +[export] +def has_path : bool { + return has_env_variable("PATH") +} diff --git a/tests/aot/_standalone_handle_fixture.das b/tests/aot/_standalone_handle_fixture.das new file mode 100644 index 0000000000..9248e86ee4 --- /dev/null +++ b/tests/aot/_standalone_handle_fixture.das @@ -0,0 +1,10 @@ +options gen2 + +// reaches fio_core only through a handled type in a signature - no extern is called + +require daslib/fio + +[export] +def is_open(f : FILE?) : bool { + return f != null +} diff --git a/tests/aot/test_standalone_emit.das b/tests/aot/test_standalone_emit.das index 09167aea5f..2f6e050b36 100644 --- a/tests/aot/test_standalone_emit.das +++ b/tests/aot/test_standalone_emit.das @@ -3,8 +3,11 @@ options no_aot options stack = 1_048_576 require daslib/aot_standalone +require daslib/aot_cpp require daslib/ast_boost +require daslib/strings_boost require daslib/fio +require math require daslib/rtti require strings require dastest/testing_boost public @@ -221,5 +224,98 @@ def test_standalone_emit(t : T?) { // nolint:STYLE038 - flat list of emit subc remove(path_join(out_dir, "_standalone_novars_fixture.das.h")) } + t |> run("a context reaching no C++ module beyond builtin registers nothing") @(t : T?) { + let files = generate_standalone_files(t, "_standalone_novars_fixture", out_dir) + t |> success(find(files.header, "standalone_modules.h") < 0, "no registry header for the registry-free tier") + t |> success(find(files.header, "class Standalone : public Context \{") >= 0, "the context derives from Context alone") + t |> success(find(files.source, "DECLARE_MODULE") < 0, "no module is declared") + } + + t |> run("an extern call links and registers its module, builtin modules first") @(t : T?) { + let files = generate_standalone_files(t, "_standalone_fio_fixture", out_dir) + let source = files.source + t |> success(find(source, " // require fio_core\n") >= 0, "fio_core is linked") + t |> success(find(source, "fio_core - compile time only") < 0, "fio_core is not pruned") + t |> success(find(source, "aot_builtin_fio.h") >= 0, "the fio AOT header is included") + t |> success(find(files.header, "standalone_modules.h") >= 0, "the header includes the registry") + t |> success(find(files.header, "class Standalone : public StandaloneModuleScope, public Context \{") >= 0, "the module scope is the first base, ahead of Context") + t |> success(find(source, "DECLARE_MODULE(Module_FIO);") >= 0, "Module_FIO is declared") + let builtin_row = find(source, "\{ \"$\", &::register_Module_BuiltIn, 0 \}") + let strings_row = find(source, "&::register_Module_Strings") + let fio_row = find(source, "&::register_Module_FIO") + t |> success(builtin_row >= 0, "the builtin module is a row at rank 0") + t |> success(strings_row > builtin_row && fio_row > strings_row, "strings registers before fio_core, whose constructor requires it by name") + t |> equal(brace_balance(source), 0, "unbalanced braces in the generated C++") + } + + t |> run("a handled type alone links its module") @(t : T?) { + let source = generate_standalone_source(t, "_standalone_handle_fixture", out_dir) + t |> success(find(source, " // require fio_core\n") >= 0, "fio_core is linked for the FILE handle") + t |> success(find(source, "DECLARE_MODULE(Module_FIO);") >= 0, "Module_FIO is declared") + } + + t |> run("a module only dead code reaches stays compile-time-only") @(t : T?) { + let files = generate_standalone_files(t, "_standalone_dead_fio_fixture", out_dir) + t |> success(find(files.source, "fio_core - compile time only") >= 0, "fio_core is pruned: only an unused function names FILE? and calls its extern") + t |> success(find(files.source, "DECLARE_MODULE") < 0, "no module is declared") + t |> success(find(files.header, "StandaloneModuleScope") < 0, "the context has no module scope") + } + + t |> run("the function table has one row per emitted function, numbered from zero") @(t : T?) { + let source = generate_standalone_source(t, "_standalone_fio_fixture", out_dir) + let total_pos = find(source, "context.totalFunctions = ") + t |> success(total_pos >= 0, "totalFunctions is emitted") + let digits_from = total_pos + length("context.totalFunctions = ") + let total = to_int(slice(source, digits_from, find(source, "/*totalFunctions*/;", digits_from))) + let rows = length(split(source, ", FunctionInfo(")) - 1 + t |> equal(rows, total, "one row per slot") + t |> success(find(source, "\{0, FunctionInfo(") >= 0, "rows are numbered from zero") + t |> success(find(source, "\{{total}, FunctionInfo(") < 0, "no row past the table") + } + + t |> run("registration lists the builtin modules first with non-decreasing ranks") @(t : T?) { + let input = path_join(get_das_root(), "tests/aot/_standalone_fio_fixture.das") + var names : array + var ranks : array + using() $(var cop : CodeOfPolicies) { + standalone_cop(cop) + ast_gc_guard() { + standalone_aot(input, out_dir, false, false, cop) $(var program : ProgramPtr) { + for (entry in standaloneModuleRegistration(program)) { + names |> push(string(entry.mod.name)) + ranks |> push(entry.rank) + } + } + } + } + t |> success(length(names) >= 2, "the fixture registers more than the builtin module") + t |> success(!empty(names) && names[0] == "$" && ranks[0] == 0, "the builtin module is first, at rank 0") + t |> success((names |> find_index("strings")) < (names |> find_index("fio_core")), "strings precedes fio_core") + for (i in range(1, length(ranks))) { + t |> success(ranks[i] >= ranks[i - 1], "ranks never decrease") + } + remove(path_join(out_dir, "_standalone_fio_fixture.das.cpp")) + remove(path_join(out_dir, "_standalone_fio_fixture.das.h")) + } + + t |> run("DEFAULT_MODULE_ORDER mirrors register_builtin_modules_impl") @(t : T?) { + var registrar : array + fopen(path_join(get_das_root(), "src/builtin/modules.cpp"), "rb") $(fr) { + if (fr != null) { + for (line in split(fread(fr), "\n")) { + let at = find(line, "NEED_MODULE(") + if (at >= 0) { + registrar |> push(slice(line, at + length("NEED_MODULE("), find(line, ")"))) + } + } + } + } + t |> success(!empty(registrar), "src/builtin/modules.cpp lists the builtin registrations") + t |> equal(length(registrar), length(DEFAULT_MODULE_ORDER), "the same number of modules on both sides") + for (i in range(min(length(registrar), length(DEFAULT_MODULE_ORDER)))) { + t |> equal(registrar[i], DEFAULT_MODULE_ORDER[i], "the same module at position {i}") + } + } + rmdir(out_dir) }