Context finds functions and globals by name and by mangled hash through sealed perfect-hash tables; standalone exes and AOT contexts adopt emitter-sealed constant data

CMakeLists.txt

@@ -1154,6 +1154,8 @@ include/daScript/simulate/simulate_visit.h
 include/daScript/simulate/simulate_visit_op.h
 include/daScript/simulate/simulate_visit_op_undef.h
 include/daScript/simulate/sim_policy.h
+include/daScript/simulate/name_lookup.h
+src/simulate/name_lookup.cpp
 src/simulate/data_walker.cpp
 include/daScript/simulate/data_walker.h
 src/simulate/debug_print.cpp
@@ -1767,6 +1769,7 @@ SET(DAS_SIMULATE_INCLUDES
     include/daScript/simulate/simulate_fusion_op2_set_perm.h
     include/daScript/simulate/simulate_fusion_op2_vec_settings.h
     include/daScript/simulate/simulate.h
+    include/daScript/simulate/name_lookup.h
     include/daScript/simulate/simulate_nodes.h
     include/daScript/simulate/simulate_visit.h
     include/daScript/simulate/simulate_visit_op.h
@@ -1861,6 +1864,7 @@ install(FILES
     ${PROJECT_SOURCE_DIR}/src/simulate/debug_info.cpp
     ${PROJECT_SOURCE_DIR}/src/simulate/escape_string.cpp
     ${PROJECT_SOURCE_DIR}/src/simulate/heap.cpp
+    ${PROJECT_SOURCE_DIR}/src/simulate/name_lookup.cpp
     ${PROJECT_SOURCE_DIR}/src/simulate/runtime_array.cpp
     ${PROJECT_SOURCE_DIR}/src/simulate/runtime_iterator.cpp
     ${PROJECT_SOURCE_DIR}/src/simulate/runtime_table.cpp

daslib/aot_standalone.das

@@ -150,14 +150,81 @@ def private writeGlobalVarInfos(var helper : AotDebugInfoHelper?; infos : array<
     }
 }
 
+//! the sealed lookup, written as the constant arrays the generated constructor adopts: a NameLookup::StaticTable
+//! named `<prefix>_table` and the four arrays it points at
+def private writeLookupTables(var tw : StringBuilderWriter; prefix : string; lookup : NameLookup?) {
+    write(tw, "    static const NameLookup::Entry {prefix}_entries[] = \{\n")
+    for (i in urange(name_lookup_mnh_slots(lookup))) {
+        write(tw, "        \{0x{name_lookup_entry_mnh(lookup, i):x}ull, {name_lookup_entry_value(lookup, i):d}u, {name_lookup_entry_index(lookup, i):d}u, {name_lookup_entry_next(lookup, i)}, 0\},\n")
+    }
+    write(tw, "    \};\n")
+    write(tw, "    static const NameLookup::NameSlot {prefix}_names[] = \{\n")
+    for (i in urange(name_lookup_name_slots(lookup))) {
+        write(tw, "        \{0x{name_lookup_name_hash(lookup, i):x}ull, {name_lookup_name_head(lookup, i)}, 0\},\n")
+    }
+    write(tw, "    \};\n")
+    write(tw, "    static const uint32_t {prefix}_mnh_disp[] = \{")
+    for (i in urange(name_lookup_mnh_buckets(lookup))) {
+        write(tw, " {name_lookup_mnh_disp(lookup, i):d}u,")
+    }
+    write(tw, " \};\n")
+    write(tw, "    static const uint32_t {prefix}_name_disp[] = \{")
+    for (i in urange(name_lookup_name_buckets(lookup))) {
+        write(tw, " {name_lookup_name_disp(lookup, i):d}u,")
+    }
+    write(tw, " \};\n")
+    write(tw, "    static const NameLookup::StaticTable {prefix}_table = \{ {name_lookup_mnh_buckets(lookup):d}u, {name_lookup_mnh_slots(lookup):d}u, {name_lookup_name_buckets(lookup):d}u, {name_lookup_name_slots(lookup):d}u, {name_lookup_count(lookup):d}u, 0u, {prefix}_mnh_disp, {prefix}_name_disp, {prefix}_entries, {prefix}_names \};\n")
+}
+
+//! the standalone context's function lookup: its table is dense, one slot per emitted function in emission
+//! order, so the index is the position in `usedFunctions`, never the compiler's function index
+def private writeFunctionLookup(var tw : StringBuilderWriter; usedFunctions : array<Function?>) {
+    var lookup = unsafe(name_lookup_create())
+    var index = 0u
+    for (pfun in usedFunctions) {
+        name_lookup_insert(lookup, pfun.getMangledNameHash, string(pfun.name), index, index)
+        index++
+    }
+    name_lookup_seal(lookup)
+    writeLookupTables(tw, "fn_lookup", lookup)
+    unsafe {
+        name_lookup_destroy(lookup)
+    }
+}
+
+//! the standalone context's global lookup: the value is the offset InitGlobalVariable assigns at construction -
+//! the 16-byte-rounded running size of the shared or the plain globals, in declaration order - and the
+//! generated constructor verifies every global against it, so the two computations cannot drift apart silently
+def private writeVariableLookup(var tw : StringBuilderWriter; globals : array<Variable?>) {
+    var lookup = unsafe(name_lookup_create())
+    var globalsSize = 0u
+    var sharedSize = 0u
+    for (pvar in globals) {
+        let size = (uint(pvar._type.sizeOf) + 15u) & ~15u
+        let offset = pvar.flags.global_shared ? sharedSize : globalsSize
+        if (pvar.flags.global_shared) {
+            sharedSize += size
+        } else {
+            globalsSize += size
+        }
+        name_lookup_insert(lookup, hash(pvar |> get_mangled_name()), string(pvar.name), uint(pvar.index), offset)
+    }
+    name_lookup_seal(lookup)
+    writeLookupTables(tw, "var_lookup", lookup)
+    unsafe {
+        name_lookup_destroy(lookup)
+    }
+}
+
 def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var tw : StringBuilderWriter, program : ProgramPtr; lookupVariableTable : array<Variable?>; var context : Context) {   // nolint:STYLE038 - flat emitter - one write per ctor section
 
     let min_init_stack = 16384
     let stack_arg = program._options |> find_arg("stack")
     let requested_stack = stack_arg ?as tInt ?? int(program.policies.stack)
     let stack_base = requested_stack > 0 && stack_arg is tInt ? requested_stack : max(requested_stack, min_init_stack)
     let stack_size = stack_base + int(program.globalInitStackSize)
-    let usedFunctionCount = length(collectProgramUsedFunctions(program, true, false))
+    let usedFunctions <- collectProgramUsedFunctions(program, true, false)
+    let usedFunctionCount = length(usedFunctions)
     write(tw, "{cfg.class_name}::{cfg.class_name}() : Context({stack_size}/*stack*/) \{\n");
     write(tw, "    auto & context = *this;\n");
     write(tw, "    CodeOfPolicies policies;");
@@ -192,8 +259,9 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var
     write(tw, "/*(policies.threadlock_context || policies.debugger)*/ ) \{\n");
     write(tw, "        context.contextMutex = new recursive_mutex;\n");
     write(tw, "    }\n");
-    write(tw, "    context.tabMnLookup = make_shared<das_hash_map<uint64_t,SimFunction *>>();\n");
-    write(tw, "    context.tabMnLookup->clear();\n");
+    writeFunctionLookup(tw, usedFunctions)
+    write(tw, "    context.functionLookup = make_shared<NameLookup>();\n");
+    write(tw, "    context.functionLookup->adopt(fn_lookup_table);\n");
 
     if (!empty(initFunctions)) {
         write(tw, "     // start totalFunctions\n");
@@ -206,17 +274,18 @@ def writeStandaloneCtor(cfg : StandaloneContextCfg; initFunctions : string; var
         write(tw, "    for (const auto& [index, func_info, debug_info]: usedFunctions) \{\n");
         write(tw, "        InitAotFunction(context, &context.functions[index], func_info);\n");
         write(tw, "        context.functions[index].debugInfo = debug_info;\n");
-        write(tw, "        (*context.tabMnLookup)[func_info.mnh] = context.functions + index;\n");
+        write(tw, "        DAS_VERIFYF(context.functionLookup->valueByMnh(func_info.mnh)==uint32_t(index), \"standalone function '%s' sits at index %d, its emitted lookup says %u\", func_info.name.c_str(), int(index), context.functionLookup->valueByMnh(func_info.mnh));\n");
         write(tw, "        id_to_funcs.emplace_back(func_info.aotHash, &context.functions[index]);\n");
         write(tw, "        anyPInvoke |= func_info.pinvoke;\n");
         write(tw, "    \}\n");
     }
 
-    write(tw, "    context.tabGMnLookup = make_shared<das_hash_map<uint64_t,uint32_t>>();\n");
-    write(tw, "    context.tabGMnLookup->clear();\n");
+    writeVariableLookup(tw, lookupVariableTable)
+    write(tw, "    context.variableLookup = make_shared<NameLookup>();\n");
+    write(tw, "    context.variableLookup->adopt(var_lookup_table);\n");
     write(tw, "    for ( int i=0, is=context.totalVariables; i!=is; ++i ) \{\n");
-    write(tw, "        auto mnh = context.globalVariables[i].mangledNameHash;\n");
-    write(tw, "        (*context.tabGMnLookup)[mnh] = context.globalVariables[i].offset;\n");
+    write(tw, "        auto & gvar = context.globalVariables[i];\n");
+    write(tw, "        DAS_VERIFYF(context.variableLookup->valueByMnh(gvar.mangledNameHash)==gvar.offset, \"standalone global '%s' sits at offset %u, its emitted lookup says %u\", gvar.name, gvar.offset, context.variableLookup->valueByMnh(gvar.mangledNameHash));\n");
     write(tw, "    \}\n");
     write(tw, "    context.tabAdLookup = make_shared<das_hash_map<uint64_t,uint64_t>>();\n");
     program.get_ptr() |> for_each_module_no_order($(pm) {

doc/reflections/das2rst.das

@@ -454,6 +454,7 @@ def document_module_rtti(_root : string) {
         group_by_regex("Data walking and printing", mod, %regex~(sprint_data|sprint_json_at|sscan_json_at|describe|get_mangled_name)$%%),
         group_by_regex("Function and mangled name hash", mod, %regex~(get_function_by_mangled_name_hash|get_function_mangled_name_hash|get_function_address)$%%),
         group_by_regex("Context and mutex locking", mod, %regex~(lock_this_context|lock_context|lock_mutex)$%%),
+        group_by_regex("Name lookup builder", mod, %regex~name_lookup_.*%%),
         group_by_regex("Runtime data access", mod, %regex~(get_table_key_index)$%%),
         group_by_regex("Tuple and variant access", mod, %regex~(get_tuple_field_offset|get_variant_field_offset)$%%),
         group_by_regex("Lint suppression", mod, %regex~(rtti_get_source_line|rtti_is_nolint_suppressed|is_lint_suppressed|was_nolint_consumed|extract_lint_code)$%%),

doc/source/stdlib/handmade/annotation-rtti-NameLookup.rst

@@ -0,0 +1 @@
+Handled type wrapping the runtime's sealed function or global name lookup (``das::NameLookup``): a perfect-hash table over mangled-name hashes and plain names. The standalone emitters build one at code-generation time through the ``name_lookup_*`` functions and write its sealed arrays into the generated artifact as constant data.

doc/source/stdlib/handmade/function-rtti-name_lookup_count-0x9780e5f677f52ae5.rst

@@ -0,0 +1 @@
+Number of entries a sealed lookup holds.

doc/source/stdlib/handmade/function-rtti-name_lookup_create-0xdc119a2c4b28198f.rst

@@ -0,0 +1 @@
+Creates an empty ``NameLookup`` builder and returns a raw pointer the caller owns; release it with ``name_lookup_destroy``.

doc/source/stdlib/handmade/function-rtti-name_lookup_destroy-0x3fdb742796d6d4b.rst

@@ -0,0 +1 @@
+Deletes a ``NameLookup`` created by ``name_lookup_create``.

doc/source/stdlib/handmade/function-rtti-name_lookup_entry_index-0xa8e92693addbad91.rst

@@ -0,0 +1 @@
+Index stored in one entries slot - the position in the context's function or global array; ``0xffffffff`` in an empty slot.

doc/source/stdlib/handmade/function-rtti-name_lookup_entry_mnh-0x2dbf39242b1b1cfd.rst

@@ -0,0 +1 @@
+Mangled-name hash stored in one entries slot, for ``slot`` below ``name_lookup_mnh_slots``; zero in an empty slot.

doc/source/stdlib/handmade/function-rtti-name_lookup_entry_next-0xde41b4c5e481bd98.rst

@@ -0,0 +1 @@
+Entries slot of the next entry carrying the same plain name, ``-1`` at the end of the chain.

doc/source/stdlib/handmade/function-rtti-name_lookup_entry_value-0x8770c19fdc477823.rst

@@ -0,0 +1 @@
+Value stored in one entries slot - the function index or global byte offset a mangled-name probe answers; ``0xffffffff`` in an empty slot.

doc/source/stdlib/handmade/function-rtti-name_lookup_insert-0x1df6f40370eafa48.rst

@@ -0,0 +1 @@
+Stages one entry in an unsealed lookup: ``mnh`` is the mangled-name hash the runtime probes with, ``name`` the plain name (copied, so the string need not outlive the call), ``index`` the position in the context's function or global array, and ``value`` what a mangled-name probe answers - a function index, or a global's byte offset.

doc/source/stdlib/handmade/function-rtti-name_lookup_mnh_buckets-0x623f630fac05a289.rst

@@ -0,0 +1 @@
+Bucket count of a sealed lookup's mangled-name hash - the length of the displacement array ``name_lookup_mnh_disp`` reads.

doc/source/stdlib/handmade/function-rtti-name_lookup_mnh_disp-0x5f1d16a55ceea87c.rst

@@ -0,0 +1 @@
+Displacement of one bucket of the mangled-name hash, for ``bucket`` below ``name_lookup_mnh_buckets``.

doc/source/stdlib/handmade/function-rtti-name_lookup_mnh_slots-0xacc87eebdfa17415.rst

@@ -0,0 +1 @@
+Slot count of a sealed lookup's mangled-name hash - the length of the entries array the ``name_lookup_entry_*`` functions read.

doc/source/stdlib/handmade/function-rtti-name_lookup_name_buckets-0xc771560f6c131fb.rst

@@ -0,0 +1 @@
+Bucket count of a sealed lookup's plain-name hash - the length of the displacement array ``name_lookup_name_disp`` reads.

doc/source/stdlib/handmade/function-rtti-name_lookup_name_disp-0xaa40dba56bcd5663.rst

@@ -0,0 +1 @@
+Displacement of one bucket of the plain-name hash, for ``bucket`` below ``name_lookup_name_buckets``.

doc/source/stdlib/handmade/function-rtti-name_lookup_name_hash-0xaaa1df5fdee04233.rst

@@ -0,0 +1 @@
+Plain-name hash stored in one name slot, for ``slot`` below ``name_lookup_name_slots``; zero in an empty slot.

doc/source/stdlib/handmade/function-rtti-name_lookup_name_head-0x1db1092788d8cc73.rst

@@ -0,0 +1 @@
+Entries slot of the first entry whose plain name hashes into this name slot, ``-1`` when the slot is empty.

doc/source/stdlib/handmade/function-rtti-name_lookup_name_slots-0x843def8a75a15.rst

@@ -0,0 +1 @@
+Slot count of a sealed lookup's plain-name hash - the length of the name-slot array ``name_lookup_name_hash`` and ``name_lookup_name_head`` read.

doc/source/stdlib/handmade/function-rtti-name_lookup_seal-0xe859d3033156b01.rst

@@ -0,0 +1 @@
+Builds the two perfect hashes over every staged entry and freezes the lookup; panics when two entries share a mangled-name hash or two different names share a name hash, naming both entries.

include/daScript/simulate/ARCHITECTURE.md

@@ -49,6 +49,41 @@ reads as zero, so two such keys merge wherever they meet in the probe sequence.
 for the bit compare with the `0.0 == -0.0` tie, which its two distinct hashes never honoured
 anyway. `tests/language/table_vector_keys.das` covers both patterns.
 
+## Function and global lookup
+
+A `Context` finds a function or a global by mangled-name hash and by plain name through one
+object each, `functionLookup` and `variableLookup` (`name_lookup.h`, shared between a context and
+its forks and clones the way the functions array is). A simulated program builds them:
+`Program::buildMNLookup` / `buildGMNLookup` insert every entry and seal once, and the sealed blob is
+owned by the object and freed with it. A standalone exe and a standalone AOT context adopt them:
+the emitter (`modules/dasLLVM/daslib/llvm_exe.das` and `daslib/aot_standalone.das`, repo root)
+builds the same object at code-generation time through the `name_lookup_*` builtins of the
+`rtti_core` module, writes the sealed arrays into the artifact as constant data in the word layout
+the header pins, and the generated constructor hands that `StaticTable` to `adopt` - nothing is
+built or allocated at startup and the object owns nothing. The standalone C++ constructor also
+verifies every global's runtime offset against the emitted table, since the emitter computes those
+offsets with `InitGlobalVariable`'s rule rather than reading them back. An insert or an adopt after
+the seal stops the program, and a seal that finds two entries on one mangled-name hash, or two
+different names on one name hash, fails and names both entries - the same footing the runtime
+already gives every 64-bit string hash.
+
+The seal builds two perfect hashes (compress-hash-displace over the distinct keys, five keys per
+bucket, five percent empty slots), so a lookup is one probe and one 64-bit compare with no
+collision chain: `fnByMangledName` and `globalOffsetByMangledName` - the latter on the hot path of
+every global access by hash in all three tiers - read one entry, and a by-name lookup hashes the
+string, reads one slot, and walks the same-name chain the seal linked in function-index order.
+`findFunction(name, isUnique)` answers from the head's link, `findFunctions` is the walk, and a
+missing key of either kind answers `NOT_FOUND` / `-1` without touching a name string. The
+entry's `value` is what the hash probe hands back - a function index, a global's byte offset -
+and `index` is the position in `functions` / `globalVariables`, which is what the by-name API
+returns. Measured against `das_hash_map` on 8 000 keys the hash probe is 2.2 ns against 5.8 and
+the by-name probe 25 ns against 41, in one eighth and one half the memory; sealing both hashes
+for 8 000 entries takes single-digit milliseconds, paid once per `Program::simulate` and never at
+the startup of an artifact that adopts. A fresh object, and one whose seal failed, points at a
+static empty table and answers every probe with a miss, so a context that never simulated, a
+compile that failed at the seal, and a standalone exe between its creation and its adopt all
+answer `NOT_FOUND` rather than reading through a null pointer.
+
 ## Sanctioned hot-path additions
 
 The ledger the checklist's hot-path rules route to. Each entry: what was added, where, why

include/daScript/simulate/REVIEW.md

@@ -1,7 +1,8 @@
 # Simulate Headers Code Review Checklist
 
 **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc:
-`ARCHITECTURE.md`. A diff changing a `debug_info.h` struct layout applies
+`ARCHITECTURE.md`. A diff that changes a `debug_info.h` struct layout, or removes, renames or
+retypes a public member of a struct or class under this folder, applies
 `skills/internal/abi_break_sweep.md` too. A diff that changes or removes a name under this
 folder that a `daslib/*.das` file spells out - a struct or member the AOT C++ emitter writes
 into generated code, a flag or field a daslib predicate reads - applies `daslib/REVIEW.md`
@@ -30,14 +31,18 @@ checklist on its own.
   template under this folder that generated code runs for every evaluated expression. An added
   load, branch, call, copy, or counter, a direct call becoming indirect, a static dispatch
   becoming virtual, or an unboxed value becoming a boxed round-trip is that defect unless the
-  PR names the check showing the shipped build's codegen unchanged - the burden is the
-  author's, because a diff cannot show optimized codegen.
+  PR names the check showing the shipped build costs no more: its codegen unchanged, or a
+  measurement of the new code against the code it replaces - a diff cannot show optimized
+  codegen.
 
 - **A diff that adds work to the hot path - whether or not the shipped build flattens it -
   lands its entry under `ARCHITECTURE.md`'s sanctioned hot-path additions in the same diff:
   what was added, where, why correctness required it, and the alternative that was rejected.**
-  A change that costs more only under a relaxed-math or otherwise non-default compiler flag
-  states which flavor and how much in its PR description.
+  Replacing a hot-path body with code that performs the same per-evaluation operations - no
+  load, branch, call, copy, or counter the old body did not have - and measures no slower on
+  the build the repo ships is not added work. A body that gains one of those operations is
+  added work, even at no measured cost. A change that costs more only under a relaxed-math or
+  otherwise non-default compiler flag states which flavor and how much in its PR description.
 
 - **A diff that changes the layout of a `debug_info.h` struct - a field added, removed,
   reordered, or retyped, or a base changed - states a per-consumer verdict (updated / no

include/daScript/simulate/aot_builtin_rtti.h

@@ -89,6 +89,24 @@ namespace das {
     DAS_API vec4f rtti_contextVariableInfo ( Context & context, SimNode_CallBase *, vec4f * );
     DAS_API int32_t rtti_contextTotalFunctions(Context & context);
     DAS_API int32_t rtti_contextTotalVariables(Context & context);
+    struct NameLookup;
+    DAS_API NameLookup * rtti_name_lookup_create ();
+    DAS_API void rtti_name_lookup_destroy ( NameLookup * lookup );
+    DAS_API void rtti_name_lookup_insert ( NameLookup * lookup, uint64_t mnh, const char * name, uint32_t index, uint32_t value );
+    DAS_API void rtti_name_lookup_seal ( NameLookup * lookup, Context * context, LineInfoArg * at );
+    DAS_API uint32_t rtti_name_lookup_count ( NameLookup * lookup );
+    DAS_API uint32_t rtti_name_lookup_mnh_buckets ( NameLookup * lookup );
+    DAS_API uint32_t rtti_name_lookup_mnh_slots ( NameLookup * lookup );
+    DAS_API uint32_t rtti_name_lookup_name_buckets ( NameLookup * lookup );
+    DAS_API uint32_t rtti_name_lookup_name_slots ( NameLookup * lookup );
+    DAS_API uint32_t rtti_name_lookup_mnh_disp ( NameLookup * lookup, uint32_t bucket );
+    DAS_API uint32_t rtti_name_lookup_name_disp ( NameLookup * lookup, uint32_t bucket );
+    DAS_API uint64_t rtti_name_lookup_entry_mnh ( NameLookup * lookup, uint32_t slot );
+    DAS_API uint32_t rtti_name_lookup_entry_value ( NameLookup * lookup, uint32_t slot );
+    DAS_API uint32_t rtti_name_lookup_entry_index ( NameLookup * lookup, uint32_t slot );
+    DAS_API int32_t rtti_name_lookup_entry_next ( NameLookup * lookup, uint32_t slot );
+    DAS_API uint64_t rtti_name_lookup_name_hash ( NameLookup * lookup, uint32_t slot );
+    DAS_API int32_t rtti_name_lookup_name_head ( NameLookup * lookup, uint32_t slot );
 
     __forceinline Context  & thisContext ( Context * context ) { return *context; }