diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28532b0..25a381c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,8 @@ name: Push gem on: - push: - tags: - - "v*" + pull_request: + branches: ["main"] jobs: push: @@ -20,9 +19,5 @@ jobs: - uses: actions/checkout@v6 with: persist-credentials: false - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - ruby-version: ruby - - uses: rubygems/release-gem@v1 + + - run: ./zig/download.sh && ./zig/zig run scripts/tp_test.zig diff --git a/scripts/stdx/bit_set.zig b/scripts/stdx/bit_set.zig new file mode 100644 index 0000000..d3a9d1c --- /dev/null +++ b/scripts/stdx/bit_set.zig @@ -0,0 +1,157 @@ +const stdx = @import("stdx.zig"); +const std = @import("std"); +const assert = std.debug.assert; + +/// Use a dynamic bitset for larger sizes. +pub fn BitSetType(comptime with_capacity: u9) type { + assert(with_capacity <= 256); + + return struct { + // While mathematically 0 and 1 are symmetric, we intentionally bias the API to use zeros + // default, as zero-initialization reduces binary size. + bits: Word = 0, + + pub const Word = for (.{ u8, u16, u32, u64, u128, u256 }) |w| { + if (@bitSizeOf(w) >= with_capacity) break w; + } else unreachable; + + const BitSet = @This(); + + pub fn is_set(bit_set: BitSet, index: usize) bool { + assert(index < bit_set.capacity()); + return bit_set.bits & bit(index) != 0; + } + + pub fn count(bit_set: BitSet) usize { + return @popCount(bit_set.bits); + } + + pub inline fn capacity(_: BitSet) usize { + return with_capacity; + } + + pub fn full(bit_set: BitSet) bool { + return bit_set.count() == bit_set.capacity(); + } + + pub fn empty(bit_set: BitSet) bool { + return bit_set.bits == 0; + } + + pub fn first_set(bit_set: BitSet) ?usize { + if (bit_set.bits == 0) return null; + return @ctz(bit_set.bits); + } + + pub fn first_unset(bit_set: BitSet) ?usize { + const result = @ctz(~bit_set.bits); + return if (result < bit_set.capacity()) result else null; + } + + pub fn set(bit_set: *BitSet, index: usize) void { + assert(index < bit_set.capacity()); + bit_set.bits |= bit(index); + } + + pub fn unset(bit_set: *BitSet, index: usize) void { + assert(index < bit_set.capacity()); + bit_set.bits &= ~bit(index); + } + + pub fn set_value(bit_set: *BitSet, index: usize, value: bool) void { + if (value) { + bit_set.set(index); + } else { + bit_set.unset(index); + } + } + + fn bit(index: usize) Word { + assert(index < with_capacity); + return @as(Word, 1) << @intCast(index); + } + + pub fn iterate(bit_set: BitSet) Iterator { + return .{ .bits_remain = bit_set.bits }; + } + + pub const Iterator = struct { + bits_remain: Word, + + pub fn next(it: *@This()) ?usize { + const result = @ctz(it.bits_remain); + if (result >= with_capacity) return null; + it.bits_remain &= it.bits_remain - 1; + return result; + } + }; + }; +} + +test BitSetType { + var prng = stdx.PRNG.from_seed_testing(); + inline for (.{ 0, 1, 8, 32, 65, 255, 256 }) |N| { + const BitSet = BitSetType(N); + + var set: BitSet = .{}; + var model = try std.DynamicBitSetUnmanaged.initEmpty(std.testing.allocator, N); + defer model.deinit(std.testing.allocator); + + for (0..1000) |_| { + switch (prng.enum_uniform(std.meta.DeclEnum(BitSet))) { + .Word => { + const bit_size = + comptime if (N == 0) 8 else @max(8, try std.math.ceilPowerOfTwo(u16, N)); + assert(BitSet.Word == std.meta.Int(.unsigned, bit_size)); + }, + .Iterator => {}, + .is_set => { + if (N > 0) { + const bit = prng.int_inclusive(usize, N - 1); + assert(set.is_set(bit) == model.isSet(bit)); + } + }, + .count => assert(set.count() == model.count()), + .capacity => assert(set.capacity() == N), + .full => assert(set.full() == (model.count() == N)), + .empty => assert(set.empty() == (model.count() == 0)), + .first_set => assert(set.first_set() == model.findFirstSet()), + .first_unset => { + var it = model.iterator(.{ .kind = .unset }); + assert(set.first_unset() == it.next()); + }, + .set => { + if (N > 0) { + const bit = prng.int_inclusive(usize, N - 1); + set.set(bit); + model.set(bit); + } + }, + .unset => { + if (N > 0) { + const bit = prng.int_inclusive(usize, N - 1); + set.unset(bit); + model.unset(bit); + } + }, + .set_value => { + if (N > 0) { + const bit = prng.int_inclusive(usize, N - 1); + const value = prng.boolean(); + set.set_value(bit, value); + model.setValue(bit, value); + } + }, + .iterate => { + var it_set = set.iterate(); + var it_model = model.iterator(.{}); + while (it_model.next()) |next| { + assert(next == it_set.next()); + } + assert(it_set.next() == null); + assert(it_set.next() == null); + }, + } + } + } +} diff --git a/scripts/stdx/bounded_array.zig b/scripts/stdx/bounded_array.zig new file mode 100644 index 0000000..f5d6d3a --- /dev/null +++ b/scripts/stdx/bounded_array.zig @@ -0,0 +1,292 @@ +const std = @import("std"); +const stdx = @import("stdx.zig"); +const assert = std.debug.assert; + +/// A version of standard `BoundedArray` with TigerBeetle-idiomatic APIs. +pub fn BoundedArrayType(comptime T: type, comptime buffer_capacity: usize) type { + return struct { + buffer: [buffer_capacity]T = undefined, + // Its not clear whether the best type of count is u32, usize, or u`log(buffer_capacity)`. + // Use an ugly internal name here and expose `pub fn count` as an API. + count_u32: u32 = 0, + + const BoundedArray = @This(); + + pub inline fn from_slice(items: []const T) error{Overflow}!BoundedArray { + if (items.len <= buffer_capacity) { + var result: BoundedArray = .{}; + result.push_slice(items); + return result; + } else { + return error.Overflow; + } + } + + pub inline fn count(array: *const BoundedArray) usize { + return array.count_u32; + } + + /// Returns count of elements in this BoundedArray in the specified integer types, + /// checking at compile time that it indeed can represent the length. + pub inline fn count_as(array: *const BoundedArray, comptime Int: type) Int { + comptime assert(buffer_capacity <= std.math.maxInt(Int)); + return @intCast(array.count_u32); + } + + pub inline fn full(array: BoundedArray) bool { + return array.count_u32 == buffer_capacity; + } + + pub inline fn empty(array: BoundedArray) bool { + return array.count_u32 == 0; + } + + pub inline fn get(array: *const BoundedArray, index: usize) T { + assert(index < array.count_u32); + return array.buffer[index]; + } + + pub inline fn slice(array: *BoundedArray) []T { + return array.buffer[0..array.count_u32]; + } + + pub inline fn const_slice(array: *const BoundedArray) []const T { + return array.buffer[0..array.count_u32]; + } + + pub inline fn unused_capacity_slice(array: *BoundedArray) []T { + return array.buffer[array.count_u32..]; + } + + pub fn insert_at(array: *BoundedArray, index: usize, item: T) void { + assert(!array.full()); + assert(index <= array.count_u32); + stdx.copy_right( + .exact, + T, + array.buffer[index + 1 .. array.count_u32 + 1], + array.buffer[index..array.count_u32], + ); + array.buffer[index] = item; + array.count_u32 += 1; + } + + pub fn push(array: *BoundedArray, item: T) void { + assert(!array.full()); + array.buffer[array.count_u32] = item; + array.count_u32 += 1; + } + + pub fn push_slice(array: *BoundedArray, items: []const T) void { + assert(array.count_u32 + items.len <= array.capacity()); + stdx.copy_disjoint(.inexact, T, array.buffer[array.count_u32..], items); + array.count_u32 += @intCast(items.len); + } + + pub inline fn swap_remove(array: *BoundedArray, index: usize) T { + assert(array.count_u32 > 0); + assert(index < array.count_u32); + const result = array.buffer[index]; + array.count_u32 -= 1; + array.buffer[index] = array.buffer[array.count_u32]; + return result; + } + + pub inline fn ordered_remove(array: *BoundedArray, index: usize) T { + assert(array.count_u32 > 0); + assert(index < array.count_u32); + const result = array.buffer[index]; + stdx.copy_left( + .exact, + T, + array.buffer[index .. array.count_u32 - 1], + array.buffer[index + 1 .. array.count_u32], + ); + array.count_u32 -= 1; + return result; + } + + pub fn resize(array: *BoundedArray, count_new: usize) error{Overflow}!void { + if (count_new <= buffer_capacity) { + array.count_u32 = @intCast(count_new); + } else { + return error.Overflow; + } + } + + pub inline fn truncate(array: *BoundedArray, count_new: usize) void { + assert(count_new <= array.count_u32); + array.count_u32 = @intCast(count_new); // can't overflow due to check above. + } + + pub inline fn clear(array: *BoundedArray) void { + array.count_u32 = 0; + } + + pub inline fn pop(array: *BoundedArray) ?T { + if (array.count_u32 == 0) return null; + array.count_u32 -= 1; + return array.buffer[array.count_u32]; + } + + pub inline fn capacity(_: *BoundedArray) usize { + return buffer_capacity; + } + }; +} + +test BoundedArrayType { + const capacity = 8; + const Array = BoundedArrayType(u8, capacity); + const Model = std.ArrayListUnmanaged(u8); + const swarm_count = 10; + const action_count = 1_000; + + const gpa = std.testing.allocator; + + var array: Array = .{}; + var model: Model = try .initCapacity(gpa, capacity); + defer model.deinit(gpa); + + var prng = stdx.PRNG.from_seed_testing(); + + for (0..swarm_count) |_| { + const swarm_weights = prng.enum_weights(std.meta.DeclEnum(Array)); + for (0..action_count) |_| { + const action = prng.enum_weighted(std.meta.DeclEnum(Array), swarm_weights); + switch (action) { + .count => assert(array.count() == model.items.len), + .count_as => assert(array.count_as(u8) == model.items.len), + .full => assert(array.full() == (model.unusedCapacitySlice().len == 0)), + .empty => assert(array.empty() == (model.items.len == 0)), + .get => { + if (model.items.len > 0) { + const index = prng.index(model.items); + assert(array.get(index) == model.items[index]); + } + }, + .slice => assert(std.mem.eql(u8, array.slice(), model.items)), + .const_slice => assert(std.mem.eql(u8, array.const_slice(), model.items)), + .unused_capacity_slice => { + assert(array.unused_capacity_slice().len == model.unusedCapacitySlice().len); + }, + .insert_at => { + if (model.items.len < model.capacity) { + const index = prng.int_inclusive(usize, model.items.len); + const value = prng.int(u8); + + array.insert_at(index, value); + model.insertAssumeCapacity(index, value); + } + }, + .push => { + if (model.items.len < model.capacity) { + const value = prng.int(u8); + + array.push(value); + model.appendAssumeCapacity(value); + } + }, + .push_slice => { + var buffer: [capacity]u8 = undefined; + const count = prng.int_inclusive(usize, model.capacity - model.items.len); + for (0..count) |index| buffer[index] = prng.int(u8); + const slice = buffer[0..count]; + + array.push_slice(slice); + model.appendSliceAssumeCapacity(slice); + }, + .swap_remove => { + if (model.items.len > 0) { + const index = prng.index(model.items); + + const a = array.swap_remove(index); + const b = model.swapRemove(index); + assert(a == b); + } + }, + .ordered_remove => { + if (model.items.len > 0) { + const index = prng.index(model.items); + + const a = array.ordered_remove(index); + const b = model.orderedRemove(index); + assert(a == b); + } + }, + .resize => { + const count_old = model.items.len; + const count_new = prng.int_inclusive(usize, capacity); + + model.resize(gpa, count_new) catch unreachable; + array.resize(count_new) catch unreachable; + if (count_old <= count_new) { + for (count_old..count_new) |index| { + const value = prng.int(u8); + model.items[index] = value; + array.buffer[index] = value; + } + } + }, + .truncate => { + const count_new = prng.int_inclusive(usize, model.items.len); + array.truncate(count_new); + model.resize(gpa, count_new) catch unreachable; + }, + .clear => { + array.clear(); + model.clearRetainingCapacity(); + }, + .pop => { + const b = model.pop(); + const a = array.pop(); + assert((a == null and b == null) or (a.? == b.?)); + }, + .capacity => assert(array.capacity() == model.capacity), + .from_slice => { + var buffer: [capacity]u8 = undefined; + const count = prng.int_inclusive(usize, model.capacity - model.items.len); + for (0..count) |index| buffer[index] = prng.int(u8); + const slice = buffer[0..count]; + + array = Array.from_slice(slice) catch unreachable; + model.clearRetainingCapacity(); + model.appendSliceAssumeCapacity(slice); + }, + } + } + } +} + +test "BoundedArray.insert_at" { + const items_max = 32; + const BoundedArrayU64 = BoundedArrayType(u64, items_max); + + // Test lists of every size (less than the capacity). + for (0..items_max) |len| { + var list_base = BoundedArrayU64{}; + for (0..len) |i| { + list_base.push(i); + } + + // Test an insert at every possible position (including an append). + for (0..list_base.count() + 1) |i| { + var list = list_base; + + list.insert_at(i, 12345); + + // Verify the result: + + try std.testing.expectEqual(list.count(), list_base.count() + 1); + try std.testing.expectEqual(list.get(i), 12345); + + for (0..i) |j| { + try std.testing.expectEqual(list.get(j), j); + } + + for (i + 1..list.count()) |j| { + try std.testing.expectEqual(list.get(j), j - 1); + } + } + } +} diff --git a/scripts/stdx/debug.zig b/scripts/stdx/debug.zig new file mode 100644 index 0000000..5b68dd1 --- /dev/null +++ b/scripts/stdx/debug.zig @@ -0,0 +1,65 @@ +const std = @import("std"); + +/// Utility function for ad-hoc profiling. +/// +/// A thin wrapper around `std.time.Timer` which handles the boilerplate of +/// printing to stderr and formatting times in some (unspecified) readable way. +pub fn timeit() TimeIt { + return TimeIt{ .inner = std.time.Timer.start() catch unreachable }; +} + +const TimeIt = struct { + inner: std.time.Timer, + + /// Prints elapsed time to stderr and resets the internal timer. + pub fn print(self: *TimeIt, comptime label: []const u8) void { + const label_alignment = comptime " " ** (1 + (12 -| label.len)); + + const elapsed_ns = self.inner.lap(); + std.debug.print( + label ++ ":" ++ label_alignment ++ "{}\n", + .{std.fmt.fmtDuration(elapsed_ns)}, + ); + } + + pub fn print_if_longer_than_ms( + self: *TimeIt, + threshold_ms: u64, + comptime label: []const u8, + ) void { + self.if_longer_than(label, threshold_ms, false); + } + + pub fn backtrace_if_longer_than_ms( + self: *TimeIt, + threshold_ms: u64, + comptime label: []const u8, + ) void { + self.if_longer_than(label, threshold_ms, true); + } + + fn if_longer_than( + self: *TimeIt, + comptime label: []const u8, + threshold_ms: u64, + backtrace: bool, + ) void { + const elapsed_ns = self.inner.lap(); + if (elapsed_ns > threshold_ms * std.time.ns_per_ms) { + std.debug.print(label ++ ": {}\n", .{std.fmt.fmtDuration(elapsed_ns)}); + if (backtrace) std.debug.dumpCurrentStackTrace(null); + } + } +}; + +/// Utility for print-if debugging, a-la Rust's dbg! macro. +/// +/// dbg prints the value with the prefix, while also returning the value, which makes it convenient +/// to drop it in the middle of a complex expression. +pub fn dbg(prefix: []const u8, value: anytype) @TypeOf(value) { + std.debug.print("{s} = {any}\n", .{ + prefix, + std.json.fmt(value, .{ .whitespace = .indent_2 }), + }); + return value; +} diff --git a/scripts/stdx/flags.zig b/scripts/stdx/flags.zig new file mode 100644 index 0000000..f75a8c8 --- /dev/null +++ b/scripts/stdx/flags.zig @@ -0,0 +1,1482 @@ +//! The purpose of `flags` is to define standard behavior for parsing CLI arguments and provide +//! a specific parsing library, implementing this behavior. +//! +//! These are TigerBeetle CLI guidelines: +//! +//! - The main principle is robustness --- make operator errors harder to make. +//! - For production usage, avoid defaults. +//! - Thoroughly validate options. +//! - In particular, check that no options are repeated. +//! - Use only long options (`--addresses`). +//! - Exception: `-h/--help` is allowed. +//! - Use `--key=value` syntax for an option with an argument. +//! Don't use `--key value`, as that can be ambiguous (e.g., `--key --verbose`). +//! - Use subcommand syntax when appropriate. +//! - Use positional arguments when appropriate. +//! +//! Design choices for this particular `flags` library: +//! +//! - Be a 80% solution. Parsing arguments is a surprisingly vast topic: auto-generated help, +//! bash completions, typo correction. Rather than providing a definitive solution, `flags` +//! is just one possible option. It is ok to re-implement arg parsing in a different way, as long +//! as the CLI guidelines are observed. +//! +//! - No auto-generated help. Zig doesn't expose doc comments through `@typeInfo`, so its hard to +//! implement auto-help nicely. Additionally, fully hand-crafted `--help` message can be of +//! higher quality. +//! +//! - Fatal errors. It might be "cleaner" to use `try` to propagate the error to the caller, but +//! during early CLI parsing, it is much simpler to terminate the process directly and save the +//! caller the hassle of propagating errors. The `fatal` function is public, to allow the caller +//! to run additional validation or parsing using the same error reporting mechanism. +//! +//! - Concise DSL. Most cli parsing is done for ad-hoc tools like benchmarking, where the ability to +//! quickly add a new argument is valuable. As this is a 80% solution, production code may use +//! more verbose approach if it gives better UX. +//! +//! - Caller manages ArgsIterator. ArgsIterator owns the backing memory of the args, so we let the +//! caller to manage the lifetime. The caller should be skipping program name. + +const std = @import("std"); +const stdx = @import("stdx.zig"); +const builtin = @import("builtin"); +const assert = std.debug.assert; + +const Allocator = std.mem.Allocator; + +arena: std.heap.ArenaAllocator, + +const Flags = @This(); + +pub fn init(gpa: Allocator) Flags { + return .{ + .arena = std.heap.ArenaAllocator.init(gpa), + }; +} + +pub fn deinit(flags: *Flags, gpa: Allocator) void { + assert(std.meta.eql(flags.arena.child_allocator, gpa)); + flags.arena.deinit(); + flags.* = undefined; +} + +/// Format and print an error message to stderr, then exit with an exit code of 1. +fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn { + const stderr = std.io.getStdErr().writer(); + stderr.print("error: " ++ fmt_string ++ "\n", args) catch {}; + // NB: this status must match vsr.FatalReason.cli, but it would be wrong for flags to depend on + // vsr. The right way would be to parametrize flags by this behavior, and let the caller inject + // the implementation of fatal function, but let's be pragmatic here and just match the behavior + // manually. + std.process.exit(1); +} + +fn oom(_: error{OutOfMemory}) noreturn { + fatal("error: out of memory when parsing cli", .{}); +} + +/// Parse CLI arguments for subcommands specified as Zig `struct` or `union(enum)`: +/// +/// ``` +/// const CLIArgs = union(enum) { +/// start: struct { addresses: []const u8, replica: u32 }, +/// format: struct { +/// verbose: bool = false, +/// @"--": void, +/// path: []const u8, +/// }, +/// +/// pub const help = +/// \\ tigerbeetle start --addresses= --replica= +/// \\ tigerbeetle format [--verbose] +/// } +/// +/// const cli_args = parse_commands(&args, CLIArgs); +/// ``` +/// +/// `@"--"` field is treated specially, it delineates positional arguments. +/// +/// If `pub const help` declaration is present, it is used to implement `-h/--help` argument. +/// +/// Value parsing can be customized on per-type basis via `parse_flag_value` customization point. +pub fn parse(flags: *Flags, comptime CLIArgs: type) CLIArgs { + comptime assert(CLIArgs != void); + + const arena = flags.arena.allocator(); + + var args = std.process.argsWithAllocator(arena) catch |err| oom(err); + if (!args.skip()) fatal("executable name missing", .{}); + + return parse_flags(arena, &args, CLIArgs); +} + +fn parse_commands( + arena: Allocator, + args: *std.process.ArgIterator, + comptime Commands: type, +) Commands { + comptime assert(@typeInfo(Commands) == .@"union"); + comptime assert(std.meta.fields(Commands).len >= 2); + + const first_arg = args.next() orelse fatal( + "subcommand required, expected {s}", + .{comptime fields_to_comma_list(Commands)}, + ); + + // NB: help must be declared as *pub* const to be visible here. + if (@hasDecl(Commands, "help")) { + if (std.mem.eql(u8, first_arg, "-h") or std.mem.eql(u8, first_arg, "--help")) { + std.io.getStdOut().writeAll(Commands.help) catch std.process.exit(1); + std.process.exit(0); + } + } + + inline for (comptime std.meta.fields(Commands)) |field| { + comptime assert(std.mem.indexOfScalar(u8, field.name, '_') == null); + if (std.mem.eql(u8, first_arg, field.name)) { + return @unionInit(Commands, field.name, parse_flags(arena, args, field.type)); + } + } + fatal("unknown subcommand: '{s}'", .{first_arg}); +} + +fn parse_flags(arena: Allocator, args: *std.process.ArgIterator, comptime CLIArgs: type) CLIArgs { + @setEvalBranchQuota(5_000); + + if (CLIArgs == void) { + if (args.next()) |arg| { + fatal("unexpected argument: '{s}'", .{arg}); + } + return {}; + } + + if (@typeInfo(CLIArgs) == .@"union") { + return parse_commands(arena, args, CLIArgs); + } + + assert(@typeInfo(CLIArgs) == .@"struct"); + + const fields = std.meta.fields(CLIArgs); + comptime var fields_named, var fields_positional: []const std.builtin.Type.StructField = + for (fields, 0..) |field, index| { + if (std.mem.eql(u8, field.name, "--")) { + assert(field.type == void); + const positional_count = fields.len - index - 1; + if (positional_count == 0) @panic("expected positional fields"); + + break .{ + fields[0..index].*, + fields[index + 1 ..], + }; + } + } else .{ + fields[0..fields.len].*, + &.{}, + }; + + comptime var field_extended: ?std.builtin.Type.StructField = null; + if (fields_positional.len == 1 and fields_positional[0].type == []const []const u8) { + field_extended = fields_positional[0]; + fields_positional = fields_positional[1..]; + assert(fields_positional.len == 0); + } + + var arg_extended: if (field_extended == null) void else std.ArrayListUnmanaged([]const u8) = + if (field_extended == null) {} else .empty; + + comptime { + assert( + fields.len == fields_named.len + + fields_positional.len + + @intFromBool(field_extended != null) + + if (field_extended != null or fields_positional.len > 0) 1 else 0, // The @"--" + ); + if (field_extended != null) assert(fields_positional.len == 0); + + // When parsing named arguments, we must consider longer arguments first, such that + // `--foo-bar=92` is not confused for a misspelled `--foo=92`. Using `std.sort` for + // comptime-only values does not work, so open-code insertion sort, and comptime assert + // order during the actual parsing. + for (fields_named[0..], 0..) |*field_right, i| { + for (fields_named[0..i]) |*field_left| { + if (field_left.name.len < field_right.name.len) { + std.mem.swap(std.builtin.Type.StructField, field_left, field_right); + } + } + } + + for (fields_named) |field| { + switch (@typeInfo(field.type)) { + .bool => { + // Boolean flags must have a default. + assert(field.defaultValue() != null); + assert(field.defaultValue().? == false); + }, + .optional => |optional| { + // Optional flags must have a default. + assert(field.defaultValue() != null); + assert(field.defaultValue().? == null); + + assert_valid_value_type(optional.child); + }, + else => { + assert_valid_value_type(field.type); + }, + } + } + + var optional_tail: bool = false; + for (fields_positional) |field| { + if (field.defaultValue() == null) { + if (optional_tail) @panic("optional positional arguments must be trailing"); + } else { + optional_tail = true; + } + switch (@typeInfo(field.type)) { + .optional => |optional| { + // optional flags should have a default + assert(field.defaultValue() != null); + assert(field.defaultValue().? == null); + assert_valid_value_type(optional.child); + }, + else => { + assert_valid_value_type(field.type); + }, + } + } + } + + var counts: std.enums.EnumFieldStruct(std.meta.FieldEnum(CLIArgs), u32, 0) = .{}; + var result: CLIArgs = undefined; + var parsed_positional = false; + next_arg: while (args.next()) |arg| { + comptime var field_len_prev = std.math.maxInt(usize); + inline for (fields_named) |field| { + const flag = comptime flag_name(field); + + comptime assert(field_len_prev >= field.name.len); + field_len_prev = field.name.len; + if (std.mem.startsWith(u8, arg, flag)) { + if (parsed_positional) { + fatal("unexpected trailing option: '{s}'", .{arg}); + } + + @field(counts, field.name) += 1; + const flag_value = parse_flag(field.type, flag, arg); + @field(result, field.name) = flag_value; + continue :next_arg; + } + } + + if (fields_positional.len > 0) { + assert(field_extended == null); + counts.@"--" += 1; + switch (counts.@"--" - 1) { + inline 0...fields_positional.len - 1 => |field_index| { + const field = fields_positional[field_index]; + const flag = comptime flag_name_positional(field); + + if (arg.len == 0) fatal("{s}: empty argument", .{flag}); + // Prevent ambiguity between a flag and positional argument value. We could add + // support for bare ` -- ` as a disambiguation mechanism once we have a real + // use-case. + if (arg[0] == '-') fatal("unexpected argument: '{s}'", .{arg}); + parsed_positional = true; + + @field(result, field.name) = + parse_value(field.type, flag, arg); + continue :next_arg; + }, + else => {}, // Fall-through to the unexpected argument error. + } + } else { + if (field_extended != null) { + if (std.mem.eql(u8, arg, "--")) { + break; + } else { + fatal("unexpected argument: '{s}'; expected '-- ...'", .{arg}); + } + } + } + + fatal("unexpected argument: '{s}'", .{arg}); + } + if (field_extended != null) { + while (args.next()) |arg| { + arg_extended.append(arena, arg) catch |err| oom(err); + } + } + + assert(args.next() == null); + + inline for (fields_named) |field| { + const flag = flag_name(field); + switch (@field(counts, field.name)) { + 0 => if (field.defaultValue()) |default| { + @field(result, field.name) = default; + } else { + fatal("{s}: argument is required", .{flag}); + }, + 1 => {}, + else => fatal("{s}: duplicate argument", .{flag}), + } + } + + if (fields_positional.len > 0) { + assert(field_extended == null); + assert(counts.@"--" <= fields_positional.len); + inline for (fields_positional, 0..) |field, field_index| { + if (field_index >= counts.@"--") { + const flag = comptime flag_name_positional(field); + if (field.defaultValue()) |default| { + @field(result, field.name) = default; + } else { + fatal("{s}: argument is required", .{flag}); + } + } + } + } + + if (field_extended) |field| { + assert(fields_positional.len == 0); + @field(result, field.name) = arg_extended.items; + } + + return result; +} + +fn assert_valid_value_type(comptime T: type) void { + comptime { + if (T == []const u8 or T == [:0]const u8 or @typeInfo(T) == .int) return; + if (@hasDecl(T, "parse_flag_value")) return; + + if (@typeInfo(T) == .@"enum") { + const info = @typeInfo(T).@"enum"; + assert(info.is_exhaustive); + assert(info.fields.len >= 2); + return; + } + + @compileError("flags: unsupported type: " ++ @typeName(T)); + } +} + +/// Parse, e.g., `--cluster=123` into `123` integer +fn parse_flag(comptime T: type, flag: []const u8, arg: [:0]const u8) T { + assert(flag[0] == '-' and flag[1] == '-'); + + if (T == bool) { + if (std.mem.eql(u8, arg, flag)) { + // Bool argument may not have a value. + return true; + } + } + + const value = parse_flag_split_value(flag, arg); + assert(value.len > 0); + return parse_value(T, flag, value); +} + +/// Splits the value part from a `--arg=value` syntax. +fn parse_flag_split_value(flag: []const u8, arg: [:0]const u8) [:0]const u8 { + assert(flag[0] == '-' and flag[1] == '-'); + assert(std.mem.startsWith(u8, arg, flag)); + + const value = arg[flag.len..]; + if (value.len == 0) { + fatal("{s}: expected value separator '='", .{flag}); + } + if (value[0] != '=') { + fatal( + "{s}: expected value separator '=', but found '{c}' in '{s}'", + .{ flag, value[0], arg }, + ); + } + if (value.len == 1) fatal("{s}: argument requires a value", .{flag}); + return value[1..]; +} + +fn parse_value(comptime T: type, flag: []const u8, value: [:0]const u8) T { + assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); + assert(value.len > 0); + + const V = switch (@typeInfo(T)) { + .optional => |optional| optional.child, + else => T, + }; + + if (V == []const u8 or V == [:0]const u8) return value; + if (V == bool) return parse_value_bool(flag, value); + if (@typeInfo(V) == .int) return parse_value_int(V, flag, value); + if (@hasDecl(V, "parse_flag_value")) { + // Contracts: + // - Input string is guaranteed to be not empty. + // - Output diagnostic must point to statically-allocated data. + // - Diagnostic must start with a lower case letter. + // - Diagnostic must end with a ':' (it will be concatenated with original input). + // - (static_diagnostic != null) iff error.InvalidFlagValue is returned. + const parse_flag_value: fn ( + string: []const u8, + static_diagnostic: *?[]const u8, + ) error{InvalidFlagValue}!V = V.parse_flag_value; + + var diagnostic: ?[]const u8 = null; + if (parse_flag_value(value, &diagnostic)) |result| { + assert(diagnostic == null); + return result; + } else |err| switch (err) { + error.InvalidFlagValue => { + const message = diagnostic.?; + assert(std.ascii.isLower(message[0])); + assert(message[message.len - 1] == ':'); + fatal("{s}: {s} '{s}'", .{ flag, message, value }); + }, + } + } + if (@typeInfo(V) == .@"enum") return parse_value_enum(V, flag, value); + comptime unreachable; +} + +/// Parse string value into an integer, providing a nice error message for the user. +fn parse_value_int(comptime T: type, flag: []const u8, value: [:0]const u8) T { + assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); + + // Support only unsigned integers, as a conservative choice. + comptime assert(@typeInfo(T).int.signedness == .unsigned); + return std.fmt.parseUnsigned(T, value, 10) catch |err| { + switch (err) { + error.Overflow => fatal( + "{s}: value exceeds {d}-bit {s} integer: '{s}'", + .{ flag, @typeInfo(T).int.bits, @tagName(@typeInfo(T).int.signedness), value }, + ), + error.InvalidCharacter => fatal( + "{s}: expected an integer value, but found '{s}' (invalid digit)", + .{ flag, value }, + ), + } + }; +} + +fn parse_value_bool(flag: []const u8, value: [:0]const u8) bool { + return switch (parse_value_enum( + enum { + true, + false, + }, + flag, + value, + )) { + .true => true, + .false => false, + }; +} + +fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { + assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); + comptime assert(@typeInfo(E).@"enum".is_exhaustive); + + return std.meta.stringToEnum(E, value) orelse fatal( + "{s}: expected one of {s}, but found '{s}'", + .{ flag, comptime fields_to_comma_list(E), value }, + ); +} + +fn fields_to_comma_list(comptime E: type) []const u8 { + comptime { + const field_count = std.meta.fields(E).len; + assert(field_count >= 2); + + var result: []const u8 = ""; + for (std.meta.fields(E), 0..) |field, field_index| { + const separator = switch (field_index) { + 0 => "", + else => ", ", + field_count - 1 => if (field_count == 2) " or " else ", or ", + }; + result = result ++ separator ++ "'" ++ field.name ++ "'"; + } + return result; + } +} + +fn flag_name(comptime field: std.builtin.Type.StructField) []const u8 { + return comptime blk: { + assert(!std.mem.eql(u8, field.name, "-")); + assert(!std.mem.eql(u8, field.name, "--")); + + var result: []const u8 = "--"; + var index = 0; + while (std.mem.indexOfScalar(u8, field.name[index..], '_')) |i| { + result = result ++ field.name[index..][0..i] ++ "-"; + index = index + i + 1; + } + result = result ++ field.name[index..]; + break :blk result; + }; +} + +test flag_name { + const field = @typeInfo(struct { statsd: bool }).@"struct".fields[0]; + try std.testing.expectEqualStrings(flag_name(field), "--statsd"); +} + +fn flag_name_positional(comptime field: std.builtin.Type.StructField) []const u8 { + comptime assert(std.mem.indexOfScalar(u8, field.name, '_') == null); + return "<" ++ field.name ++ ">"; +} + +/// Fuzz parse_flag_value function: +/// +/// - Check that ok cases return a value. +/// - Check that err cases return an error with a properly formatted diagnostics. +/// - Check that the diagnostic contains specified substring +/// - Random tests with the input alphabet seeded from explicit cases. +/// - Random tests with uniform input. +pub fn parse_flag_value_fuzz( + comptime T: type, + parse_flag_value: fn ([]const u8, *?[]const u8) error{InvalidFlagValue}!T, + cases: struct { + ok: []const struct { []const u8, T }, + err: []const struct { []const u8, []const u8 }, + }, +) !void { + comptime assert(T.parse_flag_value == parse_flag_value); + + const test_count = 50_000; + const string_size_max = 32; + + const gpa = std.testing.allocator; + var prng = stdx.PRNG.from_seed_testing(); + + for (cases.ok) |case| { + const string, const want = case; + assert(string.len > 0); + + var diagnostic: ?[]const u8 = null; + const got = try parse_flag_value(string, &diagnostic); + assert(diagnostic == null); + try std.testing.expectEqual(want, got); + } + + for (cases.err) |case| { + const string, const want_message = case; + assert(string.len > 0); // Empty value are rejected early. + + var diagnostic: ?[]const u8 = null; + if (parse_flag_value(string, &diagnostic)) |value| { + std.debug.print("expected an error, got value: input='{s}', value={}", .{ + string, + value, + }); + return error.TestUnexpectedResult; + } else |err| switch (err) { + error.InvalidFlagValue => { + try parse_flag_value_check_diagnostic(string, diagnostic); + if (stdx.cut(diagnostic.?, want_message) == null) { + std.debug.print( + "expected diagnostic to contain substring='{s}' diagnostic='{s}'", + .{ want_message, diagnostic.? }, + ); + return error.TestUnexpectedResult; + } + }, + } + } + + var corpus: std.ArrayListUnmanaged(u8) = .empty; + defer corpus.deinit(gpa); + + for (cases.ok) |case| try corpus.appendSlice(gpa, case[0]); + for (cases.err) |case| try corpus.appendSlice(gpa, case[0]); + for (0..5) |_| try corpus.append(gpa, prng.int(u8)); + + std.mem.sort(u8, corpus.items, {}, std.sort.asc(u8)); + + const alphabet = unique(corpus.items); + + var string_buffer: [string_size_max]u8 = @splat(0); + for (0..test_count) |_| { + const string_size = prng.range_inclusive(usize, 1, string_size_max); + const string = string_buffer[0..string_size]; + assert(string.len > 0); + if (prng.boolean()) { + for (string) |*c| c.* = alphabet[prng.index(alphabet)]; + } else { + for (string) |*c| c.* = prng.int(u8); + } + + var diagnostic: ?[]const u8 = null; + if (parse_flag_value(string, &diagnostic)) |_| { + assert(diagnostic == null); + } else |err| switch (err) { + error.InvalidFlagValue => try parse_flag_value_check_diagnostic(string, diagnostic), + } + } +} + +fn parse_flag_value_check_diagnostic(string: []const u8, diagnostic: ?[]const u8) !void { + const message = diagnostic orelse { + std.debug.print("expected a diagnostic: string='{s}'", .{string}); + return error.TestUnexpectedResult; + }; + if (!(message.len > 0 and + std.ascii.isLower(message[0]) and + message[message.len - 1] == ':')) + { + std.debug.print("wrong diagnostic format: string='{s}' diagnostic='{s}'", .{ + string, + message, + }); + return error.TestUnexpectedResult; + } +} + +fn unique(sorted: []u8) []u8 { + assert(sorted.len > 0); + + var count: usize = 1; + for (1..sorted.len) |index| { + assert(sorted[count - 1] <= sorted[index]); + if (sorted[count - 1] == sorted[index]) { + // Duplicate! Skip to the next index. + } else { + sorted[count] = sorted[index]; + count += 1; + } + } + + return sorted[0..count]; +} + +// CLI parsing makes a liberal use of `fatal`, so testing it within the process is impossible. We +// test it out of process by: +// - using Zig compiler to build this very file as an executable in a temporary directory, +// - running the following main with various args and capturing stdout, stderr, and the exit code. +// - asserting that the captured values are correct. +// For production builds, don't include the main function. +// This is `if __name__ == "__main__":` at comptime! +pub const main = + if (@import("root") != @This()) {} else struct { + const CLIArgs = union(enum) { + empty, + prefix: struct { + foo: u8 = 0, + foo_bar: u8 = 0, + opt: bool = false, + option: bool = false, + }, + positional: struct { + flag: bool = false, + + @"--": void, + p1: []const u8, + p2: []const u8, + p3: ?u32 = null, + p4: ?u32 = null, + }, + extended: struct { + flag: bool = false, + @"--": void, + rest: []const []const u8, + }, + required: struct { + foo: u8, + bar: u8, + }, + values: struct { + int: u32 = 0, + size: stdx.ByteSize = .{ .value = 0 }, + boolean: bool = false, + path: []const u8 = "not-set", + optional: ?[]const u8 = null, + choice: enum { marlowe, shakespeare } = .marlowe, + }, + subcommand: union(enum) { + pub const help = + \\subcommand help + \\ + ; + + c1: struct { a: bool = false }, + c2: struct { b: bool = false }, + }, + + pub const help = + \\ flags-test-program [flags] + \\ + ; + }; + + fn main() !void { + var gpa_allocator = std.heap.GeneralPurposeAllocator(.{}){}; + const gpa = gpa_allocator.allocator(); + + var flags = Flags.init(gpa); + defer flags.deinit(gpa); + + const cli_args = flags.parse(CLIArgs); + + const stdout = std.io.getStdOut(); + const out_stream = stdout.writer(); + switch (cli_args) { + .empty => try out_stream.print("empty\n", .{}), + .prefix => |values| { + try out_stream.print("foo: {}\n", .{values.foo}); + try out_stream.print("foo-bar: {}\n", .{values.foo_bar}); + try out_stream.print("opt: {}\n", .{values.opt}); + try out_stream.print("option: {}\n", .{values.option}); + }, + .positional => |values| { + try out_stream.print("p1: {s}\n", .{values.p1}); + try out_stream.print("p2: {s}\n", .{values.p2}); + try out_stream.print("p3: {?}\n", .{values.p3}); + try out_stream.print("p4: {?}\n", .{values.p4}); + try out_stream.print("flag: {}\n", .{values.flag}); + }, + .extended => |values| { + try out_stream.print("flag: {}\n", .{values.flag}); + for (values.rest) |arg| try out_stream.print("arg: {s}\n", .{arg}); + }, + .required => |required| { + try out_stream.print("foo: {}\n", .{required.foo}); + try out_stream.print("bar: {}\n", .{required.bar}); + }, + .values => |values| { + try out_stream.print("int: {}\n", .{values.int}); + try out_stream.print("size: {}\n", .{values.size.bytes()}); + try out_stream.print("boolean: {}\n", .{values.boolean}); + try out_stream.print("path: {s}\n", .{values.path}); + try out_stream.print("optional: {?s}\n", .{values.optional}); + try out_stream.print("choice: {?s}\n", .{@tagName(values.choice)}); + }, + .subcommand => |values| { + switch (values) { + .c1 => |c1| try out_stream.print("c1.a: {}\n", .{c1.a}), + .c2 => |c2| try out_stream.print("c2.b: {}\n", .{c2.b}), + } + }, + } + } + }.main; + +test "flags" { + const Snap = stdx.Snap; + const module_path = "src/stdx"; + const snap = Snap.snap_fn(module_path); + + const T = struct { + const T = @This(); + + gpa: std.mem.Allocator, + tmp_dir: std.testing.TmpDir, + output_buf: std.ArrayList(u8), + flags_exe_buf: *[std.fs.max_path_bytes]u8, + flags_exe: []const u8, + + fn init(gpa: std.mem.Allocator) !T { + // TODO: Avoid std.posix.getenv() as it currently causes a linker error on windows. + // See: https://github.com/ziglang/zig/issues/8456 + const zig_exe = try std.process.getEnvVarOwned(gpa, "ZIG_EXE"); // Set by build.zig + defer gpa.free(zig_exe); + + var tmp_dir = std.testing.tmpDir(.{}); + errdefer tmp_dir.cleanup(); + + const tmp_dir_path = try std.fs.path.join(gpa, &.{ + ".zig-cache", + "tmp", + &tmp_dir.sub_path, + }); + defer gpa.free(tmp_dir_path); + + const output_buf = std.ArrayList(u8).init(gpa); + errdefer output_buf.deinit(); + + const flags_exe_buf = try gpa.create([std.fs.max_path_bytes]u8); + errdefer gpa.destroy(flags_exe_buf); + + { // Compile this file as an executable! + const path_relative = try std.fs.path.join(gpa, &.{ + module_path, + @src().file, + }); + defer gpa.free(path_relative); + + const this_file = try std.fs.cwd().realpath( + path_relative, + flags_exe_buf, + ); + const argv = [_][]const u8{ zig_exe, "build-exe", this_file }; + const exec_result = try std.process.Child.run(.{ + .allocator = gpa, + .argv = &argv, + .cwd = tmp_dir_path, + }); + defer gpa.free(exec_result.stdout); + defer gpa.free(exec_result.stderr); + + if (exec_result.term.Exited != 0) { + std.debug.print("{s}{s}", .{ exec_result.stdout, exec_result.stderr }); + return error.FailedToCompile; + } + } + + const flags_exe = try tmp_dir.dir.realpath( + "flags" ++ comptime builtin.target.exeFileExt(), + flags_exe_buf, + ); + + const sanity_check = try std.fs.openFileAbsolute(flags_exe, .{}); + sanity_check.close(); + + return .{ + .gpa = gpa, + .tmp_dir = tmp_dir, + .output_buf = output_buf, + .flags_exe_buf = flags_exe_buf, + .flags_exe = flags_exe, + }; + } + + fn deinit(t: *T) void { + t.gpa.destroy(t.flags_exe_buf); + t.output_buf.deinit(); + t.tmp_dir.cleanup(); + t.* = undefined; + } + + fn check(t: *T, cli: []const []const u8, want: Snap) !void { + const argv = try t.gpa.alloc([]const u8, cli.len + 1); + defer t.gpa.free(argv); + + argv[0] = t.flags_exe; + for (argv[1..], 0..) |*arg, i| { + arg.* = cli[i]; + } + if (cli.len > 0) { + assert(argv[argv.len - 1].ptr == cli[cli.len - 1].ptr); + } + + const exec_result = try std.process.Child.run(.{ + .allocator = t.gpa, + .argv = argv, + }); + defer t.gpa.free(exec_result.stdout); + defer t.gpa.free(exec_result.stderr); + + t.output_buf.clearRetainingCapacity(); + + if (exec_result.term.Exited != 0) { + try t.output_buf.writer().print("status: {}\n", .{exec_result.term.Exited}); + } + if (exec_result.stdout.len > 0) { + try t.output_buf.writer().print("stdout:\n{s}", .{exec_result.stdout}); + } + if (exec_result.stderr.len > 0) { + try t.output_buf.writer().print("stderr:\n{s}", .{exec_result.stderr}); + } + + try want.diff(t.output_buf.items); + } + }; + + var t = try T.init(std.testing.allocator); + defer t.deinit(); + + // Test-cases are roughly in the source order of the corresponding features. + + try t.check(&.{"empty"}, snap(@src(), + \\stdout: + \\empty + \\ + )); + + try t.check(&.{}, snap(@src(), + \\status: 1 + \\stderr: + \\error: subcommand required, expected 'empty', 'prefix', 'positional', 'extended', 'required', 'values', or 'subcommand' + \\ + )); + + try t.check(&.{"-h"}, snap(@src(), + \\stdout: + \\ flags-test-program [flags] + \\ + )); + + try t.check(&.{"--help"}, snap(@src(), + \\stdout: + \\ flags-test-program [flags] + \\ + )); + + try t.check(&.{""}, snap(@src(), + \\status: 1 + \\stderr: + \\error: unknown subcommand: '' + \\ + )); + + try t.check(&.{"bogus"}, snap(@src(), + \\status: 1 + \\stderr: + \\error: unknown subcommand: 'bogus' + \\ + )); + + try t.check(&.{"--int=92"}, snap(@src(), + \\status: 1 + \\stderr: + \\error: unknown subcommand: '--int=92' + \\ + )); + + try t.check(&.{ "empty", "--help" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: '--help' + \\ + )); + + try t.check(&.{ "prefix", "--foo=92" }, snap(@src(), + \\stdout: + \\foo: 92 + \\foo-bar: 0 + \\opt: false + \\option: false + \\ + )); + + try t.check(&.{ "prefix", "--foo-bar=92" }, snap(@src(), + \\stdout: + \\foo: 0 + \\foo-bar: 92 + \\opt: false + \\option: false + \\ + )); + + try t.check(&.{ "prefix", "--foo-baz=92" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --foo: expected value separator '=', but found '-' in '--foo-baz=92' + \\ + )); + + try t.check(&.{ "prefix", "--opt" }, snap(@src(), + \\stdout: + \\foo: 0 + \\foo-bar: 0 + \\opt: true + \\option: false + \\ + )); + + try t.check(&.{ "prefix", "--option" }, snap(@src(), + \\stdout: + \\foo: 0 + \\foo-bar: 0 + \\opt: false + \\option: true + \\ + )); + + try t.check(&.{ "prefix", "--optx" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --opt: expected value separator '=', but found 'x' in '--optx' + \\ + )); + + try t.check(&.{ "positional", "x", "y" }, snap(@src(), + \\stdout: + \\p1: x + \\p2: y + \\p3: null + \\p4: null + \\flag: false + \\ + )); + + try t.check(&.{ "positional", "x", "y", "1" }, snap(@src(), + \\stdout: + \\p1: x + \\p2: y + \\p3: 1 + \\p4: null + \\flag: false + \\ + )); + + try t.check(&.{ "positional", "x", "y", "1", "2" }, snap(@src(), + \\stdout: + \\p1: x + \\p2: y + \\p3: 1 + \\p4: 2 + \\flag: false + \\ + )); + + try t.check(&.{"positional"}, snap(@src(), + \\status: 1 + \\stderr: + \\error: : argument is required + \\ + )); + + try t.check(&.{ "positional", "x" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: : argument is required + \\ + )); + + try t.check(&.{ "positional", "x", "y", "z" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: : expected an integer value, but found 'z' (invalid digit) + \\ + )); + + try t.check(&.{ "positional", "x", "y", "1", "2", "3" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: '3' + \\ + )); + + try t.check(&.{ "positional", "" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: : empty argument + \\ + )); + + try t.check(&.{ "positional", "x", "--flag" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected trailing option: '--flag' + \\ + )); + + try t.check(&.{ "positional", "x", "--flag", "y" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected trailing option: '--flag' + \\ + )); + + try t.check(&.{ "positional", "--flag", "x", "y" }, snap(@src(), + \\stdout: + \\p1: x + \\p2: y + \\p3: null + \\p4: null + \\flag: true + \\ + )); + + try t.check(&.{ "positional", "--", "x", "y" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: '--' + \\ + )); + + try t.check(&.{ "positional", "--flak", "x", "y" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: '--flak' + \\ + )); + + try t.check(&.{ "required", "--foo=1", "--bar=2" }, snap(@src(), + \\stdout: + \\foo: 1 + \\bar: 2 + \\ + )); + + try t.check(&.{ "required", "--surprise" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: '--surprise' + \\ + )); + + try t.check(&.{ "required", "--foo=1" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --bar: argument is required + \\ + )); + + try t.check(&.{ "required", "--foo=1", "--bar=2", "--foo=3" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --foo: duplicate argument + \\ + )); + + try t.check(&.{ + "values", + "--int=92", + "--size=1GiB", + "--boolean", + "--path=/home", + "--optional=some", + "--choice=shakespeare", + }, snap(@src(), + \\stdout: + \\int: 92 + \\size: 1073741824 + \\boolean: true + \\path: /home + \\optional: some + \\choice: shakespeare + \\ + )); + + try t.check(&.{"values"}, snap(@src(), + \\stdout: + \\int: 0 + \\size: 0 + \\boolean: false + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--boolean=true" }, snap(@src(), + \\stdout: + \\int: 0 + \\size: 0 + \\boolean: true + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--boolean=false" }, snap(@src(), + \\stdout: + \\int: 0 + \\size: 0 + \\boolean: false + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--boolean=foo" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --boolean: expected one of 'true' or 'false', but found 'foo' + \\ + )); + + try t.check(&.{ "values", "--int" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected value separator '=' + \\ + )); + + try t.check(&.{ "values", "--int:" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected value separator '=', but found ':' in '--int:' + \\ + )); + + try t.check(&.{ "values", "--int=" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: argument requires a value + \\ + )); + + try t.check(&.{ "values", "--int=-92" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected an integer value, but found '-92' (invalid digit) + \\ + )); + + try t.check(&.{ "values", "--int=_92" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected an integer value, but found '_92' (invalid digit) + \\ + )); + + try t.check(&.{ "values", "--int=92_" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected an integer value, but found '92_' (invalid digit) + \\ + )); + + try t.check(&.{ "values", "--int=92" }, snap(@src(), + \\stdout: + \\int: 92 + \\size: 0 + \\boolean: false + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--int=900_200" }, snap(@src(), + \\stdout: + \\int: 900200 + \\size: 0 + \\boolean: false + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--int=XCII" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected an integer value, but found 'XCII' (invalid digit) + \\ + )); + + try t.check(&.{ "values", "--int=44444444444444444444" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: value exceeds 32-bit unsigned integer: '44444444444444444444' + \\ + )); + + try t.check(&.{ "values", "--int=-0" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected an integer value, but found '-0' (invalid digit) + \\ + )); + + try t.check(&.{ "values", "--int=+0" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --int: expected an integer value, but found '+0' (invalid digit) + \\ + )); + + try t.check(&.{ "values", "--size=-0" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: expected a size, but found: '-0' + \\ + )); + + try t.check(&.{ "values", "--size=+0" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: expected a size, but found: '+0' + \\ + )); + + try t.check(&.{ "values", "--size=_1000KiB" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: expected a size, but found: '_1000KiB' + \\ + )); + + try t.check(&.{ "values", "--size=1000_KiB" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: expected a size, but found: '1000_KiB' + \\ + )); + + try t.check(&.{ "values", "--size=1_000KiB" }, snap(@src(), + \\stdout: + \\int: 0 + \\size: 1024000 + \\boolean: false + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--size=3MiB" }, snap(@src(), + \\stdout: + \\int: 0 + \\size: 3145728 + \\boolean: false + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--size=44444444444444444444" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: value exceeds 64-bit unsigned integer: '44444444444444444444' + \\ + )); + + try t.check(&.{ "values", "--size=100000000000000000" }, snap(@src(), + \\stdout: + \\int: 0 + \\size: 100000000000000000 + \\boolean: false + \\path: not-set + \\optional: null + \\choice: marlowe + \\ + )); + + try t.check(&.{ "values", "--size=100000000000000000kib" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: size in bytes exceeds 64-bit unsigned integer: '100000000000000000kib' + \\ + )); + + try t.check(&.{ "values", "--size=3bogus" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: invalid unit in size, needed KiB, MiB, GiB or TiB: '3bogus' + \\ + )); + + try t.check(&.{ "values", "--size=MiB" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --size: expected a size, but found: 'MiB' + \\ + )); + + try t.check(&.{ "values", "--path=" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --path: argument requires a value + \\ + )); + + try t.check(&.{ "values", "--optional=" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --optional: argument requires a value + \\ + )); + + try t.check(&.{ "values", "--choice=molière" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: --choice: expected one of 'marlowe' or 'shakespeare', but found 'molière' + \\ + )); + + try t.check(&.{"subcommand"}, snap(@src(), + \\status: 1 + \\stderr: + \\error: subcommand required, expected 'c1' or 'c2' + \\ + )); + try t.check(&.{ "subcommand", "c1", "--a" }, snap(@src(), + \\stdout: + \\c1.a: true + \\ + )); + try t.check(&.{ "subcommand", "c2", "--b" }, snap(@src(), + \\stdout: + \\c2.b: true + \\ + )); + try t.check(&.{ "subcommand", "c1", "--b" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: '--b' + \\ + )); + try t.check(&.{ "subcommand", "c2", "--a" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: '--a' + \\ + )); + try t.check(&.{ "subcommand", "--help" }, snap(@src(), + \\stdout: + \\subcommand help + \\ + )); + try t.check(&.{ "subcommand", "-h" }, snap(@src(), + \\stdout: + \\subcommand help + \\ + )); + + try t.check(&.{"extended"}, snap(@src(), + \\stdout: + \\flag: false + \\ + )); + try t.check(&.{ "extended", "--" }, snap(@src(), + \\stdout: + \\flag: false + \\ + )); + try t.check(&.{ "extended", "--flag", "--" }, snap(@src(), + \\stdout: + \\flag: true + \\ + )); + try t.check(&.{ "extended", "a" }, snap(@src(), + \\status: 1 + \\stderr: + \\error: unexpected argument: 'a'; expected '-- ...' + \\ + )); + try t.check(&.{ "extended", "--", "a" }, snap(@src(), + \\stdout: + \\flag: false + \\arg: a + \\ + )); + try t.check(&.{ "extended", "--flag", "--", "a" }, snap(@src(), + \\stdout: + \\flag: true + \\arg: a + \\ + )); + try t.check(&.{ "extended", "--", "a", "b" }, snap(@src(), + \\stdout: + \\flag: false + \\arg: a + \\arg: b + \\ + )); + try t.check(&.{ "extended", "--flag", "--", "a", "b" }, snap(@src(), + \\stdout: + \\flag: true + \\arg: a + \\arg: b + \\ + )); + try t.check(&.{ "extended", "--", "--flag" }, snap(@src(), + \\stdout: + \\flag: false + \\arg: --flag + \\ + )); + try t.check(&.{ "extended", "--", "--", "--" }, snap(@src(), + \\stdout: + \\flag: false + \\arg: -- + \\arg: -- + \\ + )); +} diff --git a/scripts/stdx/huge_page_allocator.zig b/scripts/stdx/huge_page_allocator.zig new file mode 100644 index 0000000..a7e7541 --- /dev/null +++ b/scripts/stdx/huge_page_allocator.zig @@ -0,0 +1,115 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const stdx = @import("stdx.zig"); +const Allocator = std.mem.Allocator; + +const log = std.log.scoped(.allocator); + +const page_allocator_vtable = std.heap.page_allocator.vtable; + +/// Like `std.heap.page_allocator`, but on Linux applies `MADV_HUGEPAGE` to +/// allocated regions so that the kernel may back them with 2 MiB transparent +/// huge pages, reducing TLB pressure for large allocations. +/// +/// Only `alloc` is intercepted. `resize` and `remap` inherit the VMA flags +/// (including `VM_HUGEPAGE`) set on the original mapping, so they need no +/// additional `madvise` call. +/// +/// On non-Linux targets this is identical to `std.heap.page_allocator`. +pub const huge_page_allocator: Allocator = .{ + .ptr = std.heap.page_allocator.ptr, + .vtable = if (builtin.target.os.tag == .linux) &vtable else page_allocator_vtable, +}; + +const vtable: Allocator.VTable = .{ + .alloc = alloc, + .resize = page_allocator_vtable.resize, + .remap = page_allocator_vtable.remap, + .free = page_allocator_vtable.free, +}; + +fn alloc(context: *anyopaque, n: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 { + const ptr = page_allocator_vtable.alloc(context, n, alignment, ra) orelse return null; + // This is just a hint, so if it fails we can safely ignore it. + std.posix.madvise(@alignCast(ptr), n, std.posix.MADV.HUGEPAGE) catch { + log.warn("Transparent Huge Pages (THP) are disabled.", .{}); + }; + return ptr; +} + +const testing = std.testing; +const assert = std.debug.assert; + +/// Checks /proc/self/smaps for the "hg" VmFlag on the mapping containing `ptr`. +fn verify_address_is_huge_page(ptr: [*]const u8) !bool { + assert(builtin.target.os.tag == .linux); + const addr = @intFromPtr(ptr); + + var file = try std.fs.openFileAbsolute("/proc/self/smaps", .{}); + defer file.close(); + + const content = try file.readToEndAlloc(testing.allocator, 10 * 1024 * 1024); + defer testing.allocator.free(content); + + var lines = std.mem.splitScalar(u8, content, '\n'); + + // Find the mapping header that contains our address. + while (lines.next()) |line| { + if (parse_mapping_range(line)) |range| { + if (addr >= range.min and addr < range.max) { + // Scan subsequent lines for VmFlags within this mapping. + while (lines.next()) |detail| { + if (stdx.cut_prefix(detail, "VmFlags:")) |rest| { + return stdx.cut(rest, "hg") != null; + } + } + } + } + } + return false; +} + +fn parse_mapping_range(line: []const u8) ?struct { min: u64, max: u64 } { + const addr_range, _ = stdx.cut(line, " ") orelse return null; + const addr_hex_min, const addr_hex_max = stdx.cut(addr_range, "-") orelse return null; + const addr_min = std.fmt.parseUnsigned(u64, addr_hex_min, 16) catch return null; + const addr_max = std.fmt.parseUnsigned(u64, addr_hex_max, 16) catch return null; + return .{ .min = addr_min, .max = addr_max }; +} + +test "huge_page_allocator: basic alloc and free" { + const slice = try huge_page_allocator.alloc(u8, 4096); + defer huge_page_allocator.free(slice); + + @memset(slice, 0xab); + try testing.expectEqual(@as(u8, 0xab), slice[0]); +} + +test "huge_page_allocator: large THP-eligible allocation" { + // 4 MiB — large enough for THP promotion on Linux. + const size = 4 * 1024 * 1024; + const slice = try huge_page_allocator.alloc(u8, size); + defer huge_page_allocator.free(slice); + + @memset(slice, 0xcd); + try testing.expectEqual(@as(u8, 0xcd), slice[size - 1]); + + if (builtin.target.os.tag == .linux) { + // Verify that MADV_HUGEPAGE was applied by checking VmFlags in /proc/self/smaps. + // The "hg" flag means the process requested hugepages via madvise — this is + // deterministic regardless of whether the kernel actually promoted the pages. + try testing.expect(try verify_address_is_huge_page(slice.ptr)); + } +} + +test "huge_page_allocator: as ArenaAllocator backing" { + var arena = std.heap.ArenaAllocator.init(huge_page_allocator); + defer arena.deinit(); + + const alloc1 = try arena.allocator().alloc(u8, 1024); + const alloc2 = try arena.allocator().alloc(u8, 2048); + @memset(alloc1, 1); + @memset(alloc2, 2); + try testing.expectEqual(@as(u8, 1), alloc1[0]); + try testing.expectEqual(@as(u8, 2), alloc2[0]); +} diff --git a/scripts/stdx/iops.zig b/scripts/stdx/iops.zig new file mode 100644 index 0000000..30d304e --- /dev/null +++ b/scripts/stdx/iops.zig @@ -0,0 +1,134 @@ +const std = @import("std"); +const stdx = @import("stdx.zig"); +const assert = std.debug.assert; + +/// Take a u8 to limit to 255 items max (maxInt(u8) == 255). +pub fn IOPSType(comptime T: type, comptime size: u8) type { + const Map = stdx.BitSetType(size); + + return struct { + const IOPS = @This(); + + items: [size]T = undefined, + busy: Map = .{}, + + pub fn acquire(self: *IOPS) ?*T { + const i = self.busy.first_unset() orelse return null; + self.busy.set(i); + return &self.items[i]; + } + + pub fn release(self: *IOPS, item: *T) void { + item.* = undefined; + const i = self.index(item); + assert(self.busy.is_set(i)); + self.busy.unset(i); + } + + pub fn index(self: *const IOPS, item: *const T) usize { + const i = @divExact( + (@intFromPtr(item) - @intFromPtr(&self.items)), + @sizeOf(T), + ); + assert(i < size); + return i; + } + + /// Returns the count of IOPs available. + pub fn available(self: *const IOPS) usize { + return self.busy.capacity() - self.busy.count(); + } + + pub inline fn total(_: *const IOPS) usize { + return size; + } + + /// Returns the count of IOPs in use. + pub fn executing(self: *const IOPS) usize { + return self.busy.count(); + } + + pub const Iterator = struct { + iops: *IOPS, + bitset_iterator: Map.Iterator, + + pub fn next(iterator: *@This()) ?*T { + const i = iterator.bitset_iterator.next() orelse return null; + return &iterator.iops.items[i]; + } + }; + + pub const IteratorConst = struct { + iops: *const IOPS, + bitset_iterator: Map.Iterator, + + pub fn next(iterator: *@This()) ?*const T { + const i = iterator.bitset_iterator.next() orelse return null; + return &iterator.iops.items[i]; + } + }; + + /// Iterates over all currently executing IOPs. + pub fn iterate(self: *IOPS) Iterator { + return .{ + .iops = self, + .bitset_iterator = self.busy.iterate(), + }; + } + + /// Iterates over all currently executing IOPs. + pub fn iterate_const(self: *const IOPS) IteratorConst { + return .{ + .iops = self, + .bitset_iterator = self.busy.iterate(), + }; + } + }; +} + +test "IOPS" { + const testing = std.testing; + var iops = IOPSType(u32, 4){}; + + try testing.expectEqual(@as(usize, 4), iops.available()); + try testing.expectEqual(@as(usize, 0), iops.executing()); + + var one = iops.acquire().?; + + try testing.expectEqual(@as(usize, 3), iops.available()); + try testing.expectEqual(@as(usize, 1), iops.executing()); + + var two = iops.acquire().?; + var three = iops.acquire().?; + + try testing.expectEqual(@as(usize, 1), iops.available()); + try testing.expectEqual(@as(usize, 3), iops.executing()); + + var four = iops.acquire().?; + try testing.expectEqual(@as(?*u32, null), iops.acquire()); + + try testing.expectEqual(@as(usize, 0), iops.available()); + try testing.expectEqual(@as(usize, 4), iops.executing()); + + iops.release(two); + + try testing.expectEqual(@as(usize, 1), iops.available()); + try testing.expectEqual(@as(usize, 3), iops.executing()); + + // there is only one slot free, so we will get the same pointer back. + try testing.expectEqual(@as(?*u32, two), iops.acquire()); + + iops.release(four); + iops.release(two); + iops.release(one); + iops.release(three); + + try testing.expectEqual(@as(usize, 4), iops.available()); + try testing.expectEqual(@as(usize, 0), iops.executing()); + + one = iops.acquire().?; + two = iops.acquire().?; + three = iops.acquire().?; + four = iops.acquire().?; + try testing.expectEqual(@as(?*u32, null), iops.acquire()); +} diff --git a/scripts/stdx/mlock.zig b/scripts/stdx/mlock.zig new file mode 100644 index 0000000..3dc4241 --- /dev/null +++ b/scripts/stdx/mlock.zig @@ -0,0 +1,92 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const os = std.os; + +const stdx = @import("stdx.zig"); + +const MiB = stdx.MiB; + +const log = std.log.scoped(.mlock); + +const MemoryLockError = error{memory_not_locked} || std.posix.UnexpectedError; + +const mlockall_error = "Unable to lock pages in memory ({s})" ++ + " - kernel swap would otherwise bypass TigerBeetle's storage fault tolerance."; + +/// Pin virtual memory pages allocated so far to physical pages in RAM, preventing the pages from +/// being swapped out and introducing storage error into memory, bypassing ECC RAM. +pub fn memory_lock_allocated(options: struct { allocated_size: usize }) MemoryLockError!void { + switch (builtin.os.tag) { + .linux => try memory_lock_allocated_linux(), + .macos => { + // macOS has mlock() but not mlockall(). mlock() requires an address range which + // would be difficult to gather for non-heap memory that is also faulted in, + // such as the stack, globals, etc. + }, + .windows => try memory_lock_allocated_windows(options.allocated_size), + else => @compileError("unsupported platform"), + } +} + +fn memory_lock_allocated_linux() MemoryLockError!void { + // https://github.com/torvalds/linux/blob/v6.12/include/uapi/asm-generic/mman.h#L18-L20 + const MCL_CURRENT = 1; // Lock all currently mapped pages. + const MCL_ONFAULT = 4; // Lock all pages faulted in (i.e. stack space). + const result = os.linux.syscall1(.mlockall, MCL_CURRENT | MCL_ONFAULT); + switch (os.linux.E.init(result)) { + .SUCCESS => return, + .AGAIN => log.warn(mlockall_error, .{"some addresses could not be locked"}), + .NOMEM => log.warn(mlockall_error, .{"memory would exceed RLIMIT_MEMLOCK"}), + .PERM => log.warn(mlockall_error, .{ + "insufficient privileges to lock memory", + }), + .INVAL => unreachable, // MCL_ONFAULT specified without MCL_CURRENT. + else => |err| return stdx.unexpected_errno("mlockall", err), + } + return error.memory_not_locked; +} + +fn memory_lock_allocated_windows(allocated_size: usize) MemoryLockError!void { + // Windows has VirtualLock which works similar to mlock with an address range. + // It would be difficult to gather the addresses of non-heap memory that is also + // faulted in, such as the stack, globals, etc. SetProcessWorkingSetSize can be + // used instead to lock all existing pages into memory to avoid swapping. + const process_handle = os.windows.kernel32.GetCurrentProcess(); + var working_set_min: os.windows.SIZE_T = 0; + var working_set_max: os.windows.SIZE_T = 0; + + if (stdx.windows.GetProcessWorkingSetSize( + process_handle, + &working_set_min, + &working_set_max, + ) == os.windows.FALSE) { + working_set_min = allocated_size; // Count bytes allocated so far. + working_set_min += 64 * MiB; // 64mb buffer room for stack/globals. + working_set_max = working_set_min * 2; // Buffer room for new page faults. + } + + if (stdx.windows.SetProcessWorkingSetSize( + process_handle, + working_set_min, + working_set_max, + ) == os.windows.FALSE) { + // From std.os.windows.unexpectedError(): + const format_flags = os.windows.FORMAT_MESSAGE_FROM_SYSTEM | + os.windows.FORMAT_MESSAGE_IGNORE_INSERTS; + + // 614 is the length of the longest windows error description. + var buffer: [614:0]os.windows.WCHAR = undefined; + const buffer_size = os.windows.kernel32.FormatMessageW( + format_flags, + null, + os.windows.kernel32.GetLastError(), + os.windows.LANG.NEUTRAL | (os.windows.SUBLANG.DEFAULT << 10), + &buffer, + buffer.len, + null, + ); + + log.warn(mlockall_error, .{std.unicode.fmtUtf16Le(buffer[0..buffer_size])}); + return error.memory_not_locked; + } +} diff --git a/scripts/stdx/prng.zig b/scripts/stdx/prng.zig new file mode 100644 index 0000000..87b69b0 --- /dev/null +++ b/scripts/stdx/prng.zig @@ -0,0 +1,677 @@ +//! TigerBeetle standard Pseudo Random Number generator. +//! +//! Import qualified and use `prng` for field/variable name: +//! +//! ``` +//! prng: *stdx.PRNG +//! ``` +//! +//! The implementation matches Zig's `std.Random.DefaultPrng`, but we avoid using that directly in +//! order to: +//! - remove floating point from the API, to ensure determinism +//! - isolate our test suite from stdlib API churn +//! - isolate TigerBeetle from the churn in the PRNG algorithms +//! - simplify and extend the API +//! - remove dynamic-dispatch indirection (a minor bonus). + +const std = @import("std"); +const stdx = @import("stdx.zig"); +const assert = std.debug.assert; +const math = std.math; +const Snap = stdx.Snap; +const module_path = "src/stdx"; +const snap = Snap.snap_fn(module_path); +const KiB = stdx.KiB; + +s: [4]u64, + +const PRNG = @This(); + +/// A less than one rational number, used to specify probabilities. +pub const Ratio = struct { + // Invariant: numerator ≤ denominator. + numerator: u64, + // Invariant: denominator ≠ 0. + denominator: u64, + + pub fn zero() Ratio { + return .{ .numerator = 0, .denominator = 1 }; + } + + pub fn format( + r: Ratio, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + if (r.numerator == 0) return writer.print("0", .{}); + return writer.print("{d}/{d}", .{ r.numerator, r.denominator }); + } + + pub fn parse_flag_value( + string: []const u8, + static_diagnostic: *?[]const u8, + ) error{InvalidFlagValue}!Ratio { + assert(string.len > 0); + if (string.len == 1 and string[0] == '0') return .zero(); + + const string_numerator, const string_denominator = stdx.cut(string, "/") orelse { + static_diagnostic.* = "expected 'a/b' ratio, but found:"; + return error.InvalidFlagValue; + }; + + const numerator = std.fmt.parseInt(u64, string_numerator, 10) catch { + static_diagnostic.* = "invalid numerator:"; + return error.InvalidFlagValue; + }; + const denominator = std.fmt.parseInt(u64, string_denominator, 10) catch { + static_diagnostic.* = "invalid denominator:"; + return error.InvalidFlagValue; + }; + if (denominator == 0) { + static_diagnostic.* = "denominator is zero:"; + return error.InvalidFlagValue; + } + if (numerator > denominator) { + static_diagnostic.* = "ratio greater than 1:"; + return error.InvalidFlagValue; + } + return ratio(numerator, denominator); + } +}; + +test "Ratio.parse_flag_value" { + try stdx.Flags.parse_flag_value_fuzz(Ratio, Ratio.parse_flag_value, .{ + .ok = &.{ + .{ "0", .zero() }, + .{ "3/4", ratio(3, 4) }, + .{ "10/100", ratio(10, 100) }, + }, + .err = &.{ + .{ "1/0", "denominator is zero" }, + .{ "0/0", "denominator is zero" }, + .{ "3", "expected 'a/b' ratio, but found" }, + .{ "π/4", "invalid numerator" }, + .{ "3/i", "invalid denominator" }, + .{ "4/3", "ratio greater than 1" }, + }, + }); +} + +/// Canonical constructor for Ratio. Import as `const ratio = stdx.PRNG.ratio`. +pub fn ratio(numerator: u64, denominator: u64) Ratio { + assert(denominator > 0); + assert(numerator <= denominator); + return .{ .numerator = numerator, .denominator = denominator }; +} + +pub fn from_seed(seed: u64) PRNG { + var s = seed; + return .{ .s = .{ + split_mix_64(&s), + split_mix_64(&s), + split_mix_64(&s), + split_mix_64(&s), + } }; +} + +pub fn from_seed_testing() PRNG { + comptime assert(@import("builtin").is_test); + return .from_seed(std.testing.random_seed); +} + +fn split_mix_64(s: *u64) u64 { + s.* +%= 0x9e3779b97f4a7c15; + + var z = s.*; + z = (z ^ (z >> 30)) *% 0xbf58476d1ce4e5b9; + z = (z ^ (z >> 27)) *% 0x94d049bb133111eb; + return z ^ (z >> 31); +} + +fn next(prng: *PRNG) u64 { + const r = std.math.rotl(u64, prng.s[0] +% prng.s[3], 23) +% prng.s[0]; + + const t = prng.s[1] << 17; + + prng.s[2] ^= prng.s[0]; + prng.s[3] ^= prng.s[1]; + prng.s[1] ^= prng.s[2]; + prng.s[0] ^= prng.s[3]; + + prng.s[2] ^= t; + + prng.s[3] = math.rotl(u64, prng.s[3], 45); + + return r; +} + +test next { + var prng = from_seed(92); + var distribution: [8]u32 = @splat(0); + for (0..1000) |_| { + distribution[prng.next() % 8] += 1; + } + try snap(@src(), + \\{ 134, 134, 117, 121, 117, 128, 131, 118 } + ).diff_fmt("{d}", .{distribution}); +} + +pub fn fill(prng: *PRNG, target: []u8) void { + var i: usize = 0; + const aligned_len = target.len - (target.len & 7); + + // Complete 8 byte segments. + while (i < aligned_len) : (i += 8) { + var n = prng.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + target[i + j] = @as(u8, @truncate(n)); + n >>= 8; + } + } + + // Remaining (cuts the stream). + if (i != target.len) { + var n = prng.next(); + while (i < target.len) : (i += 1) { + target[i] = @as(u8, @truncate(n)); + n >>= 8; + } + } +} + +test fill { + const size_max = 128; + var buffer_max: [size_max]u8 = undefined; + var prng = from_seed(32); + + var distribution: [8]u32 = @splat(0); + for (0..size_max + 1) |size| { + // Check that the entire buffer is filled, by filling it over a couple of times + // and checking that each byte is non-zero at least once. + var non_zero: stdx.BitSetType(size_max) = .{}; + for (0..3) |_| { + const buffer = buffer_max[0..size]; + @memset(buffer, 0); + prng.fill(buffer); + for (buffer, 0..) |byte, i| { + distribution[byte % 8] += 1; + if (byte != 0) non_zero.set(i); + } + } + for (0..size) |i| assert(non_zero.is_set(i)); + } + + try snap(@src(), + \\{ 3120, 3084, 3089, 3103, 3092, 3120, 3074, 3086 } + ).diff_fmt("{d}", .{distribution}); +} + +/// Generate an unbiased, uniformly distributed integer r such that 0 ≤ r ≤ max. +/// +/// No biased version is provided --- while biased generation is simpler&faster, the bias can be +/// quite high depending on max! +pub fn int_inclusive(prng: *PRNG, Int: anytype, max: Int) Int { + comptime assert(@typeInfo(Int).int.signedness == .unsigned); + if (max == std.math.maxInt(Int)) { + return prng.int(Int); + } + + comptime assert(@typeInfo(Int).int.signedness == .unsigned); + const bits = @typeInfo(Int).int.bits; + const less_than = max + 1; + + // adapted from: + // http://www.pcg-random.org/posts/bounded-rands.html + // "Lemire's (with an extra tweak from Zig)" + var x = prng.int(Int); + var m = math.mulWide(Int, x, less_than); + var l: Int = @truncate(m); + if (l < less_than) { + var t = -%less_than; + + if (t >= less_than) { + t -= less_than; + if (t >= less_than) { + t %= less_than; + } + } + while (l < t) { + x = prng.int(Int); + m = math.mulWide(Int, x, less_than); + l = @truncate(m); + } + } + return @intCast(m >> bits); +} + +test int_inclusive { + var prng = from_seed(92); + for (0..8) |max_usize| { + const max: u8 = @intCast(max_usize); + var distribution: [8]u32 = @splat(0); + for (0..100) |_| { + distribution[prng.int_inclusive(u8, max)] += 1; + } + for (distribution[0 .. max + 1]) |d| assert(d > 0); + for (distribution[max + 1 ..]) |d| assert(d == 0); + } + + var distribution: [8]u32 = @splat(0); + for (0..1000) |_| { + const n = prng.int_inclusive(u128, 7); + distribution[@intCast(n)] += 1; + } + try snap(@src(), + \\{ 123, 127, 115, 125, 125, 139, 111, 135 } + ).diff_fmt("{d}", .{distribution}); + + var large: u32 = 0; + var small: u32 = 0; + for (0..1000) |_| { + if (prng.int_inclusive(u64, math.maxInt(u64) / 2) > math.maxInt(u64) / 4) { + large += 1; + } else { + small += 1; + } + } + try snap(@src(), + \\large=506 small=494 + ).diff_fmt("large={} small={}", .{ large, small }); +} + +// Deliberately excluded from the API to normalize everything to closed ranges. +// Somewhat surprisingly, closed ranges are more convenient for generating random numbers: +// - passing zero is not a subtle error +// - passing intMax allows generating any integer +// - at the call-site, inclusive is usually somewhat more obvious. +pub const int_exclusive = @compileError("intentionally not implemented"); + +/// Given a slice, generates a random valid index for the slice. +pub fn index(prng: *PRNG, slice: anytype) usize { + assert(slice.len > 0); + return prng.int_inclusive(usize, slice.len - 1); +} + +test index { + var prng = from_seed(92); + + var distribution: [8]u32 = @splat(0); + for (0..100) |_| { + distribution[index(&prng, &distribution)] += 1; + } + try snap(@src(), + \\{ 9, 13, 13, 11, 10, 16, 16, 12 } + ).diff_fmt("{d}", .{distribution}); +} + +/// Generates a uniform, unbiased integer r such that max ≤ r ≤ max. +pub fn range_inclusive(prng: *PRNG, Int: type, min: Int, max: Int) Int { + comptime assert(@typeInfo(Int).int.signedness == .unsigned); + assert(min <= max); + return min + prng.int_inclusive(Int, max - min); +} + +test range_inclusive { + var prng = from_seed(92); + for (0..8) |min| { + for (min..8) |max| { + var distribution: [8]u32 = @splat(0); + for (0..100) |_| { + distribution[prng.range_inclusive(usize, min, max)] += 1; + } + for (distribution, 0..) |d, i| { + assert((d > 0) == (min <= i and i <= max)); + } + } + } +} + +/// Returns a uniformly distributed integer of type T. +/// +/// That is, fills @sizeOf(T) bytes with random bits. +pub fn int(prng: *PRNG, Int: type) Int { + comptime assert(@typeInfo(Int).int.signedness == .unsigned); + if (Int == u64) return prng.next(); + if (@sizeOf(Int) < @sizeOf(u64)) return @truncate(prng.next()); + var result: Int = undefined; + prng.fill(std.mem.asBytes(&result)); + return result; +} + +test int { + try test_bytes_int(u8, snap(@src(), + \\{ 134, 134, 117, 121, 117, 128, 131, 118 } + )); + try test_bytes_int(u64, snap(@src(), + \\{ 134, 134, 117, 121, 117, 128, 131, 118 } + )); + try test_bytes_int(u128, snap(@src(), + \\{ 130, 143, 107, 135, 111, 119, 132, 123 } + )); +} + +fn test_bytes_int(Int: type, want: Snap) !void { + var prng = PRNG.from_seed(92); + var distribution: [8]u32 = @splat(0); + for (0..1000) |_| { + distribution[@intCast(prng.int(Int) % 8)] += 1; + } + try want.diff_fmt("{d}", .{distribution}); +} + +/// Returns true with probability 0.5. +pub fn boolean(prng: *PRNG) bool { + return prng.next() & 1 == 1; +} + +test boolean { + var prng = PRNG.from_seed(92); + var heads: u32 = 0; + var tails: u32 = 0; + for (0..1000) |_| { + if (prng.boolean()) heads += 1 else tails += 1; + } + try snap(@src(), + \\heads = 501 tails = 499 + ).diff_fmt("heads = {} tails = {}", .{ heads, tails }); +} + +/// Returns a Word with a single randomly-chosen bit set. +pub fn bit(prng: *PRNG, comptime Word: type) Word { + comptime assert(@typeInfo(Word) == .int); + comptime assert(@typeInfo(Word).int.signedness == .unsigned); + return @as(Word, 1) << prng.int_inclusive(std.math.Log2Int(Word), @bitSizeOf(Word) - 1); +} + +test bit { + var prng = PRNG.from_seed(92); + var hits: [8]u32 = @splat(0); + for (0..1000) |_| { + const word = prng.bit(u8); + assert(@popCount(word) == 1); + hits[@ctz(word)] += 1; + } + try snap(@src(), + \\{ 134, 134, 117, 121, 117, 128, 131, 118 } + ).diff_fmt("{any}", .{hits}); +} + +/// Returns true with the given rational probability. +pub fn chance(prng: *PRNG, probability: Ratio) bool { + assert(probability.denominator > 0); + assert(probability.numerator <= probability.denominator); + return prng.int_inclusive(u64, probability.denominator - 1) < probability.numerator; +} + +test chance { + var prng = PRNG.from_seed(92); + var balance: i32 = 0; + for (0..1000) |_| { + if (prng.chance(ratio(2, 7))) balance += 1 else balance -= 1; + if (prng.chance(ratio(5, 7))) balance += 1 else balance -= 1; + } + try snap(@src(), + \\balance = 46 + ).diff_fmt("balance = {d}", .{balance}); +} + +/// Like enum_weighted, but doesn't require specifying the enum up-front. +pub fn chances(prng: *PRNG, weights: anytype) std.meta.FieldEnum(@TypeOf(weights)) { + const Enum = std.meta.FieldEnum(@TypeOf(weights)); + return enum_weighted_impl(prng, Enum, weights); +} + +test chances { + var prng = from_seed(92); + var count: struct { a: u32 = 0, b: u32 = 0, c: u32 = 0 } = .{}; + for (0..1000) |_| { + switch (prng.chances(.{ .a = 1, .b = 3, .c = 2 })) { + inline else => |tag| @field(count, @tagName(tag)) += 1, + } + } + try snap(@src(), + \\a=166 b=475 c=359 + ).diff_fmt("a={} b={} c={}", .{ count.a, count.b, count.c }); +} + +pub fn error_uniform(prng: *PRNG, Error: type) Error { + const errors = @typeInfo(Error).error_set.?; + return switch (prng.index(errors)) { + inline 0...(errors.len - 1) => |i| @field(Error, errors[i].name), + else => unreachable, + }; +} + +/// Returns a random value of an enum. +pub fn enum_uniform(prng: *PRNG, Enum: type) Enum { + const values = std.enums.values(Enum); + return values[prng.index(values)]; +} + +test enum_uniform { + const E = enum(u8) { a, b, c = 8 }; // 8 tests that the discriminant is used properly. + + var prng = from_seed(92); + var count: struct { a: u32 = 0, b: u32 = 0, c: u32 = 0 } = .{}; + for (0..1000) |_| { + switch (prng.enum_uniform(E)) { + inline else => |tag| @field(count, @tagName(tag)) += 1, + } + } + + try snap(@src(), + \\a=318 b=323 c=359 + ).diff_fmt("a={} b={} c={}", .{ count.a, count.b, count.c }); +} + +pub fn EnumWeightsType(E: type) type { + return std.enums.EnumFieldStruct(E, u64, null); +} + +/// Returns a random value of an enum, where probability is proportional to weight. +pub fn enum_weighted(prng: *PRNG, Enum: type, weights: EnumWeightsType(Enum)) Enum { + return enum_weighted_impl(prng, Enum, weights); +} + +fn enum_weighted_impl(prng: *PRNG, Enum: type, weights: anytype) Enum { + const fields = @typeInfo(Enum).@"enum".fields; + var total: u64 = 0; + inline for (fields) |field| { + total += @field(weights, field.name); + } + assert(total > 0); + var pick = prng.int_inclusive(u64, total - 1); + inline for (fields) |field| { + const weight = @field(weights, field.name); + if (pick < weight) return @as(Enum, @enumFromInt(field.value)); + pick -= weight; + } + unreachable; +} + +test enum_weighted { + const E = enum(u8) { a, b, c = 8 }; // 8 tests that the discriminant is used properly. + + var prng = from_seed(92); + var count: struct { a: u32 = 0, b: u32 = 0, c: u32 = 0 } = .{}; + for (0..1000) |_| { + switch (prng.enum_weighted(E, .{ .a = 0, .b = 1, .c = 2 })) { + inline else => |tag| @field(count, @tagName(tag)) += 1, + } + } + + try snap(@src(), + \\a=0 b=318 c=682 + ).diff_fmt("a={} b={} c={}", .{ count.a, count.b, count.c }); +} + +/// Return a distribution for use with `random_enum`. +/// +/// This is swarm testing: some variants are disabled completely, +/// and the rest have wildly different probabilities. +pub fn enum_weights( + prng: *PRNG, + comptime Enum: type, +) EnumWeightsType(Enum) { + const fields = comptime std.meta.fieldNames(Enum); + + var combination = PRNG.Combination.init(.{ + .total = fields.len, + .sample = prng.range_inclusive(u32, 1, fields.len), + }); + defer assert(combination.done()); + + var weights: PRNG.EnumWeightsType(Enum) = undefined; + inline for (fields) |field| { + @field(weights, field) = if (combination.take(prng)) + prng.range_inclusive(u64, 1, 100) + else + 0; + } + + return weights; +} + +/// An iterator-style API for selecting a random combination of elements. +pub const Combination = struct { + total: u32, + sample: u32, + + taken: u32, + seen: u32, + + pub fn init(options: struct { total: u32, sample: u32 }) Combination { + assert(options.sample <= options.total); + return .{ + .total = options.total, + .sample = options.sample, + .taken = 0, + .seen = 0, + }; + } + + pub fn done(combination: *const Combination) bool { + return combination.taken == combination.sample and + combination.seen == combination.total; + } + + pub fn take(combination: *Combination, prng: *PRNG) bool { + assert(combination.seen < combination.total); + assert(combination.taken <= combination.sample); + + const n = combination.total - combination.seen; + const k = combination.sample - combination.taken; + const result = prng.chance(ratio(k, n)); + + combination.seen += 1; + if (result) combination.taken += 1; + return result; + } +}; + +test Combination { + var prng = from_seed(92); + + const pool: [7]u8 = "abcdefg".*; + var result: [3]u8 = undefined; + var result_count: usize = 0; + + var e_taken_count: u32 = 0; + for (0..1000) |_| { + result_count = 0; + var combination = Combination.init(.{ .total = pool.len, .sample = 3 }); + for (pool) |x| { + if (combination.take(&prng)) { + result[result_count] = x; + result_count += 1; + } + } + assert(combination.done()); + assert(result_count == 3); + + e_taken_count += @intFromBool(std.mem.indexOfScalar(u8, &result, 'e') != null); + } + + try snap(@src(), + \\e_taken_count = 432 expected_value=428 + ).diff_fmt("e_taken_count = {} expected_value={}", .{ e_taken_count, 1000 * 3 / 7 }); +} + +/// An iterator style API for selecting a single element out of the given weighted sequence, +/// without a priori knowledge about the total weight. +pub const Reservoir = struct { + total: u64, + + pub fn init() Reservoir { + return .{ .total = 0 }; + } + + pub fn replace(reservoir: *Reservoir, prng: *PRNG, weight: u64) bool { + reservoir.total += weight; + return prng.chance(ratio(weight, reservoir.total)); + } +}; + +test Reservoir { + var prng = from_seed(92); + const animals: []const []const u8 = &.{ "walrus", "kiwi", "capybara", "platypus" }; + var kiwi_count: u32 = 0; + + for (0..1000) |_| { + var reservoir = Reservoir.init(); + var pick: ?[]const u8 = null; + for (animals) |animal| { + if (reservoir.replace(&prng, animal.len)) pick = animal; + } + assert(pick != null); + kiwi_count += @intFromBool(std.mem.eql(u8, pick.?, "kiwi")); + } + + var total_weight: u64 = 0; + for (animals) |animal| total_weight += animal.len; + const expected_value = 1000 * "kiwi".len / total_weight; + + try snap(@src(), + \\kiwi_count = 141 expected_value=153 + ).diff_fmt("kiwi_count = {} expected_value={}", .{ kiwi_count, expected_value }); +} + +pub fn shuffle(prng: *PRNG, T: type, slice: []T) void { + for (0..slice.len) |i| { + const j = prng.int_inclusive(u64, i); + std.mem.swap(T, &slice[i], &slice[j]); + } +} + +test shuffle { + var prng = from_seed(92); + var g_first_count: u32 = 0; + + for (0..1000) |_| { + var buffer = "abcdefg".*; + shuffle(&prng, u8, &buffer); + g_first_count += @intFromBool(buffer[0] == 'g'); + } + + try snap(@src(), + \\g_first_count = 152 expected_value=142 + ).diff_fmt("g_first_count = {} expected_value={}", .{ g_first_count, 1000 / 7 }); +} + +test "no floating point please" { + const path = try std.fs.path.join(std.testing.allocator, &.{ + module_path, + @src().file, + }); + defer std.testing.allocator.free(path); + + const file_text = try std.fs.cwd().readFileAlloc(std.testing.allocator, path, 64 * KiB); + defer std.testing.allocator.free(file_text); + + assert(std.mem.indexOf(u8, file_text, "f" ++ "32") == null); + assert(std.mem.indexOf(u8, file_text, "f" ++ "64") == null); +} diff --git a/scripts/stdx/radix.zig b/scripts/stdx/radix.zig new file mode 100644 index 0000000..b79f6a5 --- /dev/null +++ b/scripts/stdx/radix.zig @@ -0,0 +1,336 @@ +//! Stable, non-allocating, out-of-place LSD radix sort over unsigned integer keys. +//! Sorts `values` in ascending order by `key_from_value`, using `values_scratch` +//! as an equally sized, disjoint swap buffer. Keys must be an unsigned `Int`. +//! The sorted result is in the original buffer `values`. +//! The implementation builds per-pass histograms, skips trivial passes (all items +//! in one bucket), and uses a fixed digit width (8 or 11 bits based on `Value` size) +//! to reduce the number of passes. Buffers are swapped after each non-trivial pass; +//! if the number of such passes is odd, results are copied back so `values` holds +//! the output on return. + +const std = @import("std"); +const assert = std.debug.assert; +const stdx = @import("stdx.zig"); + +/// Stable, ascending radix sort for unsigned integers. The sorted result will be in `values`. +pub fn sort( + comptime Key: type, + comptime Value: type, + comptime key_from_value: fn (*const Value) callconv(.@"inline") Key, + values: []Value, + values_scratch: []Value, +) void { + comptime { + assert(@typeInfo(Key) == .int); + assert(@typeInfo(Key).int.signedness == .unsigned); + } + + assert(stdx.disjoint_slices(Value, Value, values, values_scratch)); + assert(values.len == values_scratch.len); + assert(values.len <= std.math.maxInt(u32)); + + if (values.len == 0) return; + if (values.len <= 32) { + return std.sort.insertion(Value, values, {}, struct { + fn lessThan(_: void, a: Value, b: Value) bool { + return key_from_value(&a) < key_from_value(&b); + } + }.lessThan); + } + radix_sort(Key, Value, key_from_value, values, values_scratch); +} + +fn radix_sort( + comptime Key: type, + comptime Value: type, + comptime key_from_value: fn (*const Value) callconv(.@"inline") Key, + values: []Value, + values_scratch: []Value, +) void { + const count: u32 = @intCast(values.len); + // Heuristic: use more bits for larger value sizes to reduce the number of passes. + const radix_bits_heuristic = if (@sizeOf(Value) >= 128) 11 else 8; + const radix_bits = @min(@bitSizeOf(Key), radix_bits_heuristic); + const radix_passes = stdx.div_ceil(@bitSizeOf(Key), radix_bits); + const radix_partitions = 1 << radix_bits; + const radix_mask: u32 = radix_partitions - 1; + + const BitsKey = std.math.Log2Int(Key); // Used to shift the key for each pass. + const Histograms: type = [radix_passes][radix_partitions]u32; + comptime assert(@sizeOf(Histograms) <= 200 * stdx.KiB); + + // Create histograms per radix pass in a single iteration over `values`. + const histograms = blk: { + var histograms: Histograms align(64) = @splat(@splat(0)); + + for (values) |*value| { + const key = key_from_value(value); + inline for (0..radix_passes) |pass| { + const pass_bit_offset: BitsKey = @intCast(pass * radix_bits); + const partition_id: u32 = @intCast((key >> pass_bit_offset) & radix_mask); + histograms[pass][partition_id] += 1; + } + } + break :blk histograms; + }; + + var source: []Value = values; + var target: []Value = values_scratch; + var target_offsets: [radix_partitions]u32 = @splat(0); + + inline for (0..radix_passes) |pass| { + // Determine if a pass is trivial if exactly one partition has all `count` elements. + const pass_trivial: bool = for (histograms[pass]) |partition_count| { + if (partition_count == count) break true; + } else false; + + if (!pass_trivial) { + // Build prefix sums. + var next_offset: u32 = 0; + for (0..radix_partitions) |partition_id| { + target_offsets[partition_id] = next_offset; + next_offset += histograms[pass][partition_id]; + } + + // Partitioning pass. + const pass_bit_offset: BitsKey = @intCast(pass * radix_bits); + for (source) |*value| { + const key: Key = key_from_value(value); + const partition_id: u32 = @intCast((key >> pass_bit_offset) & radix_mask); + + target[target_offsets[partition_id]] = value.*; + target_offsets[partition_id] += 1; + } + std.mem.swap([]Value, &source, &target); + } + } + + // Copy the values back into the input buffer `values`. + if (values.ptr != source.ptr) { + stdx.copy_disjoint(.exact, Value, values, values_scratch); + } +} + +const ratio = stdx.PRNG.ratio; + +pub fn TestValueType(comptime Key: type, comptime value_length: usize) type { + return struct { + const Value = @This(); + + x: Key, + y: u32, // y ensures that values are distinct for the purpose of checking stability. + padding: [value_length]u8 = @splat(0), + + inline fn key_from_value(value: *const Value) Key { + return value.x; + } + + fn compare_x_ascending(_: void, a: Value, b: Value) bool { + return a.x < b.x; + } + + fn compare_x_descending(_: void, a: Value, b: Value) bool { + return a.x > b.x; + } + }; +} + +test "radix_sort: smoke" { + const Value = TestValueType(u8, 0); + var values: [5]Value = .{ + Value{ .x = 3, .y = 0 }, + Value{ .x = 2, .y = 0 }, + Value{ .x = 3, .y = 1 }, + Value{ .x = 1, .y = 0 }, + Value{ .x = 5, .y = 0 }, + }; + const values_expected: [5]Value = .{ + Value{ .x = 1, .y = 0 }, + Value{ .x = 2, .y = 0 }, + Value{ .x = 3, .y = 0 }, + Value{ .x = 3, .y = 1 }, + Value{ .x = 5, .y = 0 }, + }; + var values_scratch: [5]Value = undefined; + radix_sort( + u8, + Value, + Value.key_from_value, + &values, + &values_scratch, + ); + try std.testing.expectEqual(values_expected, values); +} + +// Ascending order + stability against a (x,y) baseline, with a large Value +// payload to exercise the 11-bit radix path (since @sizeOf(Value) >= 128). +test "radix_sort: ascending & stable on many duplicates" { + const Key = u32; + const Value = TestValueType(Key, 128); // >=128 so radix_bits heuristic picks 11 + const allocator = std.testing.allocator; + + const n: usize = 2048; + + const values = try allocator.alloc(Value, n); + defer allocator.free(values); + + const scratch = try allocator.alloc(Value, n); + defer allocator.free(scratch); + + // Many duplicates; y = original index (used to check stability). + for (values, 0..) |*v, i| { + const k: Key = @intCast(i % 257); + v.* = .{ .x = k, .y = @intCast(i) }; + } + + radix_sort(Key, Value, Value.key_from_value, values, scratch); + + // Verify that the order is `ascending` and `stable`. + for (values[0 .. values.len - 1], values[1..]) |a, b| { + switch (std.math.order(a.x, b.x)) { + .eq => try std.testing.expect(a.y < b.y), + .lt => try std.testing.expect(a.x < b.x), + .gt => unreachable, + } + } +} + +// All keys equal → every pass is "trivial". Sort should be a no-op on values, +// keep relative order (stability). +test "radix_sort: all-equal keys preserve relative order (stability)" { + const Key = u64; + const Value = TestValueType(Key, 8); + const allocator = std.testing.allocator; + + const n: usize = 1024; + + const values = try allocator.alloc(Value, n); + defer allocator.free(values); + + const scratch = try allocator.alloc(Value, n); + defer allocator.free(scratch); + + // Fill scratch with a sentinel to detect writes. + const sentinel: Value = .{ .x = 0xFFFF_FFFF_FFFF_FFFF, .y = 0xDEAD_BEEF }; + for (scratch) |*s| s.* = sentinel; + + // All keys identical; y = original index to check stability. + for (values, 0..) |*v, i| { + v.* = .{ .x = 42, .y = @intCast(i) }; + } + + radix_sort(Key, Value, Value.key_from_value, values, scratch); + + // Verify that the order is `ascending` and `stable`. + for (values[0 .. values.len - 1], values[1..]) |a, b| { + switch (std.math.order(a.x, b.x)) { + .eq => try std.testing.expect(a.y < b.y), + .lt => try std.testing.expect(a.x < b.x), + .gt => unreachable, + } + } +} + +test "fuzz radix_sort_stable" { + inline for (.{ + .{ u3, 0 }, // Smaller than radix bits. + .{ u256, 130 }, // Largest histogram, requires multiple passes and bit heuristic. + }) |pair| { + const Key = pair.@"0"; + const value_size_min = pair.@"1"; + const allocator = std.testing.allocator; + + const Value = TestValueType(Key, value_size_min); + + var prng = stdx.PRNG.from_seed_testing(); + + const values_max = 1 << 18; // Explores uneven and even passes to test copy back. + const values_all = try allocator.alloc(Value, values_max); + defer allocator.free(values_all); + + const values_all_scratch = try allocator.alloc(Value, values_max); + defer allocator.free(values_all_scratch); + + for (0..64) |_| { + const values_count = prng.range_inclusive(u32, 2, values_max); + const values = values_all[0..values_count]; + const values_scratch = values_all_scratch[0..values_count]; + + { + // Set up `values`. + for (values) |*value| { + value.* = .{ + .x = prng.int_inclusive(Key, @min( + std.math.maxInt(Key), + values_count * 2 - 1, + )), + .y = undefined, + }; + } + + // Sort algorithms often optimize the case of already-sorted + // (or already-reverse-sorted) sub-arrays. + const partitions_count = prng.range_inclusive( + u32, + 1, + @max(values_count, 64) - 1, + ); + // The `partition_reverse_probability` is a subset of the partitions sorted by + // `partition_sort_percent`. + const partition_sort_probability = ratio(prng.int_inclusive(u8, 100), 100); + const partition_reverse_probability = ratio(prng.int_inclusive(u8, 100), 100); + + var partitions_remaining: u32 = partitions_count; + var partition_offset: u32 = 0; + while (partition_offset < values_count) { + const partition_size = size: { + if (partitions_remaining == 1) { + break :size values_count - partition_offset; + } else { + break :size prng.range_inclusive( + u32, + 1, + values_count - partition_offset, + ); + } + }; + + if (prng.chance(partition_sort_probability)) { + const partition = values[partition_offset..][0..partition_size]; + if (prng.chance(partition_reverse_probability)) { + std.mem.sortUnstable( + Value, + partition, + {}, + Value.compare_x_descending, + ); + } else { + std.mem.sortUnstable( + Value, + partition, + {}, + Value.compare_x_ascending, + ); + } + } + + partitions_remaining -= 1; + partition_offset += partition_size; + } + + for (values, 0..) |*value, i| value.y = @intCast(i); + } + + radix_sort(Key, Value, Value.key_from_value, values, values_scratch); + + // Verify that the order is `ascending` and `stable`. + for (values[0 .. values.len - 1], values[1..]) |a, b| { + switch (std.math.order(a.x, b.x)) { + .eq => try std.testing.expect(a.y < b.y), + .lt => try std.testing.expect(a.x < b.x), + .gt => unreachable, + } + } + } + } +} diff --git a/scripts/stdx/ring_buffer.zig b/scripts/stdx/ring_buffer.zig new file mode 100644 index 0000000..45899b7 --- /dev/null +++ b/scripts/stdx/ring_buffer.zig @@ -0,0 +1,511 @@ +const std = @import("std"); +const assert = std.debug.assert; +const math = std.math; +const mem = std.mem; + +const stdx = @import("stdx.zig"); + +/// A First In, First Out ring buffer. +pub fn RingBufferType( + comptime T: type, + comptime buffer_type: union(enum) { + array: usize, // capacity + slice, // (Capacity is passed to init() at runtime). + }, +) type { + return struct { + const RingBuffer = @This(); + + pub const count_max = switch (buffer_type) { + .array => |count_max_| count_max_, + .slice => {}, + }; + + buffer: switch (buffer_type) { + .array => |count_max_| [count_max_]T, + .slice => []T, + }, + + /// The index of the slot with the first item, if any. + index: usize = 0, + + /// The number of items in the buffer. + count: usize = 0, + + pub const init = switch (buffer_type) { + .array => init_array, + .slice => init_slice, + }; + + fn init_array() RingBuffer { + comptime assert(buffer_type == .array); + return .{ .buffer = undefined }; + } + + fn init_slice(allocator: mem.Allocator, capacity: usize) !RingBuffer { + comptime assert(buffer_type == .slice); + assert(capacity > 0); + + const buffer = try allocator.alloc(T, capacity); + errdefer allocator.free(buffer); + return RingBuffer{ .buffer = buffer }; + } + + pub const deinit = switch (buffer_type) { + .array => {}, + .slice => deinit_slice, + }; + + fn deinit_slice(self: *RingBuffer, allocator: mem.Allocator) void { + comptime assert(buffer_type == .slice); + allocator.free(self.buffer); + } + + pub inline fn clear(self: *RingBuffer) void { + self.index = 0; + self.count = 0; + } + + // TODO Add doc comments to these functions: + pub inline fn head(self: RingBuffer) ?T { + if (self.buffer.len == 0 or self.empty()) return null; + return self.buffer[self.index]; + } + + pub inline fn head_ptr(self: *RingBuffer) ?*T { + if (self.buffer.len == 0 or self.empty()) return null; + return &self.buffer[self.index]; + } + + pub inline fn head_ptr_const(self: *const RingBuffer) ?*const T { + if (self.buffer.len == 0 or self.empty()) return null; + return &self.buffer[self.index]; + } + + pub inline fn tail(self: RingBuffer) ?T { + if (self.buffer.len == 0 or self.empty()) return null; + return self.buffer[(self.index + self.count - 1) % self.buffer.len]; + } + + pub inline fn tail_ptr(self: *RingBuffer) ?*T { + if (self.buffer.len == 0 or self.empty()) return null; + return &self.buffer[(self.index + self.count - 1) % self.buffer.len]; + } + + pub inline fn tail_ptr_const(self: *const RingBuffer) ?*const T { + if (self.buffer.len == 0 or self.empty()) return null; + return &self.buffer[(self.index + self.count - 1) % self.buffer.len]; + } + + pub fn get(self: *const RingBuffer, index: usize) ?T { + if (self.buffer.len == 0) unreachable; + + if (index < self.count) { + return self.buffer[(self.index + index) % self.buffer.len]; + } else { + assert(index < self.buffer.len); + return null; + } + } + + pub inline fn get_ptr(self: *RingBuffer, index: usize) ?*T { + if (self.buffer.len == 0) unreachable; + + if (index < self.count) { + return &self.buffer[(self.index + index) % self.buffer.len]; + } else { + assert(index < self.buffer.len); + return null; + } + } + + pub inline fn next_tail(self: RingBuffer) ?T { + if (self.buffer.len == 0 or self.full()) return null; + return self.buffer[(self.index + self.count) % self.buffer.len]; + } + + pub inline fn next_tail_ptr(self: *RingBuffer) ?*T { + if (self.buffer.len == 0 or self.full()) return null; + return &self.buffer[(self.index + self.count) % self.buffer.len]; + } + + pub inline fn next_tail_ptr_const(self: *const RingBuffer) ?*const T { + if (self.buffer.len == 0 or self.full()) return null; + return &self.buffer[(self.index + self.count) % self.buffer.len]; + } + + pub inline fn advance_head(self: *RingBuffer) void { + self.index += 1; + self.index %= self.buffer.len; + self.count -= 1; + } + + pub fn advance_head_many(self: *RingBuffer, discard: u64) void { + assert(discard <= self.count); + if (self.buffer.len == 0) return; + + self.index += discard; + self.index %= self.buffer.len; + self.count -= discard; + } + + pub inline fn retreat_head(self: *RingBuffer) void { + assert(self.count < self.buffer.len); + + // This condition is covered by the above assert, but it is necessary to make it + // explicitly unreachable so that the compiler doesn't error when computing (at + // comptime) `buffer.len - 1` for a zero-capacity array-backed ring buffer. + if (self.buffer.len == 0) unreachable; + + self.index += self.buffer.len - 1; + self.index %= self.buffer.len; + self.count += 1; + } + + pub inline fn advance_tail(self: *RingBuffer) void { + assert(self.count < self.buffer.len); + self.count += 1; + } + + pub inline fn retreat_tail(self: *RingBuffer) void { + self.count -= 1; + } + + /// Returns whether the ring buffer is completely full. + pub inline fn full(self: RingBuffer) bool { + return self.count == self.buffer.len; + } + + pub inline fn spare_capacity(self: RingBuffer) usize { + return self.buffer.len - self.count; + } + + /// Returns whether the ring buffer is completely empty. + pub inline fn empty(self: RingBuffer) bool { + return self.count == 0; + } + + // Higher level, less error-prone wrappers: + + pub fn push_head(self: *RingBuffer, item: T) error{NoSpaceLeft}!void { + if (self.count == self.buffer.len) return error.NoSpaceLeft; + self.push_head_assume_capacity(item); + } + + pub fn push_head_assume_capacity(self: *RingBuffer, item: T) void { + assert(self.count < self.buffer.len); + + self.retreat_head(); + self.head_ptr().?.* = item; + } + + /// Add an element to the RingBuffer. Returns an error if the buffer + /// is already full and the element could not be added. + pub fn push(self: *RingBuffer, item: T) error{NoSpaceLeft}!void { + const ptr = self.next_tail_ptr() orelse return error.NoSpaceLeft; + ptr.* = item; + self.advance_tail(); + } + + /// Add an element to a RingBuffer, and assert that the capacity is sufficient. + pub fn push_assume_capacity(self: *RingBuffer, item: T) void { + self.push(item) catch |err| switch (err) { + error.NoSpaceLeft => unreachable, + }; + } + + pub fn push_slice(self: *RingBuffer, items: []const T) error{NoSpaceLeft}!void { + if (self.buffer.len == 0) return error.NoSpaceLeft; + if (self.count + items.len > self.buffer.len) return error.NoSpaceLeft; + + const pre_wrap_start = (self.index + self.count) % self.buffer.len; + const pre_wrap_count = @min(items.len, self.buffer.len - pre_wrap_start); + const post_wrap_count = items.len - pre_wrap_count; + + const pre_wrap_items = items[0..pre_wrap_count]; + const post_wrap_items = items[pre_wrap_count..]; + stdx.copy_disjoint(.inexact, T, self.buffer[pre_wrap_start..], pre_wrap_items); + stdx.copy_disjoint(.exact, T, self.buffer[0..post_wrap_count], post_wrap_items); + + self.count += items.len; + } + + /// Remove and return the next item, if any. + pub fn pop(self: *RingBuffer) ?T { + const result = self.head() orelse return null; + self.advance_head(); + return result; + } + + /// Remove and return the last item, if any. + pub fn pop_tail(self: *RingBuffer) ?T { + const result = self.tail() orelse return null; + self.retreat_tail(); + return result; + } + + pub const Iterator = struct { + ring: *const RingBuffer, + count: usize = 0, + + pub fn next(it: *Iterator) ?T { + if (it.next_ptr()) |item| { + return item.*; + } + return null; + } + + pub fn next_ptr(it: *Iterator) ?*const T { + assert(it.count <= it.ring.count); + if (it.ring.buffer.len == 0) return null; + if (it.count == it.ring.count) return null; + defer it.count += 1; + + return &it.ring.buffer[(it.ring.index + it.count) % it.ring.buffer.len]; + } + }; + + /// Returns an iterator to iterate through all `count` items in the ring buffer. + /// The iterator is invalidated if the ring buffer is advanced. + pub fn iterator(self: *const RingBuffer) Iterator { + return .{ .ring = self }; + } + + pub const IteratorMutable = struct { + ring: *RingBuffer, + count: usize = 0, + + pub fn next_ptr(it: *IteratorMutable) ?*T { + assert(it.count <= it.ring.count); + if (it.ring.buffer.len == 0) return null; + if (it.count == it.ring.count) return null; + defer it.count += 1; + + return &it.ring.buffer[(it.ring.index + it.count) % it.ring.buffer.len]; + } + }; + + pub fn iterator_mutable(self: *RingBuffer) IteratorMutable { + return .{ .ring = self }; + } + }; +} + +const testing = std.testing; + +fn test_iterator(comptime T: type, ring: *T, values: []const u32) !void { + const ring_index = ring.index; + + inline for (.{ .immutable, .mutable }) |mutability| { + for (0..2) |_| { + var iterator = switch (mutability) { + .immutable => ring.iterator(), + .mutable => ring.iterator_mutable(), + else => unreachable, + }; + var index: u32 = 0; + switch (mutability) { + .immutable => while (iterator.next()) |item| { + try testing.expectEqual(values[index], item); + index += 1; + }, + .mutable => { + const permutation = @divFloor(std.math.maxInt(u32), 2); + while (iterator.next_ptr()) |item| { + try testing.expectEqual(values[index], item.*); + item.* += permutation + index; + index += 1; + } + iterator = ring.iterator_mutable(); + var check_index: u32 = 0; + while (iterator.next_ptr()) |item| { + try testing.expectEqual( + values[check_index] + permutation + check_index, + item.*, + ); + item.* -= permutation + check_index; + check_index += 1; + } + try testing.expectEqual(index, check_index); + }, + else => unreachable, + } + try testing.expectEqual(values.len, index); + } + + try testing.expectEqual(ring_index, ring.index); + } +} + +fn test_low_level_interface(comptime Ring: type, ring: *Ring) !void { + try ring.push_slice(&[_]u32{}); + try test_iterator(Ring, ring, &[_]u32{}); + + try testing.expectError(error.NoSpaceLeft, ring.push_slice(&[_]u32{ 1, 2, 3 })); + + try ring.push_slice(&[_]u32{1}); + try testing.expectEqual(@as(?u32, 1), ring.tail()); + try testing.expectEqual(@as(u32, 1), ring.tail_ptr().?.*); + ring.advance_head(); + + try testing.expectEqual(@as(usize, 1), ring.index); + try testing.expectEqual(@as(usize, 0), ring.count); + try ring.push_slice(&[_]u32{ 1, 2 }); + try test_iterator(Ring, ring, &[_]u32{ 1, 2 }); + ring.advance_head(); + ring.advance_head(); + + try testing.expectEqual(@as(usize, 1), ring.index); + try testing.expectEqual(@as(usize, 0), ring.count); + try ring.push_slice(&[_]u32{1}); + try testing.expectEqual(@as(?u32, 1), ring.tail()); + try testing.expectEqual(@as(u32, 1), ring.tail_ptr().?.*); + ring.advance_head(); + + try testing.expectEqual(@as(?u32, null), ring.head()); + try testing.expectEqual(@as(?*u32, null), ring.head_ptr()); + try testing.expectEqual(@as(?u32, null), ring.tail()); + try testing.expectEqual(@as(?*u32, null), ring.tail_ptr()); + + ring.next_tail_ptr().?.* = 0; + ring.advance_tail(); + try testing.expectEqual(@as(?u32, 0), ring.tail()); + try testing.expectEqual(@as(u32, 0), ring.tail_ptr().?.*); + try test_iterator(Ring, ring, &[_]u32{0}); + + ring.next_tail_ptr().?.* = 1; + ring.advance_tail(); + try testing.expectEqual(@as(?u32, 1), ring.tail()); + try testing.expectEqual(@as(u32, 1), ring.tail_ptr().?.*); + try test_iterator(Ring, ring, &[_]u32{ 0, 1 }); + + try testing.expectEqual(@as(?u32, null), ring.next_tail()); + try testing.expectEqual(@as(?*u32, null), ring.next_tail_ptr()); + + try testing.expectEqual(@as(?u32, 0), ring.head()); + try testing.expectEqual(@as(u32, 0), ring.head_ptr().?.*); + ring.advance_head(); + try test_iterator(Ring, ring, &[_]u32{1}); + + ring.next_tail_ptr().?.* = 2; + ring.advance_tail(); + try testing.expectEqual(@as(?u32, 2), ring.tail()); + try testing.expectEqual(@as(u32, 2), ring.tail_ptr().?.*); + try test_iterator(Ring, ring, &[_]u32{ 1, 2 }); + + ring.advance_head(); + try test_iterator(Ring, ring, &[_]u32{2}); + + ring.next_tail_ptr().?.* = 3; + ring.advance_tail(); + try testing.expectEqual(@as(?u32, 3), ring.tail()); + try testing.expectEqual(@as(u32, 3), ring.tail_ptr().?.*); + try test_iterator(Ring, ring, &[_]u32{ 2, 3 }); + + try testing.expectEqual(@as(?u32, 2), ring.head()); + try testing.expectEqual(@as(u32, 2), ring.head_ptr().?.*); + ring.advance_head(); + try test_iterator(Ring, ring, &[_]u32{3}); + + try testing.expectEqual(@as(?u32, 3), ring.head()); + try testing.expectEqual(@as(u32, 3), ring.head_ptr().?.*); + ring.advance_head(); + try test_iterator(Ring, ring, &[_]u32{}); + + try testing.expectEqual(@as(?u32, null), ring.head()); + try testing.expectEqual(@as(?*u32, null), ring.head_ptr()); + try testing.expectEqual(@as(?u32, null), ring.tail()); + try testing.expectEqual(@as(?*u32, null), ring.tail_ptr()); +} + +test "RingBuffer: low level interface" { + const ArrayRing = RingBufferType(u32, .{ .array = 2 }); + var array_ring = ArrayRing.init(); + try test_low_level_interface(ArrayRing, &array_ring); + + const PointerRing = RingBufferType(u32, .slice); + var pointer_ring = try PointerRing.init(testing.allocator, 2); + defer pointer_ring.deinit(testing.allocator); + + try test_low_level_interface(PointerRing, &pointer_ring); +} + +test "RingBuffer: push/pop high level interface" { + var fifo = RingBufferType(u32, .{ .array = 3 }).init(); + + try testing.expect(!fifo.full()); + try testing.expect(fifo.empty()); + try testing.expectEqual(@as(?*u32, null), fifo.get_ptr(0)); + try testing.expectEqual(@as(?*u32, null), fifo.get_ptr(1)); + try testing.expectEqual(@as(?*u32, null), fifo.get_ptr(2)); + + try fifo.push(1); + try testing.expectEqual(@as(?u32, 1), fifo.head()); + try testing.expectEqual(@as(u32, 1), fifo.get_ptr(0).?.*); + try testing.expectEqual(@as(?*u32, null), fifo.get_ptr(1)); + + try testing.expect(!fifo.full()); + try testing.expect(!fifo.empty()); + + try fifo.push(2); + try testing.expectEqual(@as(?u32, 1), fifo.head()); + try testing.expectEqual(@as(u32, 2), fifo.get_ptr(1).?.*); + + try fifo.push(3); + try testing.expectError(error.NoSpaceLeft, fifo.push(4)); + + try testing.expect(fifo.full()); + try testing.expect(!fifo.empty()); + + try testing.expectEqual(@as(?u32, 1), fifo.head()); + try testing.expectEqual(@as(?u32, 1), fifo.pop()); + try testing.expectEqual(@as(u32, 2), fifo.get_ptr(0).?.*); + try testing.expectEqual(@as(u32, 3), fifo.get_ptr(1).?.*); + try testing.expectEqual(@as(?*u32, null), fifo.get_ptr(2)); + + try testing.expect(!fifo.full()); + try testing.expect(!fifo.empty()); + + try fifo.push(4); + + try testing.expectEqual(@as(?u32, 2), fifo.pop()); + try testing.expectEqual(@as(?u32, 3), fifo.pop()); + try testing.expectEqual(@as(?u32, 4), fifo.pop()); + try testing.expectEqual(@as(?u32, null), fifo.pop()); + + try testing.expect(!fifo.full()); + try testing.expect(fifo.empty()); +} + +test "RingBuffer: pop_tail" { + var lifo = RingBufferType(u32, .{ .array = 3 }).init(); + try lifo.push(1); + try lifo.push(2); + try lifo.push(3); + try testing.expect(lifo.full()); + + try testing.expectEqual(@as(?u32, 3), lifo.pop_tail()); + try testing.expectEqual(@as(?u32, 1), lifo.head()); + try testing.expectEqual(@as(?u32, 2), lifo.pop_tail()); + try testing.expectEqual(@as(?u32, 1), lifo.head()); + try testing.expectEqual(@as(?u32, 1), lifo.pop_tail()); + try testing.expectEqual(@as(?u32, null), lifo.pop_tail()); + try testing.expect(lifo.empty()); +} + +test "RingBuffer: push_head" { + var ring = RingBufferType(u32, .{ .array = 3 }).init(); + try ring.push_head(1); + try ring.push(2); + try ring.push_head(3); + try testing.expect(ring.full()); + + try testing.expectEqual(@as(?u32, 3), ring.pop()); + try testing.expectEqual(@as(?u32, 1), ring.pop()); + try testing.expectEqual(@as(?u32, 2), ring.pop()); + try testing.expect(ring.empty()); +} + +test "RingBuffer: count_max=0" { + std.testing.refAllDecls(RingBufferType(u32, .{ .array = 0 })); +} diff --git a/scripts/stdx/shell.zig b/scripts/stdx/shell.zig new file mode 100644 index 0000000..bfbae9d --- /dev/null +++ b/scripts/stdx/shell.zig @@ -0,0 +1,1225 @@ +//! Collection of utilities for scripting: an in-process sh+coreutils combo. +//! +//! Keep this as a single file, independent from the rest of the codebase, to make it easier to +//! reuse across different processes (eg build.zig). +//! +//! If possible, avoid shelling out to `sh` or other systems utils --- the whole purpose here is to +//! avoid any extra dependencies. +//! +//! The `exec_` family of methods provides a convenience wrapper around `std.process.Child`: +//! - It allows constructing the array of arguments using convenient interpolation syntax a-la +//! `std.fmt` (but of course no actual string concatenation happens anywhere). +//! - `ChildProcess` is versatile and has many knobs, but they might be hard to use correctly (eg, +//! its easy to forget to check exit status). `Shell` instead is focused on providing a set of +//! specific narrow use-cases (eg, parsing the output of a subprocess) and takes care of setting +//! the right defaults. + +const std = @import("std"); +const log = std.log; +const assert = std.debug.assert; +const builtin = @import("builtin"); + +const stdx = @import("stdx.zig"); +const MiB = stdx.MiB; +const Snap = stdx.Snap; + +const Shell = @This(); + +const cwd_stack_max = 16; + +/// For internal use by the `Shell` itself. +gpa: std.mem.Allocator, + +/// To improve ergonomics, any returned data is owned by the `Shell` and is stored in this arena. +/// This way, the user doesn't need to worry about deallocating each individual string, as long as +/// they don't forget to call `Shell.destroy`. +arena: std.heap.ArenaAllocator, + +/// Root directory of this repository. +/// +/// This is initialized when a shell is created. It would be more flexible to lazily initialize this +/// on the first access, but, given that we always use `Shell` in the context of our repository, +/// eager initialization is more ergonomic. +project_root: std.fs.Dir, + +/// Shell's logical cwd which is used for all functions in this file. It might be different from +/// `std.fs.cwd()` and is set to `project_root` on init. +cwd: std.fs.Dir, + +// Stack of working directories backing pushd/popd. +cwd_stack: [cwd_stack_max]std.fs.Dir, +cwd_stack_count: usize, + +// Zig uses file-descriptor oriented APIs in the standard library, with the one exception being +// ChildProcess's cwd, which is required to be a path, rather than a file descriptor. This buffer +// is used to materialize the path to cwd when spawning a new process. +// +cwd_path_buffer: [std.fs.max_path_bytes]u8 = undefined, + +env: std.process.EnvMap, + +/// True if the process is run in CI (the CI env var is set) +ci: bool, + +/// Absolute path to the Zig binary. +zig_exe: ?[]const u8, + +pub fn create(gpa: std.mem.Allocator) !*Shell { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + + var project_root = try discover_project_root(); + errdefer project_root.close(); + + var cwd = try project_root.openDir(".", .{}); + errdefer cwd.close(); + + var env = try std.process.getEnvMap(gpa); + errdefer env.deinit(); + + const ci = env.get("CI") != null; + + const result = try gpa.create(Shell); + errdefer gpa.destroy(result); + + result.* = Shell{ + .gpa = gpa, + .arena = arena, + .project_root = project_root, + .cwd = cwd, + .cwd_stack = undefined, + .cwd_stack_count = 0, + .env = env, + .ci = ci, + .zig_exe = env.get("ZIG_EXE"), + }; + + return result; +} + +pub fn destroy(shell: *Shell) void { + const gpa = shell.gpa; + + assert(shell.cwd_stack_count == 0); // pushd not paired by popd + + shell.env.deinit(); + shell.cwd.close(); + shell.project_root.close(); + shell.arena.deinit(); + gpa.destroy(shell); +} + +const ansi = .{ + .red = "\x1b[0;31m", + .reset = "\x1b[0m", +}; + +/// Prints formatted input to stderr. +/// Newline symbol is appended automatically. +/// ANSI colors are supported via `"{ansi-red}my colored text{ansi-reset}"` syntax. +pub fn echo(shell: *Shell, comptime format: []const u8, format_args: anytype) void { + _ = shell; + + comptime var format_ansi: []const u8 = ""; + comptime var pos: usize = 0; + comptime var pos_start: usize = 0; + + comptime next_pos: while (pos < format.len) { + if (format[pos] == '{') { + for (std.meta.fieldNames(@TypeOf(ansi))) |field_name| { + const tag = "{ansi-" ++ field_name ++ "}"; + if (std.mem.startsWith(u8, format[pos..], tag)) { + format_ansi = format_ansi ++ format[pos_start..pos] ++ @field(ansi, field_name); + pos += tag.len; + pos_start = pos; + continue :next_pos; + } + } + } + pos += 1; + }; + comptime assert(pos == format.len); + + format_ansi = format_ansi ++ format[pos_start..pos] ++ "\n"; + + std.debug.print(format_ansi, format_args); +} + +/// Opens a logical, named section of the script. +/// When the section is subsequently closed, its name and timing are printed. +/// Additionally on CI output from a section gets into a named, foldable group. +pub fn open_section(shell: *Shell, name: []const u8) !Section { + return Section.open(shell.ci, name); +} + +const Section = struct { + ci: bool, + name: []const u8, + timer: std.time.Timer, + + fn open(ci: bool, name: []const u8) !Section { + if (ci) { + // See + // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + // https://github.com/actions/toolkit/issues/1001 + try std.io.getStdOut().writer().print("::group::{s}\n", .{name}); + } + + return .{ + .ci = ci, + .name = name, + .timer = try std.time.Timer.start(), + }; + } + + pub fn close(section: *Section) void { + const elapsed_ns = section.timer.lap(); + std.debug.print("{s}: {}\n", .{ section.name, std.fmt.fmtDuration(elapsed_ns) }); + if (section.ci) { + std.io.getStdOut().writer().print("::endgroup::\n", .{}) catch {}; + } + section.* = undefined; + } +}; + +/// Convenience string formatting function which uses shell's arena and doesn't require +/// freeing the resulting string. +pub fn fmt(shell: *Shell, comptime format: []const u8, format_args: anytype) ![]const u8 { + return std.fmt.allocPrint(shell.arena.allocator(), format, format_args); +} + +pub fn env_get_option(shell: *Shell, var_name: []const u8) ?[]const u8 { + return std.process.getEnvVarOwned(shell.arena.allocator(), var_name) catch null; +} + +pub fn env_get(shell: *Shell, var_name: []const u8) ![]const u8 { + errdefer { + log.warn("environment variable '{s}' not defined", .{var_name}); + } + + return try std.process.getEnvVarOwned(shell.arena.allocator(), var_name); +} + +/// Change `shell`'s working directory. It *must* be followed by +/// +/// defer shell.popd(); +/// +/// to restore the previous directory back. +pub fn pushd(shell: *Shell, path: []const u8) !void { + assert(shell.cwd_stack_count < cwd_stack_max); + // allow only explicitly relative paths or absolute paths + assert(path[0] == '.' or path[0] == '/'); + + const cwd_new = try shell.cwd.openDir(path, .{}); + + shell.cwd_stack[shell.cwd_stack_count] = shell.cwd; + shell.cwd_stack_count += 1; + shell.cwd = cwd_new; +} + +pub fn pushd_dir(shell: *Shell, dir: std.fs.Dir) !void { + assert(shell.cwd_stack_count < cwd_stack_max); + + // Re-open the directory such that `popd` can close it. + const cwd_new = try dir.openDir(".", .{}); + + shell.cwd_stack[shell.cwd_stack_count] = shell.cwd; + shell.cwd_stack_count += 1; + shell.cwd = cwd_new; +} + +pub fn popd(shell: *Shell) void { + shell.cwd.close(); + shell.cwd_stack_count -= 1; + shell.cwd = shell.cwd_stack[shell.cwd_stack_count]; +} + +/// Checks if the path exists and is a directory. +/// +/// Note: this api is prone to TOCTOU and exists primarily for assertions. +pub fn dir_exists(shell: *Shell, path: []const u8) !bool { + return subdir_exists(shell.cwd, path); +} + +/// Checks if the path exists and is a file. +/// +/// Note: this api is prone to TOCTOU and exists primarily for assertions. +pub fn file_exists(shell: *Shell, path: []const u8) bool { + const stat = shell.cwd.statFile(path) catch return false; + return stat.kind == .file; +} + +pub fn file_make_executable(shell: *Shell, path: []const u8) !void { + if (builtin.os.tag != .windows) { + const fd = try shell.cwd.openFile(path, .{ .mode = .read_write }); + defer fd.close(); + + try fd.chmod(0o755); + } +} + +fn subdir_exists(dir: std.fs.Dir, path: []const u8) !bool { + const stat = dir.statFile(path) catch |err| switch (err) { + error.FileNotFound => return false, + error.IsDir => return true, + else => return err, + }; + + return stat.kind == .directory; +} + +pub fn file_ensure_content( + shell: *Shell, + path: []const u8, + content: []const u8, + create_flags: std.fs.File.CreateFlags, +) !enum { unchanged, updated } { + const max_bytes = 1 * MiB; + const content_current = shell.cwd.readFileAlloc(shell.gpa, path, max_bytes) catch null; + defer if (content_current) |slice| shell.gpa.free(slice); + + if (content_current != null and std.mem.eql(u8, content_current.?, content)) { + return .unchanged; + } + + try shell.cwd.writeFile(.{ .sub_path = path, .data = content, .flags = create_flags }); + return .updated; +} + +/// Creates a new temporary directory (in the project-level .zig-cache) and returns the +/// absolute path. +/// +/// It's the callers responsibility to delete the directory when done with it, e.g. +/// with `defer shell.cwd.deleteTree(dir) catch {};`. +pub fn create_tmp_dir( + shell: *Shell, +) ![]const u8 { + const root = try shell.project_root.realpathAlloc(shell.arena.allocator(), "."); + const tmp_absolute = try shell.fmt("{s}/.zig-cache/tmp/{}", .{ + root, + std.crypto.random.int(u64), + }); + assert(!try shell.dir_exists(tmp_absolute)); + try shell.project_root.makePath(tmp_absolute); + return tmp_absolute; +} + +const FindOptions = struct { + where: []const []const u8, + extension: ?[]const u8 = null, + extensions: ?[]const []const u8 = null, +}; + +/// Analogue of the `find` utility, returns a set of paths matching filtering criteria. +/// +/// Returned slice is stored in `Shell.arena`. +pub fn find(shell: *Shell, options: FindOptions) ![]const []const u8 { + if (options.extension != null and options.extensions != null) { + @panic("conflicting extension filters"); + } + if (options.extension) |extension| { + assert(extension[0] == '.'); + } + if (options.extensions) |extensions| { + for (extensions) |extension| { + assert(extension[0] == '.'); + } + } + + var result = std.ArrayList([]const u8).init(shell.arena.allocator()); + + for (options.where) |base_path| { + var base_dir = try shell.cwd.openDir(base_path, .{ .iterate = true }); + defer base_dir.close(); + + var walker = try base_dir.walk(shell.gpa); + defer walker.deinit(); + + while (try walker.next()) |entry| { + if (entry.kind == .file and find_filter_path(entry.path, options)) { + const full_path = + try std.fs.path.join(shell.arena.allocator(), &.{ base_path, entry.path }); + try result.append(full_path); + } + } + } + + return result.items; +} + +fn find_filter_path(path: []const u8, options: FindOptions) bool { + if (options.extension == null and options.extensions == null) return true; + if (options.extension != null and options.extensions != null) @panic("conflicting filters"); + + if (options.extension) |extension| { + return std.mem.endsWith(u8, path, extension); + } + + if (options.extensions) |extensions| { + for (extensions) |extension| { + if (std.mem.endsWith(u8, path, extension)) return true; + } + return false; + } + + unreachable; +} + +/// Copy file, creating the destination directory as necessary. +pub fn copy_path( + src_dir: std.fs.Dir, + src_path: []const u8, + dst_dir: std.fs.Dir, + dst_path: []const u8, +) !void { + errdefer { + log.warn("failed to copy {s} to {s}", .{ src_path, dst_path }); + } + if (std.fs.path.dirname(dst_path)) |dir| { + try dst_dir.makePath(dir); + } + try src_dir.copyFile(src_path, dst_dir, dst_path, .{}); +} + +/// Runs the given command for side effects. +/// Returns an error if exit status is non-zero. +/// +/// Supports interpolation using the following syntax: +/// +/// ``` +/// shell.exec("git branch {op} {branches}", .{ +/// .op = "-D", +/// .branches = &.{"main", "feature"}, +/// }) +/// ``` +pub fn exec(shell: *Shell, comptime cmd: []const u8, cmd_args: anytype) !void { + var argv = try Argv.expand(shell.gpa, cmd, cmd_args); + defer argv.deinit(); + + return exec_inner(shell, argv.slice(), .{}); +} + +pub fn exec_options( + shell: *Shell, + options: struct { + stdin_slice: ?[]const u8 = null, + timeout: stdx.Duration = .minutes(10), + }, + comptime cmd: []const u8, + cmd_args: anytype, +) !void { + var argv = try Argv.expand(shell.gpa, cmd, cmd_args); + defer argv.deinit(); + + return exec_inner(shell, argv.slice(), .{ + .stdin_slice = options.stdin_slice, + .timeout = options.timeout, + }); +} + +/// Runs the given command and returns its output. +/// If the output is a single line, the final newline is stripped. +pub fn exec_stdout(shell: *Shell, comptime cmd: []const u8, cmd_args: anytype) ![]const u8 { + var argv = try Argv.expand(shell.gpa, cmd, cmd_args); + defer argv.deinit(); + + var captured_stdout: []const u8 = &.{}; + try exec_inner(shell, argv.slice(), .{ + .capture_stdout = &captured_stdout, + }); + return captured_stdout; +} + +pub fn exec_stdout_stderr(shell: *Shell, comptime cmd: []const u8, cmd_args: anytype) !struct { + []const u8, + []const u8, +} { + var argv = try Argv.expand(shell.gpa, cmd, cmd_args); + defer argv.deinit(); + + var captured_stdout: []const u8 = &.{}; + var captured_stderr: []const u8 = &.{}; + try exec_inner(shell, argv.slice(), .{ + .capture_stdout = &captured_stdout, + .capture_stderr = &captured_stderr, + }); + return .{ captured_stdout, captured_stderr }; +} + +pub fn exec_stdout_options( + shell: *Shell, + options: struct { + stdin_slice: ?[]const u8 = null, + }, + comptime cmd: []const u8, + cmd_args: anytype, +) ![]const u8 { + var argv = try Argv.expand(shell.gpa, cmd, cmd_args); + defer argv.deinit(); + + var captured_stdout: []const u8 = &.{}; + try exec_inner(shell, argv.slice(), .{ + .stdin_slice = options.stdin_slice, + .capture_stdout = &captured_stdout, + }); + return captured_stdout; +} + +/// Runs the zig compiler. +pub fn exec_zig(shell: *Shell, comptime cmd: []const u8, cmd_args: anytype) !void { + return shell.exec_zig_options(.{}, cmd, cmd_args); +} + +pub fn exec_zig_options( + shell: *Shell, + options: struct { + timeout: stdx.Duration = .minutes(10), + capture_stdout: ?*[]const u8 = null, + capture_stderr: ?*[]const u8 = null, + }, + comptime cmd: []const u8, + cmd_args: anytype, +) !void { + var argv = Argv.init(shell.gpa); + defer argv.deinit(); + + try argv.append_new_arg("{s}", .{shell.zig_exe.?}); + try expand_argv(&argv, cmd, cmd_args); + + return shell.exec_inner(argv.slice(), .{ + .timeout = options.timeout, + .capture_stdout = options.capture_stdout, + .capture_stderr = options.capture_stderr, + }); +} + +fn exec_inner( + shell: *Shell, + argv: []const []const u8, + options: struct { + stdin_slice: ?[]const u8 = null, + + // Optional out parameters: + capture_stdout: ?*[]const u8 = null, + capture_stderr: ?*[]const u8 = null, + + output_limit_bytes: usize = 128 * MiB, + timeout: stdx.Duration = .minutes(10), + }, +) !void { + const argv_formatted = try std.mem.join(shell.gpa, " ", argv); + defer shell.gpa.free(argv_formatted); + + var stdin_writer: ?std.Thread = null; + defer if (stdin_writer) |thread| thread.join(); + + const Streams = enum { stdout, stderr }; + var poller: ?std.io.Poller(Streams) = null; + defer if (poller) |*p| p.deinit(); + + errdefer |err| { + log.err("process failed with {s}: {s}", .{ @errorName(err), argv_formatted }); + if (poller) |*p| { + inline for (comptime std.enums.values(Streams)) |stream| { + if (p.fifo(stream).count > 0) { + log.err("{s}:\n++++\n{s}++++\n", .{ + @tagName(stream), + p.fifo(stream).readableSlice(0), + }); + } + } + } + } + + var child = std.process.Child.init(argv, shell.gpa); + child.cwd = try shell.cwd.realpath(".", &shell.cwd_path_buffer); + child.env_map = &shell.env; + child.stdin_behavior = if (options.stdin_slice != null) .Pipe else .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + try child.spawn(); + errdefer { + _ = child.kill() catch {}; + } + + if (options.stdin_slice) |stdin_slice| { + stdin_writer = try write_stdin(&child, stdin_slice); + } + + poller = std.io.poll(shell.gpa, Streams, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + + { + defer inline for (comptime std.enums.values(Streams)) |stream| { + assert(poller.?.fifo(stream).head == 0); + }; + + var timer = try std.time.Timer.start(); + for (0..1_000_000) |_| { + const timeout_remaining = options.timeout.ns -| timer.read(); + if (timeout_remaining == 0) { + return error.ExecTimeout; + } + if (!try poller.?.pollTimeout(@intCast(timeout_remaining))) break; + inline for (comptime std.enums.values(Streams)) |stream| { + if (poller.?.fifo(stream).count > options.output_limit_bytes) { + return error.StdoutStreamTooLong; + } + } + } else @panic("exec: safety counter exceeded"); + } + + const term = try child.wait(); + switch (term) { + .Exited => |code| if (code != 0) return error.ExecNonZeroExitStatus, + else => return error.ExecFailed, + } + + inline for ( + .{ options.capture_stdout, options.capture_stderr }, + .{ .stdout, .stderr }, + ) |capture_destination, capture_stream| { + if (capture_destination) |destination| { + const stream = poller.?.fifo(capture_stream).readableSlice(0); + const trailing_newline = if (std.mem.indexOfScalar(u8, stream, '\n')) |first_newline| + first_newline == stream.len - 1 + else + false; + const len_without_newline = stream.len - @intFromBool(trailing_newline); + destination.* = try shell.arena.allocator().dupe(u8, stream[0..len_without_newline]); + } + } +} + +fn write_stdin(child: *std.process.Child, stdin: []const u8) !std.Thread { + assert(child.stdin != null); + defer child.stdin = null; + + // Spawn a thread to avoid deadlock between us writing to stdin and reading from stdout. + return try std.Thread.spawn( + .{}, + struct { + fn write_stdin(destination: std.fs.File, source: []const u8) void { + defer destination.close(); + + destination.writeAll(source) catch {}; + } + }.write_stdin, + .{ child.stdin.?, stdin }, + ); +} + +/// Run the command and return its status, stderr and stdout. +/// The caller is responsible for checking the status. +pub fn exec_raw( + shell: *Shell, + comptime cmd: []const u8, + cmd_args: anytype, +) !std.process.Child.RunResult { + var argv = try Argv.expand(shell.gpa, cmd, cmd_args); + defer argv.deinit(); + + return try std.process.Child.run(.{ + .allocator = shell.arena.allocator(), + .argv = argv.slice(), + .cwd = try shell.cwd.realpath(".", &shell.cwd_path_buffer), + .env_map = &shell.env, + }); +} + +pub const SpawnOptions = struct { + stdin_behavior: std.process.Child.StdIo = .Ignore, + stdout_behavior: std.process.Child.StdIo = .Ignore, + stderr_behavior: std.process.Child.StdIo = .Ignore, +}; + +pub fn spawn( + shell: *Shell, + options: SpawnOptions, + comptime cmd: []const u8, + cmd_args: anytype, +) !std.process.Child { + var argv = try Argv.expand(shell.gpa, cmd, cmd_args); + defer argv.deinit(); + + return shell.spawn_argv(options, &argv); +} + +pub fn spawn_zig( + shell: *Shell, + options: SpawnOptions, + comptime cmd: []const u8, + cmd_args: anytype, +) !std.process.Child { + var argv = Argv.init(shell.gpa); + defer argv.deinit(); + + try argv.append_new_arg("{s}", .{shell.zig_exe.?}); + try expand_argv(&argv, cmd, cmd_args); + return shell.spawn_argv(options, &argv); +} + +fn spawn_argv( + shell: *Shell, + options: SpawnOptions, + argv: *const Argv, +) !std.process.Child { + var child = std.process.Child.init(argv.slice(), shell.gpa); + child.cwd = try shell.cwd.realpath(".", &shell.cwd_path_buffer); + child.env_map = &shell.env; + child.stdin_behavior = options.stdin_behavior; + child.stdout_behavior = options.stdout_behavior; + child.stderr_behavior = options.stderr_behavior; + try child.spawn(); + return child; +} + +/// On GitHub Actions runners, `git commit` fails with an "Author identity unknown" error. +/// +/// This function sets up appropriate environmental variables to correct that error. +pub fn git_env_setup(shell: *Shell, options: struct { use_hostname: bool }) !void { + if (options.use_hostname) { + if (builtin.target.os.tag != .linux) { + @panic("use_hostname only supported on linux"); + } + var hostname_buffer: [std.posix.HOST_NAME_MAX]u8 = @splat(0); + const hostname = try std.posix.gethostname(&hostname_buffer); + + try shell.env.put("GIT_AUTHOR_NAME", hostname); + try shell.env.put("GIT_COMMITTER_NAME", hostname); + } else { + try shell.env.put("GIT_AUTHOR_NAME", "TigerBeetle Bot"); + try shell.env.put("GIT_COMMITTER_NAME", "TigerBeetle Bot"); + } + + try shell.env.put("GIT_AUTHOR_EMAIL", "bot@tigerbeetle.com"); + try shell.env.put("GIT_COMMITTER_EMAIL", "bot@tigerbeetle.com"); +} + +pub fn git_commit_timestamp(shell: *Shell, sha: []const u8) !stdx.InstantUnix { + assert(sha.len == 40); + + const timestamp_s = try shell.exec_stdout("git show -s --format=%ct {sha}", .{ .sha = sha }); + return stdx.InstantUnix.from_timestamp_s( + try std.fmt.parseInt(u64, timestamp_s, 10), + ); +} + +const Argv = struct { + args: std.ArrayList([]const u8), + + fn init(gpa: std.mem.Allocator) Argv { + return Argv{ .args = std.ArrayList([]const u8).init(gpa) }; + } + + fn expand(gpa: std.mem.Allocator, comptime cmd: []const u8, cmd_args: anytype) !Argv { + var result = Argv.init(gpa); + errdefer result.deinit(); + try expand_argv(&result, cmd, cmd_args); + return result; + } + + fn deinit(argv: *Argv) void { + for (argv.args.items) |arg| argv.args.allocator.free(arg); + argv.args.deinit(); + } + + fn slice(argv: *const Argv) []const []const u8 { + return argv.args.items; + } + + fn append_new_arg(argv: *Argv, comptime arg_fmt: []const u8, arg: anytype) !void { + const arg_owned = try std.fmt.allocPrint( + argv.args.allocator, + arg_fmt, + arg, + ); + errdefer argv.args.allocator.free(arg_owned); + + try argv.args.append(arg_owned); + } + + fn extend_last_arg(argv: *Argv, comptime arg_fmt: []const u8, arg: anytype) !void { + assert(argv.args.items.len > 0); + const arg_allocated = try std.fmt.allocPrint( + argv.args.allocator, + "{s}" ++ arg_fmt, + .{argv.args.items[argv.args.items.len - 1]} ++ arg, + ); + argv.args.allocator.free(argv.args.items[argv.args.items.len - 1]); + argv.args.items[argv.args.items.len - 1] = arg_allocated; + } +}; + +/// Expands `cmd` into an array of command arguments, substituting values from `cmd_args`. +/// +/// This avoids shell injection by construction as it doesn't concatenate strings. +fn expand_argv(argv: *Argv, comptime cmd: []const u8, cmd_args: anytype) !void { + @setEvalBranchQuota(5_000); + // Mostly copy-paste from std.fmt.format + + comptime var pos: usize = 0; + + // For arguments like `tigerbeetle-{version}.exe`, we want to concatenate literal suffix + // ("tigerbeetle-") and prefix (".exe") to the value of `version` interpolated argument. + // + // These two variables track the spaces around `{}` syntax. + comptime var concat_left: bool = false; + comptime var concat_right: bool = false; + + const arg_count = std.meta.fields(@TypeOf(cmd_args)).len; + comptime var args_used: stdx.BitSetType(arg_count) = .{}; + comptime assert(std.mem.indexOfScalar(u8, cmd, '\'') == null); // Intentionally unsupported. + comptime assert(std.mem.indexOfScalar(u8, cmd, '"') == null); + inline while (pos < cmd.len) { + inline while (pos < cmd.len and (cmd[pos] == ' ' or cmd[pos] == '\n')) { + pos += 1; + } + + const pos_start = pos; + inline while (pos < cmd.len) : (pos += 1) { + switch (cmd[pos]) { + ' ', '\n', '{' => break, + else => {}, + } + } + + const pos_end = pos; + if (pos_start != pos_end) { + if (concat_right) { + assert(pos_start > 0 and cmd[pos_start - 1] == '}'); + try argv.extend_last_arg("{s}", .{cmd[pos_start..pos_end]}); + } else { + try argv.append_new_arg("{s}", .{cmd[pos_start..pos_end]}); + } + } + + concat_left = false; + concat_right = false; + + if (pos >= cmd.len) break; + if (cmd[pos] == ' ' or cmd[pos] == '\n') continue; + + comptime assert(cmd[pos] == '{'); + concat_left = pos > 0 and cmd[pos - 1] != ' ' and cmd[pos - 1] != '\n'; + if (concat_left) assert(argv.slice().len > 0); + pos += 1; + + const pos_arg_start = pos; + inline while (pos < cmd.len and cmd[pos] != '}') : (pos += 1) {} + const pos_arg_end = pos; + + if (pos >= cmd.len) @compileError("Missing closing }"); + + comptime assert(cmd[pos] == '}'); + concat_right = pos + 1 < cmd.len and cmd[pos + 1] != ' ' and cmd[pos + 1] != '\n'; + pos += 1; + + const arg_name = comptime cmd[pos_arg_start..pos_arg_end]; + const arg_or_slice = @field(cmd_args, arg_name); + comptime args_used.set(for (std.meta.fieldNames(@TypeOf(cmd_args)), 0..) |field, index| { + if (std.mem.eql(u8, field, arg_name)) break index; + } else unreachable); + + const T = @TypeOf(arg_or_slice); + + if (@typeInfo(T) == .int or @typeInfo(T) == .comptime_int) { + if (concat_left) { + try argv.extend_last_arg("{d}", .{arg_or_slice}); + } else { + try argv.append_new_arg("{d}", .{arg_or_slice}); + } + } else if (std.meta.Elem(T) == u8) { + if (concat_left) { + try argv.extend_last_arg("{s}", .{arg_or_slice}); + } else { + try argv.append_new_arg("{s}", .{arg_or_slice}); + } + } else if (std.meta.Elem(T) == []const u8) { + if (concat_left or concat_right) @compileError("Can't concatenate slices"); + for (arg_or_slice) |arg_part| { + try argv.append_new_arg("{s}", .{arg_part}); + } + } else { + @compileError("Unsupported argument type"); + } + } + + comptime if (args_used.count() != arg_count) @compileError("Unused argument"); +} + +const snap = Snap.snap_fn("src"); + +test "shell: expand_argv" { + const T = struct { + fn check( + comptime cmd: []const u8, + args: anytype, + want: Snap, + ) !void { + var argv = Argv.init(std.testing.allocator); + defer argv.deinit(); + + try expand_argv(&argv, cmd, args); + try want.diff_zon(argv.slice()); + } + }; + + try T.check("zig version", .{}, snap(@src(), + \\.{ "zig", "version" } + )); + try T.check(" zig version ", .{}, snap(@src(), + \\.{ "zig", "version" } + )); + + try T.check( + "zig {version}", + .{ .version = @as([]const u8, "version") }, + snap(@src(), + \\.{ "zig", "version" } + ), + ); + + try T.check( + "zig {version}", + .{ .version = @as([]const []const u8, &.{ "version", "--verbose" }) }, + snap(@src(), + \\.{ + \\ "zig", + \\ "version", + \\ "--verbose", + \\} + ), + ); + + try T.check( + "git fetch origin refs/pull/{pr}/head", + .{ .pr = 92 }, + snap(@src(), + \\.{ + \\ "git", + \\ "fetch", + \\ "origin", + \\ "refs/pull/92/head", + \\} + ), + ); + try T.check( + "gh pr checkout {pr}", + .{ .pr = @as(u32, 92) }, + snap(@src(), + \\.{ + \\ "gh", + \\ "pr", + \\ "checkout", + \\ "92", + \\} + ), + ); +} + +/// Finds the root of TigerBeetle repo. +/// +/// Caller is responsible for closing the dir. +fn discover_project_root() !std.fs.Dir { + var current = try std.fs.cwd().openDir(".", .{}); + errdefer current.close(); // Caller is responsible for closing on success. + + for (0..16) |_| { + if (detect_project_root(current)) |_| { + return current; + } else |err| switch (err) { + error.FileNotFound => { + const parent = try current.openDir("..", .{}); + current.close(); + current = parent; + }, + else => return err, + } + } + + return error.DiscoverProjectRootDepthExceeded; +} + +fn detect_project_root(dir: std.fs.Dir) !void { + try dir.access("methodfinder.gemspec", .{}); +} + +pub const HttpOptions = struct { + pub const ContentType = enum { + json, + + fn string(content_type: ContentType) []const u8 { + return switch (content_type) { + .json => "application/json", + }; + } + }; + + content_type: ?ContentType = null, + authorization: ?[]const u8 = null, + + response_body_size_max: u32 = 512 * stdx.KiB, +}; + +pub fn http_get(shell: *Shell, url: []const u8, options: HttpOptions) ![]const u8 { + return shell.http_request(.get, url, options); +} + +pub fn http_post( + shell: *Shell, + url: []const u8, + body: []const u8, + options: HttpOptions, +) ![]const u8 { + return shell.http_request(.{ .post = body }, url, options); +} + +/// Issues an HTTP request to the given `url` and returns the response. +/// +/// The returned body is owned by the shell arena and doesn't need to be freed. +/// If the response is not 200 OK, the response body is logged and an error is returned. +fn http_request( + shell: *Shell, + method: union(enum) { get, post: []const u8 }, + url: []const u8, + options: HttpOptions, +) ![]const u8 { + errdefer |err| log.err( + "failed to HTTP {s} to \"{s}\": {s}", + .{ @tagName(method), url, @errorName(err) }, + ); + + var client = std.http.Client{ .allocator = shell.gpa }; + defer client.deinit(); + + const uri = try std.Uri.parse(url); + var header_buffer: [4 * stdx.KiB]u8 = undefined; + var request = try client.open( + switch (method) { + .post => .POST, + .get => .GET, + }, + uri, + .{ .server_header_buffer = &header_buffer }, + ); + defer request.deinit(); + + if (options.content_type) |content_type| { + request.headers.content_type = .{ .override = content_type.string() }; + } + + if (options.authorization) |authorization| { + request.headers.authorization = .{ .override = authorization }; + } + + if (method == .post) { + request.transfer_encoding = .{ .content_length = method.post.len }; + } + + try request.send(); + if (method == .post) { + try request.writeAll(method.post); + } + try request.finish(); + try request.wait(); + + const response_body_buffer_size = request.response.content_length orelse + options.response_body_size_max; + + if (response_body_buffer_size > options.response_body_size_max) { + return error.ResponseTooLarge; + } + + const response_body_buffer = try shell.arena.allocator().alloc(u8, response_body_buffer_size); + const response_body_size = try request.readAll(response_body_buffer); + assert(response_body_size <= options.response_body_size_max); + const response_body = response_body_buffer[0..response_body_size]; + + if (request.response.content_length) |response_content_length| { + assert(response_content_length == response_body_size); + } + + const response_status = request.response.status; + if (response_status != std.http.Status.ok and response_status != std.http.Status.created) { + log.err("response: {s}", .{response_body}); + return error.ResponseWrongStatus; + } + + return response_body; +} + +/// Converts an ISO8601 timestamp into seconds from the epoch by shelling out to the `date` util. +pub fn iso8601_to_timestamp_seconds(shell: *Shell, datetime_iso8601: []const u8) !u64 { + return try std.fmt.parseInt(u64, try shell.exec_stdout( + "date -d {datetime_iso8601} +%s", + .{ .datetime_iso8601 = datetime_iso8601 }, + ), 10); +} + +pub fn unzip_executable( + shell: *Shell, + zip_path: []const u8, + executable_name: []const u8, +) !void { + const zip_file = try shell.cwd.openFile(zip_path, .{}); + defer zip_file.close(); + + try std.zip.extract(shell.cwd, zip_file.seekableStream(), .{}); + + const zip_extracted = try shell.cwd.openFile(executable_name, .{}); + defer zip_extracted.close(); + + // Zig's std.zip.extract doesn't handle permissions. + if (builtin.os.tag != .windows) { + try zip_extracted.chmod(0o755); + } +} + +fn unix_to_dos_timestamp(instant: stdx.InstantUnix) struct { time: u16, date: u16 } { + const date_time = instant.date_time(); + assert(date_time.year >= 1980 and date_time.year <= 2107); + + const time: u16 = + (@as(u16, date_time.hour) << 11) | + (@as(u16, date_time.minute) << 5) | + (@as(u16, @divFloor(date_time.second, 2))); + + const date: u16 = + ((@as(u16, date_time.year - 1980)) << 9) | + (@as(u16, date_time.month) << 5) | + (@as(u16, date_time.day)); + + return .{ .time = time, .date = date }; +} + +pub fn zip_executable( + shell: *Shell, + zip_file: std.fs.File, + input: struct { + executable_name: []const u8, + executable_mtime: stdx.InstantUnix, + max_size: u64, + }, +) !void { + assert(std.mem.eql(u8, std.fs.path.basename(input.executable_name), input.executable_name)); + + var zip_file_writer = std.io.countingWriter(zip_file.writer()); + + const executable = try shell.cwd.readFileAlloc( + shell.gpa, + input.executable_name, + input.max_size, + ); + defer shell.gpa.free(executable); + + const executable_mtime_dos = unix_to_dos_timestamp(input.executable_mtime); + const crc32 = std.hash.Crc32.hash(executable); + + const executable_deflated_buffer = try shell.gpa.alloc(u8, input.max_size); + defer shell.gpa.free(executable_deflated_buffer); + + const executable_deflated = blk: { + var executable_stream = std.io.fixedBufferStream(executable); + var executable_deflated_stream = std.io.fixedBufferStream(executable_deflated_buffer); + + try std.compress.flate.deflate.compress( + .raw, + executable_stream.reader(), + executable_deflated_stream.writer(), + .{ .level = .best }, + ); + assert(executable_stream.pos == executable.len); + + break :blk executable_deflated_stream.getWritten(); + }; + + const zip_version_20 = 0x14; + const zip_unix = 0x0300; + + const local_file_header: std.zip.LocalFileHeader = .{ + .signature = std.zip.local_file_header_sig, + .version_needed_to_extract = zip_version_20, + .flags = .{ .encrypted = false, ._ = 0 }, + .compression_method = .deflate, + .last_modification_time = executable_mtime_dos.time, + .last_modification_date = executable_mtime_dos.date, + .crc32 = crc32, + .compressed_size = @intCast(executable_deflated.len), + .uncompressed_size = @intCast(executable.len), + .filename_len = @intCast(input.executable_name.len), + .extra_len = 0, + }; + + try zip_file_writer.writer().writeStructEndian(local_file_header, .little); + try zip_file_writer.writer().writeAll(input.executable_name); + try zip_file_writer.writer().writeAll(executable_deflated); + + const central_directory_file_header: std.zip.CentralDirectoryFileHeader = .{ + .signature = std.zip.central_file_header_sig, + .version_made_by = zip_unix | zip_version_20, + .version_needed_to_extract = zip_version_20, + .flags = .{ .encrypted = false, ._ = 0 }, + .compression_method = .deflate, + .last_modification_time = executable_mtime_dos.time, + .last_modification_date = executable_mtime_dos.date, + .crc32 = crc32, + .compressed_size = @intCast(executable_deflated.len), + .uncompressed_size = @intCast(executable.len), + .filename_len = @intCast(input.executable_name.len), + .extra_len = 0, + .comment_len = 0, + .disk_number = 0, + .internal_file_attributes = 0, + .external_file_attributes = 0o0100755 << 16, // Regular file, executable. + .local_file_header_offset = 0, + }; + + const central_directory_offset = zip_file_writer.bytes_written; + try zip_file_writer.writer().writeStructEndian(central_directory_file_header, .little); + try zip_file_writer.writer().writeAll(input.executable_name); + const central_directory_end = zip_file_writer.bytes_written; + + const end_record: std.zip.EndRecord = .{ + .signature = std.zip.end_record_sig, + .disk_number = 0, + .central_directory_disk_number = 0, + .record_count_disk = 1, + .record_count_total = 1, + .central_directory_size = @intCast(central_directory_end - central_directory_offset), + .central_directory_offset = @intCast(central_directory_offset), + .comment_len = 0, + }; + try zip_file_writer.writer().writeStructEndian(end_record, .little); +} + +pub fn sha256sum(shell: *Shell, file_path: []const u8) !u256 { + const buffer = try shell.gpa.alloc(u8, 512 * stdx.KiB); + defer shell.gpa.free(buffer); + + var hasher = std.crypto.hash.sha2.Sha256.init(.{}); + + const file = try shell.cwd.openFile(file_path, .{}); + defer file.close(); + + const stat = try file.stat(); + var bytes_read_total: u64 = 0; + while (bytes_read_total < stat.size) { + const bytes_read = try file.readAll(buffer); + if (bytes_read == 0) { + break; + } + + bytes_read_total += bytes_read; + hasher.update(buffer[0..bytes_read]); + } + + assert(stat.size == bytes_read_total); + + var output: u256 = undefined; + hasher.final(std.mem.asBytes(&output)); + + return output; +} diff --git a/scripts/stdx/sort_test.zig b/scripts/stdx/sort_test.zig new file mode 100644 index 0000000..aeea2cb --- /dev/null +++ b/scripts/stdx/sort_test.zig @@ -0,0 +1,112 @@ +const std = @import("std"); +const assert = std.debug.assert; + +const stdx = @import("stdx.zig"); +const ratio = stdx.PRNG.ratio; + +test "sort_stable" { + const Value = struct { + const Value = @This(); + + x: u32, // x determines the order of the values. + y: u32, // y ensures that values are distinct for the purpose of checking stability. + + fn compare_x_ascending(_: void, a: Value, b: Value) bool { + return a.x < b.x; + } + + fn compare_x_descending(_: void, a: Value, b: Value) bool { + return a.x > b.x; + } + + fn compare_xy_ascending(_: void, a: Value, b: Value) bool { + if (a.x < b.x) return true; + if (a.x > b.x) return false; + return a.y < b.y; + } + }; + + const allocator = std.testing.allocator; + + var prng = stdx.PRNG.from_seed_testing(); + + const values_max = 1 << 15; + const values_all = try allocator.alloc(Value, values_max); + defer allocator.free(values_all); + + const values_all_expected = try allocator.alloc(Value, values_max); + defer allocator.free(values_all_expected); + + for (0..256) |_| { + const values_count = prng.range_inclusive(u32, 2, values_max); + const values_expected = values_all_expected[0..values_count]; + const values = values_all[0..values_count]; + + { + // Set up `values`. + + for (values) |*value| { + value.* = .{ + .x = prng.int_inclusive(u32, values_count * 2 - 1), + .y = undefined, + }; + } + + // Sort algorithms often optimize the case of already-sorted (or already-reverse-sorted) + // sub-arrays. + const partitions_count = prng.range_inclusive(u32, 1, @max(values_count, 64) - 1); + // The `partition_reverse_probability` is a subset of the partitions sorted by + // `partition_sort_percent`. + const partition_sort_probability = ratio(prng.int_inclusive(u8, 100), 100); + const partition_reverse_probability = ratio(prng.int_inclusive(u8, 100), 100); + + var partitions_remaining: u32 = partitions_count; + var partition_offset: u32 = 0; + while (partition_offset < values_count) { + const partition_size = size: { + if (partitions_remaining == 1) { + break :size values_count - partition_offset; + } else { + break :size prng.range_inclusive(u32, 1, values_count - partition_offset); + } + }; + + if (prng.chance(partition_sort_probability)) { + const partition = values[partition_offset..][0..partition_size]; + if (prng.chance(partition_reverse_probability)) { + std.mem.sortUnstable(Value, partition, {}, Value.compare_x_descending); + } else { + std.mem.sortUnstable(Value, partition, {}, Value.compare_x_ascending); + } + } + + partitions_remaining -= 1; + partition_offset += partition_size; + } + + for (values, 0..) |*value, i| value.y = @intCast(i); + } + + { + // Set up `values_expected`. + stdx.copy_disjoint(.exact, Value, values_expected, values); + std.mem.sortUnstable(Value, values_expected, {}, Value.compare_xy_ascending); + + // Sanity-check the expected values' order. + for ( + values_expected[0 .. values_count - 1], + values_expected[1..], + ) |a, b| { + assert(a.x <= b.x); + if (a.x == b.x) assert(a.y < b.y); + } + } + + std.mem.sort(Value, values, {}, Value.compare_x_ascending); + + for (values, values_expected) |value, value_expected| { + try std.testing.expectEqual(value.x, value_expected.x); + try std.testing.expectEqual(value.y, value_expected.y); + } + } +} diff --git a/scripts/stdx/stdx.zig b/scripts/stdx/stdx.zig new file mode 100644 index 0000000..4beee6a --- /dev/null +++ b/scripts/stdx/stdx.zig @@ -0,0 +1,1134 @@ +//! Extensions to the standard library -- things which could have been in std, but aren't. +//! +//! Unlike std, the namespacing is relatively flat: `stdx.PRNG` rather than `stdx.random.PRNG`. +//! We don't care about backwards compatibility and prefer directness to scalability. Hierarchy can +//! always be introduced later, when/if stdx grows too large. + +const std = @import("std"); +const builtin = @import("builtin"); +const assert = std.debug.assert; + +pub const BitSetType = @import("bit_set.zig").BitSetType; +pub const IOPSType = @import("iops.zig").IOPSType; +pub const BoundedArrayType = @import("bounded_array.zig").BoundedArrayType; +pub const PRNG = @import("prng.zig"); +pub const RingBufferType = @import("ring_buffer.zig").RingBufferType; +pub const Snap = @import("testing/snaptest.zig").Snap; +pub const ZipfianGenerator = @import("zipfian.zig").ZipfianGenerator; +pub const ZipfianShuffled = @import("zipfian.zig").ZipfianShuffled; + +pub const huge_page_allocator = @import("huge_page_allocator.zig").huge_page_allocator; + +pub const aegis = @import("vendored/aegis.zig"); +pub const dbg = @import("debug.zig").dbg; +pub const Flags = @import("flags.zig"); +pub const memory_lock_allocated = @import("mlock.zig").memory_lock_allocated; +pub const Shell = @import("shell.zig"); +pub const timeit = @import("debug.zig").timeit; +pub const unshare = @import("unshare.zig"); +pub const windows = @import("windows.zig"); +pub const radix_sort = @import("radix.zig").sort; + +pub const Instant = @import("time_units.zig").Instant; +pub const Duration = @import("time_units.zig").Duration; +pub const InstantUnix = @import("time_units.zig").InstantUnix; + +// Import these as `const GiB = stdx.GiB;` +pub const KiB = 1 << 10; +pub const MiB = 1 << 20; +pub const GiB = 1 << 30; +pub const TiB = 1 << 40; +pub const PiB = 1 << 50; +// pub const NiB = "Some people say my love cannot be true"; + +comptime { + assert(KiB == 1024); + assert(MiB == 1024 * KiB); + assert(GiB == 1024 * MiB); + assert(TiB == 1024 * GiB); + assert(PiB == 1024 * TiB); +} + +pub inline fn div_ceil(numerator: anytype, denominator: anytype) @TypeOf(numerator, denominator) { + comptime { + switch (@typeInfo(@TypeOf(numerator))) { + .int => |int| assert(int.signedness == .unsigned), + .comptime_int => assert(numerator >= 0), + else => @compileError("div_ceil: invalid numerator type"), + } + + switch (@typeInfo(@TypeOf(denominator))) { + .int => |int| assert(int.signedness == .unsigned), + .comptime_int => assert(denominator > 0), + else => @compileError("div_ceil: invalid denominator type"), + } + } + + assert(denominator > 0); + + if (numerator == 0) return 0; + return @divFloor(numerator - 1, denominator) + 1; +} + +test "div_ceil" { + // Comptime ints. + try std.testing.expectEqual(div_ceil(0, 8), 0); + try std.testing.expectEqual(div_ceil(1, 8), 1); + try std.testing.expectEqual(div_ceil(7, 8), 1); + try std.testing.expectEqual(div_ceil(8, 8), 1); + try std.testing.expectEqual(div_ceil(9, 8), 2); + + // Unsized ints + const max = std.math.maxInt(u64); + try std.testing.expectEqual(div_ceil(@as(u64, 0), 8), 0); + try std.testing.expectEqual(div_ceil(@as(u64, 1), 8), 1); + try std.testing.expectEqual(div_ceil(@as(u64, max), 2), max / 2 + 1); + try std.testing.expectEqual(div_ceil(@as(u64, max) - 1, 2), max / 2); + try std.testing.expectEqual(div_ceil(@as(u64, max) - 2, 2), max / 2); +} + +pub const SizePrecision = enum { exact, inexact }; + +pub inline fn copy_left( + comptime precision: SizePrecision, + comptime T: type, + target: []T, + source: []const T, +) void { + switch (precision) { + .exact => assert(target.len == source.len), + .inexact => assert(target.len >= source.len), + } + + if (!disjoint_slices(T, T, target, source)) { + assert(@intFromPtr(target.ptr) < @intFromPtr(source.ptr)); + } + + // (Bypass tidy's ban.) + const copyForwards = std.mem.copyForwards; + copyForwards(T, target, source); +} + +test "copy_left" { + const a = try std.testing.allocator.alloc(usize, 8); + defer std.testing.allocator.free(a); + + for (a, 0..) |*v, i| v.* = i; + copy_left(.exact, usize, a[0..6], a[2..]); + try std.testing.expect(std.mem.eql(usize, a, &.{ 2, 3, 4, 5, 6, 7, 6, 7 })); +} + +pub inline fn copy_right( + comptime precision: SizePrecision, + comptime T: type, + target: []T, + source: []const T, +) void { + switch (precision) { + .exact => assert(target.len == source.len), + .inexact => assert(target.len >= source.len), + } + + if (!disjoint_slices(T, T, target, source)) { + assert(@intFromPtr(target.ptr) > @intFromPtr(source.ptr)); + } + + // (Bypass tidy's ban.) + const copyBackwards = std.mem.copyBackwards; + copyBackwards(T, target, source); +} + +test "copy_right" { + const a = try std.testing.allocator.alloc(usize, 8); + defer std.testing.allocator.free(a); + + for (a, 0..) |*v, i| v.* = i; + copy_right(.exact, usize, a[2..], a[0..6]); + try std.testing.expect(std.mem.eql(usize, a, &.{ 0, 1, 0, 1, 2, 3, 4, 5 })); +} + +pub inline fn copy_disjoint( + comptime precision: SizePrecision, + comptime T: type, + target: []T, + source: []const T, +) void { + switch (precision) { + .exact => assert(target.len == source.len), + .inexact => assert(target.len >= source.len), + } + + // disjoint_slices() doesn't work in comptime, because of limitations with @intFromPtr: + // https://github.com/ziglang/zig/issues/23072. + // + // It's also possible to construct slices into an array at comptime that are _not_ disjoint, + // which would violate the intention of this function, so it can't just be skipped. + assert(!@inComptime()); + assert(disjoint_slices(T, T, target, source)); + + @memcpy // Bypass tidy ban. + (target[0..source.len], source); +} + +pub inline fn disjoint_slices(comptime A: type, comptime B: type, a: []const A, b: []const B) bool { + return @intFromPtr(a.ptr) + a.len * @sizeOf(A) <= @intFromPtr(b.ptr) or + @intFromPtr(b.ptr) + b.len * @sizeOf(B) <= @intFromPtr(a.ptr); +} + +test "disjoint_slices" { + const a = try std.testing.allocator.alignedAlloc(u8, @sizeOf(u32), 8 * @sizeOf(u32)); + defer std.testing.allocator.free(a); + + const b = try std.testing.allocator.alloc(u32, 8); + defer std.testing.allocator.free(b); + + try std.testing.expectEqual(true, disjoint_slices(u8, u32, a, b)); + try std.testing.expectEqual(true, disjoint_slices(u32, u8, b, a)); + + try std.testing.expectEqual(true, disjoint_slices(u8, u8, a, a[0..0])); + try std.testing.expectEqual(true, disjoint_slices(u32, u32, b, b[0..0])); + + try std.testing.expectEqual(false, disjoint_slices(u8, u8, a, a[0..1])); + try std.testing.expectEqual(false, disjoint_slices(u8, u8, a, a[a.len - 1 .. a.len])); + + try std.testing.expectEqual(false, disjoint_slices(u32, u32, b, b[0..1])); + try std.testing.expectEqual(false, disjoint_slices(u32, u32, b, b[b.len - 1 .. b.len])); + + try std.testing.expectEqual(false, disjoint_slices(u8, u32, a, std.mem.bytesAsSlice(u32, a))); + try std.testing.expectEqual(false, disjoint_slices(u32, u8, b, std.mem.sliceAsBytes(b))); +} + +/// Checks that a byteslice is zeroed. +pub fn zeroed(bytes: []const u8) bool { + // This implementation already gets vectorized + // https://godbolt.org/z/46cMsPKPc + var byte_bits: u8 = 0; + for (bytes) |byte| { + byte_bits |= byte; + } + return byte_bits == 0; +} + +/// Similar to `std.mem.bytesAsSlice`, but allows buffers with inexact sizes, +/// returning the largest possible slice that is less than or equal to the buffer length. +/// Differently from `std.mem.bytesAsSlice` that can return `[]align(1) T`, this function +/// always `@alignCast` the result. +pub fn bytes_as_slice( + comptime precision: SizePrecision, + comptime T: type, + bytes: anytype, +) type: { + const type_info = @typeInfo(@TypeOf(bytes)); + switch (type_info) { + .pointer => |info| switch (info.size) { + .one => switch (@typeInfo(info.child)) { + .array => |array_info| assert(array_info.child == u8), + else => unreachable, + }, + .slice => assert(info.child == u8), + else => unreachable, + }, + else => unreachable, + } + + break :type if (type_info.pointer.is_const) []const T else []T; +} { + switch (precision) { + .exact => { + assert(bytes.len % @sizeOf(T) == 0); + return @alignCast(std.mem.bytesAsSlice(T, bytes)); + }, + .inexact => { + const size = @divFloor(bytes.len, @sizeOf(T)) * @sizeOf(T); + return @alignCast(std.mem.bytesAsSlice(T, bytes[0..size])); + }, + } +} + +test bytes_as_slice { + var buffer: [64]u8 = undefined; + const T10 = extern struct { content: [10]u8 }; + const T16 = extern struct { content: [16]u8 }; + + try std.testing.expectEqual( + @as(usize, 4), + bytes_as_slice(.exact, T16, buffer[0..]).len, + ); + try std.testing.expectEqual( + @as(usize, 6), + bytes_as_slice(.exact, T10, buffer[0..60]).len, + ); + + try std.testing.expectEqual( + @as(usize, 6), + bytes_as_slice(.inexact, T10, buffer[0..]).len, + ); + try std.testing.expectEqual( + @as(usize, 4), + bytes_as_slice(.inexact, T16, buffer[0..]).len, + ); + try std.testing.expectEqual( + @as(usize, 6), + bytes_as_slice(.inexact, T10, buffer[0 .. buffer.len - 1]).len, + ); + try std.testing.expectEqual( + @as(usize, 3), + bytes_as_slice(.inexact, T16, buffer[0 .. buffer.len - 1]).len, + ); + try std.testing.expectEqual( + @as(usize, 5), + bytes_as_slice(.inexact, T10, buffer[0 .. buffer.len - 10]).len, + ); + try std.testing.expectEqual( + @as(usize, 3), + bytes_as_slice(.inexact, T16, buffer[0 .. buffer.len - 10]).len, + ); +} + +/// Splits the `haystack` around the first occurrence of `needle`, returning parts before and after. +/// +/// This is a Zig version of Go's `string.Cut` / Rust's `str::split_once`. Cut turns out to be a +/// surprisingly versatile primitive for ad-hoc string processing. Often `std.mem.indexOf` and +/// `std.mem.split` can be replaced with a shorter and clearer code using `cut`. +pub fn cut(haystack: []const u8, needle: []const u8) ?struct { []const u8, []const u8 } { + const index = std.mem.indexOf(u8, haystack, needle) orelse return null; + + return .{ haystack[0..index], haystack[index + needle.len ..] }; +} + +test cut { + try std.testing.expectEqualStrings("he", cut("hello world", "l").?[0]); + try std.testing.expectEqualStrings("lo world", cut("hello world", "l").?[1]); + assert(null == cut("hello world", "x")); +} + +pub fn cut_prefix(haystack: []const u8, needle: []const u8) ?[]const u8 { + if (std.mem.startsWith(u8, haystack, needle)) { + return haystack[needle.len..]; + } + return null; +} + +test cut_prefix { + try std.testing.expectEqualStrings(" world", cut_prefix("hello world", "hello").?); + assert(null == cut_prefix("hello world", "hellnope")); +} + +pub fn cut_suffix(haystack: []const u8, needle: []const u8) ?[]const u8 { + if (std.mem.endsWith(u8, haystack, needle)) { + return haystack[0 .. haystack.len - needle.len]; + } + return null; +} + +test cut_suffix { + try std.testing.expectEqualStrings("hello ", cut_suffix("hello world", "world").?); + assert(null == cut_suffix("hello world", "hello")); +} + +/// `maybe` is the dual of `assert`: it signals that condition is sometimes true +/// and sometimes false. +/// +/// Currently we use it for documentation, but maybe one day we plug it into +/// coverage. +pub fn maybe(ok: bool) void { + assert(ok or !ok); +} + +pub const log = if (builtin.is_test) + // Downgrade `err` to `warn` for tests. + // Zig fails any test that does `log.err`, but we want to test those code paths here. + struct { + pub fn scoped(comptime scope: @Type(.enum_literal)) type { + const base = std.log.scoped(scope); + return struct { + pub const err = warn; + pub const warn = base.warn; + pub const info = base.info; + pub const debug = base.debug; + }; + } + } +else + std.log; + +/// An alternative to the default logFn from `std.log`, which prepends a UTC timestamp. +pub fn log_with_timestamp( + comptime message_level: std.log.Level, + comptime scope: @Type(.enum_literal), + comptime format: []const u8, + args: anytype, +) void { + const level_text = comptime message_level.asText(); + const scope_prefix = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; + const instant_unix = InstantUnix.now(); + + const stderr = std.io.getStdErr().writer(); + var buffered_writer = std.io.bufferedWriter(stderr); + const writer = buffered_writer.writer(); + + nosuspend { + instant_unix.format("", .{}, writer) catch return; + writer.print(" " ++ level_text ++ scope_prefix ++ format ++ "\n", args) catch return; + buffered_writer.flush() catch return; + } +} + +/// Compare two values by directly comparing the underlying memory. +/// +/// Assert at compile time that this is a reasonable thing to do for a given `T`. That is, check +/// that: +/// - `T` doesn't have any non-deterministic padding, +/// - `T` doesn't embed any pointers. +pub fn equal_bytes(comptime T: type, a: *const T, b: *const T) bool { + comptime assert(has_unique_representation(T)); + comptime assert(!has_pointers(T)); + comptime assert(@sizeOf(T) * 8 == @bitSizeOf(T)); + + // Pick the biggest "word" for word-wise comparison, and don't try to early-return on the first + // mismatch, so that a compiler can vectorize the loop. + + const Word = comptime for (.{ u64, u32, u16, u8 }) |Word| { + if (@alignOf(T) >= @alignOf(Word) and @sizeOf(T) % @sizeOf(Word) == 0) break Word; + } else unreachable; + + // NOTE: Alignment is intentionally downcast here to work around a Zig code generation bug + // described in https://codeberg.org/ziglang/zig/issues/31473. + // TODO: Remove the following two lines once the issue is fixed. + const a_bytes: *align(@alignOf(Word)) const [@sizeOf(T)]u8 = + @alignCast(std.mem.asBytes(a)); + const b_bytes: *align(@alignOf(Word)) const [@sizeOf(T)]u8 = + @alignCast(std.mem.asBytes(b)); + + const a_words = std.mem.bytesAsSlice(Word, a_bytes); + const b_words = std.mem.bytesAsSlice(Word, b_bytes); + + assert(a_words.len == b_words.len); + + var total: Word = 0; + for (a_words, b_words) |a_word, b_word| { + total |= a_word ^ b_word; + } + + return total == 0; +} + +fn has_pointers(comptime T: type) bool { + switch (@typeInfo(T)) { + .pointer => return true, + // Be conservative. + else => return true, + + .bool, .int, .@"enum" => return false, + + .array => |info| return comptime has_pointers(info.child), + .@"struct" => |info| { + inline for (info.fields) |field| { + if (comptime has_pointers(field.type)) return true; + } + return false; + }, + } +} + +/// Checks that a type does not have implicit padding. +pub fn no_padding(comptime T: type) bool { + comptime switch (@typeInfo(T)) { + .void => return true, + .int => return @bitSizeOf(T) == 8 * @sizeOf(T), + .array => |info| return no_padding(info.child), + .@"struct" => |info| { + switch (info.layout) { + .auto => return false, + .@"extern" => { + for (info.fields) |field| { + if (!no_padding(field.type)) return false; + } + + // Check offsets of u128 and pseudo-u256 fields. + for (info.fields) |field| { + if (field.type == u128) { + const offset = @offsetOf(T, field.name); + if (offset % @sizeOf(u128) != 0) return false; + + if (@hasField(T, field.name ++ "_padding")) { + if (offset % @sizeOf(u256) != 0) return false; + if (offset + @sizeOf(u128) != + @offsetOf(T, field.name ++ "_padding")) + { + return false; + } + } + } + } + + var offset = 0; + for (info.fields) |field| { + const field_offset = @offsetOf(T, field.name); + if (offset != field_offset) return false; + offset += @sizeOf(field.type); + } + return offset == @sizeOf(T); + }, + .@"packed" => return @bitSizeOf(T) == 8 * @sizeOf(T), + } + }, + .@"enum" => |info| { + maybe(info.is_exhaustive); + return no_padding(info.tag_type); + }, + .pointer => return false, + .@"union" => return false, + else => return false, + }; +} + +test no_padding { + comptime for (.{ + u8, + extern struct { x: u8 }, + packed struct { x: u7, y: u1 }, + extern struct { x: extern struct { y: u64, z: u64 } }, + enum(u8) { x }, + }) |T| { + assert(no_padding(T)); + }; + + comptime for (.{ + u7, + struct { x: u7 }, + struct { x: u8 }, + struct { x: u64, y: u32 }, + extern struct { x: extern struct { y: u64, z: u32 } }, + packed struct { x: u7 }, + enum(u7) { x }, + }) |T| { + assert(!no_padding(T)); + }; +} + +pub inline fn hash_inline(value: anytype) u64 { + comptime { + assert(no_padding(@TypeOf(value))); + assert(has_unique_representation(@TypeOf(value))); + } + return low_level_hash(0, switch (@typeInfo(@TypeOf(value))) { + .@"struct", .int => std.mem.asBytes(&value), + else => @compileError("unsupported hashing for " ++ @typeName(@TypeOf(value))), + }); +} + +/// Inline version of Google Abseil "LowLevelHash" (inspired by wyhash). +/// https://github.com/abseil/abseil-cpp/blob/master/absl/hash/internal/low_level_hash.cc +inline fn low_level_hash(seed: u64, input: anytype) u64 { + const salt = [_]u64{ + 0xa0761d6478bd642f, + 0xe7037ed1a0b428db, + 0x8ebc6af09c88c6e3, + 0x589965cc75374cc3, + 0x1d8e4e27c47d124f, + }; + + var in: []const u8 = input; + var state = seed ^ salt[0]; + const starting_len = input.len; + + if (in.len > 64) { + var dup = [_]u64{ state, state }; + defer state = dup[0] ^ dup[1]; + + while (in.len > 64) : (in = in[64..]) { + for (@as([2][4]u64, @bitCast(in[0..64].*)), 0..) |chunk, i| { + const mix1 = @as(u128, chunk[0] ^ salt[(i * 2) + 1]) *% (chunk[1] ^ dup[i]); + const mix2 = @as(u128, chunk[2] ^ salt[(i * 2) + 2]) *% (chunk[3] ^ dup[i]); + dup[i] = @as(u64, @truncate(mix1 ^ (mix1 >> 64))); + dup[i] ^= @as(u64, @truncate(mix2 ^ (mix2 >> 64))); + } + } + } + + while (in.len > 16) : (in = in[16..]) { + const chunk = @as([2]u64, @bitCast(in[0..16].*)); + const mixed = @as(u128, chunk[0] ^ salt[1]) *% (chunk[1] ^ state); + state = @as(u64, @truncate(mixed ^ (mixed >> 64))); + } + + var chunk: [2]u64 = .{ 0, 0 }; + if (in.len > 8) { + chunk[0] = @as(u64, @bitCast(in[0..8].*)); + chunk[1] = @as(u64, @bitCast(in[in.len - 8 ..][0..8].*)); + } else if (in.len > 3) { + chunk[0] = @as(u32, @bitCast(in[0..4].*)); + chunk[1] = @as(u32, @bitCast(in[in.len - 4 ..][0..4].*)); + } else if (in.len > 0) { + chunk[0] = (@as(u64, in[0]) << 16) | (@as(u64, in[in.len / 2]) << 8) | in[in.len - 1]; + } + + var mixed = @as(u128, chunk[0] ^ salt[1]) *% (chunk[1] ^ state); + mixed = @as(u64, @truncate(mixed ^ (mixed >> 64))); + mixed *%= (@as(u64, starting_len) ^ salt[1]); + return @as(u64, @truncate(mixed ^ (mixed >> 64))); +} + +test "hash_inline" { + for (@import("testing/low_level_hash_vectors.zig").cases) |case| { + var buffer: [0x100]u8 = undefined; + + const b64 = std.base64.standard; + const input = buffer[0..try b64.Decoder.calcSizeForSlice(case.b64)]; + try b64.Decoder.decode(input, case.b64); + + const hash = low_level_hash(case.seed, input); + try std.testing.expectEqual(case.hash, hash); + } +} + +/// Returns a copy of `base` with fields changed according to `diff`. +/// +/// Intended exclusively for table-driven prototype-based tests. Write +/// updates explicitly in production code. +pub fn update(base: anytype, diff: anytype) @TypeOf(base) { + assert(builtin.is_test); + assert(@typeInfo(@TypeOf(base)) == .@"struct"); + + var updated = base; + inline for (std.meta.fields(@TypeOf(diff))) |f| { + @field(updated, f.name) = @field(diff, f.name); + } + return updated; +} + +// TODO(zig): std doesn't have the statfs / fstatfs syscalls to get the type of a filesystem. +// Once those are available, this can be removed. +// The `statfs` definition used by the Linux kernel, and the magic number for tmpfs, from +// `man 2 fstatfs`. +const fsblkcnt64_t = u64; +const fsfilcnt64_t = u64; +const fsword_t = i64; +const fsid_t = u64; + +pub const TmpfsMagic = 0x01021994; +pub const StatFs = extern struct { + f_type: fsword_t, + f_bsize: fsword_t, + f_blocks: fsblkcnt64_t, + f_bfree: fsblkcnt64_t, + f_bavail: fsblkcnt64_t, + f_files: fsfilcnt64_t, + f_ffree: fsfilcnt64_t, + f_fsid: fsid_t, + f_namelen: fsword_t, + f_frsize: fsword_t, + f_flags: fsword_t, + f_spare: [4]fsword_t, +}; + +pub fn fstatfs(fd: i32, statfs_buf: *StatFs) usize { + return std.os.linux.syscall2( + if (@hasField(std.os.linux.SYS, "fstatfs64")) .fstatfs64 else .fstatfs, + @as(usize, @bitCast(@as(isize, fd))), + @intFromPtr(statfs_buf), + ); +} + +/// True if every value of the type `T` has a unique bit pattern representing it. +/// In other words, `T` has no unused bits and no padding. +pub fn has_unique_representation(comptime T: type) bool { + switch (@typeInfo(T)) { + else => return false, // TODO can we know if it's true for some of these types ? + + .@"enum", + .error_set, + .@"fn", + => return true, + + .bool => return false, + + .int => |info| return @sizeOf(T) * 8 == info.bits, + + .pointer => |info| return info.size != .slice, + + .array => |info| return comptime has_unique_representation(info.child), + + .@"struct" => |info| { + // Only consider packed structs unique if they are byte aligned. + if (info.backing_integer) |backing_integer| { + return @sizeOf(T) * 8 == @bitSizeOf(backing_integer); + } + + var sum_size = @as(usize, 0); + + inline for (info.fields) |field| { + const FieldType = field.type; + if (comptime !has_unique_representation(FieldType)) return false; + sum_size += @sizeOf(FieldType); + } + + return @sizeOf(T) == sum_size; + }, + + .vector => |info| return comptime has_unique_representation(info.child) and + @sizeOf(T) == @sizeOf(info.child) * info.len, + } +} + +// Test vectors mostly from upstream, with some added to test the packed struct case. +test "has_unique_representation" { + const TestStruct1 = struct { + a: u32, + b: u32, + }; + + try std.testing.expect(has_unique_representation(TestStruct1)); + + const TestStruct2 = struct { + a: u32, + b: u16, + }; + + try std.testing.expect(!has_unique_representation(TestStruct2)); + + const TestStruct3 = struct { + a: u32, + b: u32, + }; + + try std.testing.expect(has_unique_representation(TestStruct3)); + + const TestStruct4 = struct { a: []const u8 }; + + try std.testing.expect(!has_unique_representation(TestStruct4)); + + const TestStruct5 = struct { a: TestStruct4 }; + + try std.testing.expect(!has_unique_representation(TestStruct5)); + + const TestStruct6 = packed struct { + a: u32, + b: u31, + }; + + try std.testing.expect(!has_unique_representation(TestStruct6)); + + const TestStruct7 = struct { + a: u64, + b: TestStruct6, + }; + + try std.testing.expect(!has_unique_representation(TestStruct7)); + + const TestStruct8 = packed struct { + a: u32, + b: u32, + }; + + try std.testing.expect(has_unique_representation(TestStruct8)); + + const TestStruct9 = struct { + a: u64, + b: TestStruct8, + }; + + try std.testing.expect(has_unique_representation(TestStruct9)); + + const TestStruct10 = packed struct { + a: TestStruct8, + b: TestStruct8, + }; + + try std.testing.expect(has_unique_representation(TestStruct10)); + + const TestUnion1 = packed union { + a: u32, + b: u16, + }; + + try std.testing.expect(!has_unique_representation(TestUnion1)); + + const TestUnion2 = extern union { + a: u32, + b: u16, + }; + + try std.testing.expect(!has_unique_representation(TestUnion2)); + + const TestUnion3 = union { + a: u32, + b: u16, + }; + + try std.testing.expect(!has_unique_representation(TestUnion3)); + + const TestUnion4 = union(enum) { + a: u32, + b: u16, + }; + + try std.testing.expect(!has_unique_representation(TestUnion4)); + + inline for ([_]type{ i0, u8, i16, u32, i64 }) |T| { + try std.testing.expect(has_unique_representation(T)); + } + inline for ([_]type{ i1, u9, i17, u33, i24 }) |T| { + try std.testing.expect(!has_unique_representation(T)); + } + + try std.testing.expect(!has_unique_representation([]u8)); + try std.testing.expect(!has_unique_representation([]const u8)); + + try std.testing.expect(has_unique_representation(@Vector(4, u16))); +} + +/// Construct a `union(Enum)` type, where each union "value" type is defined in terms of the +/// variant. +/// +/// That is, `EnumUnionType(Enum, TypeForVariant)` is equivalent to: +/// +/// union(Enum) { +/// // For every `e` in `Enum`: +/// e: TypeForVariant(e), +/// } +/// +pub fn EnumUnionType( + comptime Enum: type, + comptime TypeForVariant: fn (comptime variant: Enum) type, +) type { + const UnionField = std.builtin.Type.UnionField; + + var fields: [std.enums.values(Enum).len]UnionField = undefined; + for (std.enums.values(Enum), 0..) |enum_variant, i| { + fields[i] = .{ + .name = @tagName(enum_variant), + .type = TypeForVariant(enum_variant), + .alignment = @alignOf(TypeForVariant(enum_variant)), + }; + } + + return @Type(.{ .@"union" = .{ + .layout = .auto, + .fields = &fields, + .decls = &.{}, + .tag_type = Enum, + } }); +} + +/// Constructs an `enum` type from names. +pub fn EnumType(comptime names: anytype) type { + comptime assert(names.len > 0); + const EnumField = std.builtin.Type.EnumField; + var fields: [names.len]EnumField = undefined; + for (names, 0..) |name, i| { + fields[i] = .{ + .name = name, + .value = i, + }; + } + + return @Type(.{ .@"enum" = .{ + .fields = &fields, + .decls = &.{}, + .tag_type = std.math.IntFittingRange(0, names.len), + .is_exhaustive = true, + } }); +} + +/// Creates a slice to a comptime slice without triggering +/// `error: runtime value contains reference to comptime var` +pub fn comptime_slice(comptime slice: anytype, comptime len: usize) []const @TypeOf(slice[0]) { + return &@as([len]@TypeOf(slice[0]), slice[0..len].*); +} + +/// Return a Formatter for a u64 value representing a file size. +/// This formatter statically checks that the number is a multiple of 1024, +/// and represents it using the IEC measurement units (KiB, MiB, GiB, ...). +pub fn fmt_int_size_bin_exact(comptime value: u64) std.fmt.Formatter(format_int_size_bin_exact) { + comptime assert(value < 1024 or value % 1024 == 0); + return .{ .data = value }; +} + +fn format_int_size_bin_exact( + value: u64, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, +) !void { + _ = fmt; + if (value == 0) { + return std.fmt.formatBuf("0B", options, writer); + } + + // The worst case in terms of space needed is 20 bytes, + // since `maxInt(u64)` is the highest number, + // + 3 bytes for the measurement units suffix. + comptime assert(std.fmt.comptimePrint("{}GiB", .{std.math.maxInt(u64)}).len == 23); + var buf: [23]u8 = undefined; + + var magnitude: u8 = 0; + var value_unit = value; + while (value_unit % 1024 == 0) : (magnitude += 1) { + value_unit = @divExact(value_unit, 1024); + } + + const magnitudes_iec = "BKMGTPEZY"; + const suffix = magnitudes_iec[magnitude]; + + const length: usize = length: { + const i = std.fmt.formatIntBuf(&buf, value_unit, 10, .lower, .{}); + if (magnitude == 0) { + buf[i] = suffix; + break :length i + 1; + } else { + buf[i..][0..3].* = [_]u8{ suffix, 'i', 'B' }; + break :length i + 3; + } + }; + + return std.fmt.formatBuf(buf[0..length], options, writer); +} + +test fmt_int_size_bin_exact { + try std.testing.expectFmt("0B", "{}", .{fmt_int_size_bin_exact(0)}); + try std.testing.expectFmt("128B", "{}", .{fmt_int_size_bin_exact(128)}); + try std.testing.expectFmt("8KiB", "{}", .{fmt_int_size_bin_exact(8 * 1024)}); + try std.testing.expectFmt("1025KiB", "{}", .{fmt_int_size_bin_exact(1025 * 1024)}); + try std.testing.expectFmt("12345KiB", "{}", .{fmt_int_size_bin_exact(12345 * 1024)}); + try std.testing.expectFmt("42MiB", "{}", .{fmt_int_size_bin_exact(42 * 1024 * 1024)}); + try std.testing.expectFmt("18014398509481983KiB", "{}", .{ + fmt_int_size_bin_exact(std.math.maxInt(u64) - 1023), + }); +} + +/// Like std.fmt.bufPrint, but checks, at compile time, that the buffer is sufficiently large. +pub fn array_print( + comptime n: usize, + buffer: *[n]u8, + comptime fmt: []const u8, + args: anytype, +) []const u8 { + const Args = @TypeOf(args); + const ArgsStruct = @typeInfo(Args).@"struct"; + comptime assert(ArgsStruct.is_tuple); + + comptime { + var args_worst_case: Args = undefined; + for (ArgsStruct.fields, 0..) |field, index| { + const arg_worst_case = switch (field.type) { + u8, u16, u32, u64, u128 => std.math.maxInt(field.type), + else => @compileError("array_print: unsupported type: " ++ @typeName(field.type)), + }; + args_worst_case[index] = arg_worst_case; + } + const buffer_size = std.fmt.count(fmt, args_worst_case); + assert(n >= buffer_size); // array_print buffer too small + } + + return std.fmt.bufPrint(buffer, fmt, args) catch |err| switch (err) { + error.NoSpaceLeft => unreachable, + }; +} + +/// Like std.posix version, but log unconditionally, not just when mode=Debug. +/// The added `label` argument works around the absence of stack traces in ReleaseSafe builds. +pub fn unexpected_errno(label: []const u8, err: std.posix.system.E) std.posix.UnexpectedError { + log.scoped(.stdx).err("unexpected errno: {s}: code={d} name={?s}", .{ + label, + @intFromEnum(err), + std.enums.tagName(std.posix.system.E, err), + }); + + if (builtin.mode == .Debug) { + std.debug.dumpCurrentStackTrace(null); + } + return error.Unexpected; +} + +pub fn unique_u128() u128 { + const value = std.crypto.random.int(u128); + + // Broken CSPRNG is the likeliest explanation for zero or all ones. + assert(value != 0); + assert(value != std.math.maxInt(u128)); + + return value; +} + +/// NB: intended for parsing CLI arguments where we care to preserve the user-specified unit. +/// Use `size: u64` for all other use-cases. +pub const ByteSize = struct { + value: u64, + unit: Unit = .bytes, + + const Unit = enum(u64) { + bytes = 1, + kib = KiB, + mib = MiB, + gib = GiB, + tib = TiB, + }; + + pub fn parse_flag_value( + string: []const u8, + static_diagnostic: *?[]const u8, + ) error{InvalidFlagValue}!ByteSize { + assert(string.len != 0); + + const split_index = for (string, 0..) |c, index| { + if (std.ascii.isDigit(c) or c == '_') { + // Numeric part continues. + } else break index; + } else string.len; + + const string_amount = string[0..split_index]; + const string_unit = string[split_index..]; + maybe(string_amount.len == 0); + maybe(string_unit.len == 0); + + const amount = std.fmt.parseUnsigned(u64, string_amount, 10) catch |err| switch (err) { + error.Overflow => { + static_diagnostic.* = "value exceeds 64-bit unsigned integer:"; + return error.InvalidFlagValue; + }, + error.InvalidCharacter => { + static_diagnostic.* = "expected a size, but found:"; + return error.InvalidFlagValue; + }, + }; + + const unit = if (string_unit.len == 0) + .bytes + else inline for (comptime std.enums.values(Unit)) |tag| { + if (std.ascii.eqlIgnoreCase(string_unit, @tagName(tag))) break tag; + } else { + static_diagnostic.* = "invalid unit in size, needed KiB, MiB, GiB or TiB:"; + return error.InvalidFlagValue; + }; + + _ = std.math.mul(u64, amount, @intFromEnum(unit)) catch { + static_diagnostic.* = "size in bytes exceeds 64-bit unsigned integer:"; + return error.InvalidFlagValue; + }; + + return .{ .value = amount, .unit = unit }; + } + + pub fn bytes(size: *const ByteSize) u64 { + return std.math.mul( + u64, + size.value, + @intFromEnum(size.unit), + ) catch unreachable; + } + + pub fn suffix(size: *const ByteSize) []const u8 { + return switch (size.unit) { + .bytes => "", + .kib => "KiB", + .mib => "MiB", + .gib => "GiB", + .tib => "TiB", + }; + } +}; + +test "ByteSize.parse_flag_value" { + try Flags.parse_flag_value_fuzz(ByteSize, ByteSize.parse_flag_value, .{ + .ok = &.{ + .{ "0", .{ .value = 0, .unit = .bytes } }, + .{ "1", .{ .value = 1, .unit = .bytes } }, + + .{ "140737488355328", .{ .value = 140737488355328, .unit = .bytes } }, + .{ "128TiB", .{ .value = 128, .unit = .tib } }, + .{ "1TiB", .{ .value = 1, .unit = .tib } }, + .{ "10tib", .{ .value = 10, .unit = .tib } }, + .{ "1GiB", .{ .value = 1, .unit = .gib } }, + .{ "10gib", .{ .value = 10, .unit = .gib } }, + .{ "1MiB", .{ .value = 1, .unit = .mib } }, + .{ "10mib", .{ .value = 10, .unit = .mib } }, + .{ "1KiB", .{ .value = 1, .unit = .kib } }, + .{ "10kib", .{ .value = 10, .unit = .kib } }, + .{ "1_0kib", .{ .value = 10, .unit = .kib } }, + }, + .err = &.{ + .{ "18446744073709551616", "value exceeds 64-bit unsigned integer" }, + .{ "MiB", "expected a size, but found" }, + .{ "_MiB", "expected a size" }, + .{ "10bananas", "invalid unit in size, needed KiB, MiB, GiB or TiB" }, + .{ "10GB", "invalid unit in size" }, + .{ "18446744073709551GiB", "size in bytes exceeds 64-bit unsigned integer" }, + }, + }); +} + +// Fast alternative to modulo reduction (Note, it is not the same as modulo). +// See https://github.com/lemire/fastrange/ and +// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ +pub inline fn fastrange(word: u64, p: u64) u64 { + const lword: u128 = @intCast(word); + const lp: u128 = @intCast(p); + const ln: u128 = lword *% lp; + return @truncate(ln >> 64); +} + +// For Zig 14.1 the compiler generates branchless code as is: https://godbolt.org/z/hc563WPKP +pub inline fn branchless_select(comptime T: type, flag: bool, a: T, b: T) T { + @branchHint(.unpredictable); + return if (flag) a else b; +} + +const snap = Snap.snap_fn("src/stdx"); + +test fastrange { + var prng = PRNG.from_seed(42); + var distribution: [8]u32 = @splat(0); + for (0..10_000) |_| { + const key = prng.int(u64); + distribution[fastrange(key, 8)] += 1; + } + try snap(@src(), + \\{ 1263, 1273, 1244, 1226, 1228, 1276, 1169, 1321 } + ).diff_fmt("{d}", .{distribution}); +} + +// This test shows that fastrange is not equivalent to modulo, but rather an alternative method. +// It is best used uniformly distributed hashes or random numbers across the full range. +test "fastrange not modulo" { + var distribution: [8]u32 = @splat(0); + for (0..10_000) |key| { + distribution[fastrange(key, 8)] += 1; + } + try snap(@src(), + \\{ 10000, 0, 0, 0, 0, 0, 0, 0 } + ).diff_fmt("{d}", .{distribution}); +} + +/// `status` is a waitpid() status result. +pub fn term_from_status(status: u32) std.process.Child.Term { + const Term = std.process.Child.Term; + return if (std.posix.W.IFEXITED(status)) + Term{ .Exited = std.posix.W.EXITSTATUS(status) } + else if (std.posix.W.IFSIGNALED(status)) + Term{ .Signal = std.posix.W.TERMSIG(status) } + else if (std.posix.W.IFSTOPPED(status)) + Term{ .Stopped = std.posix.W.STOPSIG(status) } + else + Term{ .Unknown = status }; +} + +comptime { + _ = @import("bit_set.zig"); + _ = @import("bounded_array.zig"); + _ = @import("flags.zig"); + _ = @import("huge_page_allocator.zig"); + _ = @import("iops.zig"); + _ = @import("prng.zig"); + _ = @import("radix.zig"); + _ = @import("ring_buffer.zig"); + _ = @import("sort_test.zig"); + _ = @import("stdx.zig"); + _ = @import("testing/snaptest.zig"); + _ = @import("time_units.zig"); + _ = @import("unshare.zig"); + _ = @import("vendored/aegis.zig"); + _ = @import("zipfian.zig"); + _ = @import("shell.zig"); +} diff --git a/scripts/stdx/testing/low_level_hash_vectors.zig b/scripts/stdx/testing/low_level_hash_vectors.zig new file mode 100644 index 0000000..9d15e46 --- /dev/null +++ b/scripts/stdx/testing/low_level_hash_vectors.zig @@ -0,0 +1,142 @@ +//! Test vectors for `stdx.inline_hash` from +//! +//! + +pub const Case = struct { seed: u64, hash: u64, b64: []const u8 }; + +pub const cases = [_]Case{ + .{ .seed = 0xec42b7ab404b8acb, .hash = 0xe5a40d39ab796423, .b64 = "" }, + .{ .seed = 0, .hash = 0x1766974bf7527d81, .b64 = "ICAg" }, + .{ .seed = 0, .hash = 0x5c3bbbe230db17a8, .b64 = "YWFhYQ==" }, + .{ .seed = 0, .hash = 0xa6630143a7e6aa6f, .b64 = "AQID" }, + .{ .seed = 0, .hash = 0x8787cb2d04b0c984, .b64 = "AQIDBA==" }, + .{ .seed = 0, .hash = 0x33603654ff574ac2, .b64 = "dGhpcmRfcGFydHl8d3loYXNofDY0" }, + .{ .seed = 0xeeee074043a3ee0f, .hash = 0xa6564b468248c683, .b64 = "Zw==" }, + .{ .seed = 0x857902089c393de, .hash = 0xef192f401b116e1c, .b64 = "xmk=" }, + .{ .seed = 0x993df040024ca3af, .hash = 0xbe8dc0c54617639d, .b64 = "c1H/" }, + .{ .seed = 0xc4e4c2acea740e96, .hash = 0x93d7f665b5521c8e, .b64 = "SuwpzQ==" }, + .{ .seed = 0x6a214b3db872d0cf, .hash = 0x646d70bb42445f28, .b64 = "uqvy++M=" }, + .{ .seed = 0x44343db6a89dba4d, .hash = 0x96a7b1e3cc9bd426, .b64 = "RnzCVPgb" }, + .{ .seed = 0x77b5d6d1ae1dd483, .hash = 0x76020289ab0790c4, .b64 = "6OeNdlouYw==" }, + .{ .seed = 0x89ab8ecb44d221f1, .hash = 0x39f842e4133b9b44, .b64 = "M5/JmmYyDbc=" }, + .{ .seed = 0x60244b17577ca81b, .hash = 0x2b8d7047be4bcaab, .b64 = "MVijWiVdBRdY" }, + .{ .seed = 0x59a08dcee0717067, .hash = 0x99628abef6716a97, .b64 = "6V7Uq7LNxpu0VA==" }, + .{ .seed = 0xf5f20db3ade57396, .hash = 0x4432e02ba42b2740, .b64 = "EQ6CdEEhPdyHcOk=" }, + .{ .seed = 0xbf8dee0751ad3efb, .hash = 0x74d810efcad7918a, .b64 = "PqFB4fxnPgF+l+rc" }, + .{ .seed = 0x6b7a06b268d63e30, .hash = 0x88c84e986002507f, .b64 = "a5aPOFwq7LA7+zKvPA==" }, + .{ .seed = 0xb8c37f0ae0f54c82, .hash = 0x4f99acf193cf39b9, .b64 = "VOwY21wCGv5D+/qqOvs=" }, + .{ .seed = 0x9fcbed0c38e50eef, .hash = 0xd90e7a3655891e37, .b64 = "KdHmBTx8lHXYvmGJ+Vy7" }, + .{ .seed = 0x2af4bade1d8e3a1d, .hash = 0x3bb378b1d4df8fcf, .b64 = "qJkPlbHr8bMF7/cA6aE65Q==" }, + .{ .seed = 0x714e3aa912da2f2c, .hash = 0xf78e94045c052d47, .b64 = "ygvL0EhHZL0fIx6oHHtkxRQ=" }, + .{ .seed = 0xf5ee75e3cbb82c1c, .hash = 0x26da0b2130da6b40, .b64 = "c1rFXkt5YztwZCQRngncqtSs" }, + .{ .seed = 0x620e7007321b93b9, .hash = 0x30b4d426af8c6986, .b64 = "8hsQrzszzeNQSEcVXLtvIhm6mw==" }, + .{ .seed = 0xc08528cac2e551fc, .hash = 0x5413b4aaf3baaeae, .b64 = "ffUL4RocfyP4KfikGxO1yk7omDI=" }, + .{ .seed = 0x6a1debf9cc3ad39, .hash = 0x756ab265370a1597, .b64 = "OOB5TT00vF9Od/rLbAWshiErqhpV" }, + .{ .seed = 0x7e0a3c88111fc226, .hash = 0xdaf5f4b7d09814fb, .b64 = "or5wtXM7BFzTNpSzr+Lw5J5PMhVJ/Q==" }, + .{ .seed = 0x1301fef15df39edb, .hash = 0x8f874ae37742b75e, .b64 = "gk6pCHDUsoopVEiaCrzVDhioRKxb844=" }, + .{ .seed = 0x64e181f3d5817ab, .hash = 0x8fecd03956121ce8, .b64 = "TNctmwlC5QbEM6/No4R/La3UdkfeMhzs" }, + .{ .seed = 0xafafc44961078ecb, .hash = 0x229c292ea7a08285, .b64 = "SsQw9iAjhWz7sgcE9OwLuSC6hsM+BfHs2Q==" }, + .{ .seed = 0x4f7bb45549250094, .hash = 0xbb4bf0692d14bae, .b64 = "ZzO3mVCj4xTT2TT3XqDyEKj2BZQBvrS8RHg=" }, + .{ .seed = 0xa30061abaa2818c, .hash = 0x207b24ca3bdac1db, .b64 = "+klp5iPQGtppan5MflEls0iEUzqU+zGZkDJX" }, + .{ .seed = 0xd902ee3e44a5705f, .hash = 0x64f6cd6745d3825b, .b64 = "RO6bvOnlJc8I9eniXlNgqtKy0IX6VNg16NRmgg==" }, + .{ .seed = 0x316d36da516f583, .hash = 0xa2b2e1656b58df1e, .b64 = "ZJjZqId1ZXBaij9igClE3nyliU5XWdNRrayGlYA=" }, + .{ .seed = 0x402d83f9f834f616, .hash = 0xd01d30d9ee7a148, .b64 = "7BfkhfGMDGbxfMB8uyL85GbaYQtjr2K8g7RpLzr/" }, + .{ .seed = 0x9c604164c016b72c, .hash = 0x1cb4cd00ab804e3b, .b64 = "rycWk6wHH7htETQtje9PidS2YzXBx+Qkg2fY7ZYS7A==" }, + .{ .seed = 0x3f4507e01f9e73ba, .hash = 0x4697f2637fd90999, .b64 = "RTkC2OUK+J13CdGllsH0H5WqgspsSa6QzRZouqx6pvI=" }, + .{ .seed = 0xc3fe0d5be8d2c7c7, .hash = 0x8383a756b5688c07, .b64 = "tKjKmbLCNyrLCM9hycOAXm4DKNpM12oZ7dLTmUx5iwAi" }, + .{ .seed = 0x531858a40bfa7ea1, .hash = 0x695c29cb3696a975, .b64 = "VprUGNH+5NnNRaORxgH/ySrZFQFDL+4VAodhfBNinmn8cg==" }, + .{ .seed = 0x86689478a7a7e8fa, .hash = 0xda2e5a5a5e971521, .b64 = "gc1xZaY+q0nPcUvOOnWnT3bqfmT/geth/f7Dm2e/DemMfk4=" }, + .{ .seed = 0x4ec948b8e7f27288, .hash = 0x7935d4befa056b2b, .b64 = "Mr35fIxqx1ukPAL0su1yFuzzAU3wABCLZ8+ZUFsXn47UmAph" }, + .{ .seed = 0xce46c7213c10032, .hash = 0x38dd541ca95420fe, .b64 = "A9G8pw2+m7+rDtWYAdbl8tb2fT7FFo4hLi2vAsa5Y8mKH3CX3g==" }, + .{ .seed = 0xf63e96ee6f32a8b6, .hash = 0xcc06c7a4963f967f, .b64 = "DFaJGishGwEHDdj9ixbCoaTjz9KS0phLNWHVVdFsM93CvPft3hM=" }, + .{ .seed = 0x1cfe85e65fc5225, .hash = 0xbf0f6f66e232fb20, .b64 = "7+Ugx+Kr3aRNgYgcUxru62YkTDt5Hqis+2po81hGBkcrJg4N0uuy" }, + .{ .seed = 0x45c474f1cee1d2e8, .hash = 0xf7efb32d373fe71a, .b64 = "H2w6O8BUKqu6Tvj2xxaecxEI2wRgIgqnTTG1WwOgDSINR13Nm4d4Vg==" }, + .{ .seed = 0x6e024e14015f329c, .hash = 0xe2e64634b1c12660, .b64 = "1XBMnIbqD5jy65xTDaf6WtiwtdtQwv1dCVoqpeKj+7cTR1SaMWMyI04=" }, + .{ .seed = 0x760c40502103ae1c, .hash = 0x285b8fd1638e306d, .b64 = "znZbdXG2TSFrKHEuJc83gPncYpzXGbAebUpP0XxzH0rpe8BaMQ17nDbt" }, + .{ .seed = 0x17fd05c3c560c320, .hash = 0x658e8a4e3b714d6c, .b64 = "ylu8Atu13j1StlcC1MRMJJXIl7USgDDS22HgVv0WQ8hx/8pNtaiKB17hCQ==" }, + .{ .seed = 0x8b34200a6f8e90d9, .hash = 0xf391fb968e0eb398, .b64 = "M6ZVVzsd7vAvbiACSYHioH/440dp4xG2mLlBnxgiqEvI/aIEGpD0Sf4VS0g=" }, + .{ .seed = 0x6be89e50818bdf69, .hash = 0x744a9ea0cc144bf2, .b64 = "li3oFSXLXI+ubUVGJ4blP6mNinGKLHWkvGruun85AhVn6iuMtocbZPVhqxzn" }, + .{ .seed = 0xfb389773315b47d8, .hash = 0x12636f2be11012f1, .b64 = "kFuQHuUCqBF3Tc3hO4dgdIp223ShaCoog48d5Do5zMqUXOh5XpGK1t5XtxnfGA==" }, + .{ .seed = 0x4f2512a23f61efee, .hash = 0x29c57de825948f80, .b64 = "jWmOad0v0QhXVJd1OdGuBZtDYYS8wBVHlvOeTQx9ZZnm8wLEItPMeihj72E0nWY=" }, + .{ .seed = 0x59ccd92fc16c6fda, .hash = 0x58c6f99ab0d1c021, .b64 = "z+DHU52HaOQdW4JrZwDQAebEA6rm13Zg/9lPYA3txt3NjTBqFZlOMvTRnVzRbl23" }, + .{ .seed = 0x25c5a7f5bd330919, .hash = 0x13e7b5a7b82fe3bb, .b64 = "MmBiGDfYeTayyJa/tVycg+rN7f9mPDFaDc+23j0TlW9094er0ADigsl4QX7V3gG/qw==" }, + .{ .seed = 0x51df4174d34c97d7, .hash = 0x10fbc87901e02b63, .b64 = "774RK+9rOL4iFvs1q2qpo/JVc/I39buvNjqEFDtDvyoB0FXxPI2vXqOrk08VPfIHkmU=" }, + .{ .seed = 0x80ce6d76f89cb57, .hash = 0xa24c9184901b748b, .b64 = "+slatXiQ7/2lK0BkVUI1qzNxOOLP3I1iK6OfHaoxgqT63FpzbElwEXSwdsryq3UlHK0I" }, + .{ .seed = 0x20961c911965f684, .hash = 0xcac4fd4c5080e581, .b64 = "64mVTbQ47dHjHlOHGS/hjJwr/K2frCNpn87exOqMzNUVYiPKmhCbfS7vBUce5tO6Ec9osQ==" }, + .{ .seed = 0x4e5b926ec83868e7, .hash = 0xc38bdb7483ba68e1, .b64 = "fIsaG1r530SFrBqaDj1kqE0AJnvvK8MNEZbII2Yw1OK77v0V59xabIh0B5axaz/+a2V5WpA=" }, + .{ .seed = 0x3927b30b922eecef, .hash = 0xdb2a8069b2ceaffa, .b64 = "PGih0zDEOWCYGxuHGDFu9Ivbff/iE7BNUq65tycTR2R76TerrXALRosnzaNYO5fjFhTi+CiS" }, + .{ .seed = 0xbd0291284a49b61c, .hash = 0xdf9fe91d0d1c7887, .b64 = "RnpA/zJnEnnLjmICORByRVb9bCOgxF44p3VMiW10G7PvW7IhwsWajlP9kIwNA9FjAD2GoQHk2Q==" }, + .{ .seed = 0x73a77c575bcc956, .hash = 0xe83f49e96e2e6a08, .b64 = "qFklMceaTHqJpy2qavJE+EVBiNFOi6OxjOA3LeIcBop1K7w8xQi3TrDk+BrWPRIbfprszSaPfrI=" }, + .{ .seed = 0x766a0e2ade6d09a6, .hash = 0xc69e61b62ca2b62, .b64 = "cLbfUtLl3EcQmITWoTskUR8da/VafRDYF/ylPYwk7/zazk6ssyrzxMN3mmSyvrXR2yDGNZ3WDrTT" }, + .{ .seed = 0x2599f4f905115869, .hash = 0xb4a4f3f85f8298fe, .b64 = "s/Jf1+FbsbCpXWPTUSeWyMH6e4CvTFvPE5Fs6Z8hvFITGyr0dtukHzkI84oviVLxhM1xMxrMAy1dbw==" }, + .{ .seed = 0xd8256e5444d21e53, .hash = 0x167a1b39e1e95f41, .b64 = "FvyQ00+j7nmYZVQ8hI1Edxd0AWplhTfWuFGiu34AK5X8u2hLX1bE97sZM0CmeLe+7LgoUT1fJ/axybE=" }, + .{ .seed = 0xf664a91333fb8dfd, .hash = 0xf8a2a5649855ee41, .b64 = "L8ncxMaYLBH3g9buPu8hfpWZNlOF7nvWLNv9IozH07uQsIBWSKxoPy8+LW4tTuzC6CIWbRGRRD1sQV/4" }, + .{ .seed = 0x9625b859be372cd1, .hash = 0x27992565b595c498, .b64 = "CDK0meI07yrgV2kQlZZ+wuVqhc2NmzqeLH7bmcA6kchsRWFPeVF5Wqjjaj556ABeUoUr3yBmfU3kWOakkg==" }, + .{ .seed = 0x7b99940782e29898, .hash = 0x3e08cca5b71f9346, .b64 = "d23/vc5ONh/HkMiq+gYk4gaCNYyuFKwUkvn46t+dfVcKfBTYykr4kdvAPNXGYLjM4u1YkAEFpJP+nX7eOvs=" }, + .{ .seed = 0x4fe12fa5383b51a8, .hash = 0xad406b10c770a6d2, .b64 = "NUR3SRxBkxTSbtQORJpu/GdR6b/h6sSGfsMj/KFd99ahbh+9r7LSgSGmkGVB/mGoT0pnMTQst7Lv2q6QN6Vm" }, + .{ .seed = 0xe2ccb09ac0f5b4b6, .hash = 0xd1713ce6e552bcf2, .b64 = "2BOFlcI3Z0RYDtS9T9Ie9yJoXlOdigpPeeT+CRujb/O39Ih5LPC9hP6RQk1kYESGyaLZZi3jtabHs7DiVx/VDg==" }, + .{ .seed = 0x7d0a37adbd7b753b, .hash = 0x753b287194c73ad3, .b64 = "FF2HQE1FxEvWBpg6Z9zAMH+Zlqx8S1JD/wIlViL6ZDZY63alMDrxB0GJQahmAtjlm26RGLnjW7jmgQ4Ie3I+014=" }, + .{ .seed = 0xd3ae96ef9f7185f2, .hash = 0x5ae41a95f600af1c, .b64 = "tHmO7mqVL/PX11nZrz50Hc+M17Poj5lpnqHkEN+4bpMx/YGbkrGOaYjoQjgmt1X2QyypK7xClFrjeWrCMdlVYtbW" }, + .{ .seed = 0x4fb88ea63f79a0d8, .hash = 0x4a61163b86a8bb4c, .b64 = "/WiHi9IQcxRImsudkA/KOTqGe8/gXkhKIHkjddv5S9hi02M049dIK3EUyAEjkjpdGLUs+BN0QzPtZqjIYPOgwsYE9g==" }, + .{ .seed = 0xed564e259bb5ebe9, .hash = 0x42eeaa79e760c7e4, .b64 = "qds+1ExSnU11L4fTSDz/QE90g4Jh6ioqSh3KDOTOAo2pQGL1k/9CCC7J23YF27dUTzrWsCQA2m4epXoCc3yPHb3xElA=" }, + .{ .seed = 0x3e3256b60c428000, .hash = 0x698df622ef465b0a, .b64 = "8FVYHx40lSQPTHheh08Oq0/pGm2OlG8BEf8ezvAxHuGGdgCkqpXIueJBF2mQJhTfDy5NncO8ntS7vaKs7sCNdDaNGOEi" }, + .{ .seed = 0xfb05bad59ec8705, .hash = 0x157583111e1a6026, .b64 = "4ZoEIrJtstiCkeew3oRzmyJHVt/pAs2pj0HgHFrBPztbQ10NsQ/lM6DM439QVxpznnBSiHMgMQJhER+70l72LqFTO1JiIQ==" }, + .{ .seed = 0xafdc251dbf97b5f8, .hash = 0xaa1388f078e793e0, .b64 = "hQPtaYI+wJyxXgwD5n8jGIKFKaFA/P83KqCKZfPthnjwdOFysqEOYwAaZuaaiv4cDyi9TyS8hk5cEbNP/jrI7q6pYGBLbsM=" }, + .{ .seed = 0x10ec9c92ddb5dcbc, .hash = 0xf10d68d0f3309360, .b64 = "S4gpMSKzMD7CWPsSfLeYyhSpfWOntyuVZdX1xSBjiGvsspwOZcxNKCRIOqAA0moUfOh3I5+juQV4rsqYElMD/gWfDGpsWZKQ" }, + .{ .seed = 0x9a767d5822c7dac4, .hash = 0x2af056184457a3de, .b64 = "oswxop+bthuDLT4j0PcoSKby4LhF47ZKg8K17xxHf74UsGCzTBbOz0MM8hQEGlyqDT1iUiAYnaPaUpL2mRK0rcIUYA4qLt5uOw==" }, + .{ .seed = 0xee46254080d6e2db, .hash = 0x6d0058e1590b2489, .b64 = "0II/697p+BtLSjxj5989OXI004TogEb94VUnDzOVSgMXie72cuYRvTFNIBgtXlKfkiUjeqVpd4a+n5bxNOD1TGrjQtzKU5r7obo=" }, + .{ .seed = 0xbbb669588d8bf398, .hash = 0x638f287f68817f12, .b64 = "E84YZW2qipAlMPmctrg7TKlwLZ68l4L+c0xRDUfyyFrA4MAti0q9sHq3TDFviH0Y+Kq3tEE5srWFA8LM9oomtmvm5PYxoaarWPLc" }, + .{ .seed = 0xdc2afaa529beef44, .hash = 0xc46b71fecefd5467, .b64 = "x3pa4HIElyZG0Nj7Vdy9IdJIR4izLmypXw5PCmZB5y68QQ4uRaVVi3UthsoJROvbjDJkP2DQ6L/eN8pFeLFzNPKBYzcmuMOb5Ull7w==" }, + .{ .seed = 0xf1f67391d45013a8, .hash = 0x2c8e94679d964e0a, .b64 = "jVDKGYIuWOP/QKLdd2wi8B2VJA8Wh0c8PwrXJVM8FOGM3voPDVPyDJOU6QsBDPseoR8uuKd19OZ/zAvSCB+zlf6upAsBlheUKgCfKww=" }, + .{ .seed = 0x16fce2b8c65a3429, .hash = 0x8612b797ce22503a, .b64 = "mkquunhmYe1aR2wmUz4vcvLEcKBoe6H+kjUok9VUn2+eTSkWs4oDDtJvNCWtY5efJwg/j4PgjRYWtqnrCkhaqJaEvkkOwVfgMIwF3e+d" }, + .{ .seed = 0xf4b096699f49fe67, .hash = 0x59f929babfba7170, .b64 = "fRelvKYonTQ+s+rnnvQw+JzGfFoPixtna0vzcSjiDqX5s2Kg2//UGrK+AVCyMUhO98WoB1DDbrsOYSw2QzrcPe0+3ck9sePvb+Q/IRaHbw==" }, + .{ .seed = 0xca584c4bc8198682, .hash = 0x9527556923fb49a0, .b64 = "DUwXFJzagljo44QeJ7/6ZKw4QXV18lhkYT2jglMr8WB3CHUU4vdsytvw6AKv42ZcG6fRkZkq9fpnmXy6xG0aO3WPT1eHuyFirAlkW+zKtwg=" }, + .{ .seed = 0xed269fc3818b6aad, .hash = 0x1039ab644f5e150b, .b64 = "cYmZCrOOBBongNTr7e4nYn52uQUy2mfe48s50JXx2AZ6cRAt/xRHJ5QbEoEJOeOHsJyM4nbzwFm++SlT6gFZZHJpkXJ92JkR86uS/eV1hJUR" }, + .{ .seed = 0x33f253cbb8fe66a8, .hash = 0x7816c83f3aa05e6d, .b64 = "EXeHBDfhwzAKFhsMcH9+2RHwV+mJaN01+9oacF6vgm8mCXRd6jeN9U2oAb0of5c5cO4i+Vb/LlHZSMI490SnHU0bejhSCC2gsC5d2K30ER3iNA==" }, + .{ .seed = 0xd0b76b2c1523d99c, .hash = 0xf51d2f564518c619, .b64 = "FzkzRYoNjkxFhZDso94IHRZaJUP61nFYrh5MwDwv9FNoJ5jyNCY/eazPZk+tbmzDyJIGw2h3GxaWZ9bSlsol/vK98SbkMKCQ/wbfrXRLcDzdd/8=" }, + .{ .seed = 0xfd28f0811a2a237f, .hash = 0x67d494cff03ac004, .b64 = "Re4aXISCMlYY/XsX7zkIFR04ta03u4zkL9dVbLXMa/q6hlY/CImVIIYRN3VKP4pnd0AUr/ugkyt36JcstAInb4h9rpAGQ7GMVOgBniiMBZ/MGU7H" }, + .{ .seed = 0x6261fb136482e84, .hash = 0x2802d636ced1cfbb, .b64 = "ueLyMcqJXX+MhO4UApylCN9WlTQ+ltJmItgG7vFUtqs2qNwBMjmAvr5u0sAKd8jpzV0dDPTwchbIeAW5zbtkA2NABJV6hFM48ib4/J3A5mseA3cS8w==" }, + .{ .seed = 0x458efc750bca7c3a, .hash = 0xf64e20bad771cb12, .b64 = "6Si7Yi11L+jZMkwaN+GUuzXMrlvEqviEkGOilNq0h8TdQyYKuFXzkYc/q74gP3pVCyiwz9KpVGMM9vfnq36riMHRknkmhQutxLZs5fbmOgEO69HglCU=" }, + .{ .seed = 0xa7e69ff84e5e7c27, .hash = 0xb9a6cf84a83e15e, .b64 = "Q6AbOofGuTJOegPh9Clm/9crtUMQqylKrTc1fhfJo1tqvpXxhU4k08kntL1RG7woRnFrVh2UoMrL1kjin+s9CanT+y4hHwLqRranl9FjvxfVKm3yvg68" }, + .{ .seed = 0x3c59bfd0c29efe9e, .hash = 0x8da6630319609301, .b64 = "ieQEbIPvqY2YfIjHnqfJiO1/MIVRk0RoaG/WWi3kFrfIGiNLCczYoklgaecHMm/1sZ96AjO+a5stQfZbJQwS7Sc1ODABEdJKcTsxeW2hbh9A6CFzpowP1A==" }, + .{ .seed = 0x10befacc6afd298d, .hash = 0x40946a86e2a996f3, .b64 = "zQUv8hFB3zh2GGl3KTvCmnfzE+SUgQPVaSVIELFX5H9cE3FuVFGmymkPQZJLAyzC90Cmi8GqYCvPqTuAAB//XTJxy4bCcVArgZG9zJXpjowpNBfr3ngWrSE=" }, + .{ .seed = 0x41d5320b0a38efa7, .hash = 0xcab7f5997953fa76, .b64 = "US4hcC1+op5JKGC7eIs8CUgInjKWKlvKQkapulxW262E/B2ye79QxOexf188u2mFwwe3WTISJHRZzS61IwljqAWAWoBAqkUnW8SHmIDwHUP31J0p5sGdP47L" }, + .{ .seed = 0x58db1c7450fe17f3, .hash = 0x39129ca0e04fc465, .b64 = "9bHUWFna2LNaGF6fQLlkx1Hkt24nrkLE2CmFdWgTQV3FFbUe747SSqYw6ebpTa07MWSpWRPsHesVo2B9tqHbe7eQmqYebPDFnNqrhSdZwFm9arLQVs+7a3Ic6A==" }, + .{ .seed = 0x6098c055a335b7a6, .hash = 0x5238221fd685e1b8, .b64 = "Kb3DpHRUPhtyqgs3RuXjzA08jGb59hjKTOeFt1qhoINfYyfTt2buKhD6YVffRCPsgK9SeqZqRPJSyaqsa0ovyq1WnWW8jI/NhvAkZTVHUrX2pC+cD3OPYT05Dag=" }, + .{ .seed = 0x1bbacec67845a801, .hash = 0x175130c407dbcaab, .b64 = "gzxyMJIPlU+bJBwhFUCHSofZ/319LxqMoqnt3+L6h2U2+ZXJCSsYpE80xmR0Ta77Jq54o92SMH87HV8dGOaCTuAYF+lDL42SY1P316Cl0sZTS2ow3ZqwGbcPNs/1" }, + .{ .seed = 0xc419cfc7442190, .hash = 0x2f20e7536c0b0df, .b64 = "uR7V0TW+FGVMpsifnaBAQ3IGlr1wx5sKd7TChuqRe6OvUXTlD4hKWy8S+8yyOw8lQabism19vOQxfmocEOW/vzY0pEa87qHrAZy4s9fH2Bltu8vaOIe+agYohhYORQ==" }, + .{ .seed = 0xc95e510d94ba270c, .hash = 0x2742cb488a04ad56, .b64 = "1UR5eoo2aCwhacjZHaCh9bkOsITp6QunUxHQ2SfeHv0imHetzt/Z70mhyWZBalv6eAx+YfWKCUib2SHDtz/A2dc3hqUWX5VfAV7FQsghPUAtu6IiRatq4YSLpDvKZBQ=" }, + .{ .seed = 0xff1ae05c98089c3f, .hash = 0xd6afb593879ff93b, .b64 = "opubR7H63BH7OtY+Avd7QyQ25UZ8kLBdFDsBTwZlY6gA/u+x+czC9AaZMgmQrUy15DH7YMGsvdXnviTtI4eVI4aF1H9Rl3NXMKZgwFOsdTfdcZeeHVRzBBKX8jUfh1il" }, + .{ .seed = 0x90c02b8dceced493, .hash = 0xf50ad64caac0ca7f, .b64 = "DC0kXcSXtfQ9FbSRwirIn5tgPri0sbzHSa78aDZVDUKCMaBGyFU6BmrulywYX8yzvwprdLsoOwTWN2wMjHlPDqrvVHNEjnmufRDblW+nSS+xtKNs3N5xsxXdv6JXDrAB/Q==" }, + .{ .seed = 0x9f8a76697ab1aa36, .hash = 0x2ade95c4261364ae, .b64 = "BXRBk+3wEP3Lpm1y75wjoz+PgB0AMzLe8tQ1AYU2/oqrQB2YMC6W+9QDbcOfkGbeH+b7IBkt/gwCMw2HaQsRFEsurXtcQ3YwRuPz5XNaw5NAvrNa67Fm7eRzdE1+hWLKtA8=" }, + .{ .seed = 0x6ba1bf3d811a531d, .hash = 0x5c4f3299faacd07a, .b64 = "RRBSvEGYnzR9E45Aps/+WSnpCo/X7gJLO4DRnUqFrJCV/kzWlusLE/6ZU6RoUf2ROwcgEvUiXTGjLs7ts3t9SXnJHxC1KiOzxHdYLMhVvgNd3hVSAXODpKFSkVXND55G2L1W" }, + .{ .seed = 0x6a418974109c67b4, .hash = 0xfffe3bff0ae5e9bc, .b64 = "jeh6Qazxmdi57pa9S3XSnnZFIRrnc6s8QLrah5OX3SB/V2ErSPoEAumavzQPkdKF1/SfvmdL+qgF1C+Yawy562QaFqwVGq7+tW0yxP8FStb56ZRgNI4IOmI30s1Ei7iops9Uuw==" }, + .{ .seed = 0x8472f1c2b3d230a3, .hash = 0x1db785c0005166e4, .b64 = "6QO5nnDrY2/wrUXpltlKy2dSBcmK15fOY092CR7KxAjNfaY+aAmtWbbzQk3MjBg03x39afSUN1fkrWACdyQKRaGxgwq6MGNxI6W+8DLWJBHzIXrntrE/ml6fnNXEpxplWJ1vEs4=" }, + .{ .seed = 0x5e06068f884e73a7, .hash = 0xea000d962ad18418, .b64 = "0oPxeEHhqhcFuwonNfLd5jF3RNATGZS6NPoS0WklnzyokbTqcl4BeBkMn07+fDQv83j/BpGUwcWO05f3+DYzocfnizpFjLJemFGsls3gxcBYxcbqWYev51tG3lN9EvRE+X9+Pwww" }, + .{ .seed = 0x55290b1a8f170f59, .hash = 0xe42aef38359362d9, .b64 = "naSBSjtOKgAOg8XVbR5cHAW3Y+QL4Pb/JO9/oy6L08wvVRZqo0BrssMwhzBP401Um7A4ppAupbQeJFdMrysY34AuSSNvtNUy5VxjNECwiNtgwYHw7yakDUv8WvonctmnoSPKENegQg==" }, + .{ .seed = 0x5501cfd83dfe706a, .hash = 0xc8e95657348a3891, .b64 = "vPyl8DxVeRe1OpilKb9KNwpGkQRtA94UpAHetNh+95V7nIW38v7PpzhnTWIml5kw3So1Si0TXtIUPIbsu32BNhoH7QwFvLM+JACgSpc5e3RjsL6Qwxxi11npwxRmRUqATDeMUfRAjxg=" }, + .{ .seed = 0xe43ed13d13a66990, .hash = 0xc162eca864f238c6, .b64 = "QC9i2GjdTMuNC1xQJ74ngKfrlA4w3o58FhvNCltdIpuMhHP1YsDA78scQPLbZ3OCUgeQguYf/vw6zAaVKSgwtaykqg5ka/4vhz4hYqWU5ficdXqClHl+zkWEY26slCNYOM5nnDlly8Cj" }, + .{ .seed = 0xdf43bc375cf5283f, .hash = 0xbe1fb373e20579ad, .b64 = "7CNIgQhAHX27nxI0HeB5oUTnTdgKpRDYDKwRcXfSFGP1XeT9nQF6WKCMjL1tBV6x7KuJ91GZz11F4c+8s+MfqEAEpd4FHzamrMNjGcjCyrVtU6y+7HscMVzr7Q/ODLcPEFztFnwjvCjmHw==" }, + .{ .seed = 0x8112b806d288d7b5, .hash = 0x628a1d4f40aa6ffd, .b64 = "Qa/hC2RPXhANSospe+gUaPfjdK/yhQvfm4cCV6/pdvCYWPv8p1kMtKOX3h5/8oZ31fsmx4Axphu5qXJokuhZKkBUJueuMpxRyXpwSWz2wELx5glxF7CM0Fn+OevnkhUn5jsPlG2r5jYlVn8=" }, + .{ .seed = 0xd52a18abb001cb46, .hash = 0xa87bdb7456340f90, .b64 = "kUw/0z4l3a89jTwN5jpG0SHY5km/IVhTjgM5xCiPRLncg40aqWrJ5vcF891AOq5hEpSq0bUCJUMFXgct7kvnys905HjerV7Vs1Gy84tgVJ70/2+pAZTsB/PzNOE/G6sOj4+GbTzkQu819OLB" }, + .{ .seed = 0xe12b76a2433a1236, .hash = 0x5960ef3ba982c801, .b64 = "VDdfSDbO8Tdj3T5W0XM3EI7iHh5xpIutiM6dvcJ/fhe23V/srFEkDy5iZf/VnA9kfi2C79ENnFnbOReeuZW1b3MUXB9lgC6U4pOTuC+jHK3Qnpyiqzj7h3ISJSuo2pob7vY6VHZo6Fn7exEqHg==" }, + .{ .seed = 0x175bf7319cf1fa00, .hash = 0x5026586df9a431ec, .b64 = "Ldfvy3ORdquM/R2fIkhH/ONi69mcP1AEJ6n/oropwecAsLJzQSgezSY8bEiEs0VnFTBBsW+RtZY6tDj03fnb3amNUOq1b7jbqyQkL9hpl+2Z2J8IaVSeownWl+bQcsR5/xRktIMckC5AtF4YHfU=" }, + .{ .seed = 0xd63d57b3f67525ae, .hash = 0xfe4b8a20fdf0840b, .b64 = "BrbNpb42+VzZAjJw6QLirXzhweCVRfwlczzZ0VX2xluskwBqyfnGovz5EuX79JJ31VNXa5hTkAyQat3lYKRADTdAdwE5PqM1N7YaMqqsqoAAAeuYVXuk5eWCykYmClNdSspegwgCuT+403JigBzi" }, + .{ .seed = 0x933faea858832b73, .hash = 0xdcb761867da7072f, .b64 = "gB3NGHJJvVcuPyF0ZSvHwnWSIfmaI7La24VMPQVoIIWF7Z74NltPZZpx2f+cocESM+ILzQW9p+BC8x5IWz7N4Str2WLGKMdgmaBfNkEhSHQDU0IJEOnpUt0HmjhFaBlx0/LTmhua+rQ6Wup8ezLwfg==" }, + .{ .seed = 0x53d061e5f8e7c04f, .hash = 0xc10d4653667275b7, .b64 = "hTKHlRxx6Pl4gjG+6ksvvj0CWFicUg3WrPdSJypDpq91LUWRni2KF6+81ZoHBFhEBrCdogKqeK+hy9bLDnx7g6rAFUjtn1+cWzQ2YjiOpz4+ROBB7lnwjyTGWzJD1rXtlso1g2qVH8XJVigC5M9AIxM=" }, + .{ .seed = 0xdb4124556dd515e0, .hash = 0x727720deec13110b, .b64 = "IWQBelSQnhrr0F3BhUpXUIDauhX6f95Qp+A0diFXiUK7irwPG1oqBiqHyK/SH/9S+rln9DlFROAmeFdH0OCJi2tFm4afxYzJTFR4HnR4cG4x12JqHaZLQx6iiu6CE3rtWBVz99oAwCZUOEXIsLU24o2Y" }, + .{ .seed = 0x4fb31a0dd681ee71, .hash = 0x710b009662858dc9, .b64 = "TKo+l+1dOXdLvIrFqeLaHdm0HZnbcdEgOoLVcGRiCbAMR0j5pIFw8D36tefckAS1RCFOH5IgP8yiFT0Gd0a2hI3+fTKA7iK96NekxWeoeqzJyctc6QsoiyBlkZerRxs5RplrxoeNg29kKDTM0K94mnhD9g==" }, + .{ .seed = 0x27cc72eefa138e4c, .hash = 0xfbf8f7a3ecac1eb7, .b64 = "YU4e7G6EfQYvxCFoCrrT0EFgVLHFfOWRTJQJ5gxM3G2b+1kJf9YPrpsxF6Xr6nYtS8reEEbDoZJYqnlk9lXSkVArm88Cqn6d25VCx3+49MqC0trIlXtb7SXUUhwpJK16T0hJUfPH7s5cMZXc6YmmbFuBNPE=" }, + .{ .seed = 0x44bc2dfba4bd3ced, .hash = 0xb6fc4fcd0722e3df, .b64 = "/I/eImMwPo1U6wekNFD1Jxjk9XQVi1D+FPdqcHifYXQuP5aScNQfxMAmaPR2XhuOQhADV5tTVbBKwCDCX4E3jcDNHzCiPvViZF1W27txaf2BbFQdwKrNCmrtzcluBFYu0XZfc7RU1RmxK/RtnF1qHsq/O4pp" }, + .{ .seed = 0x242da1e3a439bed8, .hash = 0x7cb86dcc55104aac, .b64 = "CJTT9WGcY2XykTdo8KodRIA29qsqY0iHzWZRjKHb9alwyJ7RZAE3V5Juv4MY3MeYEr1EPCCMxO7yFXqT8XA8YTjaMp3bafRt17Pw8JC4iKJ1zN+WWKOESrj+3aluGQqn8z1EzqY4PH7rLG575PYeWsP98BugdA==" }, + .{ .seed = 0xdc559c746e35c139, .hash = 0x19e71e9b45c3a51e, .b64 = "ZlhyQwLhXQyIUEnMH/AEW27vh9xrbNKJxpWGtrEmKhd+nFqAfbeNBQjW0SfG1YI0xQkQMHXjuTt4P/EpZRtA47ibZDVS8TtaxwyBjuIDwqcN09eCtpC+Ls+vWDTLmBeDM3u4hmzz4DQAYsLiZYSJcldg9Q3wszw=" }, + .{ .seed = 0xd0b0350275b9989, .hash = 0x51de38573c2bea48, .b64 = "v2KU8y0sCrBghmnm8lzGJlwo6D6ObccAxCf10heoDtYLosk4ztTpLlpSFEyu23MLA1tJkcgRko04h19QMG0mOw/wc93EXAweriBqXfvdaP85sZABwiKO+6rtS9pacRVpYYhHJeVTQ5NzrvBvi1huxAr+xswhVMfL" }, + .{ .seed = 0xb04489e41d17730c, .hash = 0xa73ab6996d6df158, .b64 = "QhKlnIS6BuVCTQsnoE67E/yrgogE8EwO7xLaEGei26m0gEU4OksefJgppDh3X0x0Cs78Dr9IHK5b977CmZlrTRmwhlP8pM+UzXPNRNIZuN3ntOum/QhUWP8SGpirheXENWsXMQ/nxtxakyEtrNkKk471Oov9juP8oQ==" }, + .{ .seed = 0x2217285eb4572156, .hash = 0x55ef2b8c930817b2, .b64 = "/ZRMgnoRt+Uo6fUPr9FqQvKX7syhgVqWu+WUSsiQ68UlN0efSP6Eced5gJZL6tg9gcYJIkhjuQNITU0Q3TjVAnAcobgbJikCn6qZ6pRxKBY4MTiAlfGD3T7R7hwJwx554MAy++Zb/YUFlnCaCJiwQMnowF7aQzwYFCo=" }, + .{ .seed = 0x12c2e8e68aede73b, .hash = 0xb2850bf5fae87157, .b64 = "NB7tU5fNE8nI+SXGfipc7sRkhnSkUF1krjeo6k+8FITaAtdyz+o7mONgXmGLulBPH9bEwyYhKNVY0L+njNQrZ9YC2aXsFD3PdZsxAFaBT3VXEzh+NGBTjDASNL3mXyS8Yv1iThGfHoY7T4aR0NYGJ+k+pR6f+KrPC96M" }, + .{ .seed = 0x4d612125bdc4fd00, .hash = 0xecf3de1acd04651f, .b64 = "8T6wrqCtEO6/rwxF6lvMeyuigVOLwPipX/FULvwyu+1wa5sQGav/2FsLHUVn6cGSi0LlFwLewGHPFJDLR0u4t7ZUyM//x6da0sWgOa5hzDqjsVGmjxEHXiaXKW3i4iSZNuxoNbMQkIbVML+DkYu9ND0O2swg4itGeVSzXA==" }, + .{ .seed = 0x81826b553954464e, .hash = 0xcc0a40552559ff32, .b64 = "Ntf1bMRdondtMv1CYr3G80iDJ4WSAlKy5H34XdGruQiCrnRGDBa+eUi7vKp4gp3BBcVGl8eYSasVQQjn7MLvb3BjtXx6c/bCL7JtpzQKaDnPr9GWRxpBXVxKREgMM7d8lm35EODv0w+hQLfVSh8OGs7fsBb68nNWPLeeSOo=" }, + .{ .seed = 0xc2e5d345dc0ddd2d, .hash = 0xc385c374f20315b1, .b64 = "VsSAw72Ro6xks02kaiLuiTEIWBC5bgqr4WDnmP8vglXzAhixk7td926rm9jNimL+kroPSygZ9gl63aF5DCPOACXmsbmhDrAQuUzoh9ZKhWgElLQsrqo1KIjWoZT5b5QfVUXY9lSIBg3U75SqORoTPq7HalxxoIT5diWOcJQi" }, + .{ .seed = 0x3da6830a9e32631e, .hash = 0xb90208a4c7234183, .b64 = "j+loZ+C87+bJxNVebg94gU0mSLeDulcHs84tQT7BZM2rzDSLiCNxUedHr1ZWJ9ejTiBa0dqy2I2ABc++xzOLcv+//YfibtjKtYggC6/3rv0XCc7xu6d/O6xO+XOBhOWAQ+IHJVHf7wZnDxIXB8AUHsnjEISKj7823biqXjyP3g==" }, + .{ .seed = 0xc9ae5c8759b4877a, .hash = 0x58aa1ca7a4c075d9, .b64 = "f3LlpcPElMkspNtDq5xXyWU62erEaKn7RWKlo540gR6mZsNpK1czV/sOmqaq8XAQLEn68LKj6/cFkJukxRzCa4OF1a7cCAXYFp9+wZDu0bw4y63qbpjhdCl8GO6Z2lkcXy7KOzbPE01ukg7+gN+7uKpoohgAhIwpAKQXmX5xtd0=" }, +}; diff --git a/scripts/stdx/testing/snaptest.zig b/scripts/stdx/testing/snaptest.zig new file mode 100644 index 0000000..292ed31 --- /dev/null +++ b/scripts/stdx/testing/snaptest.zig @@ -0,0 +1,361 @@ +//! A tiny pattern/library for testing with expectations ([1], [2]). +//! +//! On a high level, this is a replacement for `std.testing.expectEqual` which: +//! +//! - is less cumbersome to use for complex types, +//! - gives somewhat more useful feedback on a test failure without much investment, +//! - drastically reduces the time to update the tests after refactors, +//! - encourages creation of reusable visualizations for data structures. +//! +//! Implementation-wise, `snaptest` provides a `Snap` type, which can be thought of as a Zig string +//! literal which also remembers its location in the source file, can be diffed with other strings, +//! and, crucially, can _update its own source code_ to match the expected value. +//! +//! Example usage: +//! +//! ``` +//! const Snap = @import("snaptest.zig").Snap; +//! const snap = Snap.snap; +//! +//! fn check_addition(x: u32, y: u32, want: Snap) !void { +//! const got = x + y; +//! try want.diff_fmt("{}", .{got}); +//! } +//! +//! test "addition" { +//! try check_addition(2, 2, snap(@src(), +//! \\8 +//! )); +//! } +//! ``` +//! +//! Running this test fails, printing the diff between actual result (`4`) and what's specified in +//! the source code. +//! +//! Re-running the test with `SNAP_UPDATE=1` environmental variable auto-magically updates the +//! source code to say `\\4`. Alternatively, you can use `snap(...).update()` to auto-update just a +//! single test. +//! +//! Note the `@src()` argument passed to the `snap(...)` invocation --- that's how it knows which +//! lines to update. +//! +//! Snapshots can use `` marker to ignore part of input: +//! +//! ``` +//! test "time" { +//! var buf: [32]u8 = undefined; +//! const time = try std.fmt.bufPrint(&buf, "it's {}ms", .{ +//! std.time.milliTimestamp(), +//! }); +//! try Snap.snap(@src(), +//! \\it's ms +//! ).diff(time); +//! } +//! ``` +//! +//! TODO: +//! - This doesn't actually `diff` things yet :o) But running with `SNAP_UPDATE=1` and then using +//! `git diff` is a workable substitute. +//! - Only one test can be updated at a time. To update several, we need to return +//! `error.SkipZigTest` on mismatch and adjust offsets appropriately. +//! +//! [1]: https://blog.janestreet.com/using-ascii-waveforms-to-test-hardware-designs/ +//! [2]: https://ianthehenry.com/posts/my-kind-of-repl/ +const std = @import("std"); +const assert = std.debug.assert; +const builtin = @import("builtin"); +const SourceLocation = std.builtin.SourceLocation; + +const stdx = @import("../stdx.zig"); + +const MiB = stdx.MiB; + +pub const Snap = struct { + comptime { + assert(builtin.is_test); + } + + // Set to `true` to update all snapshots. + pub var update_all: bool = false; + + module_path: []const u8, + location: SourceLocation, + text: []const u8, + update_this: bool = false, + + const SnapFn = fn (location: SourceLocation, text: []const u8) Snap; + + /// Takes the path to the root source file of the current module and creates a snap function. + /// + /// ``` + /// const snap = Snap.snap_fn("src"); + /// const snap = Snap.snap_fn("src/stdx"); + /// ``` + /// + /// For the update logic to work, usage *must* be formatted as: + /// + /// ``` + /// snap(@src(), + /// \\Text of the snapshot. + /// ) + /// ``` + pub fn snap_fn(comptime module_path: []const u8) SnapFn { + return struct { + fn snap(location: SourceLocation, text: []const u8) Snap { + return init(module_path, location, text); + } + }.snap; + } + + /// Creates a new Snap. + fn init(module_path: []const u8, location: SourceLocation, text: []const u8) Snap { + return Snap{ .module_path = module_path, .location = location, .text = text }; + } + + /// Builder-lite method to update just this particular snapshot. + pub fn update(snapshot: *const Snap) Snap { + return Snap{ + .module_path = snapshot.module_path, + .location = snapshot.location, + .text = snapshot.text, + .update_this = true, + }; + } + + /// To update a snapshot, use whichever you prefer: + /// - `.update()` method on a particular snap, + /// - `update_all` const in this file, + /// - `SNAP_UPDATE` env var. + fn should_update(snapshot: *const Snap) bool { + return snapshot.update_this or update_all or + std.process.hasEnvVarConstant("SNAP_UPDATE"); + } + + // Compare the snapshot with a formatted string. + pub fn diff_fmt(snapshot: *const Snap, comptime fmt: []const u8, fmt_args: anytype) !void { + const got = try std.fmt.allocPrint(std.testing.allocator, fmt, fmt_args); + defer std.testing.allocator.free(got); + + try snapshot.diff(got); + } + + // Compare the snapshot with the zon json serialization of a `value`. + pub fn diff_zon( + snapshot: *const Snap, + value: anytype, + ) !void { + var got: std.ArrayListUnmanaged(u8) = .empty; + defer got.deinit(std.testing.allocator); + + try std.zon.stringify.serialize(value, .{}, got.writer(std.testing.allocator)); + try snapshot.diff(got.items); + } + + pub fn diff_hex(snapshot: *const Snap, value: []const u8) !void { + var buffer: std.ArrayListUnmanaged(u8) = .empty; + defer buffer.deinit(std.testing.allocator); + + try hexdump(value, buffer.writer(std.testing.allocator).any()); + try snapshot.diff(buffer.items); + } + + // Compare the snapshot with a given string. + pub fn diff(snapshot: *const Snap, got: []const u8) !void { + if (equal_excluding_ignored(got, snapshot.text)) return; + + std.debug.print( + \\Snapshot differs. + \\Want: + \\---- + \\{s} + \\---- + \\Got: + \\---- + \\{s} + \\---- + \\ + , + .{ + snapshot.text, + got, + }, + ); + + if (!snapshot.should_update()) { + std.debug.print( + "Rerun with SNAP_UPDATE=1 environmental variable to update the snapshot.\n", + .{}, + ); + return error.SnapDiff; + } + + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + + const arena = arena_instance.allocator(); + const file_path_relative = try std.fs.path.join( + arena, + // The file location is relative to the module root path. + &.{ snapshot.module_path, snapshot.location.file }, + ); + + const file_text = try std.fs.cwd().readFileAlloc(arena, file_path_relative, 1 * MiB); + var file_text_updated = try std.ArrayList(u8).initCapacity(arena, file_text.len); + + const line_zero_based = snapshot.location.line - 1; + const range = snap_range(file_text, line_zero_based); + + const snapshot_prefix = file_text[0..range.start]; + const snapshot_text = file_text[range.start..range.end]; + const snapshot_suffix = file_text[range.end..]; + + const indent = get_indent(snapshot_text); + + try file_text_updated.appendSlice(snapshot_prefix); + { + var lines = std.mem.splitScalar(u8, got, '\n'); + while (lines.next()) |line| { + try file_text_updated.writer().print("{s}\\\\{s}\n", .{ indent, line }); + } + } + try file_text_updated.appendSlice(snapshot_suffix); + + try std.fs.cwd().writeFile(.{ + .sub_path = file_path_relative, + .data = file_text_updated.items, + }); + + std.debug.print("Updated {s}\n", .{file_path_relative}); + return error.SnapUpdated; + } +}; + +fn equal_excluding_ignored(got: []const u8, snapshot: []const u8) bool { + // Don't allow ignoring suffixes and prefixes, as that makes it easy to miss trailing or leading + // data. + assert(!std.mem.startsWith(u8, snapshot, "")); + assert(!std.mem.endsWith(u8, snapshot, "")); + + var got_rest = got; + var snapshot_rest = snapshot; + for (0..10) |_| { + // Cut the part before the first ignore, it should be equal between two strings... + const common_prefix, snapshot_rest = stdx.cut(snapshot_rest, "") orelse break; + got_rest = stdx.cut_prefix(got_rest, common_prefix) orelse return false; + + // ...then find the next part that should match, and cut up to that. + const common_middle, _ = + stdx.cut(snapshot_rest, "") orelse .{ snapshot_rest, "" }; + assert(common_middle.len > 0); + snapshot_rest = stdx.cut_prefix(snapshot_rest, common_middle).?; + + const ignored, got_rest = stdx.cut(got_rest, common_middle) orelse return false; + // If matched an empty string, or several lines, report it as an error. + if (ignored.len == 0) return false; + if (std.mem.indexOfScalar(u8, ignored, '\n') != null) return false; + } else @panic("more than 10 ignores"); + + return std.mem.eql(u8, got_rest, snapshot_rest); +} + +test equal_excluding_ignored { + try equal_excluding_ignored_case("ABA", "ABA", true); + try equal_excluding_ignored_case("ABBA", "AA", true); + try equal_excluding_ignored_case("ABBACABA", "ABCAA", true); + + try equal_excluding_ignored_case("ABA", "ACA", false); + try equal_excluding_ignored_case("ABBA", "AC", false); + try equal_excluding_ignored_case("ABBACABA", "ABDABA", false); + try equal_excluding_ignored_case("ABBACABA", "ABBADA", false); + try equal_excluding_ignored_case("ABA", "ABA", false); + try equal_excluding_ignored_case("A\nB\nA", "AA", false); +} + +fn equal_excluding_ignored_case(got: []const u8, snapshot: []const u8, ok: bool) !void { + try std.testing.expectEqual(equal_excluding_ignored(got, snapshot), ok); +} + +const Range = struct { start: usize, end: usize }; + +/// Extracts the range of the snapshot. Assumes that the snapshot is formatted as +/// +/// ``` +/// snap(@src(), +/// \\first line +/// \\second line +/// ) +/// ``` +/// +/// We could make this more robust by using `std.zig.Ast`, but sticking to manual string processing +/// is simpler, and enforced consistent style of snapshots is a good thing. +/// +/// While we expect to find a snapshot after a given line, this is not guaranteed (the file could +/// have been modified between compilation and running the test), but should be rare enough to +/// just fail with an assertion. +fn snap_range(text: []const u8, src_line: u32) Range { + var offset: usize = 0; + var line_number: u32 = 0; + + var lines = std.mem.splitScalar(u8, text, '\n'); + const snap_start = while (lines.next()) |line| : (line_number += 1) { + if (line_number == src_line) { + assert(std.mem.indexOf(u8, line, "@src()") != null); + } + if (line_number == src_line + 1) { + assert(is_multiline_string(line)); + break offset; + } + offset += line.len + 1; // 1 for \n + } else unreachable; + + lines = std.mem.splitScalar(u8, text[snap_start..], '\n'); + const snap_end = while (lines.next()) |line| { + if (!is_multiline_string(line)) { + break offset; + } + offset += line.len + 1; // 1 for \n + } else unreachable; + + return Range{ .start = snap_start, .end = snap_end }; +} + +fn is_multiline_string(line: []const u8) bool { + for (line, 0..) |c, i| { + switch (c) { + ' ' => {}, + '\\' => return (i + 1 < line.len and line[i + 1] == '\\'), + else => return false, + } + } + return false; +} + +fn get_indent(line: []const u8) []const u8 { + for (line, 0..) |c, i| { + if (c != ' ') return line[0..i]; + } + return line; +} + +fn hexdump(bytes: []const u8, writer: std.io.AnyWriter) !void { + for (bytes, 0..) |byte, index| { + if (index > 0) { + const space = if (index % 16 == 0) "\n" else if (index % 8 == 0) " " else " "; + try writer.writeAll(space); + } + try writer.print("{x:02}", .{byte}); + } +} + +test hexdump { + const snap = Snap.snap_fn("./src/stdx"); + + try snap(@src(), + \\68 65 6c 6c 6f 2c 20 77 6f 72 6c 64 0a 00 01 02 + \\03 fd fe ff + ).diff_hex("hello, world\n" ++ .{ 0, 1, 2, 3, 253, 254, 255 }); +} + +test "Snap update disabled" { + assert(!Snap.update_all); // Forgot to flip this back to false after updating snapshots? +} diff --git a/scripts/stdx/time_units.zig b/scripts/stdx/time_units.zig new file mode 100644 index 0000000..9a52173 --- /dev/null +++ b/scripts/stdx/time_units.zig @@ -0,0 +1,283 @@ +const std = @import("std"); +const assert = std.debug.assert; +const stdx = @import("stdx.zig"); + +/// A moment in monotonic time not anchored to any particular epoch. +/// +/// The absolute value of `ns` is meaningless, but it is possible to compute `Duration` between +/// two `Instant`s sourced from the same clock. +/// +/// See also `InstantUnix`. +pub const Instant = struct { + ns: u64, + + pub fn add(now: Instant, duration: Duration) Instant { + return .{ .ns = now.ns + duration.ns }; + } + + pub fn elapsed(earlier: Instant, now: Instant) Duration { + assert(now.ns >= earlier.ns); + const elapsed_ns = now.ns - earlier.ns; + return .{ .ns = elapsed_ns }; + } +}; + +/// Non-negative time difference between two `Instant`s. +pub const Duration = struct { + ns: u64, + + pub fn us(amount_us: u64) Duration { + return .{ .ns = amount_us * std.time.ns_per_us }; + } + + pub fn ms(amount_ms: u64) Duration { + return .{ .ns = amount_ms * std.time.ns_per_ms }; + } + + pub fn seconds(amount_seconds: u64) Duration { + return .{ .ns = amount_seconds * std.time.ns_per_s }; + } + + pub fn minutes(amount_minutes: u64) Duration { + return .{ .ns = amount_minutes * std.time.ns_per_min }; + } + + // Duration in microseconds, μs, 1/1_000_000 of a second. + pub fn to_us(duration: Duration) u64 { + return @divFloor(duration.ns, std.time.ns_per_us); + } + + // Duration in milliseconds, ms, 1/1_000 of a second. + pub fn to_ms(duration: Duration) u64 { + return @divFloor(duration.ns, std.time.ns_per_ms); + } + + pub fn min(lhs: Duration, rhs: Duration) Duration { + return .{ .ns = @min(lhs.ns, rhs.ns) }; + } + + pub fn max(lhs: Duration, rhs: Duration) Duration { + return .{ .ns = @max(lhs.ns, rhs.ns) }; + } + + pub fn clamp(duration: Duration, clamp_min: Duration, clamp_max: Duration) Duration { + assert(clamp_min.ns <= clamp_max.ns); + if (duration.ns < clamp_min.ns) return clamp_min; + if (duration.ns > clamp_max.ns) return clamp_max; + return duration; + } + + pub const sort = struct { + pub fn asc(ctx: void, lhs: Duration, rhs: Duration) bool { + return std.sort.asc(u64)(ctx, lhs.ns, rhs.ns); + } + }; + + // Human readable format like `1.123s`. + // NB: this is a lossy operation, durations are rounded to look nice. + pub fn format( + duration: Duration, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + try std.fmt.fmtDuration(duration.ns).format(fmt, options, writer); + } + + pub fn parse_flag_value( + string: []const u8, + static_diagnostic: *?[]const u8, + ) error{InvalidFlagValue}!Duration { + assert(string.len > 0); + var string_remaining = string; + + var result: Duration = .{ .ns = 0 }; + while (string_remaining.len > 0) { + string_remaining, const component = + try parse_flag_value_component(string_remaining, static_diagnostic); + result.ns +|= component.ns; + } + + if (result.ns >= 1_000 * std.time.ns_per_day) { + static_diagnostic.* = "duration too large:"; + return error.InvalidFlagValue; + } + return result; + } + + fn parse_flag_value_component( + string: []const u8, + static_diagnostic: *?[]const u8, + ) error{InvalidFlagValue}!struct { []const u8, Duration } { + const split_index = for (string, 0..) |c, index| { + if (std.ascii.isDigit(c)) { + // Numeric part continues. + } else break index; + } else { + static_diagnostic.* = "missing unit; must be one of: d/h/m/s/ms/us/ns:"; + return error.InvalidFlagValue; + }; + + if (split_index == 0) { + static_diagnostic.* = "missing value:"; + return error.InvalidFlagValue; + } + + const string_amount = string[0..split_index]; + const string_remaining = string[split_index..]; + assert(string_amount.len > 0); + assert(string_remaining.len > 0); + + const amount = std.fmt.parseInt(u64, string_amount, 10) catch |err| switch (err) { + error.Overflow => { + static_diagnostic.* = "integer overflow:"; + return error.InvalidFlagValue; + }, + error.InvalidCharacter => unreachable, + }; + + const Unit = enum(u64) { + ns = 1, + us = std.time.ns_per_us, + ms = std.time.ns_per_ms, + s = std.time.ns_per_s, + m = std.time.ns_per_min, + h = std.time.ns_per_hour, + d = std.time.ns_per_day, + }; + + inline for (comptime std.enums.values(Unit)) |unit| { + if (stdx.cut_prefix(string_remaining, @tagName(unit))) |suffix| { + return .{ suffix, .{ .ns = amount *| @intFromEnum(unit) } }; + } + } else { + static_diagnostic.* = "unknown unit; must be one of: d/h/m/s/ms/us/ns:"; + return error.InvalidFlagValue; + } + } +}; + +test "Instant/Duration" { + const instant_1: Instant = .{ .ns = 100 * std.time.ns_per_day }; + const instant_2: Instant = .{ .ns = 100 * std.time.ns_per_day + std.time.ns_per_s }; + assert(instant_1.elapsed(instant_1).ns == 0); + assert(instant_1.elapsed(instant_2).ns == std.time.ns_per_s); + + const duration = instant_1.elapsed(instant_2); + assert(duration.ns == 1_000_000_000); + assert(duration.to_us() == 1_000_000); + assert(duration.to_ms() == 1_000); + + assert(Duration.ms(1).ns == std.time.ns_per_ms); + assert(Duration.seconds(1).ns == std.time.ns_per_s); + assert(Duration.minutes(1).ns == std.time.ns_per_min); +} + +test "Duration.parse_flag_value" { + try stdx.Flags.parse_flag_value_fuzz(Duration, Duration.parse_flag_value, .{ + .ok = &.{ + .{ "1h", .{ .ns = std.time.ns_per_hour } }, + .{ "1m", .{ .ns = std.time.ns_per_min } }, + .{ "1h2m", .{ .ns = std.time.ns_per_hour + 2 * std.time.ns_per_min } }, + .{ "1ms2us3ns", .{ .ns = std.time.ns_per_ms + 2 * std.time.ns_per_us + 3 } }, + }, + .err = &.{ + .{ "h", "missing value" }, + .{ "1", "missing unit" }, + .{ "h1", "missing value" }, + .{ "1H", "unknown unit; must be one of: d/h/m/s/ms/us/ns" }, + .{ "1h2x", "unknown unit" }, + .{ "1_0h", "unknown unit" }, + .{ "1h 2m", "missing value" }, + .{ "18446744073709551616ns", "integer overflow" }, + .{ "1844674407370955161s", "duration too large" }, + }, + }); +} + +/// A moment in non-monotonic Unix time. +/// Timestamp is relative to epoch 1970-01-1. +/// +/// See also `Instant`. +pub const InstantUnix = struct { + ns: u64, + + pub fn add(instant: InstantUnix, duration: Duration) InstantUnix { + return .{ .ns = instant.ns + duration.ns }; + } + + pub fn now() InstantUnix { + const timestamp_ns = std.time.nanoTimestamp(); + assert(timestamp_ns > 0); + assert(timestamp_ns <= std.math.maxInt(u64)); + return .{ .ns = @intCast(timestamp_ns) }; + } + + pub fn from_timestamp_s(timestamp_s: u64) InstantUnix { + return InstantUnix{ .ns = timestamp_s * std.time.ms_per_s * std.time.ns_per_ms }; + } + + pub fn date_time(instant: InstantUnix) struct { + year: u16, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + millisecond: u16, + } { + const timestamp_ms = @divTrunc(instant.ns, std.time.ns_per_ms); + const epoch_seconds = std.time.epoch.EpochSeconds{ .secs = @divTrunc(timestamp_ms, 1000) }; + const year_day = epoch_seconds.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const time = epoch_seconds.getDaySeconds(); + + return .{ + .year = year_day.year, + .month = month_day.month.numeric(), + .day = month_day.day_index + 1, + .hour = time.getHoursIntoDay(), + .minute = time.getMinutesIntoHour(), + .second = time.getSecondsIntoMinute(), + .millisecond = @intCast(@mod(timestamp_ms, 1000)), + }; + } + + pub fn format( + instant: InstantUnix, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + const datetime = instant.date_time(); + try writer.print("{d:0>4}-{d:0>2}-{d:0>2} {d:0>2}:{d:0>2}:{d:0>2}.{d:0>3}Z", .{ + datetime.year, + datetime.month, + datetime.day, + datetime.hour, + datetime.minute, + datetime.second, + datetime.millisecond, + }); + } + + pub fn to_seconds(instant: InstantUnix) u64 { + return @divFloor(instant.ns, std.time.ns_per_s); + } +}; + +test "InstantUnix format" { + const instant_min = InstantUnix{ .ns = 0 }; + var buffer: [24]u8 = undefined; + try std.testing.expectEqualStrings( + "1970-01-01 00:00:00.000Z", + try std.fmt.bufPrint(&buffer, "{}", .{instant_min}), + ); + const instant_max = InstantUnix{ .ns = std.math.maxInt(u64) }; + try std.testing.expectEqualStrings( + "2554-07-21 23:34:33.709Z", + try std.fmt.bufPrint(&buffer, "{}", .{instant_max}), + ); +} diff --git a/scripts/stdx/unshare.zig b/scripts/stdx/unshare.zig new file mode 100644 index 0000000..2d0e942 --- /dev/null +++ b/scripts/stdx/unshare.zig @@ -0,0 +1,295 @@ +//! Some tools for working with Linux `unshare` and namespaces. +//! +//! We use user, pid, and network namespaces for two purposes: +//! +//! - Processes namespaces enable all processes in the namespace +//! to be killed when the namespace's init process is. +//! - Network namespaces allow us to create an isolated loopback network. +//! +//! This code uses the Linux `unshare` syscall to create new +//! namespaces. +//! +//! The main tool here is `maybe_unshare_and_relaunch`, which provides +//! a pattern for forking a new process that is an init process in +//! its own process namespace. + +const std = @import("std"); +const builtin = @import("builtin"); +const linux = std.os.linux; +const log = std.log.scoped(.unshare); +const assert = std.debug.assert; + +// The external pid of the init process of the unshare pid namespace. +// (Its pid within the namespace is always 1.) +var child_pid: ?std.process.Child.Id = null; + +// On receiving SIGTERM, the parent must explicitly kill the pid namespace child process, since +// it would otherwise keep running. Since the child is the init process, that automatically kills +// all of its descendants too. +const trap_action = std.posix.Sigaction{ + .handler = .{ .handler = trap_handler }, + .mask = std.posix.empty_sigset, + .flags = 0, +}; + +fn trap_handler(signal: i32) callconv(.c) void { + if (child_pid) |child| { + std.posix.kill(child, std.posix.SIG.KILL) catch |err| { + log.err("error killing sandboxed process: {}", .{err}); + }; + } + std.posix.exit(@intCast(@as(i32, 128) + signal)); +} + +/// Relaunch this process with new namespaces. +/// +/// If the current process is already running with the namespaces configured as +/// requested then this function does nothing. Otherwise it configures the +/// namespaces and with them spawns a new process with the same arguments as +/// the current process, waits for it, then exits the process directly (not +/// returning from this function). +/// +/// This should generally be called immediately from `main`. +/// +/// If the `pid` option is provided then the spawned process will be the init +/// process in a new pid namespace. When it is terminated all subprocesses +/// transitively will also be terminated. +/// +/// If the `network` option is provided then the spawned process and its +/// subprocesses will have loopback network access only. +pub fn maybe_unshare_and_relaunch( + gpa: std.mem.Allocator, + options: struct { + pid: bool, + network: bool, + }, +) !void { + comptime assert(builtin.os.tag == .linux); + + if (std.os.linux.getpid() != 1) { + try linux_unshare(.{ + .pid = options.pid, + .network = options.network, + }); + if (options.network) { + try linux_ip_link_loopback(); + } + if (options.pid) { + std.posix.sigaction(std.posix.SIG.TERM, &trap_action, null); + try fork_and_exit(gpa); + } + } else { + // We are within the pid namespace. + assert(options.pid); + assert(std.os.linux.getpid() == 1); + assert(child_pid == null); + } +} + +/// Implementation of `unshare` somewhat like +/// +/// ``` +/// unshare --user --net --pid +/// ``` +/// +/// We're trying to accomplish two main things: +/// +/// - creating a new pid namespace so that all subprocesses +/// are automatically terminated when pid 1 (the forked +/// vortex supervisor) is terminated. +/// - creating a network sandbox +/// +/// Note that on recent Ubuntu's this only works if AppArmour +/// rules have been relaxed: +/// +/// ``` +/// sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0 +/// sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 +/// ``` +pub fn linux_unshare(options: struct { + pid: bool, + network: bool, +}) !void { + comptime assert(builtin.os.tag == .linux); + + // Create user namespace first. + const unshare_user_result = std.os.linux.unshare(linux.CLONE.NEWUSER); + const unshare_user_errno = std.os.linux.E.init(unshare_user_result); + if (unshare_user_errno != .SUCCESS) { + log.err("Failed to create user namespace: {}", .{unshare_user_errno}); + return error.UnshareFailure; + } + + // Create PID namespace. + if (options.pid) { + const unshare_pid_result = std.os.linux.unshare(linux.CLONE.NEWPID); + const unshare_pid_errno = std.os.linux.E.init(unshare_pid_result); + if (unshare_pid_errno != .SUCCESS) { + log.err("Failed to create pid namespace: {}", .{unshare_pid_errno}); + return error.UnshareFailure; + } + } + + // Create network namespace. + if (options.network) { + const unshare_net_result = std.os.linux.unshare(linux.CLONE.NEWNET); + const unshare_net_errno = std.os.linux.E.init(unshare_net_result); + if (unshare_net_errno != .SUCCESS) { + log.err("Failed to create net namespace: {}", .{unshare_net_errno}); + return error.UnshareFailure; + } + } +} + +/// Implementation of `ip link` equivalent to +/// +/// ``` +/// ip link set up dev lo +/// ``` +/// +/// This brings up the loopback device so that networking +/// over 127.0.0.1 works. +pub fn linux_ip_link_loopback() !void { + comptime assert(builtin.os.tag == .linux); + + // Open a netlink socket with the NETLINK.ROUTE protocol. + const sock = std.posix.socket( + linux.AF.NETLINK, + std.posix.SOCK.RAW, + linux.NETLINK.ROUTE, + ) catch |err| { + log.err("failed to create netlink socket: {}", .{err}); + return error.IpLink; + }; + defer std.posix.close(sock); + + const addr = linux.sockaddr.nl{ + .family = linux.AF.NETLINK, + .pid = 0, + .groups = 0, + }; + std.posix.bind(sock, @ptrCast(&addr), @sizeOf(@TypeOf(addr))) catch |err| { + log.err("failed to bind netlink socket: {}", .{err}); + return error.IpLink; + }; + + // Netlink definitions. + const nlmsghdr = linux.nlmsghdr; + const ifinfomsg = linux.ifinfomsg; + + const nlmsgerr = extern struct { + @"error": c_int, + msg: nlmsghdr, + }; + + const IFF_UP = 0x1; + + // Our message to the kernel - header plus interface info. + const Message = extern struct { + hdr: nlmsghdr, + ifi: ifinfomsg, + + comptime { + assert(@sizeOf(@This()) == @sizeOf(nlmsghdr) + @sizeOf(ifinfomsg)); + } + }; + + // Kernel's message to us - header plus error info. + const Response = extern struct { + hdr: nlmsghdr, + err: nlmsgerr, + + comptime { + assert(@sizeOf(@This()) == @sizeOf(nlmsghdr) + @sizeOf(nlmsgerr)); + } + }; + + var msg: Message = .{ + .hdr = .{ + .len = @sizeOf(nlmsghdr) + @sizeOf(ifinfomsg), + .type = .RTM_NEWLINK, + // ACK says to always send a response, even on success. + .flags = linux.NLM_F_REQUEST | linux.NLM_F_ACK, + .seq = 0, + .pid = 0, + }, + .ifi = .{ + .family = linux.AF.UNSPEC, + .type = 0, + // Seems to be the loopback device, not sure how + // to find this value the correct way. + .index = 1, + .flags = IFF_UP, + // man pages say use this value. + .change = 0xFFFFFFFF, + }, + }; + + const msg_buf = std.mem.asBytes(&msg); + const sent_len = std.posix.sendto(sock, msg_buf, 0, null, 0) catch |err| { + log.err("failed to send netlink message: {}", .{err}); + return error.IpLink; + }; + assert(sent_len == msg.hdr.len); + + var ack: Response = undefined; + const ack_buf = std.mem.asBytes(&ack); + const ack_len = std.posix.recv(sock, ack_buf, 0) catch |err| { + log.err("failed to receive netlink ack: {}", .{err}); + return error.IpLink; + }; + + assert(ack_len == @sizeOf(Response)); + assert(ack.hdr.type == .ERROR); + assert(ack.err.msg.pid == msg.hdr.pid); + + if (ack.err.@"error" != 0) { + log.err("netlink operation failed with errno: {}", .{-ack.err.@"error"}); + return error.IpLink; + } +} + +fn fork_and_exit(gpa: std.mem.Allocator) !void { + const args_ours = std.os.argv; + + // We get a fresh path to the exe instead of using the original + // first argument so that the exe path will be correct even if + // this process's cwd has changed relative to the original exe. + var exe_path_buffer: [std.fs.max_path_bytes]u8 = undefined; + const exe_path = try std.fs.selfExePath(&exe_path_buffer); + + const args_new = try gpa.alloc([]const u8, args_ours.len); + defer gpa.free(args_new); + + args_new[0] = exe_path; + + for (1..args_ours.len) |arg_index| { + args_new[arg_index] = std.mem.span(args_ours[arg_index]); + } + + var child = std.process.Child.init(args_new, gpa); + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + + try child.spawn(); + + // Set the global pid so that we can kill it if we receive a SIGTERM. + assert(child_pid == null); + child_pid = child.id; + + const result = try child.wait(); + switch (result) { + .Exited => |code| { + std.process.exit(code); + }, + .Signal => |signal| { + log.info("sandboxed subprocesses exited with signal {}", .{signal}); + std.process.exit(1); + }, + else => { + log.err("sandboxed subprocesses exited abnormally", .{}); + std.process.exit(2); + }, + } +} diff --git a/scripts/stdx/vendored/aegis.zig b/scripts/stdx/vendored/aegis.zig new file mode 100644 index 0000000..934235b --- /dev/null +++ b/scripts/stdx/vendored/aegis.zig @@ -0,0 +1,436 @@ +//! Vendored from Zig's 0.13.0 standard library to maintain hash stability. +//! Source: https://github.com/ziglang/zig/blob/0.13.0/lib/std/crypto/aegis.zig + +const std = @import("std"); +const crypto = std.crypto; +const mem = std.mem; +const assert = std.debug.assert; +const AesBlock = crypto.core.aes.Block; +const AuthenticationError = crypto.errors.AuthenticationError; + +/// AEGIS-128L with a 128-bit authentication tag. +const Aegis128L = Aegis128LGenericType(128); + +/// AEGIS-128L with a 256-bit authentication tag. +const Aegis128L_256 = Aegis128LGenericType(256); + +const State128L = struct { + blocks: [8]AesBlock, + + fn init(key: [16]u8, nonce: [16]u8) State128L { + const c1 = AesBlock.fromBytes(&[16]u8{ + 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1, + 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd, + }); + const c2 = AesBlock.fromBytes(&[16]u8{ + 0x0, 0x1, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d, + 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62, + }); + const key_block = AesBlock.fromBytes(&key); + const nonce_block = AesBlock.fromBytes(&nonce); + const blocks = [8]AesBlock{ + key_block.xorBlocks(nonce_block), + c1, + c2, + c1, + key_block.xorBlocks(nonce_block), + key_block.xorBlocks(c2), + key_block.xorBlocks(c1), + key_block.xorBlocks(c2), + }; + var state = State128L{ .blocks = blocks }; + var i: usize = 0; + while (i < 10) : (i += 1) { + state.update(nonce_block, key_block); + } + return state; + } + + inline fn update(state: *State128L, d1: AesBlock, d2: AesBlock) void { + comptime assert(state.blocks.len == 8); + + // Hoist lanes; this keeps the blocks in registers (see #3201). + var blocks: [8]AesBlock = state.blocks; + const tmp = blocks[7]; + + inline for ([_]usize{ 7, 6, 5, 4, 3, 2, 1 }) |i| { + blocks[i] = blocks[i - 1].encrypt(blocks[i]); + } + + blocks[0] = tmp.encrypt(blocks[0]); + blocks[0] = blocks[0].xorBlocks(d1); + blocks[4] = blocks[4].xorBlocks(d2); + + // Single spill at the end. + state.blocks = blocks; + } + + fn absorb(state: *State128L, src: *const [32]u8) void { + const msg0 = AesBlock.fromBytes(src[0..16]); + const msg1 = AesBlock.fromBytes(src[16..32]); + state.update(msg0, msg1); + } + + fn enc(state: *State128L, dst: *[32]u8, src: *const [32]u8) void { + const blocks = &state.blocks; + const msg0 = AesBlock.fromBytes(src[0..16]); + const msg1 = AesBlock.fromBytes(src[16..32]); + var tmp0 = msg0.xorBlocks(blocks[6]).xorBlocks(blocks[1]); + var tmp1 = msg1.xorBlocks(blocks[2]).xorBlocks(blocks[5]); + tmp0 = tmp0.xorBlocks(blocks[2].andBlocks(blocks[3])); + tmp1 = tmp1.xorBlocks(blocks[6].andBlocks(blocks[7])); + dst[0..16].* = tmp0.toBytes(); + dst[16..32].* = tmp1.toBytes(); + state.update(msg0, msg1); + } + + fn dec(state: *State128L, dst: *[32]u8, src: *const [32]u8) void { + const blocks = &state.blocks; + var msg0 = AesBlock.fromBytes(src[0..16]).xorBlocks(blocks[6]).xorBlocks(blocks[1]); + var msg1 = AesBlock.fromBytes(src[16..32]).xorBlocks(blocks[2]).xorBlocks(blocks[5]); + msg0 = msg0.xorBlocks(blocks[2].andBlocks(blocks[3])); + msg1 = msg1.xorBlocks(blocks[6].andBlocks(blocks[7])); + dst[0..16].* = msg0.toBytes(); + dst[16..32].* = msg1.toBytes(); + state.update(msg0, msg1); + } + + fn mac(state: *State128L, comptime tag_bits: u9, adlen: usize, mlen: usize) [tag_bits / 8]u8 { + const blocks = &state.blocks; + var sizes: [16]u8 = undefined; + mem.writeInt(u64, sizes[0..8], @as(u64, adlen) * 8, .little); + mem.writeInt(u64, sizes[8..16], @as(u64, mlen) * 8, .little); + const tmp = AesBlock.fromBytes(&sizes).xorBlocks(blocks[2]); + var i: usize = 0; + while (i < 7) : (i += 1) { + state.update(tmp, tmp); + } + return switch (tag_bits) { + 128 => blocks[0].xorBlocks(blocks[1]).xorBlocks(blocks[2]).xorBlocks(blocks[3]) + .xorBlocks(blocks[4]).xorBlocks(blocks[5]).xorBlocks(blocks[6]).toBytes(), + 256 => tag: { + const t1 = blocks[0].xorBlocks(blocks[1]).xorBlocks(blocks[2]).xorBlocks(blocks[3]); + const t2 = blocks[4].xorBlocks(blocks[5]).xorBlocks(blocks[6]).xorBlocks(blocks[7]); + break :tag t1.toBytes() ++ t2.toBytes(); + }, + else => unreachable, + }; + } +}; + +/// The `Aegis128LMac` message authentication function outputs 256 bit tags. +/// In addition to being extremely fast, its large state, non-linearity +/// and non-invertibility provides the following properties: +/// - 128 bit security, stronger than GHash/Polyval/Poly1305. +/// - Recovering the secret key from the state would require ~2^128 attempts, +/// which is infeasible for any practical adversary. +/// - It has a large security margin against internal collisions. +pub const Aegis128LMac = AegisMacType(Aegis128L_256); + +/// Aegis128L MAC with a 128-bit output. +/// A MAC with a 128-bit output is not safe unless the number of messages +/// authenticated with the same key remains small. +/// After 2^48 messages, the probability of a collision is already ~ 2^-33. +/// If unsure, use the Aegis128LMac type, that has a 256 bit output. +pub const Aegis128LMac_128 = AegisMacType(Aegis128L); + +fn Aegis128LGenericType(comptime tag_bits: u9) type { + comptime assert(tag_bits == 128 or tag_bits == 256); // tag must be 128 or 256 bits + + return struct { + pub const tag_length = tag_bits / 8; + pub const nonce_length = 16; + pub const key_length = 16; + pub const block_length = 32; + + const State = State128L; + + /// c: ciphertext: output buffer should be of size m.len + /// tag: authentication tag: output MAC + /// m: message + /// ad: Associated Data + /// npub: public nonce + /// k: private key + pub fn encrypt( + c: []u8, + tag: *[tag_length]u8, + m: []const u8, + ad: []const u8, + npub: [nonce_length]u8, + key: [key_length]u8, + ) void { + assert(c.len == m.len); + var state = State128L.init(key, npub); + var src: [32]u8 align(16) = undefined; + var dst: [32]u8 align(16) = undefined; + var i: usize = 0; + while (i + 32 <= ad.len) : (i += 32) { + state.absorb(ad[i..][0..32]); + } + if (ad.len % 32 != 0) { + @memset(src[0..], 0); + @memcpy(src[0 .. ad.len % 32], ad[i..][0 .. ad.len % 32]); + state.absorb(&src); + } + i = 0; + while (i + 32 <= m.len) : (i += 32) { + state.enc(c[i..][0..32], m[i..][0..32]); + } + if (m.len % 32 != 0) { + @memset(src[0..], 0); + @memcpy(src[0 .. m.len % 32], m[i..][0 .. m.len % 32]); + state.enc(&dst, &src); + @memcpy(c[i..][0 .. m.len % 32], dst[0 .. m.len % 32]); + } + tag.* = state.mac(tag_bits, ad.len, m.len); + } + + /// `m`: Message + /// `c`: Ciphertext + /// `tag`: Authentication tag + /// `ad`: Associated data + /// `npub`: Public nonce + /// `k`: Private key + /// Asserts `c.len == m.len`. + /// + /// Contents of `m` are undefined if an error is returned. + pub fn decrypt( + m: []u8, + c: []const u8, + tag: [tag_length]u8, + ad: []const u8, + npub: [nonce_length]u8, + key: [key_length]u8, + ) AuthenticationError!void { + assert(c.len == m.len); + var state = State128L.init(key, npub); + var src: [32]u8 align(16) = undefined; + var dst: [32]u8 align(16) = undefined; + var i: usize = 0; + while (i + 32 <= ad.len) : (i += 32) { + state.absorb(ad[i..][0..32]); + } + if (ad.len % 32 != 0) { + @memset(src[0..], 0); + @memcpy(src[0 .. ad.len % 32], ad[i..][0 .. ad.len % 32]); + state.absorb(&src); + } + i = 0; + while (i + 32 <= m.len) : (i += 32) { + state.dec(m[i..][0..32], c[i..][0..32]); + } + if (m.len % 32 != 0) { + @memset(src[0..], 0); + @memcpy(src[0 .. m.len % 32], c[i..][0 .. m.len % 32]); + state.dec(&dst, &src); + @memcpy(m[i..][0 .. m.len % 32], dst[0 .. m.len % 32]); + @memset(dst[0 .. m.len % 32], 0); + const blocks = &state.blocks; + blocks[0] = blocks[0].xorBlocks(AesBlock.fromBytes(dst[0..16])); + blocks[4] = blocks[4].xorBlocks(AesBlock.fromBytes(dst[16..32])); + } + var computed_tag = state.mac(tag_bits, ad.len, m.len); + const verify = crypto.utils.timingSafeEql([tag_length]u8, computed_tag, tag); + if (!verify) { + crypto.utils.secureZero(u8, &computed_tag); + @memset(m, undefined); + return error.AuthenticationFailed; + } + } + }; +} + +fn AegisMacType(comptime T: type) type { + return struct { + const AegisMac = @This(); + + pub const mac_length = T.tag_length; + pub const key_length = T.key_length; + pub const block_length = T.block_length; + + state: T.State, + buf: [block_length]u8 = undefined, + off: usize = 0, + msg_len: usize = 0, + + /// Initialize a state for the MAC function + pub fn init(key: *const [key_length]u8) AegisMac { + const nonce = [_]u8{0} ** T.nonce_length; + return AegisMac{ + .state = T.State.init(key.*, nonce), + }; + } + + /// Add data to the state + pub fn update(self: *AegisMac, b: []const u8) void { + self.msg_len += b.len; + + const len_partial = @min(b.len, block_length - self.off); + @memcpy(self.buf[self.off..][0..len_partial], b[0..len_partial]); + self.off += len_partial; + if (self.off < block_length) { + return; + } + self.state.absorb(&self.buf); + + var i = len_partial; + self.off = 0; + while (i + block_length <= b.len) : (i += block_length) { + self.state.absorb(b[i..][0..block_length]); + } + if (i != b.len) { + self.off = b.len - i; + @memcpy(self.buf[0..self.off], b[i..]); + } + } + + /// Return an authentication tag for the current state + pub fn final(self: *AegisMac, out: *[mac_length]u8) void { + if (self.off > 0) { + var pad = [_]u8{0} ** block_length; + @memcpy(pad[0..self.off], self.buf[0..self.off]); + self.state.absorb(&pad); + } + out.* = self.state.mac(T.tag_length * 8, self.msg_len, 0); + } + + /// Return an authentication tag for a message and a key + pub fn create(out: *[mac_length]u8, msg: []const u8, key: *const [key_length]u8) void { + var ctx = AegisMac.init(key); + ctx.update(msg); + ctx.final(out); + } + + pub const Error = error{}; + pub const Writer = std.io.Writer(*AegisMac, Error, write); + + fn write(self: *AegisMac, bytes: []const u8) Error!usize { + self.update(bytes); + return bytes.len; + } + + pub fn writer(self: *AegisMac) Writer { + return .{ .context = self }; + } + }; +} + +const testing = std.testing; +const fmt = std.fmt; + +test "Aegis128L test vector 1" { + const key: [Aegis128L.key_length]u8 = [_]u8{ 0x10, 0x01 } ++ [_]u8{0x00} ** 14; + const nonce: [Aegis128L.nonce_length]u8 = [_]u8{ 0x10, 0x00, 0x02 } ++ [_]u8{0x00} ** 13; + const ad = [8]u8{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 }; + const m = [32]u8{ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + }; + var c: [m.len]u8 = undefined; + var m2: [m.len]u8 = undefined; + var tag: [Aegis128L.tag_length]u8 = undefined; + + Aegis128L.encrypt(&c, &tag, &m, &ad, nonce, key); + try Aegis128L.decrypt(&m2, &c, tag, &ad, nonce, key); + try testing.expectEqualSlices(u8, &m, &m2); + + try assertEqual("79d94593d8c2119d7e8fd9b8fc77845c5c077a05b2528b6ac54b563aed8efe84", &c); + try assertEqual("cc6f3372f6aa1bb82388d695c3962d9a", &tag); + + c[0] +%= 1; + try testing.expectError( + error.AuthenticationFailed, + Aegis128L.decrypt(&m2, &c, tag, &ad, nonce, key), + ); + c[0] -%= 1; + tag[0] +%= 1; + try testing.expectError( + error.AuthenticationFailed, + Aegis128L.decrypt(&m2, &c, tag, &ad, nonce, key), + ); +} + +test "Aegis128L test vector 2" { + const key: [Aegis128L.key_length]u8 = [_]u8{0x00} ** 16; + const nonce: [Aegis128L.nonce_length]u8 = [_]u8{0x00} ** 16; + const ad = [_]u8{}; + const m = [_]u8{0x00} ** 16; + var c: [m.len]u8 = undefined; + var m2: [m.len]u8 = undefined; + var tag: [Aegis128L.tag_length]u8 = undefined; + + Aegis128L.encrypt(&c, &tag, &m, &ad, nonce, key); + try Aegis128L.decrypt(&m2, &c, tag, &ad, nonce, key); + try testing.expectEqualSlices(u8, &m, &m2); + + try assertEqual("41de9000a7b5e40e2d68bb64d99ebb19", &c); + try assertEqual("f4d997cc9b94227ada4fe4165422b1c8", &tag); +} + +test "Aegis128L test vector 3" { + const key: [Aegis128L.key_length]u8 = [_]u8{0x00} ** 16; + const nonce: [Aegis128L.nonce_length]u8 = [_]u8{0x00} ** 16; + const ad = [_]u8{}; + const m = [_]u8{}; + var c: [m.len]u8 = undefined; + var m2: [m.len]u8 = undefined; + var tag: [Aegis128L.tag_length]u8 = undefined; + + Aegis128L.encrypt(&c, &tag, &m, &ad, nonce, key); + try Aegis128L.decrypt(&m2, &c, tag, &ad, nonce, key); + try testing.expectEqualSlices(u8, &m, &m2); + + try assertEqual("83cc600dc4e3e7e62d4055826174f149", &tag); +} + +test "Aegis MAC" { + const key = [_]u8{0x00} ** Aegis128LMac.key_length; + var msg: [64]u8 = undefined; + for (&msg, 0..) |*m, i| { + m.* = @as(u8, @truncate(i)); + } + const st_init = Aegis128LMac.init(&key); + var st = st_init; + var tag: [Aegis128LMac.mac_length]u8 = undefined; + + st.update(msg[0..32]); + st.update(msg[32..]); + st.final(&tag); + try assertEqual("f8840849602738d81037cbaa0f584ea95759e2ac60263ce77346bcdc79fe4319", &tag); + + st = st_init; + st.update(msg[0..31]); + st.update(msg[31..]); + st.final(&tag); + try assertEqual("f8840849602738d81037cbaa0f584ea95759e2ac60263ce77346bcdc79fe4319", &tag); + + st = st_init; + st.update(msg[0..14]); + st.update(msg[14..30]); + st.update(msg[30..]); + st.final(&tag); + try assertEqual("f8840849602738d81037cbaa0f584ea95759e2ac60263ce77346bcdc79fe4319", &tag); + + var empty: [0]u8 = undefined; + const nonce = [_]u8{0x00} ** Aegis128L_256.nonce_length; + Aegis128L_256.encrypt(&empty, &tag, &empty, &msg, nonce, key); + try assertEqual("f8840849602738d81037cbaa0f584ea95759e2ac60263ce77346bcdc79fe4319", &tag); + + // An update whose size is not a multiple of the block size + st = st_init; + st.update(msg[0..33]); + st.final(&tag); + try assertEqual("c7cf649a844c1a6676cf6d91b1658e0aee54a4da330b0a8d3bc7ea4067551d1b", &tag); +} + +// Assert `expected` == hex(`input`) where `input` is a bytestring +fn assertEqual(comptime expected_hex: [:0]const u8, input: []const u8) !void { + var expected_bytes: [expected_hex.len / 2]u8 = undefined; + for (&expected_bytes, 0..) |*r, i| { + r.* = fmt.parseInt(u8, expected_hex[2 * i .. 2 * i + 2], 16) catch unreachable; + } + + try testing.expectEqualSlices(u8, &expected_bytes, input); +} diff --git a/scripts/stdx/windows.zig b/scripts/stdx/windows.zig new file mode 100644 index 0000000..a121cf9 --- /dev/null +++ b/scripts/stdx/windows.zig @@ -0,0 +1,48 @@ +const std = @import("std"); +const windows = std.os.windows; + +pub extern "kernel32" fn GetSystemTimePreciseAsFileTime( + lpFileTime: *windows.FILETIME, +) callconv(.winapi) void; + +pub extern "kernel32" fn GetCommandLineW() callconv(.winapi) windows.LPWSTR; + +pub extern "kernel32" fn GetProcessTimes( + in_hProcess: windows.HANDLE, + out_lpCreationTime: *windows.FILETIME, + out_lpExitTime: *windows.FILETIME, + out_lpKernelTime: *windows.FILETIME, + out_lpUserTime: *windows.FILETIME, +) callconv(.winapi) windows.BOOL; + +pub extern "kernel32" fn SetProcessWorkingSetSize( + hProcess: windows.HANDLE, + dwMinimumWorkingSetSize: windows.SIZE_T, + dwMaximumWorkingSetSize: windows.SIZE_T, +) callconv(.winapi) windows.BOOL; + +pub extern "kernel32" fn GetProcessWorkingSetSize( + hProcess: windows.HANDLE, + lpMinimumWorkingSetSize: *windows.SIZE_T, + lpMaximumWorkingSetSize: *windows.SIZE_T, +) callconv(.winapi) windows.BOOL; + +pub const LOCKFILE_EXCLUSIVE_LOCK = 0x2; +pub const LOCKFILE_FAIL_IMMEDIATELY = 0x1; +pub extern "kernel32" fn LockFileEx( + hFile: windows.HANDLE, + dwFlags: windows.DWORD, + dwReserved: windows.DWORD, + nNumberOfBytesToLockLow: windows.DWORD, + nNumberOfBytesToLockHigh: windows.DWORD, + lpOverlapped: ?*windows.OVERLAPPED, +) callconv(.winapi) windows.BOOL; + +pub extern "kernel32" fn SetEndOfFile( + hFile: windows.HANDLE, +) callconv(.winapi) windows.BOOL; + +pub extern "kernel32" fn ConnectNamedPipe( + hNamedPipe: windows.HANDLE, + lpOverlapped: ?*windows.OVERLAPPED, +) callconv(.winapi) windows.BOOL; diff --git a/scripts/stdx/zipfian.zig b/scripts/stdx/zipfian.zig new file mode 100644 index 0000000..9badda7 --- /dev/null +++ b/scripts/stdx/zipfian.zig @@ -0,0 +1,402 @@ +//! Zipfian-distributed random number generation. +//! +//! In the Zipfian distribution a small percentage of candidate +//! items have a high probability of being selected, while most items +//! have a very low probability of being selected. +//! It is commonly understood to model the "80-20" Pareto principle, +//! and to be a discreet version of the Pareto distribution, +//! and terminology related to both are often used interchangeably. +//! +//! Zipfian numbers follow an inverse power law, where the 1st item +//! is selected with high probability, and subsequent items +//! quickly fall off in probability. The rate of the fall off +//! is tunable by the _skew_, also called `s`, or `theta`, +//! depending on the source. +//! +//! Reference: +//! +//! - https://en.wikipedia.org/wiki/Zipf's_law#Formal_definition +//! +//! Note that it is not actually possible to select a value for +//! theta that literally follows the "80-20" rule for arbitrary set sizes; +//! the proportion of items that cumulatively make up 80% probability will +//! change as the set grows. +//! A zipfian generator that can adaptively follow the 80-20 rule is left for future work. +//! +//! In practice these probabilities often need to be spread across e.g. a +//! table's keyspace, which involves some kind of mapping step from index to index. +//! Because that mapping is non-trivial to optimize, it is also provided here. +//! +//! The algorithm here is based on +//! "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994. +//! Per the paper it is adapted from Knuth vol 3. +//! This is also the algorithm used by YCSB's ZipfianGenerator.java. +//! Note that the code listing in the paper contains obvious errors, +//! corrected here and in YCSB. +//! +//! There are two generators here, +//! both of which generate random keys from 0 to a specified maximum. +//! In the basic `ZipfianGenerator`, key 0 has the highest probability, +//! 1 the next highest, etc. +//! The `ZipfianShuffled` generator instead spreads the distribution out +//! across the key space as if it were a shuffled deck. +//! +//! The `ZipfianGenerator` allows the key space to grow, +//! but the `ZipfianShuffled` does not - maintaining the illusion of a shuffled +//! deck while growing the keyspace involves tradeoffs in the quality +//! of the distribution. A previous revision of `ZipfianShuffled` _was_ growable, +//! at the cost of not preserving a true Zipfian distribution for the long tail +//! of unlikely items. Dig that out of commit history if it's ever needed. +//! +//! Both should pass a 2-sample Kolmogorov–Smirnov test. + +const std = @import("std"); +const stdx = @import("stdx.zig"); +const assert = std.debug.assert; +const Random = std.Random; +const math = std.math; +const Snap = stdx.Snap; +const module_path = "src/stdx"; +const snap = Snap.snap_fn(module_path); + +/// The default "skew" of the distribution. +const theta_default = 0.99; // per YCSB + +/// Generates Zipfian-distributed numbers from 0 to a specified maximum. +/// +/// Many internal variables here are the same is in the paper, which I think +/// should reduce confusion if this subject needs to be revisited; the external +/// intended to be more understandable to the user and follow TigerStyle. +pub const ZipfianGenerator = struct { + theta: f64, + + /// The number of items in the set. + n: u64, + /// The Riemann zeta function calculated up to `n`, + /// aka the "generalized harmonic number" of order `theta` for `n`. + /// This is a pre-calculated factor in the probability of any particular item + /// being selected. + /// It is expensive to calculate for large but useful values of `n`, + /// but can be calculated incrementally as `n` grows. + zetan: f64, + + /// Create a generator from `[0, items)` with `theta` equal to 0.99. + pub fn init(items: u64) ZipfianGenerator { + return ZipfianGenerator.init_theta(items, theta_default); + } + + /// Create a generator from `[0, items)` with given `theta`. + /// + /// `theta` is the "skew" and is usually specified to be greater than 0 and less than 1, + /// with YCSB using 0.99, though values greater than 1 also seem to generate reasonable + /// distributions. `theta = 1` isn't allowed since it does not behave reasonably. + pub fn init_theta(items: u64, theta: f64) ZipfianGenerator { + assert(theta > 0.0); + assert(theta != 1.0); + return ZipfianGenerator{ + .theta = theta, + .n = items, + .zetan = zeta(items, theta), + }; + } + + /// Note that the variables in this function are mostly named + /// as in the reference paper and do not follow TigerStyle. + pub fn next(self: *const ZipfianGenerator, prng: *stdx.PRNG) u64 { + assert(self.n > 0); + + // Math voodoo, copied from the paper, + // which doesn't explain it, but claims it is from Knuth volume 3. + + // NB: These depend only on zetan and could be cached for a minor speedup. + const alpha = 1.0 / (1.0 - self.theta); + const eta = (1.0 - math.pow( + f64, + 2.0 / @as(f64, @floatFromInt(self.n)), + 1.0 - self.theta, + )) / + (1.0 - zeta(2.0, self.theta) / self.zetan); + + const u = random_f64(prng); + const uz = u * self.zetan; + + if (uz < 1.0) { + return 0; + } + + if (uz < 1.0 + math.pow(f64, 0.5, self.theta)) { + return 1; + } + + return @as(u64, @intFromFloat( + @as(f64, @floatFromInt(self.n)) * + math.pow(f64, (eta * u) - eta + 1.0, alpha), + )); + } + + /// Grow the size of the random set. + pub fn grow(self: *ZipfianGenerator, new_items: u64) void { + const items = self.n + new_items; + const zetan_new = zeta_incremental(self.n, new_items, self.zetan, self.theta); + self.* = .{ + .theta = self.theta, + .n = items, + .zetan = zetan_new, + }; + } +}; + +/// The Riemann zeta function up to `n`, +/// aka the "generalized harmonic number" of order 'theta' for `n`. +fn zeta(n: u64, theta: f64) f64 { + var i: u64 = 1; + var zeta_sum: f64 = 0.0; + while (i <= n) : (i += 1) { + zeta_sum += math.pow(f64, 1.0 / @as(f64, @floatFromInt(i)), theta); + } + return zeta_sum; +} + +/// Incremental calculation of zeta. +fn zeta_incremental( + n_previous: u64, + n_additional: u64, + zetan_previous: f64, + theta: f64, +) f64 { + const n_new = n_previous + n_additional; + var i = n_previous + 1; + var zeta_sum = zetan_previous; + while (i <= n_new) : (i += 1) { + zeta_sum += math.pow(f64, 1.0 / @as(f64, @floatFromInt(i)), theta); + } + return zeta_sum; +} + +/// Generates Zipfian-distributed numbers from 0 to maximum, +/// but the probabilities of each number are "shuffled", +/// not clustered around 0. +/// +/// This is used to simulate typical data access patterns in +/// some keyspace, where a few keys are hot and most are cold. +/// +/// This behaves as if it maintains a shuffled mapping +/// from every index to a different index. Internally, it is implemented +/// with a bijective "hash" function (modular‑multiplication permutation) +/// f(i) = (a * i) mod N +/// with gcd(a, N) = 1, so every original (Zipfian) index i +/// maps to a unique “shuffled” index without collisions. +/// Refer to PR #3070 for further details: https://github.com/tigerbeetle/tigerbeetle/pull/3070 +pub const ZipfianShuffled = struct { + gen: ZipfianGenerator, + a: u64, + + pub fn init(items: u64, prng: *stdx.PRNG) ZipfianShuffled { + return ZipfianShuffled.init_theta(items, theta_default, prng); + } + + pub fn init_theta(items: u64, theta: f64, prng: *stdx.PRNG) ZipfianShuffled { + var zipf = ZipfianShuffled{ + .gen = ZipfianGenerator.init_theta(0, theta), + .a = 0, // Correct a is determined in grow. + }; + + zipf.choose_shuffle_function(items, prng); + + return zipf; + } + + fn transform(self: *const ZipfianShuffled, zipf_standard: u64) u64 { + return (zipf_standard * self.a) % self.gen.n; + } + + pub fn next(self: *const ZipfianShuffled, prng: *stdx.PRNG) u64 { + const zipf_standard = self.gen.next(prng); + const zipf_shuffled = self.transform(zipf_standard); + return zipf_shuffled; + } + + fn choose_shuffle_function(self: *ZipfianShuffled, new_items: u64, prng: *stdx.PRNG) void { + if (new_items == 0) { + return; + } + + const old_n = self.gen.n; + const new_n = old_n + new_items; + + self.gen.grow(new_items); + + assert(self.gen.n == new_n); + + // We try to find an `a` so that it satisifies gcd(a,N) == 1. + // This allows us to generate a permutation with (a*zipf_standard) mod N. + // This permutation maps one index to another without holes, i.e. is bijective. + self.a = random_coprime(prng, self.gen.n); + } + + fn random_coprime(prng: *stdx.PRNG, n: u64) u64 { + // The bound is arbitrary but should be large enough to find a number that satisifies + // the requirement (see https://en.wikipedia.org/wiki/Euler%27s_totient_function). + for (0..100_000) |_| { + const a = prng.range_inclusive(u64, 1, n); + if (std.math.gcd(a, n) == 1) { + return a; + } + } else { + @panic("Did not find a random coprime (probabilistic)"); + } + } +}; + +/// stdx.PRNG intentionally doesn't support generating floats, to ensure determinism. For +/// benchmarking purposes, using floats is OK though, so we fall back to std implementation here. +fn random_f64(prng: *stdx.PRNG) f64 { + return std.Random.init(prng, stdx.PRNG.fill).float(f64); +} + +test "zeta_incremental" { + const Case = struct { + n_start: u64, + n_incremental: u64, + theta: f64, + }; + const cases = [_]Case{ + .{ + .n_start = 0, + .n_incremental = 10, + .theta = 0.99, + }, + .{ + .n_start = 0, + .n_incremental = 10, + .theta = 1.01, + }, + .{ + .n_start = 100, + .n_incremental = 100, + .theta = 0.99, + }, + }; + + for (cases) |case| { + const n = case.n_start + case.n_incremental; + const zeta_expected = zeta(n, case.theta); + const zeta_actual_start = zeta(case.n_start, case.theta); + const zeta_actual = zeta_incremental( + case.n_start, + case.n_incremental, + zeta_actual_start, + case.theta, + ); + assert(zeta_expected == zeta_actual); + } +} + +// Testing that the grow function correctly calculates zeta incrementally. +test "zipfian-grow" { + // Need to try multiple times to ensure they don't both coincidentally + // pick the likely 0 value. + var i: u64 = 10; + while (i < 100) : (i += 1) { + const expected = brk: { + var prng = stdx.PRNG.from_seed(0); + var zipf = ZipfianGenerator.init_theta(i, 0.9); + break :brk zipf.next(&prng); + }; + const actual = brk: { + var prng = stdx.PRNG.from_seed(0); + var zipf = ZipfianGenerator.init_theta(1, 0.9); + zipf.grow(i - 1); + break :brk zipf.next(&prng); + }; + assert(expected == actual); + } +} + +// Test that ctors are all doing the same thing. +test "zipfian-ctors" { + var prng = stdx.PRNG.from_seed(0); + + for ([_]u64{ 0, 1, 10, 999 }) |i| { + { + const zipf1 = ZipfianGenerator.init(i); + const zipf2 = ZipfianGenerator.init_theta(i, theta_default); + const szipf1 = ZipfianShuffled.init(i, &prng); + const szipf2 = ZipfianShuffled.init_theta(i, theta_default, &prng); + + assert(zipf1.n == zipf2.n); + assert(zipf1.n == szipf1.gen.n); + assert(zipf1.n == szipf2.gen.n); + + assert(zipf1.zetan == zipf2.zetan); + assert(zipf1.zetan == szipf1.gen.zetan); + assert(zipf1.zetan == szipf2.gen.zetan); + } + + { + const zipf1 = ZipfianGenerator.init_theta(i, 0.89); + const szipf1 = ZipfianShuffled.init_theta(i, 0.89, &prng); + + assert(zipf1.n == szipf1.gen.n); + assert(zipf1.zetan == szipf1.gen.zetan); + } + } +} + +test "zipfian-distribution" { + const max_number = 10; + + var prng = stdx.PRNG.from_seed(42); + const zipf = ZipfianGenerator.init(max_number); + + var distribution: [max_number]u32 = @splat(0); + + for (0..1000) |_| { + const n = zipf.next(&prng); + distribution[n] += 1; + } + + try snap(@src(), + \\{ 333, 170, 125, 90, 59, 61, 43, 47, 38, 34 } + ).diff_fmt("{d}", .{distribution}); +} + +test "shuffled-zipfian-distribution" { + const max_number = 10; + + var prng = stdx.PRNG.from_seed(42); + const zipf_shuffled = ZipfianShuffled.init(max_number, &prng); + + var distribution: [max_number]u32 = @splat(0); + + for (0..1000) |_| { + const n = zipf_shuffled.next(&prng); + distribution[n] += 1; + } + + try snap(@src(), + \\{ 333, 34, 38, 47, 43, 61, 60, 89, 125, 170 } + ).diff_fmt("{d}", .{distribution}); +} + +// Non-statistical smoke tests related to the shuffled hot items optimization. +// These could fail if that optimization is tweaked or if the prng changes. +// The standard zipf generator is tested, here we test the mapping of the shuffled one. +test "zipfian-shuffled" { + const max = 100; + var prng = stdx.PRNG.from_seed(0); + const allocator = std.testing.allocator; + var found = try allocator.alloc(bool, max); + defer allocator.free(found); + + for (1..max) |items| { + @memset(found, false); + var zipf = ZipfianShuffled.init(items, &prng); + + for (0..items) |i| { + const zipf_shuffled = zipf.transform(i); + try std.testing.expect(!found[zipf_shuffled]); + found[zipf_shuffled] = true; + } + } +} diff --git a/scripts/tp_test.zig b/scripts/tp_test.zig new file mode 100644 index 0000000..cde583d --- /dev/null +++ b/scripts/tp_test.zig @@ -0,0 +1,110 @@ +const std = @import("std"); +const assert = std.debug.assert; + +const stdx = @import("./stdx/stdx.zig"); +const Shell = stdx.Shell; + +pub fn main() !void { + var gpa_allocator = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa_allocator.deinit(); + + const gpa = gpa_allocator.allocator(); + + const shell = try Shell.create(gpa); + defer shell.destroy(); + + const token = try publish_ruby_trusted_publishing_token(shell); + assert(token.len > 0); +} + +fn publish_ruby_trusted_publishing_token(shell: *Shell) ![]const u8 { + const trusted_publishing_token = try shell.env_get("ACTIONS_ID_TOKEN_REQUEST_TOKEN"); + const trusted_publishing_url = try shell.env_get("ACTIONS_ID_TOKEN_REQUEST_URL"); + + const oidc_response = try shell.http_get( + try shell.fmt("{s}&audience={%}", .{ + trusted_publishing_url, + std.Uri.Component{ .raw = "rubygems.org" }, + }), + .{ + .authorization = try shell.fmt("bearer {s}", .{trusted_publishing_token}), + }, + ); + const oidc = try std.json.parseFromSliceLeaky( + struct { value: []const u8 }, + shell.arena.allocator(), + oidc_response, + .{ .ignore_unknown_fields = true }, + ); + + try dump_oidc_claims(shell, oidc.value); + + const rubygems_request = try std.json.stringifyAlloc( + shell.arena.allocator(), + .{ .jwt = oidc.value }, + .{}, + ); + const rubygems_response = try shell.http_post( + "https://rubygems.org/api/v1/oidc/trusted_publisher/exchange_token", + rubygems_request, + .{ .content_type = .json }, + ); + const rubygems = try std.json.parseFromSliceLeaky( + struct { rubygems_api_key: []const u8 }, + shell.arena.allocator(), + rubygems_response, + .{ .ignore_unknown_fields = true }, + ); + + return rubygems.rubygems_api_key; +} + +fn dump_oidc_claims(shell: *Shell, jwt: []const u8) !void { + var parts = std.mem.splitScalar(u8, jwt, '.'); + _ = parts.next() orelse return error.InvalidJWT; + const payload_encoded = parts.next() orelse return error.InvalidJWT; + _ = parts.next() orelse return error.InvalidJWT; + if (parts.next() != null) return error.InvalidJWT; + + const padding = (4 - (payload_encoded.len % 4)) % 4; + const payload_base64 = try shell.arena.allocator().alloc( + u8, + payload_encoded.len + padding, + ); + for (payload_encoded, 0..) |byte, i| { + payload_base64[i] = switch (byte) { + '-' => '+', + '_' => '/', + else => byte, + }; + } + @memset(payload_base64[payload_encoded.len..], '='); + + const payload_len = try std.base64.standard.Decoder.calcSizeForSlice(payload_base64); + const payload_json = try shell.arena.allocator().alloc(u8, payload_len); + try std.base64.standard.Decoder.decode(payload_json, payload_base64); + + const claims = try std.json.parseFromSliceLeaky( + struct { + iss: ?[]const u8 = null, + aud: ?[]const u8 = null, + repository: ?[]const u8 = null, + repository_owner: ?[]const u8 = null, + repository_owner_id: ?[]const u8 = null, + environment: ?[]const u8 = null, + job_workflow_ref: ?[]const u8 = null, + ref: ?[]const u8 = null, + sha: ?[]const u8 = null, + sub: ?[]const u8 = null, + }, + shell.arena.allocator(), + payload_json, + .{ .ignore_unknown_fields = true }, + ); + + var stdout_buffer = std.io.bufferedWriter(std.io.getStdOut().writer()); + const stdout = stdout_buffer.writer(); + try std.json.stringify(claims, .{ .whitespace = .indent_2 }, stdout); + try stdout.writeByte('\n'); + try stdout_buffer.flush(); +} diff --git a/zig/download.sh b/zig/download.sh new file mode 100755 index 0000000..a2736eb --- /dev/null +++ b/zig/download.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env sh +set -eu + +ZIG_MIRROR="https://ziglang.org/download" +ZIG_RELEASE="0.14.1" +ZIG_CHECKSUMS=$(cat< /dev/null; then + ZIG_CHECKSUM_ACTUAL=$(sha256sum "$ZIG_ARCHIVE" | cut -d ' ' -f 1) + elif command -v shasum > /dev/null; then + ZIG_CHECKSUM_ACTUAL=$(shasum -a 256 "$ZIG_ARCHIVE" | cut -d ' ' -f 1) + else + echo "Neither sha256sum nor shasum available." + exit 1 + fi + [ "$ZIG_CHECKSUM_ACTUAL" = "$ZIG_CHECKSUM_EXPECTED" ] +} + +if checksum_valid; then # Caching for CI. + echo "Skip downloading Zig $ZIG_RELEASE." +else + echo "Downloading Zig $ZIG_RELEASE ..." + mkdir -p ./zig/cache + # Download, making sure we download to the same output document, without + # wget adding "-1" etc. if the file was previously partially downloaded: + if command -v curl > /dev/null; then + curl --location --silent --show-error --output "$ZIG_ARCHIVE" "$ZIG_URL" + elif command -v wget > /dev/null; then + # -4 forces `wget` to connect to ipv4 addresses, as ipv6 fails to resolve on certain distros. + # Only A records (for ipv4) are used in DNS: + ipv4="-4" + # But Alpine doesn't support this argument + if [ -f /etc/alpine-release ]; then + ipv4="" + fi + + # shellcheck disable=SC2086 # We control ipv4 and it'll always either be empty or -4 + wget $ipv4 --quiet --output-document="$ZIG_ARCHIVE" "$ZIG_URL" + else + echo "Neither curl nor wget available." + exit 1 + fi + + # Verify the checksum. + if ! checksum_valid; then + echo "Checksum mismatch." + exit 1 + fi +fi + +echo "Extracting $ZIG_ARCHIVE ..." +case "$ZIG_EXTENSION" in + ".tar.xz") + tar -xf "$ZIG_ARCHIVE" + ;; + ".zip") + unzip -q "$ZIG_ARCHIVE" + ;; + *) + echo "Unexpected error extracting Zig archive." + exit 1 + ;; +esac +# NB: Keep archive for caching. + +# Replace these existing directories and files so that we can install or upgrade: +rm -rf zig/doc +rm -rf zig/lib +mv "$ZIG_DIRECTORY/LICENSE" zig/ +mv "$ZIG_DIRECTORY/README.md" zig/ +mv "$ZIG_DIRECTORY/doc" zig/ +mv "$ZIG_DIRECTORY/lib" zig/ +mv "$ZIG_DIRECTORY/zig" zig/ + +# We expect to have now moved all directories and files out of the extracted directory. +# Do not force remove so that we can get an error if the above list of files ever changes: +rmdir "$ZIG_DIRECTORY" + +# It's up to the user to add this to their path if they want to: +ZIG_BIN="$(pwd)/zig/zig" +echo "Downloading completed ($ZIG_BIN)! Enjoy!"