From 963de5007da9fbd5ab3a740d6293e1cc0a291822 Mon Sep 17 00:00:00 2001 From: Richard Patel Date: Fri, 23 Oct 2026 21:43:19 +0000 Subject: [PATCH 1/5] Remove wiredancer --- book/guide/protocol/cryptography.md | 1 - contrib/test/check_fp_fma.sh | 2 +- doc/organization.txt | 4 +- src/wiredancer/Local.mk | 5 - src/wiredancer/README.md | 169 -- src/wiredancer/WD-CHAIN.jpg | Bin 115456 -> 0 bytes src/wiredancer/WD-SV.jpg | Bin 676303 -> 0 bytes src/wiredancer/c/wd_f1.c | 430 ----- src/wiredancer/c/wd_f1.h | 114 -- .../platform/f1/constraints/cl_pnr_user.xdc | 101 - .../platform/f1/design/cl_dram_dma.sv | 545 ------ .../platform/f1/scripts/synth_cl_dram_dma.tcl | 179 -- src/wiredancer/py/ed25519_lib.py | 1001 ---------- src/wiredancer/py/point_decomp.py | 94 - src/wiredancer/py/point_mul.py | 123 -- src/wiredancer/py/ref_ed25519.py | 390 ---- src/wiredancer/py/sigverify.py | 271 --- src/wiredancer/py/wd_cocotil.py | 1678 ----------------- src/wiredancer/rtl/areset_sync.sv | 98 - src/wiredancer/rtl/dma_result.sv | 154 -- .../rtl/dual_clock_showahead_fifo.sv | 469 ----- src/wiredancer/rtl/ed25519_add_modp.sv | 61 - src/wiredancer/rtl/ed25519_mul_modp.sv | 255 --- src/wiredancer/rtl/ed25519_point_add.sv | 97 - src/wiredancer/rtl/ed25519_point_dbl.sv | 91 - src/wiredancer/rtl/ed25519_sigverify_0.sv | 309 --- src/wiredancer/rtl/ed25519_sigverify_1.sv | 139 -- src/wiredancer/rtl/ed25519_sigverify_2.sv | 95 - .../rtl/ed25519_sigverify_dsdp_mul.sv | 317 ---- src/wiredancer/rtl/ed25519_sigverify_ecc.sv | 235 --- src/wiredancer/rtl/ed25519_sub_modp.sv | 55 - src/wiredancer/rtl/key_store.sv | 107 -- .../rtl/mul_const_ED25519_L0_133.svh | 36 - .../rtl/mul_const_ED25519_L0_260.svh | 40 - src/wiredancer/rtl/mul_const_ED25519_L0_6.svh | 36 - src/wiredancer/rtl/mul_wide.sv | 324 ---- .../rtl/mul_wide_17nx26_dsp48e2.svh | 223 --- src/wiredancer/rtl/pcie_inorder.sv | 135 -- src/wiredancer/rtl/pcie_tr_ext.sv | 115 -- src/wiredancer/rtl/schl_cpu.sv | 1058 ----------- src/wiredancer/rtl/schl_cpu_instr_rom.mif | 1154 ------------ src/wiredancer/rtl/schl_cpu_instr_rom.sv | 36 - src/wiredancer/rtl/sha512_block.sv | 436 ----- src/wiredancer/rtl/sha512_modq.sv | 174 -- src/wiredancer/rtl/sha512_modq_meta.sv | 89 - src/wiredancer/rtl/sha512_msgseq.sv | 221 --- src/wiredancer/rtl/sha512_pre.sv | 198 -- src/wiredancer/rtl/sha512_round.sv | 114 -- src/wiredancer/rtl/sha512_sch.sv | 258 --- src/wiredancer/rtl/showahead_fifo.sv | 528 ------ src/wiredancer/rtl/simple_dual_port_ram.sv | 381 ---- src/wiredancer/rtl/tid_inorder.sv | 139 -- src/wiredancer/rtl/top_f1.sv | 981 ---------- src/wiredancer/rtl/wd_pkg.sv | 867 --------- src/wiredancer/sim/README.md | 3 - src/wiredancer/sim/ed25519_point_add/Makefile | 19 - src/wiredancer/sim/ed25519_point_add/test.py | 98 - src/wiredancer/sim/ed25519_point_dbl/Makefile | 19 - src/wiredancer/sim/ed25519_point_dbl/test.py | 81 - .../sim/ed25519_sigverify_0/Makefile | 26 - .../sim/ed25519_sigverify_0/test.py | 82 - .../sim/ed25519_sigverify_1/Makefile | 22 - .../sim/ed25519_sigverify_1/test.py | 114 -- src/wiredancer/sim/mul_wide/Makefile | 15 - src/wiredancer/sim/mul_wide/test.py | 65 - src/wiredancer/sim/schl_cpu/Makefile | 25 - src/wiredancer/sim/schl_cpu/test.py | 305 --- src/wiredancer/sim/sha512_modq_meta/Makefile | 25 - src/wiredancer/sim/sha512_modq_meta/test.py | 91 - src/wiredancer/sim/sha512_pre/Makefile | 19 - src/wiredancer/sim/sha512_pre/test.py | 94 - src/wiredancer/sim/top_f1/Makefile | 48 - src/wiredancer/sim/top_f1/test.py | 127 -- src/wiredancer/sim/top_f1_models/Makefile | 49 - .../ed25519_sigverify_dsdp_mul.sv | 45 - src/wiredancer/sim/top_f1_models/schl_cpu.sv | 30 - src/wiredancer/sim/top_f1_models/test.py | 132 -- src/wiredancer/test/Local.mk | 6 - src/wiredancer/test/fd_replay_loop.c | 635 ------- src/wiredancer/test/fd_replay_loop.h | 177 -- src/wiredancer/test/test_wiredancer_demo.c | 1318 ------------- src/wiredancer/test/wd_f1_mon.c | 443 ----- src/wiredancer/test/wd_f1_mon.h | 66 - 83 files changed, 2 insertions(+), 19309 deletions(-) delete mode 100644 src/wiredancer/Local.mk delete mode 100644 src/wiredancer/README.md delete mode 100644 src/wiredancer/WD-CHAIN.jpg delete mode 100644 src/wiredancer/WD-SV.jpg delete mode 100644 src/wiredancer/c/wd_f1.c delete mode 100644 src/wiredancer/c/wd_f1.h delete mode 100644 src/wiredancer/platform/f1/constraints/cl_pnr_user.xdc delete mode 100644 src/wiredancer/platform/f1/design/cl_dram_dma.sv delete mode 100644 src/wiredancer/platform/f1/scripts/synth_cl_dram_dma.tcl delete mode 100755 src/wiredancer/py/ed25519_lib.py delete mode 100755 src/wiredancer/py/point_decomp.py delete mode 100755 src/wiredancer/py/point_mul.py delete mode 100755 src/wiredancer/py/ref_ed25519.py delete mode 100755 src/wiredancer/py/sigverify.py delete mode 100755 src/wiredancer/py/wd_cocotil.py delete mode 100644 src/wiredancer/rtl/areset_sync.sv delete mode 100644 src/wiredancer/rtl/dma_result.sv delete mode 100644 src/wiredancer/rtl/dual_clock_showahead_fifo.sv delete mode 100644 src/wiredancer/rtl/ed25519_add_modp.sv delete mode 100644 src/wiredancer/rtl/ed25519_mul_modp.sv delete mode 100644 src/wiredancer/rtl/ed25519_point_add.sv delete mode 100644 src/wiredancer/rtl/ed25519_point_dbl.sv delete mode 100644 src/wiredancer/rtl/ed25519_sigverify_0.sv delete mode 100644 src/wiredancer/rtl/ed25519_sigverify_1.sv delete mode 100644 src/wiredancer/rtl/ed25519_sigverify_2.sv delete mode 100644 src/wiredancer/rtl/ed25519_sigverify_dsdp_mul.sv delete mode 100644 src/wiredancer/rtl/ed25519_sigverify_ecc.sv delete mode 100644 src/wiredancer/rtl/ed25519_sub_modp.sv delete mode 100644 src/wiredancer/rtl/key_store.sv delete mode 100644 src/wiredancer/rtl/mul_const_ED25519_L0_133.svh delete mode 100644 src/wiredancer/rtl/mul_const_ED25519_L0_260.svh delete mode 100644 src/wiredancer/rtl/mul_const_ED25519_L0_6.svh delete mode 100644 src/wiredancer/rtl/mul_wide.sv delete mode 100644 src/wiredancer/rtl/mul_wide_17nx26_dsp48e2.svh delete mode 100644 src/wiredancer/rtl/pcie_inorder.sv delete mode 100644 src/wiredancer/rtl/pcie_tr_ext.sv delete mode 100644 src/wiredancer/rtl/schl_cpu.sv delete mode 100644 src/wiredancer/rtl/schl_cpu_instr_rom.mif delete mode 100644 src/wiredancer/rtl/schl_cpu_instr_rom.sv delete mode 100644 src/wiredancer/rtl/sha512_block.sv delete mode 100644 src/wiredancer/rtl/sha512_modq.sv delete mode 100644 src/wiredancer/rtl/sha512_modq_meta.sv delete mode 100644 src/wiredancer/rtl/sha512_msgseq.sv delete mode 100644 src/wiredancer/rtl/sha512_pre.sv delete mode 100644 src/wiredancer/rtl/sha512_round.sv delete mode 100644 src/wiredancer/rtl/sha512_sch.sv delete mode 100644 src/wiredancer/rtl/showahead_fifo.sv delete mode 100644 src/wiredancer/rtl/simple_dual_port_ram.sv delete mode 100644 src/wiredancer/rtl/tid_inorder.sv delete mode 100644 src/wiredancer/rtl/top_f1.sv delete mode 100644 src/wiredancer/rtl/wd_pkg.sv delete mode 100644 src/wiredancer/sim/README.md delete mode 100755 src/wiredancer/sim/ed25519_point_add/Makefile delete mode 100755 src/wiredancer/sim/ed25519_point_add/test.py delete mode 100755 src/wiredancer/sim/ed25519_point_dbl/Makefile delete mode 100755 src/wiredancer/sim/ed25519_point_dbl/test.py delete mode 100755 src/wiredancer/sim/ed25519_sigverify_0/Makefile delete mode 100755 src/wiredancer/sim/ed25519_sigverify_0/test.py delete mode 100755 src/wiredancer/sim/ed25519_sigverify_1/Makefile delete mode 100755 src/wiredancer/sim/ed25519_sigverify_1/test.py delete mode 100755 src/wiredancer/sim/mul_wide/Makefile delete mode 100755 src/wiredancer/sim/mul_wide/test.py delete mode 100644 src/wiredancer/sim/schl_cpu/Makefile delete mode 100644 src/wiredancer/sim/schl_cpu/test.py delete mode 100755 src/wiredancer/sim/sha512_modq_meta/Makefile delete mode 100755 src/wiredancer/sim/sha512_modq_meta/test.py delete mode 100755 src/wiredancer/sim/sha512_pre/Makefile delete mode 100755 src/wiredancer/sim/sha512_pre/test.py delete mode 100755 src/wiredancer/sim/top_f1/Makefile delete mode 100755 src/wiredancer/sim/top_f1/test.py delete mode 100755 src/wiredancer/sim/top_f1_models/Makefile delete mode 100755 src/wiredancer/sim/top_f1_models/ed25519_sigverify_dsdp_mul.sv delete mode 100755 src/wiredancer/sim/top_f1_models/schl_cpu.sv delete mode 100755 src/wiredancer/sim/top_f1_models/test.py delete mode 100644 src/wiredancer/test/Local.mk delete mode 100644 src/wiredancer/test/fd_replay_loop.c delete mode 100644 src/wiredancer/test/fd_replay_loop.h delete mode 100644 src/wiredancer/test/test_wiredancer_demo.c delete mode 100644 src/wiredancer/test/wd_f1_mon.c delete mode 100644 src/wiredancer/test/wd_f1_mon.h diff --git a/book/guide/protocol/cryptography.md b/book/guide/protocol/cryptography.md index 8ab2ed7530c..37fb4676d4a 100644 --- a/book/guide/protocol/cryptography.md +++ b/book/guide/protocol/cryptography.md @@ -75,7 +75,6 @@ Used to sign various consensus and p2p messages. - A reference implementation using [fiat-crypto](https://github.com/firedancer-io/firedancer/blob/main/src/third_party/fiat-crypto/curve25519_64.c) - Handwritten constant-time signing routines (`fd_curve25519_secure.c`). - Handwritten high-performance AVX512-IFMA backend for Intel/AMD CPUs -- Hardware-accelerated signature verification via FPGA offload ([Wiredancer](https://github.com/firedancer-io/firedancer/tree/main/src/wiredancer)) ## Curve25519 diff --git a/contrib/test/check_fp_fma.sh b/contrib/test/check_fp_fma.sh index 9a647093d1c..75743697833 100755 --- a/contrib/test/check_fp_fma.sh +++ b/contrib/test/check_fp_fma.sh @@ -36,7 +36,7 @@ fi # Top-level object directories where FP FMA instructions are allowed: # non-consensus code (apps, tiles, networking, monitoring, utils). -ALLOW_DIRS="app disco discof discoh util waltz tango wiredancer third_party" +ALLOW_DIRS="app disco discof discoh util waltz tango third_party" # Specific objects in denied directories with reviewed, intentional # FMA use. diff --git a/doc/organization.txt b/doc/organization.txt index 6d2c5ffdddb..cd8f69e137d 100644 --- a/doc/organization.txt +++ b/doc/organization.txt @@ -56,9 +56,7 @@ Firedancer source tree ├── util/ C language environment, system runtime, common data │ structures and various utilities (math, bits, rng ...) │ - ├── waltz/ Networking - │ - └── wiredancer/ FPGA modules + └── waltz/ Networking Important files diff --git a/src/wiredancer/Local.mk b/src/wiredancer/Local.mk deleted file mode 100644 index 4a14373580b..00000000000 --- a/src/wiredancer/Local.mk +++ /dev/null @@ -1,5 +0,0 @@ -ifdef FD_HAS_WIREDANCER -$(call make-lib,fd_wiredancer) -$(call add-hdrs,c/wd_f1.h) -$(call add-objs,c/wd_f1,fd_wiredancer) -endif diff --git a/src/wiredancer/README.md b/src/wiredancer/README.md deleted file mode 100644 index 21bf4dd5b00..00000000000 --- a/src/wiredancer/README.md +++ /dev/null @@ -1,169 +0,0 @@ -# WireDancer # -This directory includes all resources to build and use Wiredancer functionalities. - -## Supported Platforms ## -* AWS-F1 series - -## Available Functions ## -* SigVerify - -# Using WD inside FD # - -WD adopts an asynchronous API. In fact WD uses the same Tango mcache mechanism to publish results back into FD. All requests to WD are sent with a push model, and results are pushed back into mcaches. - -## WD API ## - -- `wd_init_pci(wd_wksp_t*, uint64_t slots)` - - Initializes PCIe interfacing with FPGA card(s). Multiple cards can be specified as bits in the `slots` argument. For example, `slots=3` asks for first and second slots to be initialized. -- `wd_free_pci(wd_wksp_t*)` - - Frees PCIe resources. - - - - - - - - - - - - - - -# # -# Building WD # - -## AWS-F1 Series ## - -+ To build for AWS-F1 series EC2 instances, you need an EC2 build machine. Details to provision such instance are provided in [AWS-FPGA github page](https://github.com/aws/aws-fpga). - -+ Inside the build machine, clone AWS-FPGA git repo - - `git clone https://github.com/aws/aws-fpga` - -- Follow the repo's instructions to build `/hdk/cl/examples/cl_dram_dma` - -+ Copy all files from `/wiredancer/platforms/f1/build` directory into `/hdk/cl/examples/cl_dram_dma` replacing existing files. - -+ Rebuild `/hdk/cl/examples/cl_dram_dma` with the same instructions from AWS repo as before. - - -# # -# Running WD # - -## AWS-F1 Series ## - -+ To run FD with WD support, you need an EC2 F1 machine. - -+ Inside the machine, clone AWS-FPGA git repo - - `git clone https://github.com/aws/aws-fpga` - -+ Install the SDK inside the repo: - - `source $AWS-FPGA/sdk_setup.sh` - -+ Load WD image on FPGA slot-0 - - `sudo fpga-load-local-image -S 0 -I agfi-01051ff14d1bba4e0` - -+ Make FD with WD support - - `./deps.sh` - - `source activate-opt` - - `make MACHINE=linux_gcc_wd_f1 -j` - -+ Configure FD - - `sudo build/linux/gcc/x86_64/bin/fd_shmem_cfg reset` - - `sudo build/linux/gcc/x86_64/bin/fd_shmem_cfg fini` - - `sudo build/linux/gcc/x86_64/bin/fd_shmem_cfg alloc 32 gigantic 0 alloc 512 huge 0` - - `sudo build/linux/gcc/x86_64/bin/fd_shmem_cfg init 0700 $USER ""` - - `sudo build/linux/gcc/x86_64/bin/fd_shmem_cfg query` - -+ Configure app-frank - - `sudo ./build/linux/gcc/x86_64/bin/fd_frank_init_demo frank 1-6 ./build/linux/gcc/x86_64 /tmp/solana.pcap 0 0 1 0` - -+ Run app-frank - - `sudo ./build/linux/gcc/x86_64/bin/fd_frank_run frank "1-6"` - -+ Run wd-monitor - - `sudo taskset -c 7 build/linux/gcc/x86_64/bin/wd_frank_mon frank --duration 0` - -+ Run fd-monitor - - `sudo taskset -c 7 build/linux/gcc/x86_64/bin/fd_frank_mon frank --duration 10e12 --dt-min 1e7 --dt-max 1e7` - - - - - - -# # -# WD-SigVerify # - -SigVerify is the verification process of [ED25519](https://en.wikipedia.org/wiki/EdDSA). This is a computationally intensive operation. In order to match SigVerify's throughput with the rest of the FD system, many high performance cores are required. However WD.SigVerify uses hardware acceleration to achieve 1Mps throughput using only one FPGA. Table below shows the throughput of a single core running FD.SigVerify on various architectures, and the number of cores required to reach a throughput of one million per second. - -| Architecture | Throughput | Cores/Cards | -| - | - | - | -| Skylake 2.4GHz | 30 Kps | 33 | -| AWS-F1 | 1000 Kps | 1 | - -## WD.SigVerify API ## - -WD.SigVerify utilizes an asynchronous API: software pushes requests to the accelerator, and the accelerator pushes results to software. - -- `wd_ed25519_verify_init_req` - - Initializes request API. By default all signature failures are dropped and no response is sent back to software. This feature is programmable here. All fields pertaining to response's tango.mcache are also provided here. -- `wd_ed25519_verify_init_resp` - - Initializes response API. -- `wd_ed25519_verify_req` - - Pushes a new verification request to the accelerator. In addition to regular FD.sigverify parameters, details of the mcache to be populated in response are also provided here - -## WD.SigVerify Design ## - -In the design of WD.SigVerify, given hard requirements for area and latency, the goal is to maximize throughput. Due to the nature of FPGA devices, clock speed is, on average, ~1/10 of the host processor (for example 2.4GHz CPU vs 250MHz FPGA). This leads to higher latency for a single transaction. However, taking advantage of pipelining and parallelism, WD compensates for the lost latency by overlapping the processing of multiple transactions at the same time, to reach very high (>1Mps) throughput. - -### Batchless Parallelism ### -Unlike GPU-style acceleration, WD.Sigverify does not require multiple transactions to be batched together to reach its maximum throughput. All requests are processed independently, yet in parallel. As such, transaction arrival times do not impact system throughput. - -### Pipeline Design - -Every transaction goes through multiple steps (detailed further in this document), with each step posing unique challenges when it comes to hardware implementation and acceleration. The main goal is to form a single chain of pipelined blocks, that provides maximum throughput, while the latency and area are kept at a reasonable and practical level. - -Figure below shows the overall pipeline design of WD.SigVerify. It is composed of a chain link of pipelines, with each pipeline having its own unique throughput and latency characteristics highlighted in the figure. -![WD-SigVerify Architecture](WD-SV.jpg) - -We solve the queueing problem of concatenating pipelines of different latency and throughput by utilizing a credit-based chain link system. All links follow the diagram below: - -“” - -- Chain links do not backpressure previous link. -- Chain links do not accept backpressure from the next link. -- All links have an input fifo for the pipeline inside them. -- All links include a throttle (rate limiter). -- A link's rate limiting is determined by three metrics and three parameter thresholds: - - `A`: Number of transactions inside the fifo of the next link. - - `B`: Number of transactions inside the pipeline. - - `C`: Number of cycles since last data was popped from the fifo. - - wait if `A >= T0` - - wait if `A+B >= T1` - - wait if `C < T2` - -This pipeline design allows for arbitrary number of registers along the connections between links. This is especially required for timing closure when crossing SLRs in a multi-SLR FPGA, such as the VU9P present in AWS-F1 systems. This design also allows to create m-n connections between chain links, however this is not applicable to WD-SV. - - -Pseudocode below outlines an algorithmically optimized ED25519 verification procedure, broken into four steps: -- SHA - - `h = sha512_modq(sig_l + pub + msg)` -- SV0 - - `A = point_decompress(pub)` - - `R = point_decompress(sig_l)` - - `r = ` -- SV1 - - `T = dsdp_mul(A, G, sig_h, -h)` -- SV2 - - `return r & point_equal(T, R)` - -`SHA`: This step is a straight forward SHA-512 of the message and modular reduction. We implement this steps as a fully pipelined module, with a variable latency (based on message size) of `1-3.5` microsecond at `250MHz`. Since the latency of this step is dependent on the message size, output can be out of order. - -`SV0`: This step includes many data-dependent control flows. As such, we opt to create a custom processor/scheduler to execute necessary instructions for each transaction. The sequence of instructions to be executed is extracted from a reference python model, utilizing the `Expr` class, details of which are provided inside `ed25510_lib.py`. A single processor can accommodate `22` pending transactions at a time, with a latency of `101` microseconds at `250MHz`. Accordingly, we require five instances of the processor to achieve a throughput of `1.08` million per second. - -`SV1`: This step includes the optimized double-scalar double-point multiplication, similar to the algorithm mentioned [here](https://cryptojedi.org/peter/data/eccss-20130911b.pdf). This algorithm is very regular in terms of dataflow, and all the if-statements can be accommodated with simple multiplexers. However, the dynamic execution includes many invocations of modular-multiplications, hence creating a fully-pipelined implementation is required to keep latency reasonable. Our implementation achieves `~96` microseconds of latency at `266MHz`, and a throughput of `1.04` million per second. - -`SV2`: This final step includes two modular-multiplications as is required in point equality checks in twisted-edwards coordinates. We employ one modular-multiplier unit in this step, and use it in two subsequent cycles, achieving a latency of `72` nano seconds at `250MHz`, and a throughput of `125` million per second. - diff --git a/src/wiredancer/WD-CHAIN.jpg b/src/wiredancer/WD-CHAIN.jpg deleted file mode 100644 index ac85b17ddf1577cfdc0d2ed3b471884b64879638..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 115456 zcmeFa2|QKrzCXTXo{~A^mLW<~WK1^6l!PQ@+Lg=+nYGy>WDcQJwjl|zGe@$`lq6H8 zVw*Cvw_&%~({J@X_nv#c-*fK&b?!O$oO{pxf7h$Et(Ip!p3n1q*5~;Q@6Tt`Mrb%- z-&q4=1AvYWpgT)<7652dK!Sddn==5IngWLb0AK+a>39Ho=n85AeFy`Lzgz>rSvrwF zUSFV-`?ofG03gN1b^L3>pO^-7m*4k3k=Fd*~S$nV4Bv+1Q~5)%$=wboBIl80Z-p zfBqS|Q0Vsn12-ejf#bSNycXA(C4Kmm?xy6jNS!Wk=D*N~msY-hJA#!>Kybg1u*^YO zxkHCfsHm!`YiR17(Kj$OGCpg0(dv@*eb* zO?#N0k@@IxR$l(|f)|A^i;62MtEy{i-_+H&w6?W(bauV}&_6IZ^ksNtbZlyRW_IrT z{KDcAVPkV^oA?9V+5JfuG@pMU3;O&6Wxt_|8=`9u0|Pw+^G~|y_5}VUoST91z;PxX zT?^)GKD?4jcUkyOr{tD5vq~vn!1G_f-Nz;%ePT+6@RPJ(DEsROi};%;`y*lhLDx8- z2kiOfpx?8Ho{^rOo{@CU0pVT2An}u)ROsR zr(mm|GtVriXh8Uuf|v22-#d+M!_f2cezCYD*PN@R92?KZUpTM;37hRem=mh`LYNlW z6Z9e3fL^UwF7ll+(92l(;1lmJAp!Y59yd4M2i_K&jg!52yZNG6(=4FrgXl9wDWdV# z->oQ>TACldnPdF&j2D>4kCtpz&Y;Qs-NZo9V%iqX0%~`Vra8Ccs{#duecShDmlblv z`W`rVfs0^}5L1~2P^|N?T39D6cu1`YGZxqu*%a{l8>rXryXVUn&Byl();^mkR1wO# zB=4psyllIj8-6pikO1%8HETh~XdeV0V{}zMJ(~*EoWJlUZbaTx`cnC0A;-NR3hwV0 zx~e=F2dursbW=pAZ0#gw8#Dks6c~yB?7dqEMiH#7?PZN)+m27! zTq!#$DqwUimd+(5A}uL=&&SgdI?QE#h}c>hfVV|Qmb+h9P@bB2=TkWU_(k>?q4Fh3 z>l8BznSa-sq(U7)bD-SOdrW60qZR!!h=B(`t$o?$=Ip6%(ob`Yaxq}E+t;;H_~JGT z0E@@sFQSv-B4MIoUm8X5>#a0Ee7RAi-yrtISC#B%FTdK1YgO2PjR}^F9TI(Q#q6yC z?0sszWr^sXC-d*5qi*)@trVr3`X-keLPzW_rW^t1X#nGG$}=ioP{w?VHdlEg7#Vtf zRjj?^DM7EWq^MQWuFLpgB-tYWz{>Y#f%4Uj}Cq5%c;Kg34h3zBQHC0z9 zy|{kpP8CnLr`AoquACeBhmVWCyeb0AtpY3%PbRC>$o#xSFOYRw9?Q`5qfvz*ko{f0 zB=mrzDzU$))llVQS?s$I{sg0o%s;52zS>}oQwTJy3>dqmFjw0k z$7SH!ecETaDs6!i2pb%pVK`>?T z*qR3ajx{trZVKX-+Q0G=$>jO>h(kVO|Xzulp`lp z`-IjPFD+>;>*;vz=O%aLM--}`2R5GZxMkbD2d=<2b(<|RYu9pWo-7cFlZ|aKWJ;V# zW_PD@m)Cj6zZ6K?WvK{q>sP2K-hF7yvel`wvr1C&5VpRL!j)}$jPcLfL_aACYHhDo z9X_U>DwLbSsXLRZW*2?w0*eGLQU^Bpj=!C(2gc&C_o$qMR0*=)(DgnfJikMxzW#~N zMaGDiY`KpgKZ>$4zw#)Y?jTkQX$pUF zuJu4I2l7Lm(C$V!n_S-ixf4$7|KGn&m%%!*9F1Z~%l_QYRGAI`1RZ3YWC`V%oG7?w%-# zcbzTgwrLNYm3`3kVkGX6Zo(mpqzLQ_1VH8hkK>D}^H!h>&Qq{uxHcu2*X+e*UAH4C zOq}J}(ig0~nT0xZPDP7vz0lY@+qcyrhu)U<)xBd`W zw%90U$Y!XEB9EV^vOCfMSuPx!#Gg+C`j?9-6_g_E?0c#-4H!vB(SYZsIHyrmH?}^ha3%wej|@_{`s6}{2FLA*UT?$-G52A#(Rg=2%3uV4KoLiC=I)t3l=J#1 z8mF2cOvr3*WEf~rD*{m_^<4hf6Ilgf#dq!|+IBl1Mk9;7h;Opo?;UdNuo3*z4>B(eAi99|BrEdeFWSQ7 zv%@N4+&TTvv9C4#UK>hpv>tJjcIk~XA2C#n0;Y@?IK!|sp!Ff#d3{&3;R+2vt-yEK z!nOG!3ax{>FM?(yw`DBDP5+>61+F%?qQ!o#Bf=9)p|GB;i6zPBkq?vXpYyO8h2TBL zP`EeV;KL;vz&h_e-#X1{WgZ^D-tuMc2m0-s4ngPOrkjD72Dl3fJP2z-kG4*~$Rylp z+5L_!S+-HKQ+OV7kJ#}|_`6_jpgd>8xz6sC@b0TF>azPudebzZ850fd@LJq~_#e~R z@dCB^_S_PNAKr6(U@^{3F$$VBKQCmYE-hLl5stJb?KdIqLtX>IDJ&yL-Y;?NznFzaz6b9+=nZ&Sx>SwSi@rXuP`LNd$ z^ZE!4r*w{$cvzQA;7GV!%TYexaCv+hF$`a|<~&-%?bWKiJ>5+@2)!1QOqXiG-QfMp@92U~8UcH`D+VD{apx z)cEFm>gim*>lzGpvaO3HB{q<2lgC<`esh4z4)coC18V166l94S6wKB$58To-;g)F;F5X4&Gi{}bEp|r9 zd`__^8EWQ+%;@aoaM!WWYvu(@%EC0mM&a$KeGLZ`(_gvoYc5=@bn(zXn%APJff|of z4d$g32u)*!X#lPUzTr(}r2*wOaLUno3>J$k$x@jpB5}C=__ES1o>H_>Yf0Ca?>h07 zeznmFZZ1bZoeavzu2|vMX9QA8$w6S}^Ui58?G|vc{`@olScMx|>O*Tsk4kEtWquSd zT-*I@w&;Kn#e$ND#i7xawH>G!;j+7sIl{0g&PzBvUg;K+k>bLHmbSLY-MEA}z-P>= zZf(ds2S56St+fS(OA6!i z>t8&}T@1{cz<6h52qTs3>`V`|=yRO4|CW5Um{qnv9R9=$lnQJ|@PK+VwXuy1OBeY2 zk@`-jMr(XOYGyqBz%Fafe&)uN<3DO-;c}SQ*q`&_CH`|>Uce~_>M^|#(cdb;W#SIH zL-Tfp&FHvCeI=mIZTFp#dEiMp*<%a#i~KuCfaISrBnbOOih*4(Dr--Xt?T4#>(+PI z@zd-gVi$tB^7LQYSB6Dk*|0Z~qj|$7X+TX_^)xD8f4;xq4u;3$SjXI%lp{ruu1*7j z9lsJfJ48O5<|}H3LB>eAcBX$Bs#)2OQ=t9vEq^Yqpy&xnY>-y1+RNv6W(kN)NR`(| z@VaGrtujJyCZ~f@`8N*-ap#(ELt|gQAM~?aE{}KYGhWq!RltN$I5fOsvVC%o=@g3= zR{>~1l#3f#%944gYJI7KZbNy0I>R!$A$5&`H}RDoR}*~1x@jz?l?yy%79A$yUfJn? zXUg6K(O;M)x^uzU$>|Qe<}G0L`ePxXP8({qQ2IqkiGMGf!%wab>j`PhoQ8IwkGpFz zf&A2GFCnFESYUXcdg`Iv3^Co*GyYTQ4BLzDIY*Snb*t=vYf^17x=3R1Nfo|!7W`l$2P{D__q%&RD7WCwS4^7dR z=pIJUVOkZ(1gR|V`SwkK7@{-bM2RopmiM{ZnW0kgBhT1QK)Tj7hOXd_MZ<&P@T-md z)E*dvHVc^F=AZ~#PT8V35n#bplesE4p3$hf+Y$1i9dZVCvj)XnqSp1pxIi*b4Dl^k zH;uZVglQAl3V4NzDo^mre0%rR zJo)HV|I62QG4HA znfhSqlmvqdA6{L&Dyk?%vB?XEKgEEDW177Zv`J)Y z0n?Y))=Y9Uq;GKKK73##vCoXbddowfkB&@0HD3+ei?jn{2;e}6ARvC z1XMb48w&6FIQNE)P8!eTu~;uhPdG_s8Nhc=8@86RPnVE&@nt1bcZ=*6riV!vKHQOl zr)Gw^WXPUTs0>+?dZ8aS(x(rgRA@jo*9tF5pZW*+H8mR2%q#4LklzfA zGj8tGd>_(vEA4W&8cX7Exl!_Mm-8f*%?+~2*lf*K1i_OYp|9jvWVB&JZEZ6`t{1%7 zT|U!Mdr$^^=FD@j6_GOP*g*Ecib2IxsZx6%MXxUv(ASQQ2UdQ1x5R zle`@a3s1r?sFg9H5Ux2~-#a_uwJ~Hq{bBb_}TEG{h`dRp$^ODsKkzV2-; zPiNq5dyaA`2`7%{5xPJ`kR&$W1~OpgSm0i29 zd}hh`u=L1+Fu)W3b3 zsw;ZL>=VcRd)}jVkrbUi9auIBZ-yRu7}nLuJh-l9?eA3_wA_C1AU5kOFK20k-tnxlF>x6PO{x;Xv4D>FX7@J?^gxFezQd#8>YUF@N%{bl0QYq*Q4Fg-vG(}l z1y;YJj;%L-8y~}NMyOnGDsO6XtU{R%nlpb#kog=2h+PAgO|X5~8{LWLEvD*J=wDr4 zym4-Gr)6?wRVMzHpzB*1kpPVgqx7kU^ORvSABxJRP_mE)Dw=_ML=^XXp_tW-39%vJ z4dukf>*H@5uAkfdz}Jfpz%2B~0p#s_e?>BbURrDY zCbH300nXP=ltt8IrL&D?i*mT^{sjYuf8Wx7`PW|hS6l+9uF_XDU^u;#8t*>&oBsn; zmJ-raN;Y=(F7gCqNxB$7Mygy1=!jYKC5F&|R7U&&wfz~yBu05TTvz{pxBA2Xgw=06 zL#b=j23D!;AVvU>0hL{=Mbzv-CTM{4?JU5BhfWhP(%Sd(MXjF(>re{A=)aPU!PE?k zrM_jg0JfTaRr~H=jT1nmXY0LHQa!&!%KnGLP2m3obS?d6B$ch11{^ejqFTF@e)v|s zFbgfnh#Dk-Oi_>sA<69gcei+qluFNCPn6q-cOiZq8nMD$6*uNXfx8 zz)>|HQhO9iwnMOTs9!G>*81h&W2pZ%&;B!z!@u+7-UWi1*&E)Pk4kf`(=)XM*sH{!TerQ3wB1NbvJLr4RsF6$8U!_6HyYpYJcbiK z0F-li4Q9CBRS~=iF>*j=NPAyUO>dk0EqXHsr@Xh};&d5fPR0*-o1Ar67fi!^sNWEH za|n%rt)Ky`jCsgIc-6bP?wa~*kDWq&j^^_ZX3DJHIvAUw8a*v-E^s^{Uo`c+As-!y zTc6LI?vH1fe^4fs%>@r@g;iOTl@r&}OS_9-VD`oy&F9-EBK_|17YU80=PabJmbGZj z^3Zee?#k;h@>%@bY268Df!|smot(*Rm&Zgxiutz5IC|f>LN{a9hs)O<*oc}`7>Ly; zHs3x4=5Uq2%xjp0-K?v9H0;YHKhhXyc0WWQ=EW5^ zo0Cfc8*b)b+mCM5MT?~ht;t?uFnJnYC>aiSSjdY5@NrvSNgvottk$e z4=_;5i=1_8o{}(@Oub)&8Cv?Ku4IWx>$4TQN;yWy4CHZ&-=-ihD_ho>`Rk7k-wxUN-yv!p9@JbsPwS>Eg8 zxrY<@9B)`)*ut#fwVhN}V-k-T=ZR!c?# z-{9BVCu71C?MnSLr=r^w101hC8 zW7^?jZee1LkM&AlZARD2g}_t97Xz|;YMH;(Y73IRK|Mn8hZ%Fwa+;#n&V40aklo3< z!h~sMb@TeP(T}UEU-Q+~Kgkn3aBwouX&NwH8}kC{+!m~~ttBAhl!I=*nu}k3aANY< zoyE00v|rEDs{rI@!r zb47_SYF1X}`LUflBTp-xk`IP!<+o!iW5~Sg7L-y+f>*4TsS9(vTeXMnW^PSYN~}LT z)s@|b7yI$SijO!>G-cz#2)w}NdtH8zi$Nnn@IEGY;KVy6p`*9T%5GfFjoAON^1W0! zTX8X~^iw*-$IK4JopVt!l%z%|D&D7RMH(VGLt69;BmxQ?*j|EEYnL1@Jv_P-78$0A zZ!DPcVhI{I9P;B7ibEip@37n2yf*Lrlol{Ty})l^N<<6z!Ky2F zuD<)6w5RiKd<;A5?t14t)bYvkA1j-LN@5kZ4pf*DbM_Qz5>T^Ps$Xs0a}$?7{2_5n zMSlLPQtrNvp1UzOUrr3|(vhy9nx)Cy_)MI0H9>ERIo3ba#0sBY)HtZ1qUR-Tt@@gC zSdle)|2b1(=X1xstFvWYA z29%;KRmhd7=40472|>hmD}q`@M>#kA80ms9Y{iO@4-*Wds0#1-wD-O%&m>*AQb;=D zJ#L`Yk^bVtPME`IQfyO)8UJGSrs3)nNP~V%KvCO0ZE1jUcDU9PqBN*Rf{~9`6QKNr zmGWshEz5&B#6I`hLq#w0rBB0;=Oo?>;x76+i_B@{l6CDIgmE{jA`gH%V%&6?iG~fUY zfb^2>LtnBo zOaQ`o&^4-l?YV|8>C{BrU-$gnEmPWnvp?bp)ZHa^zviLye8X_}>F`wN$~y?fH%cg6 zy^`Nb*d&$`E|t2tO|wXbDt-AjBb?Ztqa*P++jTaHJv2P{_z{;w5voVOO&$sxK~x&X z!ExC%Kve6{v{@8Y=3Ux(?4`jIr_P?1YKXpLX!WM{aUNT$=iaFMR@TO<>B|qemgU>v z8^9oB7=H1h0K=P*vy;IL%9u$Y6jI{SP>Z1jsM8K?gnF_G2+Ktp80o(un8kQjBMzOP zf{l9$gobCo4V=Ma=Gm#NTrI?9Ql;m&L}&@0Du8gpNGH8bW!kdVdNHv7Gr@RLsuuRpk3mkjuyigG4v zfkpW7=`ShT@9>>%W2m?jB)7zd%EjSB-YFmXR6FI)e;I4~iPAwSWBI5D$hRSpUlP_Q zki7@=bUXS47b;-jq3+yOGq9>$y)Cj)#}v%W7M`jknJnw8wq;0C4f_-ZP%^3euOOB2 z95)I1BJsui6A&eq-wDkMjF zsc_wGWFjD08^r_>ciira=1q)iT+$xT0Z1u`D*qW&6<0HAFZDB;nQV%WZ~Jt$dU&Klw-aKa{=3k5Y!=ZTT%oXTU>EJIPVk8A$BQ4Jb|O z;7Sq>x6mfQ-;Qrlf=z*hYdw6}TfL+ap%TPE+R!FaS>ZhKZCxZLFL1WJfZcE?E= z=~HDLH;R8;XAU6?mQ(5=AV?0P#<1;_Su3g{DN`_d%R^w|i~uo-NGxl2^jIFp6J8}o zoNn9qbYJnf@lTj&{8ToU5JS`=1h!(?+J=Qc73H*i@h)L>E3DaASebmREF*_mt) zycw0XkK_f)6SXrtv>CvDliH7==}yfQdb6Ml9% z+U3Z~MP%gR$pRSO5WAzi0Bx%9B9xpf{igDaAo<1VR%$1L1GVtvFlC8SiJBeIhJ0Kj zr6s6E5S=I z|Fd?mRcPluNFW$O`@a{2#c9fWXWu9Le;YuMZ@YA^~H1K&_0Lt7cf zRD=c?*bi^1{AIPoGpq+&rTvT~PaP7jZMXM5gSL(SWNOn>dLZR$ntyABi*E!|zsCL? zu9b{N*WXeMVF%&#Q|lC-Zu#Q%>%K8J;kCyy z)gOzLgCAUe^)~V#Q-7tR`Ci@C2p#R8&_HN}f0B`P8-=$+?QoG{(BSPMGW9CUDyb~c zg#;o2u}#Q@ln%ja$m!Dzt@JD$kD|Ww$5Qrw3WvMX0B6jO8axU`JPhS_>2ikYqMC2> zpO38)z;_MgcN9QbJbaqDlN}~@g{ir?gKo z`tfJu@5K*HZnluyb*WV-C_j8h2@{PX%3g*tZzV8o7*G;Le(I74U%B`Mvc*>A=%MVZ zBzP6Nm2!C4lJcawo845TE#z5SR`j!%lK0ZATIN;@eY;KBBKr15pK1cSGpXV7J4Mi( zJqha!sck~QbV-&-o+Nwa*4-#_pjwI3qj7_w?wF)$$w$XqwH_TdNY zlRnnb7&+kLiqjw-WMMylU+9lVygD9JJ-Y>e_75srDNAvk9kfNw0oV zGVWAaI$=(4!E@sBjQPtP0B8dI0P85^Y04wGOLE+9{xX(ZTWlfgd(qK8nOUL*rKpAS zdGqC!{d7%?iFe-d(anx2HhD#aiQ=t(v?g*jWyxelZR;?8ermHoiE zG>4jW`jE{BwODh%f{lly9{UHMSPsPUwB$Ob^~&9th|7_8fl7Gcif+)Q2lv)d_1|6-<*{>z%uYh*fv`-LlMk$_Th?c(49H{y^ zNJxFjqyHxkoWO}7Iz#f2a(>x%IE*jI?82bebGNdixBTQ;Uu0QpxTG51t5g;@mQHh+ z)CYo?$x4AMP@#b{Ud?FG%x#i<{T-J9cm?0x+Q{92XO3x*dhQmse^9`wuD|rA0`B#* zTY@+9OkDe}`$*zs%%!=;R46Z^Hky=`;0x}Hlc!>Owa zEy(xF<*p|3q3v30Yts=9RPl7;j!UA7}W`h4#E+}t`=R_1%c0pO}NKY<>a9~uxx ztjR#|HpmnNm2sXLGr2KzWa{m{x8eIczn6F(0hsrHVhXX&>C+JkIUkd^Agodt<9Cuh z^yJysQ3|C{=J@5(sR)&^luluwI4++~2iPlr7Y}QK40r^4wfsBBsoJ|$X6%T`!gQ@& zXS5fTh`z>Io+W+3(wq5~&x5_{{7VSqPKsTO*&Ew%44z)L+f=QB5_|N1C{4 z@a=-pEaMe3Z(aTlvNouJe=-FhM@?N0QZvHm4%UReTnsoiCPXeBXNZ=$(5$BV**D>- zzslmjMN1G*Vfag2J4y&a2$;GCg%S!r4=+%{yiv`U;j{05I+PIghSW?w9Z_a7cP-Ua ze3RPYX$Y;Vt@YA&7!;^!7D3JSt;1KG2T@!4)nS-F3BvqoK<2;in)#oA0snn_`Mb84 zza|9ypSJneRC)gK;Pl^l@^_y6PhXn90GXku{o^6?zw_kpJo#Va$$xyM$KQGKcb@#O z@#H_giskP-`8!Yk*Ld=0$+pV%ZBKcVQA%MjXW^4)*?#fjt8YH3aX-D`44Kta3U~~K z=_VF02`A+2jfjJ4F#hSSd@9c*82tWLv8`yt7fxzH@XmUHlov7;Gl}`MC`4dAzZQD0?vq8_Nohmu+lI}dR6Xj3c zZ_;%ykRvZ#J1(!GcM2-qbZ>J1O{c*y0r2fP!WmU?Gb(D(*LZJ8aCyC>_#80ytxYduq&`Mc_ao{I86u#HtccFO9tVaPgF2#Eq^37I`+ zQH^T-NK-BLMx!HZ`FHLQsDH)ZI9rmfbdf6r*xr@H9H;}K6+GQL%|@21Crl)AB{*CR zcT)&uX;3!GO;wZ+SY3I(@6G$JCJ6f&5+~o~k4c6iL|Pg%Bie!<6DRwi%1MHD-3-SU zEhqKFW=#C}nqd!Z19tR|wo570dG9sQfu5RcaG60dlQr>k!|~+cQkx*unex&33B`{F8s+auOFSjL~u zCzvxA^&yT0!HNBpbgetn`?ch!5V5-o-%I=@qFIDzx|Gj$vZV+`97$-KvND#&DcezdSXE=d?Y_tl#;`{r1A zZG5>_uW`*kRZYD3gOSvN>z;3XR_qGaCWAI-2G$SMyPg--m#jN|N44K);2nU-i_7&W zUJM*C@pKC2Y-$zd zMNZhNYHXJ!UAZ2id}OUWZ`)adZu?-kbRISv5s#{x@8p!CvSk*MRr<)GFULV?FuyfJ z;f0}ZJiUiMXJK~q@r`}o4Ud>A?nI)MFMey&{wqUn} zAW(YKeW)CGxF(xC`nNZefBg)7&fnNi*}=NJ_f7{3#B*Z3b_%h}F^a;SLCqMFWg7XRAa+kCmBEmDg6^}09S!*PJ=CAJ*AZjgzIT>U(?tJ&&OP`5H~IKV8O_Su|ijz8IdT47H|K=r%9RnPC|ZW$jub1fz1 z!FdDS9_!yu(m!*AJVJPY#tQ}lqntQQQt4f$_FWnc#8p~{=s z4HYC*1Qny;2!-iwh7*FMs4Rqe419?6ckJL(e zb4Erc3`2@ovkz+49WOi|3>vP3HEdVmCH+nw6u{O(y|)A*o$b2QF!f<8f|vS!vLY}M z?tu|@r?TSDV+t3BHC40yS)B$sa)lVmTdR?eW=~d1XoVk>Urh$3I?3tqI27@qJ!ZO`aGBbk289u7wt(MK2?04`WXPd3Aa}VH{%oBFNDXhj{aJ1a@f$kfzn-pb z%2B8QRF5E8Lvd!E!sQ@DUPtYSC;w)$P$pI8%!EXeU06q$1|`{_i(|1-ZTQ5hJA${} z|A_4Q&6&)9&XRDjGDQ}pXn>##S z{yE9(KUSb6mu8f0-}Evsz}!D z4#(@Glb~3VZ;{yyn)Ae^6s=c$69V$W84}K4IdDEbkbsDgfv5^43{&D|ArOS;^6n{u z3puX7mlAA;TB!Mnq-4pY0Wv>25yT^Cib8D;*J8jgf)lD-_8tbxo2}|kdqtqwPavWZ zTjvJR&jhjbUh#X=R4@@yXHY@RfgNFrD4~bAwlZf*WO8rNRz3lX4 zr)l8D0mx*Ly=T8zf9qkN4q@~?`mjf=OSEUt;psCM&G+!j#9lA?i81lK-Ut)dVm=W` zfy#c>C_`1eFKzhBQZ+ANbGag!n#kh#L;cv9t_4jOD12qt*e%{m4&d?di z{Hf?gz&|MB>M<8`kCTyv_dMeiz0RpsnETU;b#f0yzeGY8e-I==$x zzZTL6-m7kMNX?>bwBsmuCI*J=)`4B3_DtdpVN5|WQ;q4biSS)Yp` z%h9hWj!Mnsg+aDe<@&wGBgXq`2UX+)V$APz97fpGMa;OaBrVu8J6+{c0r_Vpch31^ zC`n^*YFsU500ByU)J#B>@4h1ISgnMtBh(}g zQ(mr???6EAm+n*tECdJIF`0M_r5jpDptcO2AO;msiim0yxnTVqgvcvthVM$5=5T#k z2M-UEA9E#QmzRxbz(4{&c$Xwd?Lr7*Jx$n8vD*;R#b66yxq^@(VJAWIPc9DD{Pn2ZcV1@mQ|0Z2L$ z1WapIYEU4lkQK}lP%QqdW*7VYy6-D&+uB>@W?W94yJumie}wzA>P~^hgIo=7y{p`a zJoIbS{WbVa%!t}W(1+kaU0a-j1UZ|-^K%7L03t}ey#PND-Iu~JDGJ_}yK z*_)XUZY@*92h82755|7Ka_gvU)WiLbu>p7YNqBxTo__)N4%?4vmZSkL>rq&4q%uMN zEWt3=3)7(Ra@;-Ovt6&ttQIkq^*RsBszqFkxe3>NogZ+g7FqG!P;0d)hY=9Q&i3OBnATTruP1~DtsFpzGGB5Sq}%+4a>dd$YLPhy z8kxu^2e!HHXNy+XSL45@@O|zs8GWA8mEj{VRdSZC;(;#14RKDCpw%*BPuM;%W~R5j zmz^ww?~>eXvyP=Vsr9a{wQJaXO$JUkD5`9>u5TxL6z&p>*P zTJ$?X*M#~ltiXDX>(B8Azfcqg?Fh}**qiHuWd4CF)Lkr_%Ape|ZZKmAQ>|d|EzWMt zZDp0uqG{$~YM^0vgb=^Bogjy{4hZ2m8A8EW!kww@f6}o$8k?gF;Js#XqOBdI!qFbHZ zi^9caYZDCbNf4OZXu$oHAVgarA-Jbd4^jYKDcOc6YkGZ%k0Q0TK^>i&QzIPcjIvqE za|nf;fv8f*PTei}wouC{HWkfkOH#BSFnJjIxm;2soTEE>$Zx*S!l z)bqJ{`~zN2XR5`^ML#1Q-30xG*jsvo5?3JDP77KtTK#rlob_R!acBo-m^5qMkJ zj#@GUwEGOes1Lqb4?q#@oV zA)DI2<{}(W=g51x*DG+}*Ux6G!55iNd#lxV<+dKBWBzfF)LIL}Tda>LxAvJpHRW0k zp=Q|cn8ZIRs3n~-x%0w4I8f?6kKv;USD5RB>YSN+d$1u{;P(CkO4?Sin64|*`f67w zYgK~-e2>Qv9QUx|B(+VO&HDP8K%4rPr(%fr=III2pP4uFXOc&4mYdNvjI}F|kVctl zUp;KjCsQy)OJS~6{ksj5l{>G@GGs}OEyH-&SmZa;tySC|nCpw1pVan`}ub(H8C@oNT5Z?5m1rsb<}RR2}lyopzkj zwsPg}(my32jDBl`EkkzKpC@epwIu#+CMrUK3qkh(@aHa300e?M5eEzNF zJWqw}oewvlH)X~5Gj>bA6e^@eNACOWb*?{KEOIW? z>NizWC=NMXw86jN%vgme{k^ZY&+iZ3 z)G`!QDeds`zgJ4DXZvfM3HFe3s7N(gPr|NTf$!RS{q#w3&rr{z+uG4vpH%uBdi-vJjg{k_!@(N<^nR5x`yy+ z)Q0M0ijaqcE~I2YXX+Kw5lHktf^TKDK)E;Tzjn&J3u}n{EzKAN7nvK?4Xcg>I5 zLh&p0w{i;}W&656-oRzc$6iN#!c;ZP^wrd15@foEENsph$Wk$_@*4j6gVET$V zN)c0b0*^w;w5}c>Yl|P~J{jrwBk@7H9+jP+q!5(c=9M4T-^lVBeZgMovs>2Iu|p}j zwXsPzkg6LmMtprd(yP) z*|R!rQCA|>J_4WmHUd}Sq;uq-A~79#7C+yDi45EOHd()aHu#m_+%=iDN+W*hhIbFR zIgEwL#m4OWox4-NqV%*6QWD`#@cr($J+vi2tEt6xXU`22u>|iEiG!6p2h@1K))=ol z6ybkN!CKbueNkG(ax};mfHo6IA?x#1aU&nB)s;@T3w8A}#!7~3aE)Sc8R?{xNN#*# zi`ZccA4-%*{NoOvQ?o11bq`%0V3f3=z&l^+A$(?3gHNs<=-82U>czYKiSWwN zNnNp?hH%luzAmBjEL~l=4Gky}$kzXhjzASlXh2emefxATKUmnBRrO*AS(nZJWC0nyQQ!82 zk7edSK#n>b_N>lp<;tyNQ6lYiB1&^4otMmYb1vg=YPvkG ztUY!zz*gq6u5|1PIyag20?WBz3h7siL%ycuw)lx%B-<{R^y2F;A!5KfO#{Pe6W2A4_r)v{!KDBbbFSs!Hm z^2xl?^eOgtZ~KR8lLS7=Zw~Mo=X|;f7I|wOTkZvjdnA+49#Vd|oST3G0Hi=y=I!<^ z)pwowGmag8$=5W^lV!i{zU!XdFpt#%h&0ez5<_UcMi}D`(otx<1k4rj@r`m*qTrTe zA74&feVGsUDDqx^9O3J1GnY)`9`2*_kB#)zkv7sbA*Uhso-qn4e13r;yS$&;d>-ZI z^?}2L_oj%^Mom{@$Af_eqv_@5Q7pTUdE;L3_E0V7^!{n0_V8+FO_~oFIN}uz19{us z($dnwO*jd%`Aj;%MD|el&MJ`&2-X~iy)JJmH7@rXOzKWT5A0a$F1y`s6!W;xeLo8r z>F<}S=POpKyf3i-QxhxzVgB`G^1taPXV9AXRer5-OXb$6c4^k5@quwZ!g748F8u3A z5Wf$V;y>st3Y11B2OCWTQ(FTWpQlU`607aXhT(%wqJk>wv6>%`>_OPw4igX3dV0-# z+WVXIl3G%p(^(irIRY~>==B9=TGi!geAUJEe%$C^z!leV)j00_``p7r_=oe5wCT(s zBMh`Aq9#(5@>I0;;SarDymjiO;1}4TRQE41ZM&mFS{tR%?a25v6-KP*paPYE>JK)k zikih0pS;y_%>GmaO$tR)8hEa0nt+OzWSdWyoWSQ{SoV?oM%sftOjr4sn&lUD&=K}) zqB8bRs3Zz)lp3E*l(daLW+WG{E0OTW5Kchw@GZ~fAEK@N_npPb;%}wc{wLY`XP@eS ze5>E}{~zew;X0PTw@$yok^Xvr+>;ti4mGl7T{5B$$;s@vCUeCo!|7D4qj^DHCWD9C z(Le0%ws^3KW=3^r*UG}>>44a##u{&*iW=(LJAJ}1-&Vw7z*u#cepNl zgUL6PjJfGLtCp%Wv-#ufXKO^ur*l_!H}80QA6}WK+t!PFZ!_FC^MPzi_3xbMtlsEz zsGX2~1(xPOF!tPiHpuGk-H8~MCeHEIM_{eqLmHrBj=9Chb-sp)=upzy_mISC;^@oM z%4?H;K9Hbt@_gv=Cdp5_@OwyrOXXn;yl?u6x;*vybi9&`uHJmAyh|p(ymIwzS2Q{)K zxOgaaWOf_L_>&l{>*3<9XBCS& zW)xL!n%pd6XyI2cHfzDMrgq9)@xs~$$d|dU-^JUWX6 zx_u9;#6H{rJ_z|tW2ThMVI5JQwuT(kiW|5`$Z|T~?61W}dwtXBmh2ap%O9%5V2j=;cJT{sL= za|ts(q{RzD{KJ^#1D?o}U;^zzj5k(;g`lq*#JHIXPnJ2xl4ce`bIE9ORc3JYc85%> z&Kc$2<#oLizNc=D939&rF=Y6Lev7~w+vXOtZn|2SzKe{knP2?@!Bs8c2r}_!B|0{Z z6_tFtm~x?6`p{CGo%e;dnYu!W>N_As_%$nX2f)#XyAU&u01I45o~u$}vV(km zGOY*fs&-ob0vj`bfe(cjsRSDYpp_iiEIW86H0wfKWlL31eGWCwFgVL+cxBQOdju&N(1_vWFd0X6GpMwDeDvMF2@|imj zLq5(A6=$DrQHrX5G=k@`l7(rUq8_zUe8B!f;i zm5Ww>aT)iP;=$^56B zxDqPFeW2p+r~oA5t^$H4wxGh;~&&h zJxRbP=U~sSS!FASw4tWAcl+}mpFR<-&Ua>Yzt#e-5AiK>|A*lrzYRb64MrjW_|4zt z8U87~UxlS0C<$cz;tBz=q<#NaVo5PW=SS(DW&ekNNk4%9f5dgCKOm#%4@w&`)I9`{ z#F$a|?E4+uwDK5vmIcgjmO%f`o9SDit#l&pvwZ9D{NNDaMf|VS|MyD--eCF%gnrmc zMnB#>=VRE99szJ!dw)6&Kb?l(hNtFX>U0#z`0Eb~pHq(HI;PiWoq3d6G59J*rc^a_ z>-wEB9M6vzU%PkC-Fgk|_xE#uPfpVJiY4FC)6Bn81ixh;U51nONlUwY+iAkZaKK&J{QQt21LbTtl{dtp~N8`H# zbwzKVa5Ltk$dLCoi-ikY7;by1|FyD2oXhCX>hqZGNWk!Zr+NyQ-1o?3sa||0k$J9q zMPmJHi>hA?;rH(ad0U6;3+%Q!G|z0@jal#yLE~16&3tzV3>^)k-xW$Tf148a^~2() z*Z3KO>of!6`rWBv?>Y=DQ?LKpUf}l^&0+h}_tSd+XIXEjt&c;8 z)RRZ`!qui;?GOAc`v`XfF`NNXm3Th<^h*F28Rx!dnZeBtax;SjzQ7ij5Dn44+`tC{ z&xJP090X~^_nT2)V3>Hi!#7*bkPs;QsK)&Aum1l&#Z$lpZ_9Z!h_2UB4kdmR&d?}7 z!Y^}5PO!sk?Q_s*q9pS%5#8(^YduH+$WpCwmN-r-yD`?KjUD%{8Zfn%zQ9U75Xk3)@}9c{{^n~W9i zKAYrmITbE@@ubXcWF-vY0mz@fbc|T~>ga-ro4K-{fsC1vZ zXX>B1tylie`ccF^DziPeWA-JiV)bOTE1HIE7!MHR)pW3u=BI#_w6h6&-}kBv&y>(B-7Sj;S&*M5>zfgPkj(YwIgaLte8IZ+F7xLT^@ESpW`w`OjM_TC zW9w&?S3(>R5Kyg!>nm7V25n&JU^$?zMB%$d};M47#4;@w)gI z(=u1&ZEYoUwhFUus$um|+k)8&C!5aSgqOZ6DP;pG3wupx9IwxjA+w}Y$uI0*D>a^i zJH#f<%m!Cn+8|y&gHB+Wy+7t0_r63GO6Ksjws>&Bn?D1mU2YP6PfsScZZ2Tb@!~N9 zGkF%vmFWnOZ1~Rg4nSlQe=*sAU_-v=4*z`wV}!v!UY!+@<&wKvwV5|)7Spb^0l3%&ytzJqq$O(1g2b`j-q{HXv%9E6I(IH#XS{`*D= z0ZM3wGcjw>T+mL?j==ne7JOKnWq*%kMQ5*O1($Mf?+DA#!KVU->8DoRj=6kQOk@Q7 zLEIM@TQ!f*G}W@AmyQ~a;;!2p4uvx}yJvkU>2qJ#D|#h+!bDX4wUMMfi(GGY$p7sS zRek?qq<{bQP@l-N)MLK;gnl0R%^dspj}pb3xxSU`I|2{H*rxcSY9!XZrfX+mH@-RO z>M^TM?6eWpyKJG=WOIsWC@fs_0#^_3pQI`??tg(freuvn?jwy62UJQ-oXuMEQY1Er z@~XyfJ>Ij~LdYBDj(mfyQvn|oceSG9DNDh+)U^Y3mbM||<`l~zmAh=j^QTUDN2V6Q z#H-CYGxtbK+7uTR6?M&?&;P3MoZDe@OTJr5O1}2VlVKPeY{MED`V<;eeH&o{Nb8s_ zpdn-!Q^5456p`jS_Hp6P8Tac&&7a)5b{6J%ZpzN;Adtxe0bPwo(xzgcA{FirX-J~-1m{uC0!72^fp-$U zPQq4SaaI{IXiy2oi1{fG?ShWtYDSg#l_fgzB+bnV8=ka^Duzc2DP-4(uhV|d()s?W zDGwCCpX?pxi4gEfqiSm!cH};)tSq}76EpB4WZbLa2>(7U&c3^JamIscPCraZkBSbr z&tpD#?JLW8oigM>a)2*qQ42My#$>S!iewU~I%P4#gvi=*uZ+`j? z&=%+;5)FL3H?5#D=EJ38Mw>agk+vVx?Ef|GXQK|%evb+O8wh}BtaM12Y+R4bClQbv zebPr3!-}JheeC>@Sy9>PqO80{!FO-aMy8}F}*^MRT7Bgn1TLO{L5uLs>a8o$6I z`$8}^8kq5BG{D~a5=hLXGv)&et48rFj{}5S&VJfnOoswFleumN1GV~|0o4T!Ve0# zVSZqqvnD~>K;;Zs&@)szhGBI;Z|*j)GpBH*8km$++|hqFkYK9n&|eZ{#Ijphr+C@( zyI0~T2u`w6a$vbiS3=>tL*Xl5a9{D!pT|M1e&!&(wFwd@AFsva1H%d9qd1ZksAr%_n!yyQAJ0x#amyZP7*SSZXN%1o zxVBWPdSScnwthb&$|uk);QFGK366nXOY@f!6dMU^QETJ& zmcM?vLxkb~dO?YoAYzgiI8voUI@F~YV}NpYwE5V=5p`v(Atqf zd@mu|HM@|j+RW-jEs2%cxIg$|hFicB?SNcMKsEmvKmXD3bH~s4`9CXucK8hiRNp(2 zE0N?sh^h34Gmo}KH6%SAGMX499?%>UO;fXpJ{)VFb)UN)3n?8*$+SSX53$F!^~wuZ z9tpV9aUzzgkfyrEUkR){)tw$WxHtAAPZ!nATBI?T-leJ8?e0~9HA`zNOrjvABzg)d z2r=Ve%Gg|vdyE-Qf`7b#@El zl6sqg*E&h|A|11q(ni$}N%}@Sww_RMwu(r(8LmPxw22C0349(Iqw2u&xvQQ;wj>1- zkKmP4M^x50Ut`|oNT9h_s-w@r8^w-TQ_}{h8c%}j(Shh!*kU{=ZXi|=1Vt7J#{Czn zyTWss3Ddlx1|wzxAI3Iv2SPm>w5VJw*sUyS zq-KBR`A+}JXzh#buhdnar19^kAVcua_)GH19L)8may1ZYa8`g;rz2A7_M$xe<(~17 zyBK03QBDGz3OsjGk!d4}li7-A_l%BUD(6>p7Y&Nfru+JC5>6#fc?7C2-HHfkW8YHF z5@5a-8IN~kc~3Nzp<&s7O^g48?X^2E!jQ@-3oO9%kx(RmKmG2 z#OA5T3>{XG+IV-h{v~*V#TgbV2UUYnMHi%9a6(JIE>huik`V3o(`GeS&3zSX${Bn3 z-P(R$$G7Xo`NTZrlFMj^LBMkWV~^_(JBeY35a62 z3)$MA0+FHM8BC)4!weZXo$U}|w6PJs6A3EZRL@!y6vk-Ar@CdX8enf(Zdo@k6{N94 z_e#a>0Dt@=a96I?n@qBvTJ{lAuXNTuTkD{#l(&Ur`rVEj_5+wPWHx3s#sPJmR1_8r z#MsmzlibHS_Pp!z&fD&=wmhuwS8>B?LVrOk=iF^OvA$%wJ|h%ii@YhpRYrE1C-@8M z8J%-DSJkPgG^65TdeY}{sM?-WlU!w1=?bg&(Rmm_Kot(&kD@YChNBowzS`zTtH{+^ zckA4XxHcE3t~KPEcz9deOH)NO3g!>4zebz!yjsW^%??G-q%9nl@$@YMrIsEjKZ@=y zM(?MAq1{}QSMitm*V|Az_{U^=li>c`LQzu3b$nuKsO*2y2^<7(Oo$=7FsSQ)}Ie@Oc$8jC`2`U zHC+_BQ((#n*MftjOKYO9V+9n^3M~{w4be25bU6N?)$u151n+Tny$|s%pH7y{6D#r~ zXNBL(vrr4E){IO)ZrW-3-f}2@SR7&2rKngeTvT?xJ5GLrm%w+NhgJ2^qeHKrYZl&% zbVCM`@*C-o(t#(MAJ%|bR$j(Gg0s-I$#Mj3UgjoT7+T`l*K%wiEB4EJ>|=E>_J+%B z_i+;y)ZKuxQv_t=sN13TFEBll8QVf1&Y36|Qe;Ki1jQ|j?=&HGwmRTi*vvHem=813 z(Aqv}q!+|U^RzX6^LMs9bZSs3IMdX~!RIYM&Mpd_(lcpXWxQiG-Z*ConA@Oix*Q|f z1JteSD5Kkn*s$5ppeMMoxYd!@uDNOID3#Wr4=%HBpB}t)G3H4?eGE2b3*HTgXOM0# zH#V5|JCC&2AvV7C$DQonMUtIEe>!7vh#J>Kn$@hesA_K5YsS{S5;gg6!wo(IkrLLy zulivSAb6ZL$1GdBAyIvGaIy&rv@i?7e}3l;5QaW}D&w!^Ie$WjR-i@yY7XB2O0WIh zth7}Ooeof6^|i(d{H*#x;EgteG?-=sV|bbvi1RwA{L4Tib_Kt2D}OX+)na+cVjL5trRc?qZH1mVK&l z0L~10LD#0t*O>-XZ^|dLg??7y8MN@ZwP!Nya8dHd(+}0X*(dhye_Erl-MY?3hV!Gs zdn_c!cAb8781sfOLJHNg&#->KxBh;QrL)3&3F5?N^=xlN-_aO_y!V5rFLDNPAJBzg zHvh<=iD6z^s$pnCe)a&wpU+ti#7!ztO(7*p7y&pvCzR|Pk{1JuTQvzSc|Ha1*=IiV z?XVEaJ4W6=I`Ufo)UoMjp#)cBqPT6yqT8D;`A_keo{NKxzh-+63O}8h zhr=33pFaaN$cFVF+aYN=da@S|AjoGQe1T;odd`7l0rxrx*9gyVkF3-jt`hl#?6Jcd z`^?NJjh8?73}>=k$v^mXhJT;ki{aI#u)btWjSQWA3>8KyV5X--i-hH{(D^wnV}Ez? zl1I);8Lb>vmJgex@1874)Q63+2J1?E0LGM~Bkwcm4SuvZDqht{AhGly4yoKWs&JF- zb;d=w6CG5a21)wl7^1l7X5@VdyeDqW5ssd2=iXA$D5FT|md;{({rod-kjq)Wvr;de z1Yyn;8xLw653;h#xvKwuBFJ{D%q7j7t&-6PkS&XKh<-sTfpNWBt(~%ydNsehcyKkl zt-8pq1Qujser&0zwSMRJ)&TV|#Wbsp;zHX+VTkNQ%oB7`ZyO4zeCc1CJ7N^l8d<$3 z?W6F+cUyvuRTK;}%gbLJ@!#P5+RuUp2zWKdI(vM)P@!mXviMN6PP%aM^Rm|-;`>D> zPSKW<%@fWwjij>L>KiKbRx`n)0XVh>QH<};#px*fol&13i@NaIlK7_XVEUobn7P|p z_Ys(Z=H7u6221UjkKxSgu6m_=xc`>5Ty~aEEI_^RW z`HboHTvBrw>b7iQ^=@@z)ysYUrgkv>LMEHG?jDwoPhCPpEd4RB#FIVz?Y zC5#ZJ6iAowF*ygJ7HVtCF)YB8Azn;lxK?#Ma1C? zFg2)19XNP351e&&^;L4^ISxr!yVu939-0LD_D%Iyl7a*UHw38`Bw1fq7rorzt}6^g zC{v}V@JQ^2&mZ_h9!&(Q6}ix5jh!>E!WLIwZ0|NN^AiInG(^9`^H=i%9QY(I%I`?a zsLQrCMxCI5?5;$ggm+Qyc7|`>#RM6#p8Lq4-;X>N6ocp-^ou z!c)gDT{^Gd<&mVo=?Sy4bE;6 zJw?}9+$nn8HrL$aRk7njj)Lr1g@fIt0CXURa2sJazbsY>Xor{?#L{}a33Dx7gq$?P zAmjEy2Zrs-lJv;Shn_WGm1=gAHr@I5&_~rLTa{!Y(I;W{Fqi^4{U{?JF${|Q;k6(> z3M2SNvp;IvIQW4mL%uwiws_o8G8!?ohWo)hvKqO5rqcE?eLFccl_D~J)~BG)plh^O zj#J1u>ruH~dNei8}GcrvL` z1XV{0OA~@()Ay+rhlLllXrCHV2zcl)QWYYn7qi)Ri2uc<9m4NTEn&0rxK`c7`?q=9weWNr4`s{HgAxDzlSiH>6BTxH#tj(S*hN-#mUJ2jo4IuZ z$^9z+5qKeHcGU=q=saSZRF7TzVX@rquD^DD`2lqP(~;ba9r;pnv6l_UCO({1pq)k! zAviG?m3(!{i_soRHs;tm_0GZl#0ysCQu{~NVfH%QGSWIH0;7e718UAO2EJN=*{7r2 zt0f93d={Oq3s&&b7TJu%X;FiT3@h{IwBr@Z+>)H$xchhq)Gf3KBZbK`4Hwd~w(-vT zV5vaNcy{`!%jJftQB6N^#g#Xc<3Uayy|)x>jdP z37&E!)Ao^7i`J?ALKD@glKL^ZUFt8|IL;e2>YDN|H|oNr(S7*h#0c7E2wBTiqsRv9 zRAfvimw!l7aNPE&zI+!?>qJAIdXqs?R7lsJwvkAQ`C;TN0tn&U>O>R+-KK{FC(?Ft zyPcbC>Q<`q8*tnq38Pf5zRu`XqcbrzdI(f>2ykyq34?+o?Li*mHpX}|^Et{c>8P=1 zNat?Zfa}}arCr|+M`j7NqZK})_Vs-xaX3+{CCP2+?io{w$}*+pMR$kO1jU7AB14bQ zaltTt#XrC`&}Y8(204uiPh+8}gfbByNaWP_*~N&WT&mR+MRH7O@qF2fnOY6G#F%87 z9Ce0~$z%g1G|#*vKAT=2k}vJCyESd-8^NYy!gxmWkX-8Ryr8>k%x4Wd?WqIHSAv&1 zn3L&4m^ZCi!0U`~Vx0Y_iueCG)3fBxWPe1{1sp0Yz%x_YH~Cpn^d1MYKuO286Qoh) zWeRIb4hp;%u`tc+k@S{y*M&bnIffAi{T_nxh0I_OQVQ1N`79ka?v3YERj1{|RsHg4 z?xI_wq@MZc{uj?y$jPva=9#_A}SOw>}NFw7w}m; zTTU^JDvqe~JzeF}Wj)lk$Xhm>bma)!`PUoRKvBSTW;4758H#WQK(E+aNZ~+vrw4-F z#1@58o5m59TRUI9n&ojlefz47)BJ@iMveQZ{up@@*;Hi!J3^ZAeFzPhP^tr;Odo^N zMjK8axhFXwLdf*K6ojDfPb@SMs7Yp7c}Plg7f$s+{K}5?Yuqk>sP?p@`zrIunDD#vr=JQ-?*OYl8aI zPfn}f9_%=vwUJNaMBYHek&KHruuol*@cSO-v2{jxC9>rdwN)Nt# zmZPrBPUN^aY!9%_c|CrJG75(j+h35loR_g-DwW)$dz3wLvU9gMFdL4hvF*q|W`FEN zY=iFZ8IazP2>*x(gbOYpqp|Fj+7xd>iMfbZ>geuAiK4!_Y1b5wYsu&|*fz_XvXY(0 z+llea_1fe@K-^2@9lc-*pXlYfZ0m8NZ%7u7@0Y74`h;#k3u*FmzZ4QV89I zDd`KqTtY&|h>^9gi?s!ro8Zs<(`g6yYmrKqXj*t#Bc1vF0WY_18?Ce?fh} z+5Q2Mu5WPDrwlfn!$Dy8iI*@yLxoAj^Go z{*~7>|6ZhP8S#U{=htkjeu%{WDx$pdVk@_Q%8LB~atoLr3dCplss4G2ku?Y?c1V_1 zgO+|?MrOvJ&ume2w$C5x9NcVo<}FK6LQ_+l!e`7Fe5RSKLJcVIQbBKrj=*_Z(`EAe zi@MI}jX$kF9I)=9t~dXgNo|FCkWU^U<*P7w22|)AU^N;Khb^%%;f{bMa2?5uK82*m z8qjsRh6Nexs%7a<@k>(GAlV;=Lf7FJj6q*xWg5_Mm;Zhb%Q!dWpZP*%d6S>Xk_G0Y z9vrBt2Ot-upp=gVUoQjn`k3NXG#-!?Q35+&RDf=L>EG?S&mDspp<-e9_j(kKceY@q zto%ckUM6bo6U|Wvp1M(nG-SO+4(m~%$qJr%r&Gx zUHsc^7N~4OR4p@}*dSGXu zhh24gJ*{=eT~T`t8{X<0u-;2%rd9JZn~{b18u0_id(Z+tswBeVwbu00gUx)4s=G?s zYLa(Oyo6W-t~}TB!~tJ&<*@x>gT#j+%<8Pdyp9+}y=(z5NC7+`d}P z;6+A%wH0O?MdqkvCd1g-f3+plsHub{#-!~ukN_oRBwO07;|o2+Z6*#00VC0rpLAS z+cWEG0Ce2TPtyiHK86@KG4B&W|4i#ea3khV{0tazczaAlGA3*n?nX6dX>YEUv?En~ z+a9^HC>NKYhfB`y4&4=s(zlWe1jgO+H_e&n-#xi`AAr}t{chkZwqbsOD~Bij+sT7O zf<85+_zTG8EhR`u&>wuk;N`5?vFan+w{tA|+qvV9`EE48YgsWqW^DDKfuumnuukT~ zDUnMOYbo2By-|vXPFsXlbh@*|+Xac6p#DTu)MY^UhbqmNX zSb_N!=uM)>o-HE>Q%^Fm)vKn>fjYe2@HA$F)7~h2zpHGx^G%E4p@K_Nj%QYt?HgvP zhxxPpLNxNP;v0Ws-w#A3S9XyCxnc4tZjdMz1dQ>ySdb}x-S>mVt9#XQ)n6{c_UQ`@ z1Y|oIz>YW~1Mo#IrGZRdpH35&gYo+dDxGZ$3+@TOOZ4<6xv@q!=?M+q+ReY9LFehC zMnHo!L+HS$A6`hykSSBvx#d}CT~$?a>22i;&y^7!qc_Cs1?TOB#>D!nm|Y0+5frcu z#in%*Oj+m)%$l+`CAO zquQDy4LLbeu~4#=qwA4erOKyKP6gZ6(Qin?Hf?S@;MD+hMq9xrcmn+vz90?q=EGwq zU(F=*A;KQVK3%;UVkeVc49N#rPU}dJhhj6&){0-;^p@CP&zB|6eu_Aye{w zK=k9rxMzX9Z;d5)&8&#F-J(H{Tv@r1a_;Q@gXBxcL`Gi2QzBQJ_<4OADbl0fArCs~2ph*NJ;e`M{b%^E&Zg~dOyX>tRMZ`I8^S->28zA#4xrQw3LX^% z+#U-2MH*kJ;I+OReu(Th7W>=s*_D-D(sp-O#N;HNQ>|O|w(k@8NH%0Ko+IAtmjdJ7 zkn(l0V0awW<-B>QA^cjv8q5x45uVPAqEVmt)#_J+u zOm}KsI%d?9v~3>&X5fbGs^?opfzx>hhTH2@0`P_E(d4nw(0k=)@s9qOT`@A0GqCtJ zYwo)od&RQbR*hBpT?OBv9wSCa2e|UP$vVYkZvw?xJy|)`9p&-yERlFS&L}u*_2@yR zn7xtV3@p$U;Q?B8@O<1XJ5Am1Bn5Sox!vcQ68uPy$5DHfjCXIT-X!htF_!_9!n36l z_hQ!glW2Z^GL*?im44@XJSS7+v~uRC_$h0(+}R2X4>%)1n)t-H;P z-mi6Dz6DaDOl8n@3>1n?Qv(=jE{G8gD1YRAQ7YV`JEOSxu8NDR@yk)8ghTrj3g|Z) zWmXO3`b8sqNOXS9VefhskX~9(=NLv^Cvp<0IUxMoX0q+U>CvD~_Lm=9Jc@auAMi~0 zf~p~lOyF4#AZcGC>c=$_T>vFfV|o!I+q%u2K_=0$T}H&irSDP~J-&Sbxz$&EGyuV_ z!^v#N3V}Nl_hoLHzn&&o(Kn&L+A-ZaF|nqi-EH&fheG47=SX{YX_wT;(1rYR_|!W zy}>~WZ1lS-5no`_i-sWW{oWZ&*Oll5jOnGb2mR^`OcWH;D_wMi3u-BNrW@?MT~Y-z zIq#uwI;WgI$Xi0X^LPYM9G|2q)GE-%`{>&DeG2s}$dln7E!Or%cvIzxHY%tazRRt7 zF29ie5`#|8hoW55ONG#>=RGD)`J*X(~AX(Jli2*Wsj@(sQOEDN>_|zaQ6bv zAKL<&_PNs#%qF}z!m-8LX>phuu_2o~Y-ruQh;xgtb<}0C2H4&1lKtv#o%o`5Zp(I? z`;Lj`#s$v4r)@l&c3;#5{aAKO7o(4Xp;7?7YPuw$~PJm(5@WsHE1XG_uIba3(0vnYJ($aABf8e|ve};IN zn2@1JDUg3B6Ki288MKa}@EOTu$CCv|&1XK)w9ojkj22(FbS+jcy?RY5d{i^xqii_q zfofS0#J;sxTO0(>29`8nNMg}@(Ot>FgP0}-6BYMn=)jA+Z7LmcO0!;Sgcmlj!&6q9 zMR$KV3)W25SBx8AN<{Y;jx|(`VI!{bS0|;De4M6D!cXIJ^G?5QxoWSPZD@UGbxc{a z(3rR(?k%XOo!}loi~xqOD)SXu6S5yxxLDcux#hHUQK-=|ey>|o!l#90Hk-dY(A_tz zQVWMhV}7zLl(;XX-Lopc3T?ZgM!LudBMQFD{$!!1%fN{RwQPXq_860gUc1f8K@!Slli za6!5V;|`sj+}>EyY>#{IRoLjwc_rgz)RykfEFS$sr7y=G;9ASRs_T~fC}GsBDmzeVaq;u8~|jDO`hlqp}`ad@-aMK8j)Wi8w;x z^oq>c>DYI9@af^$E4 z=}~S-b{@>&TGH~z3NV4E!)fSW7r^z7hA&y*y8 zB-ApcJB!&zO4r-PD$TmwdxEmHZWIy*tb1Riybp~k#z@eShw#TJ^U-KHaax4=Yzs)GZ4`(6(oaBh=r&&RIBZ&3W*UqLt z0YuUy!G~{jbDS^7Z`V1|*1!4k?&&l3!I*bk)tVs4ZG$u5=aGe^6(P*EN2Q?Ysb)W2 zt^yKu9&YiWb78zxGP_t8b}|tglyg0^uJ*1TXlB=Plfup0>A%$~Fa=2gps2PYnEbB`shU>87W6tYww@t4Hc983 ztVIg?Y1+DFIu1!wyrLA<7p=4wPG8>`LlaJtd7#BxgCpU-3$e>h*_buh0)qg*k&y>oRPe2^wxCZ^E#(oich2>5LbMiyGi5y=kd5hA3z*(*fn7H$qG z`cSuAD9BqKEG;g%l3kVHEq3l=ATk|lU`@_@!vELkh%XSB;@l0_w6dS zHaZy3g=X`ysN^SS9*7_i70wuWxIDZnb6o4eNs>q2Y#hJA8Mm__L-?hKLH~ss@?XVq ze#d=3QdnN8Jq!&0-{r?K&i>`bSUz$5NoOFz5nf^rpg;5DU3fi&BZAmmGZDCpvN<54id(&TYG z-;9V}QP6JwH=L_$)y$4)CVxfnvf;&uSq`!<^>z^%_+?^K5@H?EVKaGL#pzNHl8y3M z%3j8u4q=g5fSH2^H-_`Ku!3lf_i&zhv?BCmvF?+jx#bzJT%VE3gal=d>oAxGzwqZ5 z3-`I-IQ>Ev>pPF(|1(GE*M{O(eg~w)|4R){O|=ZO8(uhH^98mMcbzHcGjBqkct+mR zR9q305h~2Fts17q^OUW#TqOiAR=vsZQfapB?oXo)v6i#fq~>Z0t?sya2dX(U(Mo50aj#`|{McAqdV5A?9L)XB|&kfXIzCCB@P zQM-bI9EjR)ZcXlfyVgw?URLyvo0KT}HAr;0ov1RB3#3kQ@Mdp;h-_n&;^UVmL;ZU@ z6|-2%ce7lHq*uwaP~<=mNOMQGf>1Ma=Cj){cX+kr^DfV9&x)1~zEgr43gaex>#kQG zEf~M;x-*$J&y=Y7njm>i@&*2W#5_|Ca2EGePz}DouTFia!D|U_oaYBvWB2ZXO|9PsPWA$(P7@p>OGuM5l$$lWpKFG2fgpyi| zOA}8FX6@G>-%miiA5@iR0Z`8j9AqK67V(fg$U=^tz!2UbK+;5X9z#9~k|qr~OH#NG zm=Oayk16#uC9{^q=F@n(f+9kiCaPmGm!Wu>xM1tS8%__+mUpGogz1C(pD&uD}Gyj zeTkskzICB%y^skH+$&Vok4!C?_Qz8XLPxX0ZeC^NmD_6Cdfg{Psn^$~jWo zHBR>#__N8`{4X%Blt(@lAXk5VCvzi+aP%G*hS^i2sbN@ZaY7-s&}wGj-2%a0JHLr- zaN$=L^WPwU_219HUm14%z1x^CegB=FCF~(S84oF_G@v3SRx`3@cOh0o(`U&O$wT7L z?hrqer}o@iYo0v3bLnzZ&hvm(yPIDu;GW=zqp&<^DY_oHuU@!X63RSD(Mr^~c5I*| z+h6;`btCa+H{n#0!zIo!;TP?PV2mffV?Ymk$}NCt$_3R!eF=@s2q)5tZ5VT14(%Qz z4L=^}w+mo+_WA(UmCu1}aC7d|Yaork1{5ULr;2`o89uzveeQR-GZu`ce)GX ziO!xHipI!X>Bua?IKrT~>(#YKt#36XF7X_m*nIv81{_0iFSIDc_q30zIp4#}I*f016TxO(2`s zBSozsUrT47xsalgMb#Za$6nqNkx#vR=-G6dLJqWtcEC@VabL+(lN3B87gqP|eLJ6b z6z6fh@uHf@b2roXd#|}o)CFRc>({JHk<_dBwf? zgLU)^e1H6K(oDZ&PQRZJnL?)T{Y2O@mC{mySIZ@=Yjb^cJCCJwu))2}2iF!oKwz%d zq*zf;K!a=q_!<&!Q~izgceZL$vv*%uw|klR=giOxiakA? z*v`0m2J^locln!5&iXjz_58cb01e}y5NhB9y#-ani5WWot zXea-Cs|CazETn)B(sCR{5=)3QLh{-NPX-J|T6NSDJ9{lU&X`<2lo)k-EiCxfbLrmd z=U|i-Ak%ssK{Bo$2^keR4@%E!l$$3|TdqoeQa5m`u)mX2;W~29qvdbd?IHv~rQu7*f2nc!c!2sAyrWA{4yb8v;jR}KZRkBKQ5tcmq^r8s z%~58~l6}h+T5R{Zr8^Cp=0(gGz?8UF(Zn1jRJ%Oec?WzFDvxU4ZtCrwcik+Ivn{5k zzp&!fm_))`x-X*&VUO)EZQMlHf*xQnE>D~7`N_u4Ta|g6@1~l~pWS)eRiT`big20- z1rWtbiDeja+}zxh3bzrfx!qLHY(ItrtA7g!F&f3 z7`GmA6FsKy%sprp9lbmq+ih#azv~mhh2!m+rhGji#kTQnS>nxFtB`Gt^i3M0*W zvH76BPZyLfw23>l3JtSA#Wp=(kEr~x>mtoy=4+t^q=>2eG{W7VWC~#%)m!l5PUHhp zEJ3}eSZ0r5YtO?FmN14*VRb?GIl;aDb>3!ix<2>zj(Z9p(=*xidZ6yp+r9N-=L&(= z+I>%%G#GBOk-09KVpj8%egN9)tI^`kDf0RD#o@IfA-Vl5W#_2iwPFcVI7rdF{=hK4 zKdyb`^SuETKmR8L*Y_LP3!c??*Ljbg3^be*n(Pcfm|rU9;5q~^0ul*>2jVJ5Jw2PI zR5K!bw4LijxI<@kNTW)J28@f_v<9-{)_wX=)&$Q*+hA%`s|BGPP!ggdi@VR2CPyJS zU3(XPSikP>s6dGY*T-2ml-OOZ^Gj4H1*At0A&Kr|6OX>qpPc4z;2usT9mL9-GY7h1 za+(&2wxuD6^8BXZ4eGc3Q7@)>D|%U#I_h+kx|kh9~g8`16Qc zdr7k#v>iSMK+^gy@@&6aXu8zcVxNHOF4+q(*shm3(l8ij-G&{yaCNAV$(~A%sKXZ< z&{bM+;%AAZGtOqtA05IaUnQ9}6~+|A$!w~4Z20E%YTffS@XWnK=^IiWzir=AYgJw^ zz1fT7)3*4AYoqwP$jBpOOb+RD2SAy6t>Z{+Uupcs9ZR4+`0Olg-CJhgB3g*TkBc%0 zNvTyY&A)RoUn6_>y@sVTrW#~31>VP_5?W^B)o zDL7+=F|}555f}3@iB0`mp66Qx)ZPV8J1Lg7`)F@~4r7IP{vY72FGYa~Q024Or~_v_ znJJoOsk6M5;!rKp_2z(!_pTkGj&oPM(pSa59j$%_4pi_-sY{auan4n`QI7RgC&4>I z(ZyRUzF_OE_c4$4FRb-l&rpGIG|8_JwV2E#ig6Yom@CDkKU3JgFQoa~*ZQLrjsE7t z{jG=ojr)OO(tqjV`<2!ll*I)9N>nQ_yG~WhLoUNgHg)BPe9#SV2Tv2VI@r^bhYMWI z^oNCbwCnx%0nS)3w1a{io)6QJ9q>C&PD>wg^fazgnl(2ob>M8_^IrARm1VOrCqoZi zXX0<`%CxSpb5q|LdN~t*z%*!zIL_;`?T}zz16TCR0J(p_^I=&!RUXY00$%@}IPL|d zHGYO-60yp$qx=M${L|89(v0^EJzor+Z;ttCY;l%A9-z8*JhF>6F+m)OQIQp`a`lY! zrgFH9@1U|S!-{B8r$#TZ>zBvLbj|kN`?#q;3Gu02JBtNj^sJB(4k+R|0?Yc? zja^zwzPV2~HsBT6A3@>Uv(lM&2egKLg?vZLkRl5 zBZ&2MBMOqvpE8zNkGLTy zv4#{xIPb*H5LQ=7GQtd8AG;2U0CMJtI)D%Q z9NX%uX0-4)S^mMXk1tCK9@WJ@yk(IpZ40~N9@vVO0;Xdt21phLkP1hkf!Z%HL7X`u zDNQP|7qodAjys)kYmKL~qlk=Gw4=*=L*kWw$-w6F2f+kUKzdUU zP+|d55kd=HLI(vwy7UATL|Ot80x90<{`cDFTy?Lr_c>?%cb|Kg&$FH^@q0otzxmBE z#yj5ezN#1KYV+_FR?s3T1dYMZ`*9i2IK)Z@2b3xndKipwxaV(PuevE$kXxR**Xe{_ zLgnRKy0A_hjYp4h`dllVeDES_?#4Zo$r>&3dzLFgQ>X4?3?sT9i3R)=E@yIhom@guqMD9TvgL1$M0CAijo3XPSGd0{ zZSbJZ2g$?tZ#P^_9M=ilm)s^A1KB)Xqy=PQ_LY}@mjQ%r%H|_~_`!2#%D!JfSr|gX zPjzRQc6bbs4gW9x2W^=gU1qQ!CRLCvD6_0h@=?`NQCKcH+ucKEidWNyd@lx`MR1N;>boo7k=tuwlaqn&r0Tac zNbG7=y|?^eNR8?MHk&e>P*5kQM>1syz-jxk%`?36ZX{A+i^EjSw+!bja=XrYMtseY zwsYCqw*ap(8Vps%m0<*3(ZmdpLR|)Q`BQ_wUW_hA_liz?N#S8~Ry?(~tLmbCT@Zv? z_j}9mQ`w5Edix7X1%!b?{S0YQZv6uM`u%19Mb(Pw+5DMW`k!_}`LEyhAGZzkTi^fG zXnF-4z;94*#=wL1p)?QmnVR{1l9COWHSDWs>E5J+s(L&s{)et zkcA`eI}FuXVm{R|BRkN{IHy=!yHWg56ddHm6`2CBlM6lNa4Pr|fv~x=4^& z#?*Rq8=$chV9}F6*7*Y<`9H(?PQ}|dAci$^=a;i>_s_WTCBJ!N&*JfMn|+63E5~jl zsZ45h41_{$L=_i+h9U}ShvkX` zpJAVjN0?_^O7Pxm_UbfW^~#N)WnEa&`QQ0QfA%eZXK{YpGGKK6f9R=B^kT3ZP<@cS z@Y0G!4i7vuV)wpV#yt}67nZGsWMn;Ux-+c?MnijQe0YMoj$oeP@ZXjFV6CVTx1*GZ z8>CpvSc@#(s=lYChpmsg6?^jf?%BtpJAB}^Z^FPNV*o)kS&l+sy_!GQNm24zG7Dpa z1<>2F_mmn6?@Q835K%nnp1QaD$kb~CvMxPN3q}>2n~>@Y6~|1@znO~maLmA@NgmF+ zkbGY1ZvIfGdiSG8(30o>BvSb+^zyf#Ni(;gSESwGYoEM94mKr9}a{rj=K|IGih$`?>38<4`)#xS1w zP2@e_ecJ`3Yo0#}IA+KxaN_Lwq!ZU2U?+q1y9FUJRTciZ1A{3#;k>ormtv z6X|yr=ig4G??VANDL6hCQC`ss8&?#?2Xa~%Q%8L#KYe*Pk^QXYfdTOOID{FNcI z8VRFFBNnT?c}-cP%8d3U>q*Nc23)us7tU4*ERB_!=(;_KPC=S>IvIqP$8LZT(4aE; zSD^++dL{0&bq|hc0wx0A00|_WKknLC9MKdMQ`6{0rD?SAXBZE@r?;=EZg# zzOklbTAhFw_YmqTZTfY z2gQ~MZ7imYPR+cAMU`0`2coU>g##avZOUf0c;kk{P3g?5=}hbHYL zl)uWHjKU(67KXfYlJhqdmCtgeywteo=eYN-Xbi0hoJGB01HYvBGTJy`^gTEXKyJd# z!H?o8%#0d0*rp#Cr+okanQLPra9)!mr7egOwZprhbJ>w&a{$O zyAV8zhuxbY8(~1XdR*C}3s>UR8jm1H(zAhF3PUyU{Qr@76CIIR-W0=!Es%9=$j#O~H=9(H5K6^T) zQ_r7kaIM0qIYXh;qrrcEFbsa-zcmFQiZD3TsA^uZ$AFiKl3ORzE)lbUAgELMJM7=j zY_p%|9a?^O4Z&9@O7WdqZo=?{Dw6zk0#c`p54z7nwzNwi(vEi<#20-IQM@%02EWZt z|GtY<7(lx(nbK`V&!HK+RW~(hetjnxL|EJ2N8jhzO(R7#GQZC_t$vc0Lz1sY+(C*` zmi?)k+R>FHca6r+J;xtKWVpTu zSeaqYeXeIypH|j)7Okb3JGZ${KjLFrQ2qx(#BZVE@4W&=iD=|bnhYsBmtHyz-S&B8 zkXR=X#V2ylP3%rsy!XsjqgO=#Ueb=- z=0lI~u;8;ZFii-`L9fLEstKcevg|^|O&y+~4l1kOUvyGX3yV7QN*yv^mg~NZSgEr; z6)8GgKmVO`dI*p}(eaaI2{%c+>(i#uaKgT3sLxIxU#$kuMvw@6k9E;<71PY&w=KHU=Duil?kh9d_C?Yo_erm(gIPUuxH&!!pKxgQ4+(L9_bdJ!W9`sXII7_c=-hh& zF$EZqASi7O5a!QDPsuvHZ6w+E%|sF|S0?Lu$ZL9B$qPTf30!h*^-*v*wwuN`M7~Rr zrJkg869mH+$_|!2iYG)!3S^DPdJvqb25lfmfW8AOTdog5N5gr~zSXwYc3NsMZxjgU;p92iu~OO| z=yc-EIKuZm>$kx_{apSv{aiNO$c^-8UlGH4_}AYc%{9~u{aB-1W3l}$=u1^~_mvL# zsPjC@H~i>}y{2FYY=)Uazaz{q7#{?!G4VRQS2t5#uZOnma_zNp74}oKzg7{Q=TbXp z*0eYK1SEVm$?XFUU%6YV65Tq~p zspaiKw;BdfQ2+T@6JOJsVyEg5!#bP$nc`IfMkf@;z>xGMRD&kmL)(B{Nff}kO}p8I z$O}_W7>d_X_bJ+p+g!{K7Ci9E#!bm!Uwc>*8>hmq12lJV+wVX}tMn=Kr>CL_7@DLY z`~wThDOkrzPN(A&J)JHNbDB(4tC-lB|A4~Zsn{tGPOv+JF(38!M|dv0j+N|sxLb{^FcL(io@g}Wd?d%G06aN|kn$w5CCm1S@b9D2S|Kj#XSn^=G0x5=>eh>1L^ za7jSbtogb5mZ#vegQnY?c$xy}jB21659Q_;p#njeCcCY~!Kc&EU0)Pjh2@etF4~Td zRT@5ySC?sK=yDs?0|q?=rG4EUMzr!uG)EKfTiJfG6YBFkK@GAt7GwPgSA3~EVcW{4 zrTpk4il-1km|gq^pNZi*j zvF!Pw;mhlYcsRZkUIM<{+V-+<5Dpi{Ued|tYZYeJO>S;d)MqmeS&Yrw7SmpfYt)Mr zH8PHr7eTvB@Zz$iT(^aqrnRE5Z$uLnGfoxb+lGzd2x|xfhtC$dFJ4vQEphg_-{p44 z%Si_P=dm-B+Y0yBP>$l=YSI;rRU@8h7Jum+EKV`)AIf?E8{qT{y{I3n5>OHTvjTj{ zLDSO!=kAWQ?giPod@~nolScmW(pX#LGZoke%^xFBau((zCzH4puDQ9@!vq)tH1kOE zG*BKY!|=D{Dav-Z1}06g37ehV&UtFocq_oDgc+`=Uqe&{B;UUeQK3}e<*>ylsy)Sx z+(t@l=)RSGxL>?H(X6zjw=7>;(`r0B#B!IxE*G)kjInGZb(Kz3V+YthZ&2N6z#u2; zgAv+&JwJP-fuo6oA%LI>7&Ag?U{E0Z0DMCrkz$4VF8i}eLNnMN0l7YF2HNG%EB{=j z|GpjaF?_WFK|DLIH}whA7Shc^d4kwId~8pYxdtcF=TZC;&8jbR{ya$D2u2t1MRjEPD%d; zIhXxX_6@@0I?3g)ZGSelxp&mq$3J*+#=-4E^0^MG z_UlSWG(#W;&D&~lrk@#X8&Q<2gw#ELUS{a{ud>Lszdu_jKpn<)UwZ!rI4@xVAl&Q;c*$_o7duj-(;e^cBMYYyC zagrQ{K;LPt&*-kw5W_x<{Z7y|W=IZS`~rv8p1}k~ zj&vR6>$ZP3a$4L+ZL3Fl8%x58>qB1+00NvB&NxUBJWKJOHc{0a=QtnMAfc(dQV$bv zz7}94#Psy?+aeY}%xnmryBNc0o-9W`CFZsC;>dDO1C-sW*!D_RR9@P=@mUNW4Ty36 zc1H)pUjx1q_hiQRqW2-L322WS3*I_Cp zolbt_%GNVtZIE+(p0}nfj@ihw)2Gu^Vy&?4=U-UkXC|6QkfO=MDbgq!nFM?+J#ZVs zi4;2&5#Y@mi&Nb+-ci(w3UaXr{JPude|!a*-XeK_v0Bf6xl9&I<9w_KDS+Sq^J?Xy z9OWs8(pzkmrfy&wzytuJM?7K0uY@rW0$Z4$=Xk^ZlH)UyGOVhdfxOP)E;O%t+CA zpOWL!0TE^q7LJ`8LGjR(UHobyuNWI&-d9|77j2tdD^;~p0eg+e^W*&uko}`%VM#@6kgyKYE;@MafxVWa6xJh?-;O!zDHYP04Y=X zHaHj9d}qGmqJ&u!RUX0{Z#CQTmT8<4EXj>M8p0oYQQRhS<%6N4&*A z(*HHHB}pxOKNiSnrpTw}9o)4?v83qG+j_;KL-jlKAt>{^=FphXGlWSNO5hn%V5~oz zUxA3>%_YyJ!d(((ExS6DcG9X->t}HLb@qUd4EC|J7x&e(a}zAYnTn~ex0A_5$serW ze70Y`|1NQzptlG}Fl81WhZ?#mGSG!N60F%gdl%_h#)}~KkiqR@DTm-5#HVlbH zwA|Q=Xftg#9l6zDr$Db=gkOCKKh?jN*tvU8kBYtXq2uy#8V&&+ha#*_*?edmMFl6E zFr0Dmkcr58M47>@I(6i`iJbx( z4_KRZ`1(A`UORObSv|^pD|zQs`{~fCd&gs@KYc6=lxF(c-GaM~WTH&tWlaGQu|QVE zcIWwv8n86O9J{kF7kyG5f3o!Z47jhvp=K0OkM8G|0%N6}uVAzVYHlSzn3_4#t#8j) zBO%`J&idZ(5j%`%0c`aMfz9!(hV*Qtm|e@1$ZHYv#0&jvB33MENyaQYb<>KgKRNo& zmKvU4G@@as1@*w)@o{4x-Pq1_7le}6 zqc@?;HcZ=%}+-F3rER$5dc3G}4dsx=* zS%klzT@)-tzJrx|HE4&NC7r(anBw!+jU(2WL)uNmWv2S!ltY4u3(xr)hlKm*VM_dF zCZSKM;Jk0xPh%yGwq1{Efkx>_c`?stb{2R)x6yu7T~t|ldG%CG;?5?C_*Fw6!u`}T zUZa)e+mpl-uQi5R(vT8>$+@RvT9bpfvQKtj8vny{_l!Jq?&a`o=3LDo16eptR09u4 zAxmj`-ylUSh)wf*WCNPO%hoB>aB5?EF;~wBrF^K947I<`wCP|^;$J`%o6>eJqU@Jp zL8YzUl>VSJ=#CMN1F|QwJ-BL(748ZojhWPjZJA~s(#~HnO#oS8Ek9L{CO3%ON!e8A zzW3@7Om$1Xx0mwi-3K2#aWX$;>fdqVe0WYc6A4Y@U7a!rU&39HXSeQnHDY5-5bQ6& z9+jEfRpwlmP2kME^Pzz6dHnDP1WN;f;H?b_K>%w1MTzlurs(?AixVonJT!JBeZ zPz`qKXk(H9T6yJ5=Ic+%)r|L6_qLo!oAc(?O}Zy}!Zu;GsvDS<4G@hPhHYV}1*Tlo zoP__}S|>%mDHgxX>Y~GA^se`FfI#`aD>a(!Ba{ptJp(=@{goVOP_ z&pV4<1Brz74-I_)hfX1A&rWtT+O#=THRCo;-9yt8t_w2mU@>##-#UW#o?Wm^ro{#JAZa~3KY6mcY9OLaK zYvQXqnFdFeiG9^YDFg0H`rRGwsjPCkOuarWpR|4H{I3Kv+I=(sRIZwZr2Av??VCqHD7hUMZuGBnJiR6%*T{ zzIJoDW(Zt2d8L}P)Vh3#h38p7givEb7*zMbq5bcWJm8_LtWvaEAZ9QzX7-kY`{vFn%=ic;ioq_0KTsP?m87)GI2q%hY$~xnA};2oNx_{* z5AC|~Fn5=B`yK&^nW`VUG_X04f+fl~AmI;cE`GUwUA9A0TFKVta7E>nbIdbos@kk} zx>^<$7N0OQUUVB`6v0|+j91SX@#6gQ#80v$-|KnX-3w=p5ED((q7d#S#Xw>D$ww}~ zv6FxGDlOQY4eO;@5ysCoR)4fZX1yBecJ$xvUB_|zaC_C_y(`jJf#HB6*i&wlHZ%GI zoTtY*JTBUK$4F4pWRMSDC@MwX0II^rmc~QJmO*2y zc~)ksY^4ykw-YBnbbQoEdKJIQ;V&C$JDR8tEEbwh_sd{CbVe`q<=Vs=x2L!`Ai%`s z<9kuXEIdcEcm?r`Bi&*J^gD66trZNyq>sZU-CLo zwAap%;3Ud3(|(%U8!^a_l7k#@JNq&K!DiM@LSf@!xBZXasZ`{4Mp)g@(W8n3ovHsG@ikEdEhER$2u3d=_6sB;Q z8#=5PzfY|b%`j{SxI;ZD zj#IFR-Ljc8lZGV8is3?uQvU>k;I(T#rB9zTCsTMNw@ ztJU4^uN`mzv-I)uK&IQhy-MR(*w71oaL?`$HFG+EG@auSQ)-a~)0IO@g?svxyN`Ir zXyzBRz4K@66?It>e5wE;njrTuIuJ$cEr`)|<2uS5sdVZBO~csE#LY{FishJpA>NSQ zvFc!nj}Quf4*3}Vm=9ve3g!s6dQh~faEhuT{>tf89q{HleTs&K)xIHHWAN4!I% zQ$^UK!7hI%b~Ojjj}{B^3y+;2(R}DJbwb0TX1xWqXY5grhnxJpULH&LK0PS=NxZ!z zl5_M4=k{5xHB2Lda)5PVD;_q?T8|VUA#Q7#yqHfHo_Q)6*E1<|C?`!nYy2P>?#&`JSM>dN_A2oUrJ$)h}VC=~Q_|rGApf zwUQGz!yF|UFZxAS-=LKm3nkKH?Jz%I3b300>fIkeD?Y4RQ5j3iswzGqL8AoZG zGL2^_yq5&NCR4_x*U2Nb`QCXpZ$C_avN^P`@`~bdhKtbj7LaO;+#-k2lS7?}fh{`x zwLP^5AGxWHc=T60wmh-2w6h*>+Nr!>HBNkSV%ulNJU1Jr6@3T6j|8M50DWDJX0TJ> z&1ZuG8=g%qiP%VH9xskMUE(wu=+EbH_C8{dVtf6mqc zlXVQYxKm_zGs6**+<&0^D56INrHUXPM{NM~atH7jVG?BA%S9QL{aN!bLjE|k5B2T+ zpEdu#X3hVr$o}Kcex75lgi^%4Xqnoxz{@w0ai<*c^cd^_x883I1~NA3z3{Kq!auq) z0O_IUf9>&5rl;E$_EI&Gy}Q*CrvzJM50O``5_+X~kU^53m&~EL=fiEzPotSjDCflC zbNJYQt(t&e%R0w)nsw6N&N!;Z!~j1sE`K??dI^ zhYDTBj2h?KT-`r4Gh_KcdF5n;>LYRH4pyDDMMM#7@#z$NRKySvil<;))37lHXj9q{|=EB~bM$v-Qs0{giL!TnqZ z{N;H^$pK$?6R36nH;9|G)z?LxsV|bm!R*{8(I3qk-BQEP^E}{bxe*9*eZyXVTlq$u z0K9Brrfk*A?_su|$L_`Av00I@E=46oRsUZLK44CJm5kio5=ZUtPg}4F$d)KUTll6 zRdSs+-R3sI-M_=gt?+}b<;hpq3dH(Nc5b;Av^UZIYhiZGHk3Mwc=Fx#Wj)IHG?dx) zdE8YE&6Cs1Jl>amg}1Ue*t_@g`X;d2Nv}V_L3B7@p}A1_JmKj$|x!)>vI&faf7A(e0ZnAt=X1M&rZuyOH^!=4}8h8ShHt7ULA$JG85rjEZ*A*o# z&695@Shh}jNp9?}osC0wF`9LbnBGB`m^NZ|N_(Leq}_yU*wsv)-EutGn8vKA+g8zI z`WAi!WSh^dS;=h+RX3h5jMJlD&aG#(1}q0)yY&*+C|w2@or^b_8QWQ>g8i32Otqa% zzKe-mdu?i}D_YDUF1f_U*qN2$J$Gv&Tmt#HN1u=F`vd=nX?2491Ee-_$VQZ486wUC zfrT!tUJiWO^3qoSxt+Bu+w7S>{{!J^H4*zdQzR0=tU-rFM6}2?x6R$a8Zy(H>V#@A zL#j~xb#_qBH3+I|6!db~#3M|532{co4V0K_Cnb#`t7W z_QMx6iBy9A(u z(yd#;kcTuFi{>GPG-gJZl-0zvog%Ik#J?Roy0x*imb~dBlajONNwnmHWuG@42SiZ7 zGsLthdk-;jA(7bKRClOP&hLhKhlyi$?}}jflz{8UOA-mXLZ^GV*;q3-dx3K$NH+7} zCFyO(un{@fNCbcusb4EK+S_c*#{x&UV1TwUH~?qh0D_t06rP z+^zJ<7{FQdT0~H}82y51;kvC9&(&sYW`ym^%qeFN^rhuO!d&#{2H_6v3e(A?4i&&Y2H;;+V@kY*G$hS}&e zjw%XvUk{o8yE)y~OWz==k|Y!zIS1Rc<@)g&fUALz&jNb!sRp2WEY1R5^d5nM&DXzc zNB<9Yq?rEMAoa5w58~jRJlfFNbzP}}XG=F*x z3jYAM6L+;cJ}K0c7!{*@V|T0U{vlHZ#Eqx#R`;Yh%Tk{|3`pM7rj8$~1k)FAdwEf0 zTQxksY*KhWGoYLymECH28a;dE@YPgl*3~oWI+t3N-Vc@0uF@-TPMG2O)`NKYSe=8Z zCk8)a6V@)->X1M0+V6DUgG_9H0Q_W<6Z8WROkj`;1SE7B_F3@_qMPjvAF-{31mf%5 zcX??;Or7?Ehg*T8=+f-2Su7-k{zIzvPo!4Al~aHJ^IWe@Csi`ykn<{vsKc4 z|Hv(?BhmsTr!sw8x2w~L>B3QoB*;5RfU?C*9!)!s4C>a$iDyC6=fV)6Y}q9eso%Th z!ZgEy%kRt3cg#0EXtVC#Uk2BO{oGBD&W@c(asV_YhwSh$osUm4Ull%nA89Vc;=`u0 zF-J8(HJsF8WpqyR8#E$z%?VHBy=dU7UH5e*))Q@V+sDE-N?W@c6q*lBvs_9|g244$ zUXs(z5Y~D(u8tBa^^-+d94|Tmh+*5Szk&Ts`urc!yBT}FpxAY`Q|70fqP@5#9usZ| zP|s9EPJ2fhc|3OIabUX;ld?_slK1*PoinM^UJ?wzj|j12Y7y*kpBS2+55%-c1& zu~FI&CGQun+{0PAja?A#H!JGEyd*3g3wiq1q(zMur#JH2Rgt8pj!pV4pjM zJ2ayCc$p#B>UovkC%+50``cOn&%84IgW=?FWW7~4aJnJ@hE+irTGVp`fg=jHqgIa) zFI%g9S-q@x&8sni2}xX z<8#LDBXy6xk?&=a#{_w@qI_<~#6`4TQ|hgwQr=Mc7@cTUaN!-%fb5AKSg7L&@?!9O@E?ja9-umoPi77h zZab^_p0aQWpjLJTgVxZ}R}fJ73FtCLseBa92^X{ZsWzXJtiw0>DLNzO=AjR7?LY4D zv4LE=mHl9-YN!oHG_y6ON|6;Q82_pvWGXqd^%|Dk`&b-dyaD@C3uXK17-fu432s1JA zA7}d=i1OBik>v#pN%+nA!-=N_EibXPH9QlN-t* z-9W}1M(6SPoxJ65)%LF%WsFO8fIt4$;T6CRec$80>K8y@KXuc7z6cP||8|)C5zPpu zZ@qu!TK{UfRx$196c@Ddx)>N=uQfq&r{~OFd)+cH&~%3oxvO<-dEwZ%rkdMj{6YnBf`& z-~^O+#`X=p6tq3^_Rdv{pi+Vvg;(sRgFs6n4xIR{@*~WEj(j_IYBEM{ZKEeTQb@$H zpvdzHhaU%@-q6+Xkd$EH$c-5R_JLA(G$%d32~ir52wDWEpVXvMozEKFI<42|rwWu+ zR_Pa;U$>GFPe7>xZ}t+T+0fHz93)s<#pH+v<2BdVy6=MFEwxdp3ryndFJHdog8;oB z(gnm56e}n~5oM_Q<;ZGSBWA1hXW1u|>rycv#;r_bJ~Ht%nmI|a&fC5S}s_7%jEpcO&*v;U` z12mOPIAH~jZ3B}Z-wDHgKs?0+f;v}bvC;x;(2~I<<7{^moFaz?b|)?%-nKJ_V5_qP zG)>SH49)>rnbSRp4ndj@7#Rfif^bd%vs9xPMo$?qaq2}d92JQo+Jd4@;yvq{=KqvW zI=}uODR3a=LV=A=coOnGt_;xx2qFfodn1Q zIRIRJJ2Z%XzYd0L@`@ZBQO-P-UWzD1j2LURx(@KyIc97dJ5ed(!c66Vf=iSC`Suz7 zw8p@4CLo4kiTmmP!G78fVcKv8dkfKE+Mro}BxcBt$Ql-UbjtQ{sOIJE)qTqOH6K`B zs5Cez>WLHrF=eIXj#xDs$N3ZVwXGz`H?e(CG8a$D=hsg`G!(^cH| zJD&)$TRH=oo9jdMAYT06wGG8sf8?d$0Qu@BT0rXzse{4JN?f0+i02(w!R!da7^Qu* zK5->X*@I8-ijq3>RnAX|D)c-5F2@In#NUHD5qw~+^})0Fw(bz>gASA~9bPwxi9zAP z$+6;y;D;~3SQ+;kbYY2;L}SFTAQ?r^spXn3aGsWrw6Ux0!HU>~7}(5rn98Z|b-orwUcYWh97>manlsk5e(jCRQgK<@@*Y=n!w%L< zOK;6YJ@#yyL*e7HwTWuC7$TvDQ7v`*`pS%?2HYo9>S`iCR2S52c`NDIF4m5@0=9E9 zV+?>8uNcgfDEx90YNusTQ_Jqdx2=x!Nf~#zNt<7DN^zZ%6ZE5UW5;D8P}2`sWY zUP2fdSnYF73@+kg7v`1i=;zJ-vF^RyT1!Qvqc*x3&Vg6>&+s{MqIGA9RY>j4SU6lHW&B3;VUlk zOBQvsvUK3z^O2{4pCV17GT0{ZrtG1+2WV%A&dnoTPp8J&2bP6L&xZudVp3SfWzNX# zI6*S3Kb$CoFXyS$ya&g}fkAfEowY1953*~wO#gmdS)Y4k!Q<^&sNu&6`Y~U|EmwAA z*!!I#T%*kE!s>3QtE^1MqCC*UYW177nSys12R~n%tRIz#5kX^jjfwL6dP=a@uW=p~ zn_snOaBQWXr`&2FYrq!<@EjYjt6?c(zEm2?_ z$xn&P(a2r%>(`^A{3UJX^`tjWkl(78kIfA|(UQ2&Rs!ZFL2#^Wb0yh}CAXgv{71VlSC;6y#t#ugfQjxh0n6{h3Q zAfRD>S{C*;5SzVeQ3*XTb0mDPt0m8!rY!*}Y^-ZyQ_uZP?SXaEIj`s0JOKJwXQ3 z(0Wt8FJ>;A@y*F27mfGDc5t+xR}VK5XDiiL^xVS-d0BEqJdKfna>m6FLwzVx_{ms> z(by9txrh9n?{D3U!8=OnYke}ma4Ba+S7;_oJK+c7=Z{j_zy1A}gwXcxMa(okkbt1^ zIkzHswG@iV3kU^nr*T+8Hiz5qwmvLw2K=Jvnl5t>HY0B25D^5T0>RfW@_QwyG0ovtmWge z@*oTm3BtCU2;LOz? zE4@owZ_BdioZ?)#E5F|X+0b1 zm+l8;f_5d%s)K2#5&nNTkH->$`Y?=f>fDqwy#e)=CKsh2ZCPRbBIa}$>yOGc(z#?G%rUNv0fJWOg`0-T2?ZqaMs{dSQ~4(X#uhz zsC+`Cn|)q2oulsHLgASq2}f&H76;#69D11|1zS8vTqm26FhJ9aBvd$4+Fzz$fh{cw zGu9rMy4d>p=KYs@w%4Wwo=H4+^Qa=cP?)j`7QGr6?+#}lTnKfW+S*Su5T;e{PcQ#M zD?h^f{8AG0ywWp+ajsq*6Ur4vIfPl(f9Z&4~b<-=ABNcFxgRMro-$+3}QS7oO{_(#qFI&=L6m&Zxl*d|_yS#RTRVZHdFN1S(CH z;^FbsDCbRx3RQ)kQ`+hD@QlrjRgy=u+s*RV)Blre)c-lv^nd-6GQx2V&=@fHJxx`k zNKIkl&By&QJ4VNhePrzq>k4aTT(P~=+06A^qj)AvC-qmcIiA9)!<)5qimHZ@gt)9t9gSk=s!G?@P`pozV-cym;S5arC?eBPtQT&@54$H+i+O7>aFQCX`Fy* z6|PJnDY zW}ym`IqQ(p*>m3^K;0E|>3qo-C`SuInJLT!`P({sUsv>d(bV`m<$c|7Z|*r%r#U!s zzpTIOKgLEVVcN4r7k&^@O&$SdaxHW_+2kefm2BN}m8c95&SdTkm%tGcwQE?*9nciP*A+ ziBm=F@T@(mdQatnwDbN4XCywcD}ar;>Oa^gY)>?L+#GL?f3$x_BE-oW`&Ns*eqz2sLP-XsHIYBlC^8ujb1rqA3i3%R@_R zd`uL;srkR<4coiJ>bPx?-QRPy?rpT)r2{bQs3tqv9)iuAeLIvVTq51?Hyybce4pC( z%x!b#S0V2oW7CG71E4>^rBV-3UVMXGqQJivvQw?=4uVH27Tl71p`bJ_3WaT2Y=t1V zFdETC0_i{_4bh^ruV2emEh4L_BFXyBRIyF*flQ_SQ`Ro!)4N)^kS}Rc-$NdwJDY9D zV{Q44@J!w`uB6DnY+37g#D&8>^b7mOc~Is`ol<%$N2Q4$p`Uw0r#+9b=8cspg(a-QtNY9o@U)EUGx zW7fgdDxk-KPmU6?GCl}lyt7R{lX+|l#CYd>?hVW*6l1%J$VJ9zN%xaDOM_fDUIja} zvrNpxYEK7D4k!*`6d8DAXfQGG9MPZw!FjdX;7+Z+o8-Omr57J%%+0zY_Nz`D?^RCW zmppUdpzPrXi7T(60LY^%xlKTOwT@Cx1ifwa(DCB6kxPGKpm^6+d1X$erK_*&jYEP; z+ye)3o_mfG%y=~Akh^g1^5W>L;g%!V6dMm-O(Uxe0qwz_FlywzuR74B zfw!0!Y@OU{M*~S!cyLfYVInu6I$|=SPOKyMoljUqrm?JAyDOjCkt?_3R=ZV*6^S@J z7P%v3X~>WwP8>Zqv@DmKAn9RkxyYfUz;q@2!&q5dGXd%U@vFA<*BUa|kg3KL+&9RT z@`7)WoNis%J=~9QU7;z10|b-3-ypYuO#3{z?sI_uAgX~2#1jruETc{tPTxYV0*x~` zEs6pC1{o@>04${8_pFPEpAfUzJn|91sw@MgH(BAk0Gs}##aCIX3hGT2O&Va+=fKI< z%jUN6nVtauBrA-g$LWAZUr%5^6aA>g3Y8eg1|Ff}^#+}bl%eK_@R7(>_(PQ01sX0C zJW_Yt5vULIQ*ipuU|lgF{4osX4hTUMCIsARx9FAZ?8~)TsJW6FOcbuf85k9r6Imnu z5v-xtD$3r=D|Z|$`;4l<@~yo37$M9uBFiRXe@J0FbN|pF&;f!E1Lc2FhM)_YhavXC z=w(oBz5F$&lAFfe{&}`6iW?`HVs~>%Qkt*cQOhPPxt+n-1PaRjNVsG7!&;TMM z_r++?Rh7~{thS7SY(4haF+Fe2m1ek4<#7j8<(9i|mo9NZ@8F21!LS&RXc_-s?Ol01 z)N9)xO(jcBmI;X=YlM)s7_=aX7Hcw5p(AH330cMxk~E=mlBLpanNE>J#ukaPW({LF zk)@eIGs7(2+j*YOGMv+SpL06zc|NZ{<};s}`F-a8{qFm^?)$pF*Y~CdI;> zvNPRXwqBMv6f%+;G z<4e;pG}v_Y6MSY8w2Mzo!G?(jZmZFMlXfy;Ta}|>=6$D5iOz!5k3I59LQj{oy469# zsIR7x#4i~cKXD^|!P5Br9ikRLjv^BgV0$6&Z&)3oQdH`tp#~3(-PMx%K}IltJ+ZBm z?~L3wfwTb`!vdNe=ebuupGW~=0-=okwShC=RKf?`yAG>Ba2QNk9Hx>d?KChj|MUOD zWq=0ebUEh;hEB*5idFSBwX=neW4#iLRoxplw-!2twFfV|Q6CwgoNrnDF>OF%64jy- zA^@bspde$CDoRgZ`H19iDQ}{|UAHFJ?PJiCjl#5_5keA}0kUdgZy6e%t50-OkwSet z(uPiXZ5(VU?5j3-jJj7aZ2>HZsG?}RJ4)Cg+s&iI#$QGi#wV9>aeqeIWpv#KRQ7>2 zbLj`}li!2k!k$keJrFz}y442*U3miBao2)RJ*_EBY2P~3h4;(vMvk%c+bS_vsUArx z9o6IKM0AJ5UW9=gEk6jOW{w;8zc#F+rSSC!ui%|x4fHub%G<*&-JX%g@}%8~kh*zJ zQS@_G?E>#y^?_ZJ3}ksKi`}3())m@X?3d8J-mN;Q+6NT}dRzBJ;Q31Rfq_fZiDhpL zJl#^2y4{iA?4eV2K#gp>Iio|tKerW_D<+!Z!J#QVEBTA!}U0x;W;Yb=Kcp|S^M@nN{GwJeq@eGF$$xPyvtbuOq%DOe#es}ro^QVdq zJv0>}skTVhYDj8R&drcGHE@_CYYbpqe?}w!Y7PJY_J6o`Oy>~c2izFgu-!qdAg4g5 z&4t{mQBjz*UHW)Q=dJIOMcw!!>D2?x&oe-R(1MjG+vjD(c|d1E!8QT;_X|21H7_-g zc>^@6yir&KDo|9;LbAq`KLvmB8b#<7hP|R21n96p^^a(00e4v?`_pN>B8HHFpYWh% zQZ>Ip&aCgk+ui#-He?}ZVjg}5{ohsbn6DRj)6oSU62h4PiMPw1h+XIr!jZGBi*VFG z502u@gZ-_~^CjX>=Ep8H&_blrf`v%u9zRL{FSKypcR475Ky+oVT9evVE#h;HZii;A zrRq+dy(TGZCZ&f?B);T*`0~w)+y{!#DA3xfF9y#$#Yj--&y3D*;O*>rDW9sM@kV+3 z6(j9fAHlr$(C|Hbbn_;F*5i~ZnPm-;AUDCEVFd>#eOW33@GXSR2qmp2M}xD^4kX0i z>V`j z3}>jbUm|%hN9!cQD*KOoqk$EyMay-M=0I~C^`L|d4Zm7~^|Evksv0Yq zX^?XVe~YO|9!X0M49*MY-aWsrSMM%>OTL)wYB zjsdD2CGWQUW1>}b2BN4my|7Nhz6ScTYVh%QCg>yYo|uzRyY?DMZXv)rM+tTn#fhQh zp|Z7zSTsatT>Es0fD?f_SuuSk_*rs>kf1Cc8TZI^|)hVU@9AXAmT*DJ`KTvOp_7cJc?;rzqNVqLGLW8r5sf_4=@ zi99u=TU)c=nrTGeLcT_<>A>{6({;=!4lO#YOCwh{^@?uoDXwy}alJv!Bk1ai4J$V* zUgtd>mg#b<=6D2E+AlJV`IG}90ECPcWpQmYRE@NSEnEZ zm<9w(m?g!g<+bdy+$ZU1_-L(9re6I>1kF78`Qt0KZ+t!9f?z>OUIPJL9*V3%Q%ow$ zQYG-`Ws}UW*kDfs_5E z(b*$@Nymzls^y9a?IpT--}cwg3F%GPrV|xCYMiW1Sd8w@kWLz3W5lA zMDzka;b-4aQifrHnuWN9y#ZzR&|Ln<$h|q|YAcm%J=bsGmJxMb6?_T@s9A)V8U3@R z>Yg8kA8W^YiPgS?h~VT+cdxqxQ;2H8H(?4T@2^S(&Ed8eryV=`DCV6%qg_Ycpxwx)Fl*ogiem?rK6ij+_jHwqf}*9( zq?rD-I!=j~eIhkB?KB>@ohv*5nz09~)KrqXCSM?6{xK7IZp{S->RH|b32zfxRd*VG zYw|+g?v(z7xAKfKZfL)CVi*))=cJ2XPo9)eLe9O?6!wJ09eFjl34M77;oED0OW~p} zEO?`n$vNdH`fVb;6v5P_xL-JS$)ps~j6>5ly@MV!nNX{%ChiIsDI!K>HUx~l^+8kt zTET`jxPeUrW0ejR$$RI_WUDHOx(7543zoXBo08t+m&y+JBxiy+L%$rJet7`9PPc~@ zf7{r#NQSF`r=&i7r(gU&jTL3Gt9{iO-y2<=JI)qb>(AdY0WKPO!;|s>`Pj+PL7~3w zQ?c^l2OY%CR|r=SRavi~9hbE}1uzAVR7}tVsGG>t8=EbK0%P~Keu9Xu-WB)y!4saX z0uY&q#5a1i%zE%&g@M=D<~30+=$RFL^&ptnE_y>6Px@%uj6Vr_G@+n?j@234kBz$B zJ=8g@z0WFtisGvB8O*PRcae2I?n z1*7JV(KV2F|J7vn=8wh=QWr_cK};MiRRetA9+_E)#SPpm#Bz`nCms?29P>2fj9~@d z4*%tg!ja(U;q={q%SHZ)1NR60Ac&#duk-XUR$uUza0?J;rot9jo$ITh3wiMi0TIx} zrho<5#Utte^N|F$AlQ2hn@@fX>_3A5aP73)x!eB1LAWqdh*}{?2@($I?ej-t`Z@>nN-C zhKRz4Fa7QrsCctT4G~ylK@TRR2h`|y-#X>E@)9kHelCv$B6V;v!v)AoAlxVAAdoU~gTH=x8x8E6|pE>wlRGz7f5zQr< zN$t3x*sAm(9DaZw0wF;3dD%yhgk(dm)>8Gbwq9xa08z^UglgfDr2VNo9h@Yr9FDKY zzLyQ(rBxXvqj}7iOX?YB&y?81&o!n3%wGMO8>XCZQ}zU<0AvW@uJ44zNvu4aS#a&n1e zn@i_c8$T;w>JMib0d`>wi1A}Ou++1xbf(`x4~Cc#jlS#XnXxw0GG85a&?!GfZS%Je zw(sfQDLl~uC*O_66H@!g)+Fd5%TPkP0Tb!xo#FK816#TA=lqO28??E?F>Hq+^kVFnazuvg&4nf2c3kt09)!Lrcw}~K7b3N z2dG8;hmq5&piVfL={*!XrjQ>fO+9emYl!kZtyDUx$-%|Tw91y|C0Mj9QY=sOo-BVv zJDi3bZ(<0rn-Jn9E6EB*bpIO%M?}JwksDk5_sV_B=YuySe{=1?tI-lS`5%1sPq%-x z5$0k(cMPjUk`L7J*`GM!@8#Il)wxFZCicavxV2Ih4<7VDgt@}GAvRdfg#nehY%ml2{~gmcLOBr;!se(W1K!BQIoC&se9PvZ!g4sEg*b|}5-Ra@p@ zR=!j`XSL~G6qvGdb-Zg|7dQ1d{d*#AO~|1pm8s`j4m7R2T>K&!ee$(lQH&Ejx+9_h zrf`n+=wEj9PX_H5D!?ybZ|5ywe*%h6qt5*-g3gJM~#F`OZY>#(;YzcUmvg>+76 z2O2zQ!jp^udm^@4UtCk2%Vx*(`?ASp5PvP!CZJUZ0r`}_6GPaJ89<%I7uYccaRZ2& zhA!@Eq)5mXVh{J8mJ(AR{)gQ?;y8mf%7HhesYZ24))_p6_J&OIARddKG6-~^d=5nK zG=z=C^s4~U?8);8(0=d*l0SF$m1?X>!95AjZC&!c-=) z4Z$U!`ZM3qKzE;E=nPnyGlhUZDfS428qUr_s#$)O=ita+-WqO*kG(F1LSx6`_}Kw?Cf1ax zr#3)h!JNgIB+RsAI5Mm6SHpU6-W5JgjqZ!8>6?)((VPaH-U$Q5;&d!d$G>HEQ96F5 z!i&?fI2}Lj;EP-7;>q&!L92&3D{`t3)U3`H&)$$k6rp597YrO8@mYCKS0*Y_9o4D= za)KHFp~-9(l7Qeb)CoioPZbTQ!FufjQMUQ53Ni0`uHLc;f?l>HD}B^0t6hS}{7e`6 zw?74a%8 zD~@QXsbJAAea6xtZ{SKoi*HaPKCljl)OeI(#(g!!o|PNM!&IQ#XTotM#-7;@s4}_c z@tiy6s#+rc{BgYQQocHm`F9%9bh#*nZD+oCrU#5Yh6UUnpFl*1P&=ld+W_YfX3|t zz5s`f6o5Po`~X=piM31xLz=-PAjnC90qn>tcp3{?jYlBAWUC?68Ni3)QHaG1Z}6Oj zU|N`Ss-0YkpAs&Zh2TH1wdm8FqXJ6aFU>9^Kg2`KFNej_&9`m=TZILR!Ok2&VL5F~ z&YR+1yoVnR>8~*l__-3!efd9R#wXy%y2;f)_+zz+JX&#MM<=UN1YTCC=-F_s8dWrx zGjHZ4Q1`Oqh9xM9?DzzXg$Q3LvrREi|u>3pYXE94bY zySd4XC$8fr4)z0y*P2`AgB4w&eXIxgw{ID`!>P-^_oQOh1Y*}#7d>?~`DP=Ud%Y9BI4ys z+wwm+IE>>$_QLD=U;5svQK{P?HL*%pi;9WGxFWk`EWvhz4$PvSeAbv-bHN`iSaaXl zLoN5bTj#L?*XmHVy>9f=o{WqF#%18eZ**P@E(c;-ZV@7O2Sk(t4M-=ISB@)mj%~Gq z=WOW~zjoA4srbV3JsLiTxHVdadiSR9wWT@Owp5O z|8A9S`|wZORB)NeR_{sL&>``m>EJCMO{K7k^%ek<7`}Ee1hqWp3tG zwfI@cC=D1B&BLAmFyl;php?Rg;+y-!DD~ex?*Azy{~taqbhc?xn*Naz_1`H?&5P2s zC{6$2Uc@r2xBteZg~VN18N>%*k!?&LQwv?MTlpS29e zh{<2*+11@+y75e{veJ2_3}$;jHU>|~n5+5FxBNK1qZbz3!&@0_P7Q_$z3^N`t$yDL zf0Z{7k$gVQ4ir`TTCuy|Fk+FM5nxcJ=pW!%E#B?di{dWM?c&@X`Z1orC<%*_uqX)t oikJJEteHhEThy}O)v`YcBr0{x0}|za2_(*?n0?I`X14Qx0KomLbpQYW diff --git a/src/wiredancer/WD-SV.jpg b/src/wiredancer/WD-SV.jpg deleted file mode 100644 index 471b3ba575e827529a9a6533405a91426db42f1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 676303 zcmeFa3piBm+c&&KQPPBLvYU!h2^CV3Q9@ElJJ~grWZxtS8M9=QWS4{@rbtvY*$bKM zNt)~#F&MJRFk>@@*?mj*|9Rfy{XeoBd0yO3 zZXdArsF8^gz{dyh9pyU;0Nj2c+|b+UDgYcm4r~ViKnPgQCkF6CkD#YOzhr>rKOY0Y zQ9jxKe0+*;>%X3}3;;r$0D*r!#|rv)@ga2PpRfMwU;bx&|Lcs;mi@=G*GWF(|BuIl zi)R416kzX3N7PM}ha>9x1`V~Hz}`b9#|3`=4fyNv=Edg$Yg^^RyP|*`fl>dJkH;B@ zFx-!TsF0w*I!S)MO~5iyK7LU?ZWREBKE-mrpU2OaLBIHx@e3?pAtWOCz0Bq<%3HT>*VNjhy?39E!C^xq<0B?VPoFt^&f>hKmE9Hls}7D%&K@^CZ+Ut9 z+zt#1z8i8c^!~$`N3n77kDnx@q`pXdnV#|Lbxv+xenDZ;`{Ihqs_L5By82JeEv;?s z9i3go{sHpf(D&hy(J|W8^vo=MjxoQmI4N^z-cUm5Lgts1lcNG_8_U((K<%TO8WYk^QrQ z1^m{?{&QgeYh2xc0kG`n!M|)7|8jnQ{^cu{L&plC6^n<^Dxsgps(&4;e;#WV57>Vm zT<9cx&^ZJI1O%a0kByq_qPHmcsa1|Ihbcg z$)ZM)jl%lzU6sdM)pfNz`k5{zA$eboG6Lo&Cdk*XPAL(4&$y$X^j#!>AMS@S-0+O; zcs4MLWhvHjflVg`XPAbVo0vBEdW4fYKc=>KB_os?`DF~emrzJorw5&M0^OcqTCPs@ zTy1Iad;AE%Amh^|<~6v0NFf(kZovhfDdcj2Aa5=(cog7Z9dQIgr@;W8B1^NN`_U}! zmm#cdiQW|r=gSJRO(S+VH7A}Q&w3G*g@2S-{$!A^;OSG`-fGMtE>P*l1 zF3_)lm>2U$q!U$W%Jg;Ymt5dlR456arK`x)AH5xn3!2bm)~}ynA=n841Di$Gir-V5 zkAyWO8+p$RaRFBXF`l!EpE^vBp@jtM35{AM5d!kKz&ICpXM*a_2E~2aC8rt-$m5fE z@ltbCu~Xlu!_IN0CSzbL+>N8y?jd zEc8i@IGUUmE+BXl$$y-w(Pi7)Ggc_|D>y;0!j#=m4kH$at_~M}te|FPidigKE0JbFb1~)lh;rWx*lQQ0TV|H2|v-^Nv ztyzHI0h}gsJoB@C_XeY>6w*|5^q0M(-}q;f?{jWqv@xA<>M6o%mLqNME^B3~b2G!` zC~d_Xr?gUxgp^SGPTP9<_m*X!doMqZy*|G`WZ%9lv#w!!3G{AJj9fDIj;;nx#klPf z_3fKX%Wh|sc5rp3i_4BXS5y&QF%8lP-|N)?XmG}2P@T2TfF8uoDHBDXp!Nnb_c(QQ zB&haAs-a|B3^=XkH?M6zrkQ-KWKA_PUa{i?1_Lf(7Sq#YR5eJ{t_0tKA!XaW*ECNc z?|0}RLex+j!%U^5mW|QeINxL%KDoV+@@shKO8+|>n@;7dY@x-_A2KTY@KU{)<~WOU zw+j+84$9vvY32esNm+woI}T%Pd^@}mdtm$4T2ALaC0 zbXC;m!VoaeZnvrH!{eOjL6f2oyXcb0M|{j!LvSaO+6_|Af(v{<5Deoe&S6{!jK zXn#}>$L7U{;V^))C$Cb@!tK-(zr?C_I->Rx6o37nan%`SZr`Tc<{DQjv zn6M_~XDQYm5=#Qf@xM*rh?2QLw8{*Ovy#L-gy}*`VU)dN4zgd&{P=M|vc*_?kDJ$X zE@1zvSW0w@q}dkLVLR=Hm>)!c#Dxspa6J7k`z;sfOXY|&hROKVEUl(l%Tkpz7pXX+ zU1h(!r>s@=#`B*Dbcg8`g1z57EI|R#6~@@+5L%|`Em(=!-<0oIs`Z)?cr>p~!E9C7 zP3!d&~ zIls<1M7>w29itw1G5sYzO(m_q9R&xQ!d#ac*lyAbNHC0qzGAf9eM8$H2`0qA; zDaVgWs~vo|>*-Ga_x4>=Q=QV}DF==$in9vG1@sR%Z~-A27pQy0$JT2?CSa*2%pwqw ztEy3zs@1@>NLt*oxP7)dU^HdYg!AcYELM z&aKc)*&`F1&mUzR)|tOlXQSP9!f01uvLpY3a|&Jvv_t%KiZoez3>6I2({{2-Dh$Yn zQDBAj?)}-9#OC)%iB%Sp$47HDUR*?1x>rm;weVs3ahgbl@Cq>Uy0-(}r^Qnizp5dX z_$|JG%4j7sPuK6R(}k9i6W7CHduqx0@Vqlix~8k_JkD-RGZKclis1L2?lX*SBSd)m z861l;k8CmJGx@NDrzzlyoNR5KC zLXlpOLqC9U#CBT`4A(@sRDyY!#0*P@b9d#O_N_^;XA!C*Q?^7S|Ko$k?@M28p4X_W zuWk4!IPNLIkY!qN+7Q%}lOd!$<8T(78femXO&6YSqhEaU0prSrnx+qVr}lJ5cd+o67x-5Kp1j#Tp+{?tmtFPcd*pp(6Xpe$7$dKN0`R+sOYG7KA>;iFe;wY zuClbLK890|PezdPku?a$c`UW%mUaQM3l7~7uF!krc=@GLuufgfjM8sD?hbFiMJB#y zu^8#h%g~iRp$QA^RsV`sNp%r(EWVbVVSPS_aqL)@i%8`QB@_DT(sleIG0Ay+?2YC~ z0%5$GX$`&TFw2fMSFRw*$bI2_-P`y~tC)!4gsujCYn;@%^-k5rKH#36UTd!JANWA# ztde3L<`79j98JeES@u)So7JBf4k|N*Z0?zl;lP4y$SD-c>&l)n8s!v|AM(>(F z!Y%{%UHHGw%h-iR<{9)A_KQ*xCfr+jK)GW7{#Tq$&ZJH}QS@LMyTG1#Jh8zu(M*+o z`6S;w=EBpHf+HU!SmnQ1xbPS7mtgm~Kpv_Z*)>;vVLOBE`;pdr_DZcT~PIV z^egl`)55*$St?V7|MimpMk%I>#`;@zqNN#ytQS{2B6YkNWN5D4foA9qDxoF{?v47GuqR9XZXRx}b3uZ2 zZNK^T2)pIqqPt@g#~Z(I3$=j76nVrX0GzO&>zB{J)~_9m;Xbp~tfNh{=g}&!e@NaU zCN}oFyT}k5HdqRq(XUL^zfq6swPE75V=>A{CrY+Mh3pqvNRDC zLn)(yGnFQT4e!R!&fhrT`&IBq+fvq5wufcGPDN`o1sT2r>HZY8Ax4%*D&xlYDe1gy z_v>mnNfMX-B(%(~vK zpG1-V{IFXo27KWoc5P$l;5Bj@`pGZB@V}xFo{$kM_*W<$0L}6&hK+d#N1S6z!iFg+ z9`R8blKmX-f@f@nqAkuo#$+WzY;+iRECYMQG520bGvidPgv!5<_hJ$ z3eIR5f;tG^0g?p-FU|3h$o$&!9+!__IiKlL9R&$lADGTYErv3$Gar<0Qf(Y;7S%7_6M>S5HkB2+PoVqLgz;(65>J4g^ zx6q{pEA}pBNn`OW@fflh1wjilqpm_G6X^nQ%wa!a`50ZB}IW0i*rL)tQFvy=1eZ zgd1+3Z@8N47-_r7o{-Y*`y1nePH3@sLhoU^9<8v-EG*9JB>HeO6x!wvU`?6p8tE>Y zaUGJ$R;bS}?o6?I_AMFF?EM{NR2#u*$LE`|*6r_~qll6bf~*bAp4L%FsFfN0Y$2)twz$=S4cCIm}zK0%F;Fs<&kuvQ* zRGnOML+9Ec{|x6o2ZhBLh{NAM4Lh0VZMjS#C6)DknDCEItDiHoaPRbC%mDdzWR zayd|WWjM=_Xi8zA=+|33g$W`y3=DCipQ0)HdPA9SNzu3UrJ;KY zW95)ryruWfzh@XMwLmAq^NpdYZTgXvJr{u4gjUQg#tv-BVu!JxuaiM`Uq$A1l@?m& zO_8o{2H^2Sqt%$Do#FH}eeGfzgmspUMV}qOMYYG@O5T~%uAY^7ifk%#^JV*f6{n3V zsG%*?t3H5Ry0XfbUC0Oce+z@3!A9x#StjgYgcH6yb-=6khnW!PtImbzF8%u>^rnn_ z@h~oM^;uD`A&y<()Npk2fqh`N1|7oV0sgFg>=&#{eS`p=bLk#^SqLU#T+Eu` zimdAO3@*)eD`M`SteV__*hRciYPXJ$`N0s}T@0r{VchbgSgV%))|H(sD{s$@cR6|aWZC{$Mx?oEX20H< zzh|n<3bEZb@HVdT&S+HBg^64{G3&m1E>fEJ}N14#?4Ay#JOxk zbC#BIOhrnMWQ}hTxky#JE55;8d=FC5aaa+GYoyGPv5RrdKSe~ygkQU+5gx%13j((P zHD~-2jebw(1rM=4b|Bn9#`ZHZ?a$XFrMRqU-XD5StW0!G+4&n%BJ|VI;TMOY045;= zR{`%N{L32}LEaGFM~EPvOLyzbGGH1|rfj@j&mS&39$fWqqKK*E`gv3uzVqAl(qV~S z3<=EhqSR8cPvUj-==O9mcp zh{Y6SC7~PbEqS$Je2jyb(Y~7&Le`xxaZ?#*h#b4OnYRc>P_#(7) z%x#Ic2NzEm3EEA{q7O6PJ9r9KK}mWSrrx$0eD*QY)m#%dcc-D3tWQVa>WwW+#xf8x zpqiNxmFew__q0?iDv1JrIVOpbDWI|&drEpn6ip4#@wcL%iTj97pthS}j!o@$pQ7?H zqNa1L{6}HbXZXM8fd54Zffa-77x?}j617OL))T@+Axy;QDMStzXx3|D zrw-bUDWMpHweC5+M3>-vP1q5@N5Cw%5hzJ?=yIjCQMEhhZwT%!$c{`!da!@8k zc(|k_qCBxK#>L0HddlHWU{vA*Sm)SM$S;>i4h_odsQ)Uj(}CliJ_9g!ewl+G2Rl#G z6WPUj%4kd;HHyNYHYQmnUeq$F3!plDSnXG4CY(f1#jGS4Y?h;LyYoe1-Ocp zwWJRm9vtI|NHxrd z=niWw;)LiAT^ZLchKOT-(~{w$R$nXH#_G{zYl#q;G}O1FltGBteeUp`wo zI3YLq-5-$x8PO0s8T>1l57XPn5U)gt<@ue1uNo_^PAYgg?QLUKQ;<_9d8mc`L^nHd zu5h(QE1YT$BZ|{MF$`(@D{sTbyV8*t91CyneP8Nw9L6*^Y$}S~+3d4>e7^+7ChEcr z-@@##%_{;g!37*iBAjxL0^?QRe$)3D6`K)`-22kIsyl3Kgx^#bGOxR+Z!36UJEvBj z+r3l*vK<9oti=F;3=3c(eV9irZ7s)$*PaX9+B9j(`E;xB;f`$4dG|@%K31ph;8lNw z;SyM47CUbIlLZsQJ4OgYESQ4k2cwSixwyGT_sxV5FgI1q&~?fHV!@2_;?3`rB>!Z= zFfyEO>>}4j?V#W5K*OnU)7ERHa0k@o_9<&`FQ$=(QbD~E3F6uk@7>xbTQrK?vSj!@ z%@ZoZZ(K6D&O0bV#2|hNTA@4?Pk0Cy4}Ga9;v{Dkq8|~7*-s@@+QQE!RurGh_Hlcu zt0yI8o}YP2B0Lj{fWVy>{V27Xd~6YOea` zh8UgKWc2!tlDbfshtKZ$D;$_Fnd0X*TE8 z64Yj}<1Zb*n}nBGdpAKGY z&Ulm;&~;0#;zpOW+@Qq>o=3(EK$tDzo)#K;qRgP^eF>5=oy03+_A^_RNV*ReK)MOC zJ|k&>Em%>!CwoiS%O5hq`C;loRiE15ebYH~7;C@3Tz_ZRz8|*3yz*uKh-c0!wA$Yw z%`j`1UYd<5=8IFCsy*)4hrNdt8lOmKguVeG`p?mDMLfMLp0Wp}+U}-X*cGZTI5dCT z9l3-fWU+(vARY)W0XmQ*yx)*#(|XdwcjSlta9LN>T$8jdKk^gt$kFb3lhXCc`m#>- zpQghXWhTEQg2X`HD6Fs$8PkHh4k`HYT=SouV8BR6$ zccF5^S34-{T8wf5Ua-*5wDzxb8y&m@Aq2_BZiTKaN){gy9VZ>BHYIEz!6QEyVZG z5Qm5nMem~aeqipgvh8)Txc#x9)cW`i=Wx~IrMS4LPw^Lf-%MY;?ez@yRcFa?2)*nl z0Z~Gip`}-m?nzplWSwcg$pzNdUs|1j?0PFR(A+hcy{w_Zb{u)Ft!tXP)1z+bC|(~@ zY$QQYUzrk1^BV99)r1qxPL$;~xoLWMTKW$JS8Ee-U0y$>1uvt#!; zJ|J(HNGqOdO#cX7g*fj-vavr}oBTocTv(+Q?w;nG}*^ zL6*GVXMWE0}q(U;ydowHmRe_BO*dwCOTe0 zR#H?w=92r4{crWYBx>jw34Yd=Q{PwZg)Jme2_4@C_Mp7TN_4)MXP4rtTaUv zPm!|rjnEr#sxRDCpM7{#x+l+Vug!G85|MU0stdVzGBUQ4Swj-lQ?u+jNE0{V5IKbV ze(K|=wYxH#t!=D)obbCJ3N}pNgoeLV2r~z=#9Qd0?EEq{v}1dzHq&^tI2e03s}$kk zb#3mpQ1e&P=AIXh0=^t#%LSnIY4OJ|%9zvMO$W2tiQq)qRPJ}v$i#QIDdcIvrKun4dq^9= z@bY4*!Lob6ihd4ZS~qGOKNFP)I?hWo1>@&RBX7-vl*{-f?Gs8^3H1I!K%Ni^Cdg<0WrX>PAbj1OhKu2uh4WX+{?3Gcq57gJv}d zI9jwH6{cg9E*Ru_{NRSx3*<_~<m*SpUr^ip*30@!OFH`A7@Gh+$&xTYKzs+`VdsW&iC7GR-nX~)>6f|Eo+i{Uisg#(72svI<)d1$h66_GK zn(CzzkF$xq39A2wQulLUigFg!Kvg)R;vp(n8=-EVPxTx6Fl}Sv(N7v5sfxDNeZU1? z_=r+Aw_dz9oc5Fax{EIu{;53DVy9@41X+TR@(3r`)!mt28rLh&X$r4b%`wr2lt)_z z2)`$;Dwu}`Rmv9!WEmo^&Ts;+}Ir+&Vka4>Azm;r535yEC^bEMPK}y`8JU( z?6lw-C2$-0#GM|hwe{lr6(4J^S1cQfcW7w1Xy^Yos*G1)`98g$m?NV!JlF-zMn z+ffTi;g?|_l$qNS_C2&yYnw`$2DX;=k!gRGYsCHo#%w|3cc9SE_%IC53m(#=`|o;0}_YvLYK|X&o502 z5(vCh&89_lhX&#XgoHtDLQ;oe;B|~2d{3nkXgj|i*#$g)c;=K~u zdf%zyBwI{nQiV^WREAQg(mJ(YvpN-$M)+seEEOIBWs5aDYV5nEPBiU-<*}^@3C$!# z)$a2KTjJ?1cZ>20e0J#POm+B5b?4tgvsTMZEfLRuMKF{>%1Wj^+7`-`9~Y#@wBRyl ztdGlm>&ov9VaL53u{r$MQulFTQaDM?3+eb&m#XoQ4OguUoVOX!nwdt6+sn>XjaZy&FD#UzSX;`6_KY$ z+FX0LI>>ae)5dhokEq<2pT9)knHH%P{^D*sOqpXEW4?haIs3}{v0cN*Td%QH)S_H& zTMtx1R0~y#bne+Vx&eGJM+63g4$*#19e_mEr0DJiFDR55+Sy zDn}t1)y1=>ngvmKOQ_U_jsLlSUC3+6V*&n8I;i-T{(=Hm_Cq&tRnOD)X;+DaFS?Sr*=DG|0-J0sbOuJWAmhf{`Vst4>EQB7 zDYLM%B_Bdg6+Ns`DQY!}mX=63;~AdqP>Bd<>Tv7!B>8Jc%k2Dn$WdC*5TUvo znVXR=$>MEqXKamB?T(*$pOb9r)MnecQ|$0rf5iZex@WNGh%1OLJZJsszMjGgGsY2o z|5oM!O51QZrk=ioofhaiRXR#5l=qq*Q5jue^+B?^vy9y83mWW)nmrCRGK~`@AUo@o zF+GeKs`j!oN90Y}xvRw8*EQ}Wu;I^^-QW8Acl={GV++)-5(vBNlz$5eo z(MHgRKr`aWA0|SnYAwbg186r)2h9F?Pp6H&bXe&(6PI;|9rnrAT}#znRG6+a*I)jB z>yEh%HJ8?t_nsVx8h?}FZQb7<>@}czajf`eMe`>`&F^~$_jhiZIyC4YFL*#@My*Vr zT?9HtNnjdC%dY9Ecq2~aw4o@o!lq`MSVt~xyKZ@&GfKW#2}HV6^c9eh4hPThBS+mu z=6r|`;%pqzE|Y7&)RwQ07`rlb#Yg1H<3NI!C+u z#OcMD2I+#8uH*%W=z^Y)=a!wjB@jsX?8AEVqZr^e{k0}eiPO!Xt9=iBf-%cHH3ssb zPxpUNs5Q;DwXpAbP-GoI^raj+{IIU*;y&pGII^Rda}n%I>gH#&)LmzVB*!YeCoWx!k`+8XqY{12 z5%{^S+2Yn_f9`*-!5m4}wYz?%bSR%C&+O<#!Z{Gl4!8-bNLia$H@#`^ht*koX4Urb7BK(h3@>OH*bkxG% zVJX67Goo*wM0&Lu+|r3IleZ8&lxq0Wc*nVZ)z0M`uf{oi^B@qS^i{@h|F^uM%3|x? z02CMaWULEm>^+n@5zZX!o9WYZH1z-OBX|Je2>IZs1+U0!TA`@KQ(v&E0f%1~2TFc? za#iV(l1DD~U@`#pBm5un%D?vfD$`I?>}f8r#Vvyibh?;ATND!UW3YZt?tQ1*1Njli+0u^z8 zZC^YGv=tGy2G3UtQggbiz}(>AxWv9LwYQ2#c4(5uGtLd{sY)Lo*z14!y6N)a_A>ux zFmCf-tKD>JAFP(t=}?P+tCzy4$nx8mXqp(D0@GW7!K}|8>opMdOf34kA9cWIzNBO1 z1Gp1GJSLmG-Im=G(z}>Jc2A@7@h*g5EVUSM6?+d{lM+HGZPk4BGUwQp#BN<1;+@$9CAa~y zE`2q|*d7y)%A3eg)ybUdW{L=>XQOIyYx;eGI z+o*GQ@}gEM-=9r#uAg0B+TG$LpO1gsWLQzX%|lTsse{jVI5Lnf#{Qs+aJ@@A5uM?E zNq1Wo!`>}@%C1z~y~p~S)agy)a;`76^`+&ao*X%#&jG={7bG!!|5Z#UN3SFRxu>DB zrS^Gu_~3??lZHFt^y%-|nqm@TGgu8{oZ|uoQg<`eu1;m@I-oqBpD^jmuL#YmIdHsc z7qkzJ_gBH$dMI)b970fXA(X3*>@psRNUy-b7;5SgNO?V3W&h<( zDJo}yTj2!`Rk^f4dKx2|d~Y+;FzX9WD=m@#L3^BZUZEm;WZ+!wE8LGLx;4W}PZ&dh z!#L|pFC1?Vd(j=1UxR3IQ1I(wIYc)C=sQeDNXC8irdgNbfP6@$in@R)S;T6fM)goX zk>x}!mm_R4@}C#aeLq#N3-HOsJkZmIHcR{7$OR6c1*s_r)>~X0)Ql_}Ipqa4s<}J? zP3=@=TqbxCWGHux6KKZ)2j3O4`#OgWV=x+}GW-(>&N5YX!tZ zJ7EM2jy9i=I?vEnEj`dH;&T>sy-GiLqlsUe(I9n9@4)3So5Qd4)ZPySF}kVxiX z?B(9uuzdZg?dr@^_gcNQVg}$0Mf`Z!8)Adx15frU2e?ukF`yy4M7~h;sL(7t;|DSZ znLo!8;{p)gC8hMR45$~LLkSc@fNKr&RccT*crmGW0I;;$Jh^>Wt;Jz;s0I9+n?0r{ z+Fey=Z8wdn{7}iW+?XtAc{x_v-csJ=+$o*4*PSV#680UO(m#Of3ah!LCqj#Rt_AI8 z+GFJxUl8>K%)J0L_k`c7j60d>Whz7cQ53A6*D?Fb#na8tMTGL`&4CXUJEH5t*UQV< zTOR2PSaVR`aoO#yDgLYO4-5z&Q7SHuzI0bYZDMeHN=E%vWq)8W=2XX^6iZTrj47f1 z=(`X|t*ne!cR*CRl-$E@Hn{7iQ*NFfBpRSwza>cf^@`J#GQe{hlOw#I3if}`_Uy+7 z7Oy6)om4$vyXBdKq}A2_@}2cLCJq4Y!M%t!+Yl{U5%gVjqurSoru$%P{Ng!26Q$FL zN&J2|V+ajZlt9G@VX6ohg2Ip8-US>#?eb*N$ll`zvyNspe(pHNF)Vc*D>o2&;eZx< z2CO*0lMU?vcnsC)%iQXu+@EPL$jaw5x0bRN=J8HF3GOQJZQbtT2WKCJ2MF~A z=*X~EF}MKq*{m$D>JY{|24aJ4IesctYmPos&eXnM(|D&#Dg3GDgJjqQA!x=)cVEAo zY-RJOEq-dBdfQ&e%azMKzl61cS~5d(xbFqBaJU-Qg{Ylr>4V-8d06#vv6D2 z+~y!U=TVPrX`=119V-Qv3H7ZtnMDxsQ0poL)0Q+soQ%YI9)MeZ?e}%OHdBAu=wVMM zK6d-@b+12Vnr@RlqBLbJ(S>P%=creZx@0S5a3Okfrha~E?{&STI*P-y>)kaDr`ehH z7>jM+aFw`h`1`&s{(5Qu5i{VbU{P4AMU)sznn4(t3^r$NyE}#ucGfr_I-zB8-!$l6 z|5c%a&ufnsd9HWeCYQj*-)ak( zx9)8{fG$Aj!$vgv%&L1}WI_d87;Q@(HzE?M3S8U=H9}>2?YHdfQv+6CjrWT>Xsh%|c( z_U*wUR&A!OdFDU2Kg+eemR%|adz}C*@YA85i2=_TG4y8)avEzxF~dbN0+rF-^UVZA zW#rd(#kEU5(x~~mW$2c$g28Gj!Q@ybKQe$sJv`$6-6Q#4o*pltxOYh3VMlYN5WWjjadyT&Y%} zm8q|Do{B!$Ei~YdSOGyqh6;?rPlJ(%12GD$!_71MvtZT4w1T*SW}>wdrFZJ!x*JB{ z9RyU~Dd`j#>2K(Z8cD|QYe!=eEt#dtXARBW8^yj^hX1(%mb z{Z(7{3MziS@@>_i#T2DqouAVTFHo;iDMBMwyrGF#uem$$-8V6bbLI;9X1tVI_TeY} z2R-EtEYg z*2UkpBl;yD3w!0J^bpX9NIA8JU1HRyvf%cA5o7+(WF3BuZ~wgapX>7fHycG01XKcHz8=QCa;a6CmpFeh~$o>$&`h{nGcSiW8kSsA-&av(j zT;N?hglC^;VXz%vusI#ous~2+PrRE!>N|I@3{I_X?D>4C^?YS|w7c-~=j&1H_L{+UhP%A*k|p3 zm)`MN(dmTaxmfd7hsgHk_C0Y zO5SS|)^AL%JbwO3zmi<=L(3FZ_Ry*H07HyDv>NR}Tn0rjZDpz*EMwXhr}*-+wUR%y zvhwb(N4=f!+I{#6|2DCdT@PNtJ}p;N@`t%X@8Ck~{1y`O@q(-=dMX^Xp2lH3g|Tjx zB`~%@>(omsEL6+I4Mqk7Us4~Q_=F`U+^kt>6NZ1#TpgJu9Q2z?y`4B1b`aXhGSa3k^cY_`6 z+UL5DJ(v$#M%!KYa#Iz4we-^!8{lnkgWrzyDmS^Jjr+StJXO_C2dk-G%lc-r`s=6X z(M=s2WAD^$%6Fj|bA(>ePFvHoD{ykG?QKx&=ZNmu zxzO_-tS|#k3QEvm1IBkS;@(H+YLTL|C2stOS~@grZOYo}NA9n%>?z*9VMlGSKnGc> zI*OqhAEf!=ke^0ht^YNx5ykx$=522=)w1MQrjBvO+kGXYW!sF8vY;K~0{l7c7dAjS zcP-NBg5h9PA9A&=sr~b=s;X+enq418&F5YeDQ%KGvF?J@}PQCJgL4}usIPH*SgU?#)*wWmh$Ym+|Ij*5i z)b4;siU4aKfM3BwpV*62T^yNwvx_6)6e@bE#u{OnI1cy%=IJ;>j{A6`}O!S zUqIv=P#tDdi0PoPNV?&6DE@Z(yq!4a3AuN}`{d~FV?!WRr$Sb=kYYh~T@ttTw;(UC#b8|C*Y zCwA_t_FJzH=eGPST{PY2#S+K&M}0&_&fyhGRR@k=W}Yc?VbkMZ_GDFzt+xEQt6J9H zq;Gia07C!AFSB+TKhv(b0Xupn!;%ZEvZE(5La7VV=&=2&=Ul>+`1xFM?w+oKZ#&b@ zl8dz1xgFEcsF&jymh32+!ACA2HU7L?FjRB&X2VPKTH?*&-tbx`VO_s*-i1~^rUP>W z3v#r<)y&2HG?PYR|(5l_q z>l00jKc8~%3U~e(`E1Z)48;;P?#IzPswd^pXNtpaDo8h{wDet-6HYo|Tm5Ju^V~%G zSHuBB?+%s(so#sW0iPdS8pEf*l? zxBM{!vPq%B@%|K+PJX<7N5t&Cs!%N6;U$i!#7f^Y!^b$w6ho^b*5jSAAs2*;!U#ET zr}i`ZjLv3h_FemKq4#iywO*#+X(TZM)Im}Yr?JfXj<13%R(2#`c=ck7NteC1uuzRp)$Wy> z)+;^mkBLDxAy{IhfyUu#vo7JvXy=YClc%Vj)irlCvKxGEC0;2IWOqAV*7v30K4TfG z?N3~!pu9KvC3YFbE-X|s1I`4y99Pb?_CTff$PxL;?q#)bAb#f zT9RPuG3G8XylA`oH3={VElW4+#34ypm2T9o(*(PScb{_HoM1wq(%uC>0W+A^Z>)bf z;#^;vhWQ3rDn8t^CNg|?X6D`vkq6~uI0(>19m@OF;JX6PerPic>fKJ=Bt~^9Fqb?+ zCM=_G{?W-n`XFnmz!uS|Ly^LFXz7e;)6lqUBxbNb{wxKBEbjWoDtS6Pm2`H# zVzOLuCe9xri|EUO=Mie4ggc9+(ZYdwCnH6S2PE1C%N#ESm~7c|_v>NpM%h^5?|9+x z;ir9e9f&^2LnvTWSQv)btVa8t>XJ+)BSDB@m+31LcK>fxbDvu2>3?c z3&6q+pH zv}?(WkRe)^lsq{hCcWX*)P6hVRnL<5&EUT0(xG`4@Bxja_2S(7p$|R^_6t%6ckkXj zT`OPn>h_ZjyQU?`h3Djg4Ao$)ym7=)$nPcZYSUF^!oN8sxJHP1r`oj4lmxu5_uo+% zd_!kh`zKgN-6Zk}o@%TffLY6kpe0trMRlzmBwwKU9b7gCmyYEa6&4aUm_}r-vDzF= zmqTZk-(7#^tKb(e667iR(qz;g_3m%e!ryDR(%JTaWMhz#cF%Ay+rdoPQQqv>IW&@b ziqst--LAxVe&Z|v#5z07iX0{l*f{S zkH-!~+Y2VWVeCiUbA7j)G_tX*IaMyHy-^JqD|cxIFR9CqL0%HPg1oDz+L8Y7$4Q6_ zQEEBrp_Fw=GtT#0vgKoghqVt+m~AP`D#zRyuWKxy}2 zZ4!XdV-Wi9UJ3H{I34Z!LMdcVK;f;{g04vWxtY zB%H8Oej+x#W{xol-Z(aAeBk4g&TX6#{(N)?^9J-?9$M7>bXP-*E6mau`Yvw=ZF0<% z|I}@HXny5^Zso>l(W%2PG}f4IqNvUlhNI12G88Lxh9k6M3M%!p-?yp>_1$y3?1aQI zx4SS>r_nx4Yfe)WyI>FzTDJMeEBKR*S0h)HKH3yzE>e1BIe*Fq;QDgFPs(R;BlUv_ zN3*qNs6K+2_kDW*!At3k6JL%5?aJ<3o*mVw*p&1HPt42!UCp{7yFXV5(2T=%&0pGX z+hFCLuu@rUdCkzSv0d4EgO~1lZ^FOELo_G9o*ag70eHvoqP853I`J<5tj4ExK6CN1 z`o3YII;ByOPzbf^*;c@e_NK3}5+XhEuzW-aD5<#jmPqDMw#TueU2nudAlzQ~S= zz33Nt=7GMR3_Ae&04YRe%4tQG4vq3iL7ou?;hpG@*5kLI?()C#@Y}u}50x!~a^>+l zxj;X{%!P5iIbU58^BIx1z1Q-~8a%b7>eOM%`t1)JUfSfHO5dwz^L+i01847Pu|JL` zM7+ekBs#alzFYNH%h9ywQlkyFu@0`|V|8*!NT~jtVoEKf=hF)PF$#>SVFM~BfOy-I zXn=CA3$PaUA-O-4bWl|aKQVQ??>Zo_PFWPbB5BpV@oCV8;mNa_Iq&aRbTd&^?#8n9 z2gnsKy9=lZN%F2Jc1 zZ$SH9h7Sxq++38R^=emV9FAx5Ri>?cVw6a3DaAe4>m<2dOq|1tU2UWX_%9cuwTq zvUku|r~=J^)wqFci0?PbxIhaAs}EPiC z+Hp@E8tKPBdNf+X;MF6ED80Vz!h?{FNbRp z*UfUS#I*405f5{(BV^%20^mX#2wsLHa}?NEc+kRo;#_rdl~VHem8U4)HX=o+JRwFi7-5H#+aTc&TJ*cbv{vDh3n$sz2SMWm)PZ0~*{Huw$ z_F5Z*wKed+a}BUQf*4!z6V{mSiKI({Lgk_kzH@bD4%A>osdMyLhLc9EM)YdYaO^Xn zl{_dhlM_mWbOuw51+Q&u3~Mp@I&u|2eWHJ_nDi4i(@GTw`yo`*gpMExXzLH5jN+$9 zXZ2%Oo|Zn!^k1oG20lIV2V3%=_7*F=QC3NWUf%4Ap)NUNT zHRYcD8!hPR0lNY@g~wd2dDwy81#WP^q@S<{@quBSaVxfD7Z)4(t88uC051ujXlRzA zU+*xH)PPtB8D95z8;RJT#DHroUdc>Ng+(a zrSRQ3$ePwDo&P3(?~SVx_9qN^?itD$;=chniVaf{*N-1Np0*r*g@xy041KTc)X|uo znbk^9SJzvobG}`0WxU$t*gkw4vM+(N*_e#xM?gNYx>++cV8McIC?BSsdjG{xlvN5s@<=nR!W8Q{mnI|jFMjy?n#virW z^W|-*eT$=ltGxAL_UJn+h9asCFy*V$WKSQkPyk4CF{X)KHVZw{c40y3+vTQS{tl_&pNU zVJ>{NH43{OBx-L{Jv)hkN9eO1>`vS+8S5iE6C0Kd`qsG>9E@M0K`ar7UPQJ1m>?Ax zMvkoe(YF}KT~F~=eEVF>JG9rjWH6>c_h|ejjKr38n(8yks)SCB7?v`bAQOl_Nrmq2 zpY#hBx4li7rKpQzP8}WFV26DYZ&^D=)A45C<6NV|vO>Om{3djBy)cpRFoB^GNiiQG z-6aVbAgNj8@82BWCXZ;{ST3O!GhTH0_{$q_-6C<`j_PE-AtQwO0f{?^qL?#{+zI^Z z2mudgWVd&PjBiK}-+`KXu8rHjH>69e@(yi2SWl{ZQI!v#?W@>UWI<#DJj#ge~ z%*q*j`i`3Yq2ypO?7Rb+nzU=?RY}xJUZ^?matQJiawHBMl8Oj03_p$MlK_@9gQtni z^QtD^-qIs^HA(OJfk}lV`(yFb-24%ubSugjZPx=#jhqf%3+@343d-KTpXfoqXJ>afrKk9FzVWtWB6?$nMQ6@j6+O)l+X%ydAuv=U zvU`y4wToVVuzATyJrNRurP}#iu3j290DeV`58#n)Vw`%xvSZ^b6ut6lN)DnUP6qo2 zW(24d#UvDj9z1`};J9x!EPja%^->q8f*^RMGPMK4>}lwlXv!PEW;e@V)nO~25=}=# zh4$mko6R3Cr-Yk0NxK-hCC=FdYH&8IFwJsU3LW;0iBDnyMwnBB=6(YARTh{D+F4;2 z7Vmn4Dv@RkXSacs3)6|yMEvwPQ!(`uPEcdlMei2?teE)KR_f06tf+7GCL?bG4dT7% zz^|hnjHfcq_7hUv9t~-QqnhIwXxWY08pJTuO3%>~U7*0Kd0&vmy&?P}3&y;s7Ws#vi-bJ7HBGBq0 z7d}k18Sak{!i2TiPO^QgG^kanV-W8TEPwCT=1xDRN;=;I&Z4f({*!{A?Amzdhrkd- zF+llklP<*-P1b+qZ_oH~yu8y)iI*M)s|ZhEqCN!n4$+3IlEBywvW_urV>r^79Yv|% zJKxL}m1pzF6%DX6Nr>>HS}y%R=Jjh2rba=R0W@=~RTuj7+k7giE}9UcRJ zvo-%g{77#hc9^tHo-xIA4(xTN4w8rtx14ITJu+=u-kV;E^)VpWC_Ej!H4AV%fzZf3 zk^n^JWhYTsw!UIb1nKFLw3N}kB)5=QJpd6dcqr4$W-a6Qg4^rBX#9lTNf>bp$0`$z zswS$-@{2*7DZce={beO$;8Pn8jRl=T@MDlA3>)hGYJz}wD23W`-!&)SIYF^vPZF|w zP)w<-SB80t1+Em{%sj|4WOzwC2lo_YpW{1P?;8YLpg_uFMu*|i1nN-%)UEEpl7x$Q%Vlov0T-6&u9Ub^k==Wqlj(A~|mbH{jE^Pvgvu>H{fxCvQ@5)DqN&M$YZ?EYL*)LM9Trg2dM*k^zzHJDn!5sjP)A|H5r zVct)0y8%b;!#0*oez^G;;IaK6!88NxU{o z^l%=?_s$~!5~kXRXDITp9w~+Wg#B2s1;yH_Tv{)O4|EWeZsF+aQ-oE`TkIGx7uHZ? zQXrB;(1zIRX(32c_h%IrW6Kn7>Hyt`aWe?;cuzb_DQ?SE%X@v#2mRLnyBei4S6?<~ z_GQz)&Ad+pCz?2XMq~O;XkY<8cgI^A$&z=Ly<-*jrS+!4`(*i&**lg$Ve9af&`2y+ zyrKiTOxhZlGJPoeEh8S1`w7FUIJ(OZEiUlkyCgZcsuTjp8lXaaeKV>))jt6mg`}s? z*ScMiG+EenLA0_^eoiPxM4?rWTbq3k`xU-{0=W@FsstFO+;0jd{f#I|_??ae5gxM{ zR!%QG;6At1c|2h(QFaNC@F6)MDaf%5{f}jl@mDZf0$*J$EM+v3>WDXY+dR{0`d)k@ zK*xOO-U$<)FA%i|Ag57_GK@zn(*!cAFC0{mrlIPC6@JXCKR^vsEHZ~>N1){`hY?$? z4@UKhEWLTa=e9RrSN&TNU_|}EsAy}6N;|T zzP_sd`~b68)3HH1ad0nZG108N)kyC8UOjjmfohMFV7LK-z}s*#Ma0er&C-5ignrp5 z%-`OsUDv5t@o=(HN@c9%5eo3{ME46EsigqXz<}ga z3YNI!ha1mcxOcUduFK&wr0LcixJFiYWP@hZl1bfY{&(q8!U ztcy$mY9myA2q4DcA;=kN&PT3ss}mp2A6+1gWTKm#=s`{_3)c30I2%hS+Qxa1aFo5ScphqW2)>DYXk!^ z2xAEYg7ok%12j7O%PKB<_I!TFjdV<_g7w>-#l=UqiTifPZ=ATS2OGhHCS7ejOP>AA z6)A?Vs!n9$XC79}1jwR^IuoQOe7j(O7F~#qgItHoazYnVyIGbI0(sNV)ioaMt+w-j z-LUSS8NxwjFS-9EcMo(qAzTErg~2<54%bzG@5Bf?9wAk1^uVj+Yl)(upO!v_{wXLK zFa;KJz!@Zup~X%e)7XFs?35eW_`zp&*RwBsH*~+aTzvTu(Jak)^@vIjG!g>`>Jx<( zJI0{f4i8q+jY}=jRwf5_4Nq?JITI{);H1v#7q3NKI^(yZeG~mGtsz`ZZehv$U@sG8 z33KK2=3G7>g{)M)K!(rtE`S|jJ%`Z%I^q^|JQW$OJ~6*P&tZZoZlgd(G{3qi)B=@n zdvU;u`G9NH1>9pHqq-pk%RRO56V{3nn%*SH>?ywa8@iUct5Sutso)oA80<2`stoe0 z>f#@tQe$8AX>Bo1!QEGRx|kHOXP6qpy2^f!*hS*B z{e|FD-amtMnTz!ua0_x5zb5TBuveHsE{YG%Wh^nyyks5gKP4Q*?*rm{Oy+YiP=Qo_1Lk4~0u)2TTfvZl ztu7NKjHDWZ((va8P|XC`NWfRLYDD1}a^TSX?HDJIGAmaes-J5w=6>0Fa?(g8mp|g< zuU@K*ClFUtT0-}?(kmI!qs$@SbBqh4Y*Qziz_$`zU0q4VZ_a+rO{q!dGUU@vU(FTn zW%b$8ov1&Kc44%s%vxFL9tjXCj&;oBsc)9~ayfK;+~-5}BUioNKMO%7pc%>yESv_w zkFEXyE!C;F9fTZ>z9x>P7MX_~L$u=DSA_Nf)=$C;0tE^POT8ALzU%ZHI)TbhWu>{* z2r@+raqL~kV5`nmzHB`BDP2|jpuViP(A2h~MOVxO69wuPuP_$@y2|t%*jJT7Y1AM5 ze9(?>oWrR9gyD1N*t+L>=_;rOTh>-vj#UsjfE|p2rpm?%`3^9SA9AD=Ie&I@|7^`Q4mSDR%xxIa1JoIU$uTU zHZXjV1t)u_^u}oQ!FPnnz6}OHmL=9S2JgRtUWeJx=~kE8VUC~Py+trt?1$Ih;4_a~ z%nr3%b;O*8 z2XaRdYwNL!?e;}{ySg?eWGkTbl8B#U-H#vk+p78K@KRC;(jJ5h)CNL9g~}KzAuU_V zt!I~$pr!xzoy37Fy~lIP8kM85`I78jydXOrA?(1~#MnB54;Pz$=EQv>M*oFXx#G-+ zlv%@>h4OcF?b$z~*SGfBzW`nS8B<{Ge{1XGe_{>Tj`OfYCjhY7<`=Lz6&w$X0I<0Z zN3}-YMj+YG03*b)6%>Lml;9lFPv(C(^WmW(Je6DIC+xs_H=f&vWpzPN{Yy+DFk#-h zFk+PeK5W(B(@MtIz4RXGdD?}4Yw!JHYs2@Jk2x(ciI3lG2X6eKMQ=MX@o{c|Ko4eo zrwuQUw1!rwys-;>-Haz%3=Bx^&TF#YXCIZK^2$|lrFyqk<~7t4zhAZUYH-irI zQ}p~7jjz++#ZyFCPPVgfFoJZjZQsGyC32cL>nn${1F@$|Y5e)RP1mr5{bwIZi^t>BfT+;$9>G8$LH6JKPt^+#C+-Ff@_GJ&^V zy{er<=gwI$>m#^#>~f6c@zs*a50!X-ENd3bimgx4fuAt%R*6s8x@vEaY*SC(&qC>& z&fT6g=()%CMSXy%$D&!fx-*s|b#0r2pGjrdXbWYFi{rc>5pE`o?tK_Dk&*7sRo2TA zsihz4Mo1b_=R0dpO_iMeSa|E3xT>Fv7K}gX7&+^nG?eSkDw?>cJWj|YfPOMvo$-h% z!zga|OTV4Va61FQUhlQb&%221w?nI6YTVnBB~YTUx6$N^4r%5+K@VUsawBV9=t{J} z-n$nkho(^d<+O_7;9}L4n;Z@KmX-tkWg!N-99|UDhjD)_BK;*e@E5(*c|J`U!b1Zl zm*dU}$(%jDvEC%`6Lj(n3j$N4g-`)`CGSs|7pR~Od7a98`)Je^XIPpXzb^ed+^unv z>x0>Cr4ZgjxlGs-!99!tYFG_QhO-^Kov*^pOPYRg=uM(Rg5+m<$p*8SakU|Z@F-XL zQjq2|lBw!oV#h|INS2vI-W+CME^)|}mp{pVQ01|3i{UX239E`ZMxCoBMN}y;aHEZw zJdIlalfxD*5?=(96k2VQzdYMA^U;qD`s`7WpqMAU2e)Tpq9Wr= zI`}G?s(GU>~;?{-Nmex5Be+EgV=X$Kiy*NwL3XZdA91Vz|Hg zy1H*DQPR$o}To<4Z1h2PPoj{dtA=B{#qu{wIW!{eB^||1t(Ia{cW7 zmxAr@;iyX>4!WaFLf_at>p(W56Mj`}=~EW##AA<-2W#caa0K&ni&#jSP3}9!@Vx zpyGh4vv)>jc3EDVyz;jxPoK{a-RYSfyNPR?3+eqfpo)PE;}41fZF;lW*B4K{UClWseiAT89x~*aYk3QaeAUDMe_!mEYUwo6~2A)~_>D<4fF!Ws80h?P*3#HKnijrso&XoCXn>aU+Ki6Q?iMD$^gC_Ir1MyqZ66C%_aN(D742qdXtsV5~)p zHWfk9&g%>P{pCqogK8?tLsHxpRBvoGW;z)vM4j119e(`T?4zs5 zpt5}LJ<~4RD2yP83^CYtQUR$pT`-)mWuoK0QRk&hJLB%CG^bw8LVW75*22C9>CK!% z44R4~yVbRV7Vc-NPuwUWf8sNzr!x;<@n)%iqMqqX68#9(Dq)+DizX}yuxrslS-gHX z_eypz2yAcQ#V<;{)$;H$S8iO}UC%tE-gPjn*XwxF2y;p1o!5kuQZ=c_l#(#!+NGvi zG#5o2YM5=F88<(Fn~RxuOj5}ll410=G4X0XWDAB#kOBi_C*-;tv=#J4;bXXRt%N7kAkrYw3LRixrjxtW{a9g^P=L+ z)kcR?IfF!VWyIA4H4#x)*9YVP5zQ@JocO6t>(9>%z@Qzt&Cm!4yG6@2c{+zA3~U5r zRpnr&@F+Rvl_(x`p^E9%TK1HqOulp680dEKP`t)u#Db;FocL-iufbD#9nL%;c%EaA zCIlR+d3A`3FR@GLs@E!IY~1S7oP76}#) zNfhCPP~t&ezFIyhUa^tm=8lS?>n`~YNXRqSF?N8kOW?waG6KlDVM1|Qwe%Wcd6+`! zqv)GC_odM(CT>@*@_roQa*oV$Pt%Ru-xR#HwnuE$Im5ktM{!ZKT!bQL+9YI8!g-Yw zAxIpGp^jqRLGXjsuLd@OBk%Axz$#RN8Y=<^IQ5T^ixEA9Q=FX`IO7cMI(~-4VAZdB}AbJ@8nR8>6>ibgAPwa-7xDM?&nrjc`5YIt;3n)u&j>R8$?RV8d({4Q&^^dD-N{Rt7due(O2Q zR}cHocJEXz@3=Imu<}4|-4X0@_;(UnqZW!GZ8LC!M^mDIc|4UzxaN&SNy(JhWLnD<5%OTByetT9hq#{n5j7+OavTj9#D_W(3G zylCOfzzb>R1Fq}2Ar9{n^Ohmg(Y-9;4$I2}oegMztRthNQfFL?E8h>kqCY&-|7Dy1 znPfg`v9qTl_TaL+S$SY@m$?z_4B}tu{gdy`GIu>9vyWI5bAK29Os$KtdX@O#c5m3E z*J}0?Scr)Lw2e>@HvN4-OFU0e5RR^E2LkuxvJ>B()Z z+JV`~P^Mj`3s{Y!Z{I*|9VRw>H@^1L&hXknyG+mmEXa}coagZ_&XG?GHt2$8+9 zJ-0T#f_T!i14VRuXU*Xn89qlG)NlDPjj}@mHjx}Sr9WB#sU6NZMNJu=9w0~F*^k?7 zia9+9UA8y!blW%Kme&3{b)b|octZ19Po-qYX3lx!NH+W(Alr*G<{TMA)GV{~v(zmF zgVf%4#2qO>yyK$tdD;^NhXRpCIVqQxdg3~94sP;^;I{xe8Z7q=PCLAyA%bGC8QU<1 zbH=-Z=xt zUNA&^-RrFRw=w^*+Mgd*o1KnyBSmrK?tjWW#PEF*80PSBXH!s6j$`w+-El|cl1z@Y z79F-JeGG$*Y+f4Du;|=>NVC(p-FrvpB6Cv`UyxT|&+ZEm+M5BL6!tpi$p&i}?7a1c z)sb61vKJ{1fk)ON)s|EJqZ`JL#ST+DL7^J#eZq zw4x1dh5Eh+|Af6@BM8S4J24Ur3)*#kJIjn-C{a<~d{7x>a|%4Z`w)Ck z%_#c~OZOVYlaa+xbgUD9B;nL^eVvJ2su{NeY6dj~0>XS`-tSzkdoDfS>n_=0fB5O5 zNn&X9_DkXG#}okS2(*XRe`7S$RK{9o+FfbfqgDCr5BTO+q=BN=ehQ^voBC7nrsDYB z44$`xnd4P+_^$~Jh4AOy2{XYLyn^@!zNYo$d{8)fS999L@Y^Nn-l@tzM8_6gX{Ykg zZP^J`vKUMiBZnGRPa2TbF6Fe{e6!V|pz@4kF#=jOuow?F@!>_MhM3Q-YM{OfvbI8; zlLtlxnc%z&wvI*iFa2lKcI*(LNTy^akGcoQ^Y{avoI??nbi@j9BU85gXUrxX39t>d+UDEIs5+RyHBqiKHbPSGwGnM#0=QQ^PZEw zb{O6;F%!l-6qrD-WIt6kP=48bO($&ukvpnp_^>e6{r$#AcO#WrGWtyJe47id6UsyN zW8<thb?c?lCVJZDwg8b;WR5HSouQwcSa zw&5pPPd{*)3nIE&Tv}T9Wm%(qP7KwJDalygGKN7a0C|P%M{>47^@0qCVWEQg3JKaq z52XRgQT;=m4PMB-zxncMP0~hNJw1u3!|+rf zw|@rN7Y0rak|sFX7>N4Q{?a`$g_J45+=AM z+(wDJ>Y}z9Y{&|XnR!bpa_T?2Y-s9QN>t?T+Y1h7pwGRboKe8WeTPR7l?DhCc%lym z&s@qNt%wi~VUe6JQlTIvb+lQtv;Jgp*7xI0*^>^I1)FxJT;Vs`JeOiGwVi)vu1{!- z%4&cWL&<99bSm5fzIkzUB0p^=hfTdauxi_e+QQ+%NHNUbAS42nOsRG8<+6@^yU(W? zFD9L_kq1f3!#17?C6)-2E=LobnR_~o5=ZPIvSaHx%EgY3K^qU;x9p5ny_+<4d;jdq zmWr9vXB77N!QVIl!O7eUzR#zv6`P-bv*K^p8%#3Yne2CAaIb=}EjX5pCD1$Bu@ykE z4P#U(WIyDISXY_a+@YNum!=qRqVN4y!KO%Eo>{7Zh8?^sB0dSf15xb+iALn!@H~vl zXr8VGt)|Z;-zW&XD~9p>obvU6Ob`<=|YNb8d2V=<&hu@X_r# zC8cq$5q1NiZ{s(=&m6gH+b9V@i*t0%UX66?N`_6gsfx~MagCQ*0Y$VT@ri6TcznLL zQnZ+842F$YpShtS)+8Dq&Hvi_hT6^pCyC~LWLaakB=heRe5Lme^u4y5SPJ0qg3NaP zPr&G_&F~Q<>W*R4R-Fr^)zh{lUVZMPhFMZp9Qt~RTRKAXq~5@|_1CZ5-VM_L4gvwl z2gES)qyL0OkOUXKH!^ONhjwV)*tqZZRq}K00e_Xj|2~Cgd?rA&6NpN+(EH_@DXdCh z!SV9$!bx=sMV3!Pqjl)28BoPGmJ@FMs>jNmJoY7aL4fp+85Zos z-^k^Q&5Z?;-6p#PeFlJJ_qUTda`Ct#i9wOCwIIvnhTt~IL)L|LXWgFWD&Ax~xAk}T96PvNO|E<#apVi6X87Hu|A?c#& z05v|FJPL6S8@^yH)CxC>mJp*IgMOHAS@4z*IesNCC7OCz^-HY)fMlmb;ZO;3CUMj& z!^(*I6BcVA9$0EWyZHG<$``-8Nn+-C6-^#(vi|uOTCSn`{Qx*D-V{9D8^F#NGkAn( zfY{ruaB=UENu7;X;Pg)^)Zhi&t?z}%elPkoV9sC6+(8qMTw)&^Uwj$tR#)$(Z#Sy+ z+Wdz2KK!NIj@+|3o}60nNW~B?qWSpK|AV4rDpGZ~$^)TV;){ynBvOV>Pf#LyU22-kX7Twkni11nz z*CdIrN|u2pH$=48o z_5-B*5R#t5F!Z8%ooM?`U9=Ss(23}Oj5F0Va7*^NF2}F5^k~WO&vVTXTpYO|t=k@CH7cZg;kDW18(mS_+ zGWi{h2Q1i)sTRmR;#P~gL+5N~XDsCg2USl6_^Jk+CEmT2eR2g;DmHM`~3o9N*%5yJB77M9iwlUxO`v0C)|q7FjSC zju3P7KozL=oMOmRY(&Wj{?{u5C%P$N8J?ag6`S{kM{#|7^(BMAlO)JWFHJ^wPi^K^M-=d2o~KRYpoOCjpD1=l^TR!`2o+y(C6+Ebtl zSZl~>Cq~Y&0bqZQ$vp!U-e{VfUglXq<)#CWwCqpVrC2E%ZYdHCz{$AwKNtG^Zvnvk zt*z1aGp=}fhB9hJj2t;MQG?^1_D?&Jb!aJrXcaVmQ_6cV<>W=%^JWH^99&CPIUXpE z?cu3|OJg+I$#WQc>Vk76b-Z9pUQT_9|JKTiYK6;N%I}30McQL+~ z62_@90h6XPgpw}xSr(If$DQZZXKk2Wabf&Ifen?x5GoT#J&ug3(r08b_e^n=8L@AN z7_oK--)8ZaS3YPy5Px9fC-uR5AM{!c8;-I00QK=tPK)2!0R7>Pj>qb;%e4uS98KDA zejdX|v^QB*1FeubWL%MLeCPV%-5a@ntoANyqgoPXc&MghreP2|&bJ<+LFI|se`p<= zVW#7$OFT1Flzy{gtE^Ir;@vrTfWU%3khbgkADxiDHBoDS|KCx#Uz?}3dHS^mY#VXK zIWtpv*e)FR+p4XTl9g4F0oQ^({MCJIzTH(T5K}2S(AX-vl!3hgRiCUtk6A^M-h)TLW3%S8qkc&IjKtA*?-4z-T4BQ>hwjr=*csaRe)NbRz9ty5E$LOK z%)Om$ZhmiYE&0RwYJb9k?c-Gi_?85QB0i4gLF1uYy`#Ysir7YVWd+m3yWTfn4O!g0 zY9n*xRm$Z6Vr9ftecQ=D*vR=k2sdQa{j0yjks0!%&96?3DA8}RY~VzZbDNNbF6rYJ zHMlI|-myooKe^UzDh~C zQ&7*A<{M_NF-@JY-SG7kgzE65GPyCQvb?~{ZdL$f{#kllwGZyj3m(U6FC792-=FLi zUFc3AwTlhx1XO3kR~YUdgB>|j4#5fZ7XzP_lE=|A%X>bYUBO=QMZH=2{pniU=+=Vw zTJT<51OK@-U^|h++O|OMjpyvfBg@yAPbrB+|8KHS3nDjPmT|OB%+nYNx*2Nr#iqEF$7@UEb8KYqXI#D@W$VA> zGPqA@F^`nra+&X0)vmwgG8@h-x`AAV)qwj_RsR3(Cj3hp=64;1_aZL?UD;jBEG^)a z_Y&KNq~;?_I3)ISfD0wfB)p&Dv|S|8X!H-h1%R+}RyW9+XNvvL@)Q2`y}wF`&d|5! zA~;jP16q}`oniwz@JY~3EvSm5W{v_3>eKjPD{j7W*L&v%&J8utlNuV?L$A$u`978@ zm6_!J*2+-OWj!j+_zAm^2cU084s0a;F>oHrL(v0zfdvPN)6_oDFyaz$JwMjVE8UFK zvHd2gyv*!d%eDI@JVn20;Qz@o5i}?3&9?jSQ1UT+gs z-dvZGf7HY`u_9tx>z7mJU+Ir^vyuOzRI}y?zBUePYhY~+{72US>LHwJi(Aq1Hs#bh zpg0lTs}SlCrxUBk5T-7ZZ&dAMXWeMBw5^T}KyQ0DJk|92qq;OsF|J(UkzD+CK~0hL+pw`QgvHPxn9@mDj`zxU$oZn0m@O#e2l2H}_ zRr4pCmj9G5GV<7um1K~UUHg#+W`=6bhAh*qOOx6(q8=RS_>kav%s=kV7&kX}vC69F zKW}jVS(Ccb84$=R2-^%cE7$wY`Ir~@0DnU+j$S) zIpOQK+<0>=;R*K7O%qAK(mdre~u8E1dMAzxPHVl7vz;1qis0l8I>AUMztU^cp zRjvhO23P-?ONXEd+-jBs&LZ=d*xe6W(VAWkmwV%4_BOrKlTW;N4fSG)8p+xQtSjp9 z3CIHDxms`lkVMkQ2|76!t`?KrONzf*my4q{$IY@vWYgaVZH6rn7z&wLFFm3B;YWxp zU$w5>H=dr7wjSGeC!XNCctX=kbkz;D393GT;Ab6U2r&2CV~`Ad7Rb7OP^SZy1-$98 zL642KUYRVYWv3txPnFA;;>#zw|4kq?lCH$Q3-Ex*J5cf4d>VQb9tQLjFiL*xlJCQy ze`aS@PDv7rT(&-EzI*VHwM&)C@-2c0(ginwA6u@+4+x%RsEwspB2WT=Z+>OPV?$c4 z?&{X$Sj~0jAEj4&DSxbdyaxRG`@pZY@mU+6|BGwjZ&)u$x9`Xbq&n=8?Zsf{H+nfRfaD&Q4A%sUSS378i5y zmPgt*$&961V`>Duit1)F9D576}b%| zqn7nUY&ALgonznKw(YYL$@8y2`?iH{_oaLa&iTu1xr=KS{?4l5CrldKK`Q7@t(%Lw zW^eSS>7_Gq@4IO7o#*8tkuDXH{#yj~==g$P*i>EtV8!E|1NC{G{t0XAAy|RE>hEi# zfZ`y9#6+;JGvT6wCjBmYvh4Ms~*1X;2L!C zKS)hjgS7o)NZZ;N{r_W(RLBX8-I4XJQG|l!8FBzkitPU+?ed3>IW9i$8`sDci%6|@%3eZ6_Xt$!T}f8eNA8u4%;aHvN1moESE zgiuJP!i$QxwBbQDfyS|tz`cQV}OD}lss z%=n84l>~;WOZ7q3MDwwoZ{Bd7J%tY4WFTZ}gX$D#?Q{le8c(!SIou7v6@*2GYAFZ= z*otGxxGz8}Cuuk6x)L$LKzmZW7MTpBJQQpffndd^)Np80!1X6Aga&e2I5j%ZmJ`y0 zVhF++Kps0B2oV-CfY|R5jR;7E#4*d_z&&WooI5CfjxB+eh-#nv^Dl;K6v?PnZXkeb z0Fe*FTf|w2&apZ6rYZ>*9$ML%zzEK1Agu~xv{syfn6NQqO8EER2;?#U{C;4N9_lZK z0ZFFg;LxNowe>r={YFtXz~27*FE#!18(JHQwKYLrTN8hJB-ch_Z6wwr=Gxw|wu}Dh zShY41Ya_9C#{BK{zjoUGQy#Q75^E!|mW%#t9bhdh|5HU|Z6wx4Vy$MiR=r!RW3E+n z*J|5qRem7e`tMs0YyFNt6<60rVr?Wq8}h&Z>0ko|o=^<7m$RRZuMi(42r~wbb}l;N z1gC|FN6Vn}_eWDSv{ZKm=-+NNy6eC61BpiS2l|lc`ixuyNz3f(4O!(Xs=~P{yh5Ry zB-YDDR7r#D?Erm6RR&NzzL0Lk-nE~L84SQ#JQ*x|pesEWhT7sc7bB}0Kk5XpEd~hO zGRM>QO7pcF^hFMm4)3=5s>v&{{6w%# zkby~k&2jWi zSqCAyB53>`0e#?&%OsY6h~CEjH z(AnYkk@xc~cJKP|^NN>Fa!Ipl6TVd4!G9HGsI&qdvtNw*JPn{@<_TK`;%9E`X9g-v zYnEy;BX5L#aQWD~(z4QDLUk9<(V8L^4O0! zY0d_0FA-l)lCa8ZC71W~90`t@Hdo(=nQ7lDRlB|Z;6h?%h8I1aunD>d4QSEtgU@;X zfb$}DFlw2OHU5MFtWzpI_>L5|VTT;d*Vn-gPKJfb3&buQuH^LgZ-{FiObw&VIv8XZCz^gl>L}<@(%1?AP|`zioQLBJDy+UZ zP8)t|706qxd`)GyK+GeAoasNr{aT|b{&Q)HE8KJza6HywMarI)B2BSwg1WtP#*4 z0gTOWuQA|Zf2Wqid633QcnJeWI%}{0hB5f15bBA7VfY!z)XcG3c%exV`U&IEEaiF9 z^n^`<#ocozb#(;y2d_gUU!b@z9-=T^D{mFHt?1c8^r+Q-!Yadve%tYdcEZZ@xI z`;u96-Ch1%dQ_~k+?YzH%H)^-dYbKT(r*6lw|X7{3qtYUiLM*WSi46RW>23-kDaVH z(c2Ir@zKC`u^W|!E5Na~`5LpIK<-k2Icd$195X7Izt~|>==<$N4iS|t+$5l>BQ|6cII8U`ZK$sJHA>fuk6osg&s039S_mB z>a(KXqUFrgQa5*!Q3?1POtyo-k$KAVPK6Z5>#6*kLpqK5X17m#Yj~{Fl{!s z?E>g~uIDi9Y5Fxdsb zP3*p{e9bG{u`&3&p5DH~&+ECmLM05EzXYBafCaZ3W6)G3`ZD_=!9799Xk-o+`1Bb~ z=A(b4WtZBw!qSI3Q_;Ii=fCYRy!i5a$oer@S_vLIgeuUDK!i+0V$w8d;;KYZ4X8xt#AJ)J41#kc!u zapttSaaF5!aZN&z?N^(Wdh;tI{JJD92v+hL* z!W!t8`QUF8-Uo(R(2APt@uOa!D<=1QYoD~htA0~05Wi&RqHtc!ySdT$iZnJD=uuT) zt>9n17i-Np^RoIv>1FpKJ=cfoHxZ|f#NIn7sD-mZZNxSp`CqWX&U>X9{plx6;%-_* z%7)gcq9~(x<`0e~I`*rCeb~97{YV&Ym*#8J^^kF<0XrVL3`HR}d|*JYG5YOSt|*4o zuI~{Oqo(x-H z0cAUpR|fXybz#Te7s#KdxP&EO@!vJ*61^1`%st@V4O7XoIsTT<>A68<8FN$KYmX$< z6!{*n^AXu~s!M(^M;_Jv9H`TvwsD3?MZq~U8B}Mq$kcQyLcMorX1_{6wN>G^!DSKU zAEIUChtSq-Mj`z$+1oUbyp%IGFtr9iyPwaouDyRcKv=?ivFVam`bwwXh6@S$+wL8{ zxp{|NT)*f_&?v-E7Z1yOo`T$nQ6_Y`tdIIe=o&4!s4r&r^h5O$S3+-PU@FiQ;6Y(K zn*x|;IlT^_hL$6X?01#L@a0BEjGY#~bf~+#{>-x?r{~u4%I809kUv9O_Ibp-0e+%3 zVgt!=q&0Hh&cW#X_{P%q{pQY_Vx+}%`Gh>)-&~N_^!ZWR#`CpG1_YDQzC%Q#(YZS( z2L|phG`qY}?buC^aI~vN+Hb%<<1~|sBR`^uDLrN!$sd+tCEnXp#!S0}96g1V{Do&P zC3B5boS6Q=6{WCw0rJm! z?*J7wRzl&?Xh6rL+Ll!ax_NoCqY~&rwn=@Wf~xt6TFU1zF^!)aFK!{i7)8jDQUuq* z9N@9Yr@imR1p8{&1+6ho>53sVuZB&*e#I4mxkwc%s;@roCO8v0@ z7Jk{rwk>nLs}n$y-;(>kK(J2

iA+k7yO-|tTSVr%4vOP&T%1E4h&8ToB)lBdAe^XKOSY|<>4h+yMAAMOw<#YJ~Pv5Ki4=(nuElsu^(3mYXG4r z&mS*5$Q3Vl+KitYYBAg?6&R`hjh}u-I{(c2gvK}*4&QAmN5JxF8$xV4pe*pZm{*1g zpGS=UE`=CZo`J7O2Sx*(PD4c#JV^uNts|Ec0~=qsuV&!FZ+x$IA4zjF48Gp@*k`NO zMtO;N1)k^7VZ^%M0$OC%hGGWv90WAbNUe~Jy9=`*ZqiJD&3mOx=N1Y$EBjSw);|24!- z5dg#1c7GFi5$)-p2GBoRi$mbQ!z!ac!YU^aRvE@YP%}VDgW)IB-#ikKj8IU304E4Y zlYUgd?8!>tpyZP~(^Qc*Qax>rnS9Z;_Zewt)-wdjZ zZBzJ^Fr{=~TfhrpW3dUm#r?p>JjJvl_U5yGL4H^a(+>WNG34G=2Cu^cMkfk0Hh`lF zlHpudX|3+xxQnE1SnLKk>?j3opOY4c)(s{WubMqScl#~lwn(ecX~O7ZJhr_4dcKv* z$m6xmV(0j2jd_(Bk<(w#ueKrK?g?Xenh7^=QEdq?y|*N2RSp+0dUj`6$%%+41RETe zK2yH5W+ax|h8VBwsH#2d#=iS;0-=z$_bG1I%q7P$y5FFQC8>MIO7XmAeieAU)2Nd)~Uo`3hk#BPRXs&NnyGQ5l&N?5dWtSLIJ!dwb-v zBjY)m8n~w?eL^i_%@h}B)Qmm)LX7-qmOH8_C(c}9$U{u*+OoaP0Ux_kN7Hgy^JCBo zE%;^k2%<&nMtI|aL^~ojMj-)efRiiAMk8Ge&NFkW5fLi%sR1e)R`xa5k)qi4kS;qU zBPFHdTFsj{ewpIr|8lsb%m-kTNkJC%Oyllr|TFO=+h zJV%M$MA#>6xj2;*NqSTfW-!|G?oDDxpgr1QKq~XY9S0Eyq7vg72MI};SGi9O#+UZ` z$jU^%!MUAgG;@$?X&Ao5Klf7k&}IIrT^TQqy0$vTQ_5m29V%HzTCNktBOX{@JM4U; z;$xC_jrpAaoRe)j!)H-*BvpL+06>8z82^HN`iv&m((QqaRhAIrkS55FHyP!SEs^h8 zk82ou&rY4?kJ3(*dNNg5oA!-a^vwMU?c=Ho&Wamo!aSB2-ERf5VNI6-|Bn`|&pd7J zFq5^Mbj$~9w}S$_Je#6*-h;l)UN5~~j23@^f4fd+SK??3b9s}%IqHx99s`4evS1g9=`$~bH?$M5QPR*))iONMFtwewQix_OQ9r((6Bp>ePIaFXRBs^|y-IT9y*Do6u^OK8Ou#RX&P9$R zP&z{O2=J&aEIV_EvziZ{H&7eLxlngI*nKyamWI89h(!>t8m+m{r?p}ECztmwR+%si zt!FEzrR^}wEhG6gTm@u3S*r!3Q5Q{8DwpUZn2!-{&~KM+LFeHu$m5f4wF-W&hunPM zkMQn!j2+14U|zU)?ebAkb4Iwt>8^DMz0jpN27WtX+;)SMP0QTs5@nNfz@mlzl7kJ!7$l2LgUU+AzCf zC^=x!r9n;Q?Z8zg++SC}urFc14Sk9UF!Tq08~S8K|MuTS#rgM?T>ei#m;0yqS_4Jh zKbiEJ3=JUUHa~pzF8O_Lu(NOS^lql!grC^6=PBPfps+Jbo5e+|n=SN1kBCeiFgC1X z;0#ZY#%OVnYC%iBE3t6u@domo;qkDd1>rTjp$A#etF#ELGnyOKJ+sdPtfLmw(qZJ< zHftSL^V4U$GxyA%p|(YCx9k^^-9R8lfnF3RbWY|kdQs?T)XCMVKlGyBH+yb4{??1q zE-|VSmA)e1=LPhle*9e-*kVv>VF8KqGXg{=+84~8V*VZqC?kewf^2W7pDO|;Q-KIc zUfv+;V#_uAKDt@Owa%!+DxLVFGTAqsRczrATnu|M|DC=gNFNf+1))|%C_du7>-a&8 z*N0oL?2YXXmzyX&F`E!~+jmd>rN#2CpeBkfY5+#)!W>*jNdb~G^L61T!t~#f~f+NEM9)zy=A6U2$So+7u=rVs~Be(t1PP_cfi?3u!x}v z?}fo%4GT6tRw*{*!k_q<0# zW!>DWJ;T6*{M~VtoXQiio4S0ZyFU+py6tz@MzB5!o@>3zfBs6>yz$U&qkN<4=`*iW zbUH-u?<}t1|jTFIWZ(x0=yl=M!iS&&Y2=xm=T6bFH07zTjHf{4Kjrt;=LqJ_3iEx z*Hu{%7+uJv@|esX?g-UX@kh)Tg~lPFu~Iy&QQ4mzhPDdxN_hlXwfYZiP~Iszlgw$= zSl8mQiTROqiZW{O=k!Xg&EAv7L~ovgW)J<#sf#HmGSy4l-i1lW`e{Vp*?l7tHLN^IBrQW(?(YkZog( zdhe487i~f9%(1!NS*+sLD-cppEXWV%+Yn{2HjjNhY-!4rr$V?G z$cw+-`H%6&*XK{Y)7lh{(0(jLPA`5=ht+Y@4vuc7fS#Tn2=dOYQn(GeUk7tPI-HPVXGt?)DPHp9^}e@CxprRGE!U z{khLT@D1c82lOrd3yg57H{@y%^S}~$HmkdcJlaY+kfJc>mD=>w~Ca#%AWp zuSJ(1VZ&GvhrkN5TTsGOGKC>4zFFNm1wXu8SalqF$c;DLQLt=OWYPj(+zu#Qq4-_` zdbX5InrYiZN3R#JwbnYNyq(hZ^7CJ>(dv2<%L398P)+eCUSMtr zO1Ag;4n9J&8W}_@VMLuzrSviSS#kyuX;a#V^qG=3ud8^m_n_)g1?xn$W6J*0lw0j0 z(tQS>dr?pX7*1*OhCRiG`hXzw2^!+ZI+uTo$GOT7pV3p*Fzmy0@uad%e%pQ*_`@SJ zFLYkzSBId05NwnBY;_n(ihLu@a{58%F=tAGx4igld91}acTQW?c@S z0EfG0g%^^^eM~9z5m2}=52Ku;9vwxAxh09L+>v*GYbDE2w0-h+$K5>54|?qHdJ$R{w7v0_O$rMvuHyx@8O&6CesJ2p+{HY zAnA)MfR)#)*+#McptJXP9?f3FZ=IKY?wNV#oM&A(+7@lyPY>g+Xx2MY7QJcf}SP%bDdtm)e+CbiDYSEdDZE zZmyvFP8p83!(<=5AIrLhiN->6od+v~J0&^RczM!fiBk0|Z=WPWYuIuu4bAscF<=5* zAvlLojVXNO4jb^Z9;HRbG2op|DIM5mqiT=OI-?wKs`SY1V+$wHqAL{xKZR z5p;5xTPt;L(DF$}%}s}LN7$pMbRP;kMZm_yhaygerQ(w2n_sBi_x_pl)(xA*P0j#k zRuh9$vx)YzqmQzlo^OxHp0fRtorD?Z%sZ{QVC^h;PkUWrCyf+DSnGJKxsR$$$eGk* zjf1s3)O}gM6*?T%{lT8#ObRDBwpZnn zQ7UWfG91S0*5y^PcXdyhjmf=<6@FkY>@6*uFoPVUkfO-9D3|>?;E^y^MV`7cOO2TP zfWfZ1vg$*}S~LRGZ|-A0YkBN`fi48Cfg#-NPA#JqZ&_E|YKO8dmqH$LRQWk`k1oSl z(YS0l!Ia;~O&bn^3P~JF^L0BrMqWpe+bc4q3jrFCSxKO6w-~Lgw zD)uf0+Hy9+P(w8*pRq42-~n^-@=9}a$EA|Wfvr0KHA(!u!R>G^_6{!7koS?%5VNZg z2phwvEa+kxPJB&~a*5soFMozONLO9ismc`OC6OL}#h~x+^fe*xSvYRT#&&qJw}#p6 z)7`!0$56HxO!R$S;0o$T6$mKEJGiV=np?<^Ti?cL&Z(roiDBHoagR4Xp!jACbSh*G-ymcpN)hH;GS6a~5eKNyfcc$p-zw`d=_hd&Eny)epG$0v&dO49 zQso{#iglhfncyTXkyk0&L0q*2c!K6(W84OGa(aC7U9PBKb-cmNXQrb4`p0FG8fuAJ zu*R8E-VQ(7TdXsdg=$$>Wo>L^8g8J-eEqTYgdrbG7u!pbQlysQiJSZGODtPa%fO>+ zs#oCy50!|X!hT@LY+)CGw!WZU{sXuGeT2rnn?6lc4#V0bj;}Vd4;O72Upli@Zj^s~ zq4Jfv5t+NW^SC*~vlJDLP;K1kY;O}h9$RIxCV`{zk}Sy*g>Dze_%mKvSh+7h8PwLu zi5Rb7I%Oc0&2*=O-_!f4@`5n!19*Z$bmvp@1ii65fu_=A3xe7GwgX!e=&`_7zH*)t z{osJPnCkDZuCh27cz9fP+lWoMfWN`{sHS9Sf$=A1UbzK5kw9nJobzC6=gW;Ry3F^s zkQA(Qd8i1fu$yc0YHi)F)?h~#k#RSX+bij9%H3ebJk$uf5&(7dtp`!;^xjk??v>g= z`&7k5K7YE@9r@C=7w#FuLJmq_D38sFseU%=o6OC05STRGzAJ@<+jqyL_bH^Rt*0vh z|B$sRogJVV@NhoxD*FS#gPV*Ub5p7;(e*`1OHaMO}k*UEv+oqDW@xXLF)=LN8-!H{BATZlavbQ zhU2mi&&_Z&i=`tK*L>`sJJ1U2M|8rdUiEbyuWh!uX=cD9QKg)vPom59@jb-vGr)wH zeweZK51M|L@m{*-MCOJ;k_~f_eeM0PBx?TATl2DlM3NMZBj`FgVRq&*`iO=;QN2y> z?m)e=y(0dbutKO5t-rI__R4 z+t)TKNK5@%YlwSb+%MESez=8hlBcSo9~4MySRiM8Y6J+q0)BCIbf%-L(;;x#av5IK zB#dwk8>9B_)hvSDX4%R(OTH5+xIp6LN%q8FG!%czlBBnA=IqbLW3*v)KTR33I*Cks z93)lvL-QPwk&KxgjVoWB;`n}}7IS?-pY){elitJ=Ll4h=jSIKm3lzzp47Z~FE=4Q) z+g}>MI>s5bn)eq^Y|k8>R%{L1NU4lI_U1w`@AupA3C)Tf{kb$B4{-neEYKy>e|$sY}7c!d?{+mAVfzJr;dke(83-~zn~(h5t% zTvM(cHX{|x#Tz0T1tt3pgUW+Y_&Q;`P{;+odi!}G6+v+$>$H3X2$ACzJ-+P(?PZ>>+YIoL$K zF1|M9@;#KrfOrHW*t}9ViARUMDEI9z+-_d|XsDtk=KoqricfxC%=GR3te5+~=wE*d z{e~X9ZZYih3o@2Gt5;QIYy5sina+SyP9jL zSag%MhQ8NpR68FD!^q7?%FX#!!vqt4X_H9ah42w!-`>KTv1Bosn8p)7u+>Aztr;G7 zmXie3pl6w7Yioi@nK)}O9Yoobf_Wmn3n(QTNr2|7oe&`QiUD0>eVESZn3Y2=jhPb~ocXKKywb*`Ct}qBm)nxYWU#?`LFfbnE)Hl)cCGBSj?w zcZ02zUMB1uhMmOtD@FETXh3 zqyi37M`h?1G{dUbu3Bp1>oI*l7OZT!D@8wxP?@+5)g{n7D4K zjR;A_Gm6Za_@MFWlJCb^aUp!nj=OK+J};R+2^;Y4*#k#zhcrN(jv!#^hw?p${H<@U zza_h0BYb`BE3EZ%S81fORI^P?`!3_7#1}M;el)Ilrex%U4=Hf9!COCp9(c?3N%GIi zBxiv9a9_bjVM?m--NcyoBYu7D@jtqp*#B%(3+7CfC07wf z=i%XZh#)z_K1NP9Ok?b+NcPrib#^FuX~;RBXOP!YUEGEwODjj7N-!ep&fd(?G;1C2 z!IvzB_7xU*$qH2`X~Fr(`6Lmday%&DLno$ZaNAR4rO~~GX^xRQv9eY5v{h}S+=r%P z84igh{Gmx|@{HTcR9>Dg2e8SX*9u!Sb}qP>TlwQ{g%kDOTnO9}eY=R}VI%U9QbUI*6|f3IyKCr_W^amJ|K(UPNo-I$fWN{+uCB5p{>bbkCs#o#(Zq z_7owvqi*QZgr?OD?-Uajmya*e4_Q^-AUmH8!zYZcRt>@(FflvI#uc#Uy#is(Mz1GP zm}&`p@GcqhiS~wjAj893FDt^i-{j4-UJXt?rg}kT=MW|ZMYY(Z0HdQc@DdD(wcTUZ z*VeB)t{JaZX?{a|#b*6&EbRZ^Eh&G6@o(5y|8F}1PBRji=*$37CU$)eL3`u@-+DW` z75vYBy#LDHLh27-hI9Ny+A&Ab4q!?h@n_N+(lE-`EX*~ieCSwEm3DyrTN(3fi{Sjp zdBz4*%xD*;(qfJOJ*pdaz?oc;>*D1fZXLL4t>Y@W=2=iBmUb`E?DJVI_=Ba~28cg~ z$}!p`q(Ba)h=23Nun$L?@q4EBE_1?cZgnzpqZ#RR~LX>GoGNHbrq=|4mu!vQfDuYP_)}o=q zGETBF?@@z`6ScLUn`geLj9-y2_iRI5@Ry;pI?na58mPhB&`%Oq_-M~%yU)i_Tz~wO zIhHi;;^Zx|9x2iykpYMLv$(8X8n$&mkqR9pGA&(+J}KBD%kTCgvE z%_b~VH?W3cQDZNV9dXtz`nrTy@i}%r=m&|PC8hA?l@}_#CeI1^v_4WIe*G?OcJ(&; z8fVktAr5Ps_R^NK&mP=9Zh7-|D0Dyj(6u(Ut9l*kMC=Hj`63?*&V?tde zic`7ONos_uNFNh6H(L>`d(X)gcf&7Mj@=#KKYdpopFr7F3kcENdFeu96dpiQ*8;+6 zTpmEiMP)$;R^namR7c@n?N9uiV|mj-E-{q(yf*i4J|NS(ON&AOumiU(Nb6`H ztZZp5hX5tL8uc+b93(beRO}-tH0HS!J@?feIawCDBVmvwAPb3IO56y6F7?ivG@-%f zPB88rzY%0jcD9$C<_KMPEf2ofy7{fs(d+4WF!B1&((C{S*=uADJ?$ohCvlX@6E?RL zThFchxSkQmYI2y2xfVNd_=O!wD6B9t>8glAzx0>w#QK~O| zw(Ar3ZGTWCK3FrQ+=M#PU(A2Ce~X`$|DCE_lRMWn&J{~8*hmPR3yn)bm%!pRpmP!l zP;WWp5N-4(YX7=N@bYPc55u;01Ish^!Q5b2GOeQMzD@h z@|lAeVvnR%u()aCV7=j8@TsKc%nW2^+3qo>8T$8j39{g#=~=7avEvaWE3!N21JV2i zc`VXFJm1ihQNHJtgG{KnX(P8Loj#iwzCW?53wIoKD!vf|k=Y(M{rw z3_*8^+dHkdl+A|fkH2^O>ex^<*ec^J+3I~(_oS1OefJZOw?DZ!3`v-=Fgggh2&^QF zp|?c%W2vI;87_lDzLoK^4#igX+O-E&H|OJ3!q`GDqzR`;TSSvHNJuhBNggmDI(NWX zsM@sG^`-&Y{`&g0%Df|TD@q^r{f@K?S$uoYvZ!=3{*)PG@l|<>m(~IFSvbKQ9S3Kx zccxqhYn)}fbD@He8>-o*Fmd#dzG#|KNsrh>!mbP6;dEjDRl9pesr3B!!0(2VynsbW zTS1iZwz$J1U{Dl~%WF=K8HACy7b%1C1`_byZgR^^;N+}6Bj&Xf99Q`Z!u_r;oRiS4 z|IA>ePNcSGdQ7F6Fqs(^)@zUGZ{uPh&a8FJ%OqqTq-AbUZ&6exY)t05`MXsGr3#Yh zUXIuuipPSXGTUvM^JmuySTL>m{>L7IL^52GK9+qsn{CJ;UXlUHo(Dw;)Z~9np>)_agyL%FtxD zO4uQk5Ls$%i&@+{0pB1Kl$d_Q!?J>P%vX<{Rj}@{SE#mdF&~vIFHO_ELgI zY2i}jwjsRSr|y|*+=hehb%Bex-K%+zZ8IWeLT*0V95A{8FPHh;PSTTqdEgl1 zo?emF(x>S|^+WKAA}Tz$H#jM1r~FAy=fbN+KU6sbwGULo%g`OriBVDXfQ1o^gM5apCGb z2AKcjWL}D;jio!BUAW;R@n8m$WE5<1`K8YN%Gq(iqCKfMM~j$&2sl$Z}H z?L6U}r1z;?Rvf|*0^k8(EvVd^bQ9$k3`xcnv1jRn7&;upOEE+@Y0#l4(7(V{-WwyN zIegyO7Wj$ShbZX}(oPq8NV z>Jd8}2Ti_5nGr`ES6N%7<6#H^I57Vre8aIj~TBuE)=`p`xa)dhnegzTi$pFva%4 zWYQ)4KKupW*tE00`GFCw8)T6*{Mh_0)DnW=TUg*PdaIFCQtn?QX)n51|MnGIBy=Ze zAp1>-N)fghPB27|dAGoz8oIdr#1C$Jm?&Wb(UJAHkBSRMRLxCEV&|UQi9J1tlzLfY zfjV-U5l&|Y-6f}jVYG;CnZ?7XTnm8cD>K0_;bz9XXP=>2H4X;{e{k*eJHMP^>YwSG zI`6rDCtmE(zSzSx+ou{s{!*j*3JP|k)O2VZc6HSN0Lfyx7cm4qlU;>11iigM2lS^% zu5K}s9+P=zO~!R*^@<4*?P-3gYUUqDHbhBKJ{u9MFUha3OWr@lD$I81W$ZlDe&9!z zLGk@K%$Uh+J(i7n0R7(0oR@dgP0sj(hKcNp2k}abo!~)u#6C^DTzUc7Ri+=>0B_=+ zhyLbl1PGkN^U+Ph=!LI$!62yuKlXD5t{mWiV`(}C7exA-~ zm8YjaK@RLCkA!G)gI%;61a3LYSdzl>FUX#;pKT17BblK6MU0R@L#@p0-I5BHrCdSv z*s{>_==|a>Fd@Xua&3@u-wJzAu7>HCQ;!^U_xA|o-6)7ltU9mCT}9ELG7&KNF+#o( zSs$0*JwcPWlr(oElQth15twe}Jb5Qx^ple?B%ASg=Ryct{vj!MPKGSLLFKxz)5bR( zo4oAH*D$=&!F;^FuL3Rqyy|Gq$QfovVV3N(Ci_RXv~xhaaw$*otOM}@oO_EkP%qBm z+@!52Z+?xRhf=(5`)psxf{QB z02bqfjt-%t`MHxZA5!GvfM}*vL8_O0k591NzDw_=M61R9QuCM(Cn9QrIp1iq#==l- zwr2@4kMGFc^1stHLjNi(*DLhYTwj+lseS*wET6ka0D*W}1U`nKL$d5&8l?05z{_mt zO*EKqr_P(!T)ofz{m|>Hw_(zX7kkzHyb;Erw3P;iD2-M^=ck_S2gQ>|aGvdM^x?6Y z=~ot)hGgE}yxUr;muP~N{Cc6)AWo3==-Z9`;p6;Zd;#PQa17wSM-cK`K`-Qy6_PDz zp@_jtKj_ZLIIi3N%Fa1}Q%61{veVGX!HNCt-6oSS3%4cqEIQz1(JK5ZF|2K8S_MGW zV2o~jFT<@fg3nzTflAyTkg8Aou4LcIJnQ}Ws`oAArv$Rt5`L6$VJ_Hpu7T)_m6U7Y zCgl;~V;aM!ZAXsH-yG07Vy=H=CE(g=##tEoL^1U-9bh>}1(|>>bYrnR7>^#R{AiF2 zAPYbo$LmPHnv+tLZ2hbn{R?tA@p)(ZLfFyHgPm^ZlHS!-+_-iXG7~(PGY(u zXRk=ypmJN29+FKc%4A+#aVU!O$A_G;`b^=Z@4lz3^E*SA=XJ#1^<>9f%@~n>HHrn5b9U(u`B(ZHdzgWMb!S!6L?g zLE25CHN}nH=Z|D-TyM+n@eycrtPSVTl^WB$WV{pfEbj5?{3^dMO2M~qNV1g0r}Gw9 zV(2_fkHJENuBhujuXIPgw?4R0d0mo&Ju^MJQUkO;9muAoZ*^N}9w*F1`P9`5Mvst2 zJ|@$L;_iKcGxtD(PQXAQ094vre?h|4Oz?yGuFAE@NBEPWtuZwUl+$f5_luhL-Y;D0 zVbj)MAF4Tf(2P()v8V8&Ix~f6iH%&)gX1>u^5>+iO#KSoPUrB<;SRq~3}4VWM12nx zO})=>pUH?KO7FKPy(25T61{@d=*Nuey6bL~2VyD~J>Gs3<=8cK_VcO=W$tWuUk2Mw z45^Bit04=Z%EI;JwBiY+>_CNl`<$Dv$2~U|_5J%Z$RcKX0sN&OAUtQGno6O-t%`o(P}xTs=J&PJWSJ|1NW%3wJ;hW6tz&3|MkF z-&emNT`Q2igqE4g9lY$U)S;FZq2uJ_tmPM1()L+fjZbXEpSFwjxTwpHgB%E!B6Q5n zv>;;XLdk6yRt+83xm&0BBg&KJ`nyk>@a30#JY~LJR2Zsu?)+htJ-Q_Y?ugmr3vvdQ zbKLJ2L?V9!2H*;}e`kfs2X|&?T4H)G{eqZaha86%({8*tlPllNb@eHm?1e`cnyVS) zUnZH{hH?{Rs9a?v17apNkBqoRyzURbK2Wq&?lAg>$IEy@RY%nC(0~?ueCh!b&nCThC?`qL7ibCQ;INVxLPYs8?4=WJI02)PA5&?H#&7R`C%sC}08$ z|6KDdIp!+SGv!nkpDe52Ix z;x#LNi-62PBZoVhPOhPEjzCl6XPVu&S?R2t1ZOjH80EYxoV}diskW9J@LsX4D@(FF z*ETD9PV1oI*vSh$--Ml14!Lio&6+gzhQk@D61<^wZhzm=mfFh>^7@`wjjsnaI&?s#1O)@GzZ~2X zP>Jusnm}5@uN}^5<$wJqp`c7ZcXCCJS;`adocK8}d#nG!KvPO6$WyiX(^^!^#86dw&aRuqHlrE(aN?8%mM z{75cx)HY1Wc1*34jjv3Xbin6Fj|Z2F=n~n{S!c>+ADS$>^r-VeI)Er+*DV) zRAG^a2*to|3AMf~lC0W(Wm0H(%kl4O$mFK}pnpR1H!ApdlgOhQf&c^yBZt}a0Y-tx zVNS1t^QI=pijz;U&#^eVv^VYC!^+&X*t&)2mBF6?(S|oUkP@(pQYKbCQjFtm8LVt9 zS>aW>F9x87_B>+klPZ*4mGiaH5PJB3E1BgcBmL0kZ$5N1bcnsSd-Q7(D6+Y z68HuAWMywcGMz()tV{0?bPpiSlm=J9obF6$6sxhV_uisD00-ivpx3}mK@g?rpDEJ4YX_ykNWk3F)a4Vl?foi#civqGTN8nZjZ?&s zPn$Pnukz7f2j9jJ5g=5W73|T(zy|0ZDTAj%BkL5AaKe#)j0d!{8e%G%-h54SICsikimeN-;Ptci zr~0F<1AlXp|2>BN_qjoj{uc2G0!{yqlgoWfYug}U{|`+<|9dl@=?e(6dYj=NaDoawHQ~NB0&$E z>ykt^+!i@EbaBGbn&=T4cgR2+A-ur;b-mtV`8*fHA~1+Si~)pV+|ncZ@fBs$nOpuA zx`SJtxslxQmQun7>3zy446o@0v^{(R?3h`xWWhGS*gM!UnU*vbzZ|L< zVd)j@?BFudCPn16kbe;S>PTben~<|NT}gaq&(wU=Jh)%w)IN?rW@;a=D9CN51%Yav zo%)M4{r3j&zx{8vXreRS5D|;Tvx04MSs~>I_Y%k=pchie-Skhj3G`%uK?s5WrprD1 z2VE|!D@dlF!e+0~@BKTZtwaCQaR#(Q|BL+QIJWvi`t|#nWgfmK4L$T{{e_G5*Rp%= z4hkp9%AJuq{@}#3bH{D9asdJ4>XJBK2`P&y)K!S3?`U?N;+74pzih1ZC?!P@- zeS8K*Wtq|-^mQ|*<=h(KXihq zK0mr<^nrSSXcE2^C@9_Lb1TjF!mKAN;ALguFgiIKvS9xs7;U@y+Yt{1ZdI4QhBn~t z7N^GnHapBeVt@P3LL?;dTD8CU;O(f9I+$1Y@wRAW zK16@Nb0Xrc&+aY=B;?3;^S>Z5*wV$FFcjYpKcN#NZ-*c3rjC3&__Tww#J5SrnEQoIinAil3{KeDe0B5FWGQTSpC(_O*%^k zp``5Ic943hZ(ZYjp?=;9r()q}R^fPyYjGh4Eevg_M}17chW_HE5|l|NX-R;~ z^a-qrr;I2+#}vA*kiceBSz9urKpJRoUmSZqcjnf|`PU+_SEyZ!`3O8nj~`$q_Y;>$ zCBvHg{js_PTIM51T5)M0zkBrt^qW9IK!E3B!c)z=dh!$86xiP^0A?9d$f+D&DCjw*Y>LOUUWQ9AO#D7BWUxDm@dYXucrYhr* zhgURZh^Xu%iEoyVHndGmwSaBvsY4m(E<{xvvd@5<+DIRssqh0Z>(k0@?lPg61Cm8| z2$wD3Kg?~~M+=XCr&G5K`hq^kwc01ADddQze9NdTfV*vs*UeQ?x%lTQ(wa;{Q!Qlu zPf1|5B&6-#M`Ab8sE491nhtt%vYp*8e)f*V%=Q3xg?t+GsFB~+_FcBu>@NtrzK2+= zi@l-11DVOn}UpTd;Fsq8rqUG56wt3^W20wB5i>=AUz z_ARxi*Q8@Aa1Qo_&WUa^_C$&JkN8#gZNa#L->l!9ceOmpen66)!KwWr#3U!+xlPN2 znOA(LK)IL3rt4j_)t1E)6ZL%9AG)fIJ75;`&QRSF$bn3F5H0EiZ_p3PV9CP|BP{pe zx~O~TmT&Moh&_Hlpk*2i+@Jg{xZ$v*V$a!L>w(2hd2ePl|LuH-0-v$IDPA*`b)+~T71v`)RRL{38R}~ z;JpA?H0c9jkp8I(y?0Gw3r!bl`a@oJCJkDe^_|-YMNl=5_hO%rOYAypsGIg8rSHGrJK*=Xy5n@7L~mK^PM?GsyY?gMqI zw=1R-&&=@Wj)Q)N5sXd7_2=Mg+Rj9aIasTD@owyRo1GmetaO>ws4c;E7Om{PseV6@ zfqa7K-3~A3U8^hh@0}lwvIihkXjmcNos4t^n$qaKpZK+E^_;s4zaX41>z>LYivCLY zA;(bJLA3w3Ic zUn%$%vMKUo<>0wb&Ln7arUYRoK1ey&AKT0qtJWBk-<-yi=DRL*=8f0WF8u}V1%Xh8 z{T4sLd_G_~AWCFf>!ktY;~7OMxNfC|-P3osuW~Tiy&S@TJKiE%gVtN3)ebJVy-p%M zh*PR}&m&DvG1@LcV;*yOd0K`Lo_(bF z%?cr>EoZ7OBHHC8fcEYl4PX)u(dO>^$9}s1o0>-?Fn@M;o}`b?5KLu2PNl*V3&n#~ zckbH?kFC&fQCuqJp2fA%n~g_LL(VtU*?V0McNFhATbC4~`Ic4($K~A1 zO57e&&V=r?da;WYrs_y^IOubHRiP2SxKh)yKdOyFE&Hvglp3@K+Q2~Z0ED?RqVKm6tHC8xDzaub{~Xl3(F?dYAn~9+beNUEN{>c_qY2)?e08`m zj{YbX2HD{!QP@q)FN=hAJ=y=&2#wdlH}Fv#(JM=zb$Y zFa5|}EO`w^=)*#=KXS+*tHPv&03z+@QPDr4WEVsTV(S`J9wb~cWPXdeE69A3**k** zwb!N|;64jxK@tCaz<++h|H3(FP3I$*5X#ygrjeAxsE*n0RTMsxvTtrmqu1}+czPtN zEA=NP$FlV?W&KB)8;kGyRPM(0l@5|7wzf2V?Ypu2<#A$(-f>^1{rBaLNNi0vg4BO# z9Q8Ok1;mFg4P-PSTc18T`h5s5%`>saLmG?by|eb+rjsj(anRSqPD= zpDk?cM|wxT^Eo~3Sf14JSy0)*k9#Nh@5&l;@Aeaz<~Q)bGfqgGW+Kbr8sG-xC@3ic z6U-D*Sa1q?BC@t;(nYW;vz|J&x@G%U9cEe3<^NX6r$!a95x5}uP+#DXBKleXgWV?B z;jAIaH6E!es*Y7>?zPHFuzhTtjn@o%7>2O_Ey~8dM87-%@OK|afPs2l0SWBDi&HL; z{}7>ZC0_3svHC-VCcvfNHL?4*2n~aQWeyOb+1#A(J46w}p8Ko%CMbms|0T)r?__O& zBFpcVvNcl*EXbU^8%)@ojR-}T)CYUevVP8J#yGou8Nr(`(a!R?vZ{)U($?<>EN$8T zO%+D?zmjqL*Ag)Ql#==_rB#G(Ljf=IQN8keGPZ3mR!uMF?S&m)Kffa!r|)PdeKA>p zEWCuU2%G-7Y=gvZFfAYJwh;|jMv&1Xhy$93EN$9w&x_D~1vPQSR`NZx-5-fryNUdf z9pciimkcFJls#^u{(R;^aWCV~Wd;Dgla?C9Pm%XmaIJMQ)^8n;mT-IV+D7B6>d!a5 zXZFEr8w__dGBMXiJDn_(^L9@V=*}_GZQhiUZxXWz(86va6N2EP`V>$v5GD7%JbYIJ zAD8ACUss3;xSH!$E~%yRRhf^H7rptx{G}icf4}){c7f$?9cZF0qox0XEUmV}$b!&c z5E0;J{M$c`E^~8ZNl%bVP8iD%lhi2?Ia$)b{DN$5&q1k2fIUlGCG|GHFs=-n4TfDP zbM{wQRS$F-zrgw|Tv1n2i%~@Y$B;R^=@V>Z_X{GVG_&0U-(t5NF^+ttFYOncpu=E( z?f+r#yW^T#w{3%pS?eZ79qlpjO1ev}Z56UH?T zZ^DXG`i+VyUn4J#Sq}EwnFyK(MO}4l8S=Y~ZJu~3XSS^pP)fJX0JIs+*bNwAYhW}l znM4FlX|(nq+*(@`EplSJpWkLHD-$zC3PRQb(ixV8Mly$s33Ev5O%^Y!PH>>t)lxsH z`&#coX*au{qUQ&SJ9zD?>L#lWBeu1KjfORds(E&pBGYD ze#zpRr(cO9k5}zWN*U{S(U_O>KW)r>B>PHb{b2cP!NCJ}%p8jD-`{hfbfdh^i4|g` zzCpHs8HtP!#Ts2H3%pq9p_$gXb5D`cK8cA_v!~-Rjh7h0hKt8x3y@2)M~`-VIPC;R z8Tto)LDkZ9CLXfK*?NdmK^#Ml;{l@_3cbYN%BBw223_!Z%U7r3fB$H*w)dRs=CVlK zrE8o;xsxgAZN!IhWJ^GXgN|6}&H;u@!uV)?$g?M0()J!V=;8ABw6%41gx)vbnDtI( zGW(sv$`{%LW`dyvtBW=y*1=L5a1-o>?P%sTVRh$&9aeJ%$L^=d-jdC_w`1@sNf()d zqM(ssm29vDm?o=B!H_&QbdcU`#Ydw2JX|G=)o9zR{7?adg^PL#9|II0Aee$+Vc=kT z9ZCds9u>~grLyw1$_Q>JK5zN7FuuVS8)NcGVnqOARH)-Du3EZ3!syr*PY`am+lW~axP)xcr{w?|6R#^(LHG9ff=&^i1P>_^k1 zY)gCk{z=S@S~-OT8=nH5hI28uxu3Xxy2>SJT)}o?QTI$1ySFlD|7rAMUN)=mdj&2* zZh9BPg5pUc=#lp3q-`vsFcPY~#E-XeDc*=Yev@QUBUeW}fjR@K@#`=pmLxS#3>-Sz zi`5%b=XE1Ib~3%Z_3#DXNX3CIKNZLC+G!P^^1{Ts8NoTQ^cN4AXylm3o*sM3P<|F> zod;n~sqEhNs_m+mV?ugImJ8nGL15$3MlCmi@^#@;YComdkJdBxwa`H3geM3&QW1KO$E%@3swjdnvXQ4FR(;5peJ0oO?&>W4>|TQcM3xQClb zUo`bn0_74tGF;x^LN7Mc)_lvY+lp_CZ|CMXS%ctu2ilj=o;#1BNtg``2g;R4e_z(z7vasQOhVGJ)+DFthfO@;XK zv%!m&7N%yMr#6L$$J7rbDSb}ZJZa;zb{@Di(MMoF{&3X6iyq@hPKU~@(E{TS#wxc| zOWfuCG#`@*T>CZ|$>gneC~T>GsrK={)q7O*qQSGBD)_;qspUfB@9dp6?mK(u#vd+5 z6ky3YLf2L3T!-n_c;7q4dzBe&^&h@8eC2Ev{UZD5zeo`MgNe)kn(;RO*j56(EA6Q7 z-jzORw6>=P72$^&imI;OV%U}I==>jKIED@A>B3(^8~Trt@{3?!(EuD5G@)tvEuxCk#mlZ(o;p;_ziK+QwHXL z7Z~VDK9-jC>X1x%tfO|L%5s3r!pA`7E>0k(2oKX}upeC>K3H2RO(ll=I$p#GV2u2D zANbw15>vi&`T4ely$?t3T%r&L05z$Ob-mAc$eD3LOD5MQtIy6rS)GPuO90ZKB1Wlc6pqA8+jhFFMC??_&YJSVEvK!^iQQKND|pM zJwC(r!Lc;teb)=_yaG?mJz_PuvsBG=5WESN1((r9Y0G-8AbWoI>-;d$fH?A~(tw$Z zRU1L_`P+kXqNNzxuiamFk&bqZYx@<}nH)-}3m4qZLhim-MhOXty9Db|J*$RgandfsqUCI= z9WMJ8K6W>G35HM#a;G+}HZa$I6=zYu?-3YzhBrt!FRJ%_IhjlvLQ#FL zeRm8!YhUUeTKl?pMma(x^o4gR`hcmtAO?e;K#~ZsqG`1Y!)pwQWCV17#rky z2SHW)P?s>=td>dzxPTIONN!UPcJsQSJtQxUq@KH+rxUg-q)l*-nw>vSHs{rvr2=fg zafrGS;fvyCSioM8m+G2HsMnL>J_XtL9LZy&N>(j{o6eYU>ohpuNjS{;wN~^U_8^=O z?!j$|8{Sg&YSy68C3;?H`Cd2e?N<_xoGz`|Y4VB8^=_8yc}O*D2c;*5fx0k8BlgCd=_yr9%2EULX3 z_$@6sV9Nn z9@B@nIv#m3Ys4A9Ush9e0unYfV2HtcCkVADsTn7U2a~*aNjH3Dygvkw6knZa7?Ioa z)WrP(>_N9;HKW9r54Eynum{_g3r-$gz3zH#%bP@YrSn3w@sxdYr(&1bmiT^4Cc|#0 z+u??WHvxA8Qpm;WKerD%v%hQ~+x1J)!e)}qS55Z9=s{m<*_;9004oJ|6ih#e1F(SC zh~O}jv{37*m|!6pBTL%Dn(2^_Pxp@P3R4ouGio;Rb;~tP!HYtqIYTz6HG0hR17Lv@ zltJ=E#;yY-hj`8KeL!|7qJcB+m-~(>W7p44heKZhO}@?(1v3Ni{rBFYzI?g{xsYC> z#DU>4j*P4(mT=DbffOWah`)p2lg|_B<8HZyVX}?ibC)&hO@K&VgTg~PdJaElL05&f zX(<%f4qy2Pev!o!7u%QjzxH=noHUPh;zR`mhAgTZxb?C~#G>b^H#~ED;dNx`R9nzX z)IPO5>k;oWR8XhjgSE=?kJryXJ+e2)r|o@2?V_f#t23~0oM7fxsz6nSI29*`2;6hQ zTm9CB*Pu0N(I=B4SAKCCXI9<#Cm$z8nL9d!&piL6%=8KU&z&@+O1yyl*%cr>HC8fT zWf^lFy$9ZAk6-HFdER~7T`F`YQ2&bICF3!p22#|@RSCE|E@gVOa7X6izoa^$GkpF% zb^XMT4ipd>elZ&fo$bJ8=V=%$pELHB%c#+Nj)QW0s&MpiZiHU_n0i%QUn32alif;T znBvE#YAtZ?mRzLWVdLo6oF8H}bhYd(2m`C$&02NYFW>6txCf)DEj7brKhS`b`W(sb zzq4tH$knTppr&)gB<(Tz<&Kg;Misg_`OclZ%ado19fJahK;-RB9*vt#95vbyC*_XY zU1`i}@3|PE+8&P|+@D<*zp+^>uaV+V$SyZexik_^LI%rmms0qv3pUA$n}_gi-p{*o z%9+pLWBpW;b)EKZ9mI}NtJ8tCf1vaCf7>&FI|GJ=Koduo*Bo%wD!D{0)Hr|aWALpEkASFOV?W3ZZ zqHon1!tcR))dW3O7(&v3=sotG=*PSlGqinWpXS)3l`A9 zPEZ>C=Yc<8$UiU7|M*3rxj4)aDm$1=LX}-kmrl2OV6m;v4ViTX{2tQKF^Qr{xWYQn#qY^miBH+Na9t$cjpvKUQ4 zV}oAU1?`7B;~M|F$;+3iCU7jKN&2j{>*FpZg;Kp1+$T<&jzUd|BfH`xn;efE`>nCaJk|3Q7hJF z0>xs))BJoI7F17w3DJf5yCoHLIAsIT@9~rcWN$Jn>V7h79R^x3Km-wuu|}Xyd^0l= zn$EL@Mz*f}2jBjSM_M`im(VrHU;Ou9{{836{j62|dF_6=Fn;;>f3JlSl#}BoW><>$ zcslb^t@iPRnZ_RVLH~*4A4t8oHfHeL`i(s*o!>=V0tytV#xcW+iA=s+_Ai!w?qlIq#5+i? zIff@i>?&}>G!;fY`-XV z{COY$+tGwSW(2>Ls{TLv95NM8Ifd-Y8LOX@q__?7hX!_4y`~HhZfJ+!duo&)ykXZ> zu5=k;p+e=F_+YvPZa>I|v{?pN9oTfNzpB2+~y0rFq z6W0N)n2B)7Dr6TQP9Lf=2C7P$7*T4@vr>y8RA}yr?BAStykneA*zS-_Q7qyu;%eX# z9h4RqN!Z9LBxG;C#WDS7bf+lBS!NjVihY6M=N>S}tG}%DbtzOh6ZrVkiE$sj^=YG6 zfp;;va}6j3nkL-{-U|B^mb)NsXT1>SqS}X&ks{U!^P*W-w+v5t8SH# zUx3l!(w=Ld@M{|h84Z5;hNXqU348EAX82G4HFPq77Iupaz287VGVc)S7Yu?<0UQ9* zzC>}@nf0mQ8zRU3!>T>&dv%ne^gD>Q2NPStD@<=kHS@I8VB-0tx4q z8GNt?DGZ8H=KRLxvLtu;2Yz6&K3_GoEOVP}%?_Aa-!z5?;C_-Eea&eisb@BM7-huqA*Y7yw zQU2%;5XA&7gzFAugv1O9ck)l&xPa*por=Ut?=076%XKT|FK{B4)YDGCthw@}_*_GK zP)aN<;_e4~SBG1Bt1|c-;`+o{++`GPYl>wUONQaXKjPtc_+@EF?My`DV}}TVL$PwZ z>;_&Ow{_%jXEJ=4p{>3a>`v_?RL-y~b0xBRr>tIueQiIKBprHHtD(r`$SZl?T}oSs zs$k*-ysc+;aEuTxrv_$kRqR8Cr*CatqE}e96mF5#^7fEEauPUBXGCwsS8kR(=_fnl zfzpDi1k0hZFC?nM=;OhUWrornSw)=%=dK;SUbeS9{(~^R^4imcy?37Me}8U23YYsA z(7NL>NKyR=jM$XRTOu?Tu9=i`4S-8f)4tvUpB zLQLoo5A2$YMRqNH@f7jTsyTj9Me^7w%Tp`rTQ6*m?_3nRB^jqQ(R?GWUz^s{O1xyi zRla*PJo*%TGp{z)-8nqAPFq4;8#IX1S$jRfsK6ay-U0*i_Fd+Pf^O;t5zW=OXE zL*$Tts>PPt-g_k_CZnGcITITN9GelMOf!ZZY(AnM!&=wLlE#{K2wSyjo+eKZ=%3_& zt)bqn>#Uy@YeZ8-PR@_TRnNzQ{q?ANERD69vP}A1Db(ed)J(k>ZD9RTw#oH+evO}C zTn#eK@TBntN6UD+K|@~ChgCn6KTe9FN%5>kZJ1%PQ{>VgTu;o|CdjF-fG4UFx+c6E zcYj%WCp+j*cn8JFaWEfYxje@q4u?R~fD4jpmqS@$@#}ssdhPYzJz2UkiI{`%_()mRaAZ&E8omaNB zzICgh*xi?e^@G7%PYcJ;k3v~slqT3grk81@{!f#LA4>mIW8`I4q)QBj^KKTv3S zIs4H2&C=%UH@$Aqk+z%!aHC`VQnbD{_3QXZ)=>G7BsJ)M*Sq5Ds&|KNO)lDIa*XmE zdLDOk5&N2+g+N|{syv7j;h>Lh-8Rt5PS<^w2rF?iQnagE)~KNQVtaoS+{5CzoWm0F z43?JR7ktsu-RwqJU~!dy2L12UtJ}jBX3@@6@g4*S$%`n4STf534O+eG*nqn8^~^ws zA#84)2(tNGSbwx**oHTW$D%Q>ZP4?F&vxz{kYBlc2K^IMd)H{%l$O?oz-`9WVmS3t^c$>pryZS;3nc2;ctfrDP5Fze9S41L zKej);7xQq};j`A0g4gnkcXn|ipT3D>2=R=t!T~7Y!tRF;j#Z#Zdtq!vA zN*kr0W$xK|_2!V4RrOcz_hYyXP}MP%%WyF0-wT`5F*9XS>8b7-*Lz-K?4>>2kD1nO z+iGR}@ho6tQ_LG)scatE#kmw`Y|KF!G?Hz48@UD47H)62aQ3dz(w4{nc!EN?_n0RG zvW0rIK*-GR{ZelTzOwq|WTv+#le_%mFw8%3&6yb1vIl_N@wkmY$82Qv5lgRx(sidTM&o$(?eH~x z$5i|6W%RG_e)aD4Yrv3guJ3A5n*7oh8e@9mhFX(Lp6*VImX^3a6s-sb_=|%H6LPUa zaGR?a>Q+pv+r0n5~Mm5n5q*y2y&kH9TQIo@qhsFHu*3>LbyVb_u z6u${AGu04QKJROI8%ZE&GDdJ!&h^`i;G z4aFSJcLIJQzuIpB9l4&AR4VqOxA#C&H<;j9E8-@oa7e<+N`h^_1mf!Ypia*VK`oi+ z(#KKCnYm~G2*=C+2R#RZXUTMSmblj07dc0sbG3@kh|+AVz5F4-flG!z)eZDCM82qJ z2z?<9b6UjJs*~681VQzJMNOBUT)6n6{k$okOlY|9!1LCFQ2w37;@#_xAXY7Dnqbp$ zg~!J)MJm9+59OfQ*0$^?buZAP^SQ6=IkWfPYU5!-2L*LJvR4z>89cX!TUWvvs7J8n z!55U4u*;jK&-c_RAX$X0wFNh zrBaZZ9{t>nFMsB_+B08i_mn7w(ocdlv%Rv-IvM;2z!&@rR{RfV%Ku6g{Jfj~dV#~Q zJo-D$DpaOW)*OK$P z+;5&`Y2VFf$Ez;rgVARfGtsIpA-MHts%jwIHTgi2cH50sJ0CYelw@bpFQ( z+~jY_FhAaeRdKLV{VXt((<&*TKZ5DFr+^z-ttEVUqsJCUg96&&gxXIRZnx{ASn|>;5{?&PhI@(Jp)eE z_Yf!t45VsL0al}7nQmZ=!0q~mASJMX|73a0vl)n~@xYK4zLoLafAXbfI1Qw?3O%#S zB5W`TXt3tr%McItL=yGXS*BSmnPgC{7)}tzUB@uuFfV}sJI>G0RR>j#{7%~#v&2=o zIQSX{xZ&i=Emr%0LV{}nHRg9vf1cN$*X!R~UkjojTyrw->Wc$YXGe9q4CPo^uDkV+ ze8;D^_e=Oyu%057P9oJ2TJf=2y7MzkI|hK73Fwhw`ns9nH4E9F9W$ONV3+}H*bB8` z*1aeu>wArXBB6>%Ti-!kdaw_%#QnD;&Q|AXsjw(zo)!%|s*}_s*Gh(>GY1Cy2P9{6 z00*lzkYvp7CGtL`+2rHMRpTJ8Uv@m|WLpL_Vhct59hlrw-RK!cA2`;PCFg2BF?IoZDt%6}ut{MA2Z%9|27N2fha9 zIA|A#VR5y&Q=fO!$33f5L`td5B1ug?!(D=!*+z}ZBRnrSIkU4u7SG^gQD+JLNnb4A zl)&q$w7I$G9{AZnr@VlB{)3yUF8O0Pb}6{&UE^kVh{ln0U=Q%8JJ=e zi4dE<6TURkueNQW(S(>V)OEfn$vw>Y_&3CEvt%zs`hg{xweFQ!P5QM#*A;6bqSA1% zqNM31GtRtAeNUY9qh8f{o{=yP57R3~tH?saPD&2raV3dW2hJ|j!rz~lm{k183Ujc9 zbhbcSGE0~xXGA{>wZ47==D&XdLdFT8hRZWw5T_kFK-YF(koM)Ty~(T~Q0gqfJLK2mFq{1j76+#e|FztwduYLP@+IL^5-WF$-zA1hs&jNXWk?_ zI_dZHByoAMg~p?sUdE!g+DxeI>`3swyLV-)2pgl{Kb6I$NB4!>yH$`9jFYXtUtEwM zV^x^mHg67L1z%g+i?O{p+%UC)-~v(1k^OYiJ_jje7)#w{?O3YE+YhD1J#~=haHRc z6QzgH)ma}g{LlrWlP1!mu57O8NcG*>G?dODxOApnl%}_yITEPZAxVdYi6HE>HH3I2c|pYmcguJsi|zj)5%EZo9`8u3B#VZPZ8>;_C8CYUEIy)vmRl4S8Q&H1|Y-nlp}Fuy|J=r zjV5>6J2*!3Yvkjz)?L1OY4*nxL@aLZi!!H`hky~e`&dEMoiL>>!nzi=LMwqBEcUUV{XfAA1I$%z6LrQUt9L< z1?`31sRu3OHh|K0qbpb)Tm6P*bwQ^g`-Z39Ns-R+*(zO!LI_^nLiW7_!&`eTU8mn8Ps<*v5<$X~msvwpcXpqG7(uLT{5TTnpe zpJJU_H3qozF0L1|w9^$7n*a)Lyes(z*aP*MPc_~jl8Q~qwENKbO7)EoweaNO20k5~ z2K}DpwD0qI#THpda!8i+bwF5~Ih^%b1d8YE%KL`czSch~YX=4pJ&Arkv}0>s)&&pm zEp;3BFPc2BGE%&mSZAg)kCDXvlk8C~t`+|?&BTzU)Q9QsqPjr>ng(JG|G_j>ykYac z#>qMum!EFg*fRz7s?&I=rgNHOaj+|jY&j?HB>Lu+_I29Ig~~R9!JIC(kRKOc^d7s^q!vQsVcue#pj-H7v7{_%iFD(h&@0dR1}iH1Jd53X zq>IkqO*hU^ka7N4*HyjFeeKk5YBI{d`7BUu4p_G?S4Sy7`LBxNf3Vc@WA^l8_7en% z6C5y}VF8`{LIc0HTV;eF-hyN#v_wKQ$znhdJMTx9b^Z$S`6~kY>rc^7paE&65C|j? zn2jDbEaX$u0F&vL9yi}Lv0ph(VkV8yWM@z5nVBIE(UjoX8O78LuElU=6OW5pI{D zFT5k(?#dx9gPoAg72fM^YRN?hUVqqandkfIr6Fh zm)b;w%J?kfsey)w_C2O|xi-FbzO%%xxxmTLkE5A`V4V(H_O4IHGp-aGs$%^qN^?AP zeykCdcxxx>lStc;?O{{0>T~VQE{{*F=5La@rgxJ{Ugbl$0W+;McAbXFws(*Ql0C1?7*_NukT z51^vP7M8Z@4<`eZc-Y%-Htrh&l@5S^)g2@w#+0G^mIf4We2XwNQpe9S0p(Sj;{2G{ zSo|y;fX^M0>%6=i#D27^ct*e9yx-^>(;LThMfA(t@5v1=J%h3M4~%}JYR(@ z5)aj|6oESqcWP$$5bQo|^r4DPZkLQt`}iEpoRg4L#5-ebdZfoa!%_UWt*)&zNWpUT z$MDl?@(F$GwSs%%t@X+$GmXvKI<>kLu1Z!ZR3IQ=?w}m!hn2-iMDqM-eYn2($UNWY zH)_q%`H+j**45ywoOnfh^oRK7*;o2^&Q-+GfEa!2F+-V|M!W=VWXMpJZfjOpo}?@c zMP5`@i_B@B1G z>6*pve&z%RviUx>?;9|46_gZmP`rHZsNcW{H-iTT(PRBXs%PlOba7@jgJY$PwzJJV zAp54P&-3v0+X`Q8)()vsQ9R=;t=F@()yb5~GlHx_Mho*0BAqzeux37p6J9~iygl5U z2(A?DJA;sO!6&VQcxv#fY%iLtxC!X0J zx-7W)y8Ngv6QbnIhSGQXZ~JO+kc8o8=(_vyL@J~$O)EXn=G>fMcE4|<0e5`VnpQAM z7d4WH;?TFGmeK6qMpq>zxb17Xt*H`pP4BS69!qbL-Ho35UQZ{IY^L^{S$FIs!c274 zmd*w4gJVAYefguVurBoyiJVlJJ#(r>>%RBnRdo%OYjdigMoItf^I_c-Qh39Vk<)s} zqH7U~0;1M@pI^NcV`w|ovYNSpZuRum;|bu!!UtbQ`4LLQ+TrRvjD|v6-(}RNVIqOdD&lMtaN2m&F{~3%LF}6p>~=W~gT|_$V=*JCpLVZK5-7efC{UN56W#(3riez8qg! zC9+p+iha$Lil2?4@5UbJQtZ(`OCjy=)RgQkFGHP;+Wz?5H9NB{8=HfA6P-dSizD*@ z$tQJ?vAZZ?_eWj67EVDI`)V(Dc+7m#cax;{dug!s_AW{RUnj@}Beg!*|EvRLx`$7chP8$av zUR(uYpaejl4V@4vm3SUnWl*(Qyr4tIgxezV#q>-0Z*ILL1!KGrigx973Be0j6EyZd>Yu_(=r@tM3kp58%Z@FH~9Rm^VU z)^NT+v<}q4?t4;R-QM0^-7PJ>@g$ktC#I7bb3oB0g!^e0LYW=036U>y!ZBz6Z*bJV zXK4Am=YOb83;aHg<`*Sw$2mv%8G`%(tk)3f>&VcI8DV;j_z?LWm)*AnxNQA*T$YGw zTmD^o>aXDJ|IAO!CR{I6hN&j@p-4SiXl+6FA%j;D(z4u=Z1G2&Fm=y`S zv)sZnP9TLVmE64TU@fw<$Xapk=Az1im#_)Y3WhyB}Jh2BSRx>bCn^5$m^pSE1&x zxR>cC-SIqFzAP*sMT|h>bFZT`89qDvVW2i!CBQ9T{_2S<*+QGEODAHDw;GXfjp&66 z`k|x5;wgscc7|lGIW0!2qejb)|K7TfLM?~6)tx1e$}X5*t=aeD=8W%e=D9sEnE z26J;rj3OSb?|Ma{n953FW?dH9<$k?+{$~~LRz5A$g+@Y)prQY2R#3wLe~P8(S}6GbHDGV?IOzPapxcR>^Ocl4 zv^TYty(^`?69O%XB`K$kRAUd<4@yTk^xOs!eS7)!p)JhxW%UgX1CmOVB!a$|iC4s& zy44k73B>4=z3JY~0rd#>Ps6bHd<1ANkYfjpJhx;W2BHmp47Iks8xN}_0%}}kMJo!y z00=VN5RPL-`GV^>0k&eWA%VhkfcMf|!9shDTHQkQ1{4bTO#j}?QTWNA-Rs;lTr-kq z3QoB(?xNY7@usYVu5n$RU07FMYf&Y3cZVo*8+t07Ag##1kP9#!%RfH2AKCGJrpiCG zI$~$&CmHInHZ^3_p#GNOzAx@z4e&Y)_8++tv$QWjT%gdU(?Vx^($u8<#3i|sCC4A@ zDZBASJ=LDKF&hkp`{BJKVWmUzl-#$kUqww18R2+~UM!xrxbpP&2B$Qm2BnVaH@~VH z{MKFbd(Wx-ia`9AzIY4~`zi=oX^6qHw=p*7x}s?)vNS;{T`83KvN!)M`qOG*Edm$a z_K&A*{GUzY|JgVJ>7Xd~Evq~5e5hF-kY_&LLWC-Xq4z2jPv%XZdFHlQA_5vI!!KAF zNiQcfi(knH#;mm4hTqfnr}JmPU)5oSk5a^k-l%Aa$9iq z>TdPLyCzrtMQMZB&EIh*+(cCw(VlI)L1iB`DP)EdNO`^X1ucDZU%f<{t9lw=$J(s2 zKOcy#3)V!9;PLCQ5Pcu6xfJ5VZ9V9@p=8tJplgjDyV6lRudg?}ifHECQrS2yHRDDt0j8$>#yks6{Y7wJGZP}Z>vQjmirS2dIp!+zP0{`;6w?h zDKTH7Tv5^Z(&vSg#4(`YmXs^GGs`CXpi8|*WwX~r!&hF|*Ka2qI80T#uLKfPFtF0a z@B&vbF*yhqWNP`UBr>%oO3Ef(Q}w)aunpTo_FBl|^xq+fUqK@O-K%8KO~~TmIY&w= z&5GjgpE|a6B_(%IAx-;m*$z)%E`i)Fnu_am6r9p)r34#D?F)NfacyrtIH zq+RPFk!6!OlF^ohe%gb^?5baft0Q={&$!{MJtHgS;6}=n5&URW=j)a>hOs9t)#{CL zETmd^qQdIoIkhIv z$VWRFW$=GK7R7QvjaEP-IUr*g&x1yl@r-CwhBkpA0s;fpSDmpHe1>=z*|^V!|Ky=O z$j%^9#w&;v2*M}8TT}HQ##Em`t%}9Lpqbcq;OmD!W8fo7-w<0`#Zb9xDyvt4ryhv6 zxA5N(EwPWmFv3h|K@CY6B2rF)072~}Qtnx@j5AcB6HDj?u)XsE$!k3dP))J`@o*gi zm)h=!53%2dl{HP6B(bD>SlrMzgyFtqmQW-}hl+pY%z_JITS|DkYeYpmPups~t$C3_dDy%%a)>Hm`bvEv!sQYT*QV%_>z z>^s>8rY1+P9R|&U8NyR6uI1;r*1!rv{Vz?FJb>o;AgV!!#U%|GTOqJD#~_!pkzzl-v0S>5XyjihZz{Dv^uie&>f-h%+o z+EUe7BrW{fQFjoVe>WdNFf0ErWiBdBEH&&_ig+EE&Flv8RVvOzKAnE>)WheB+;+}^ z2jX933e!*2%hQGUvw^r6wYHrTlO?mixzgsxfR>Pl{L$rGM<$PY<3)K7G<-GCFoQ==~#HVWZ@v z%r*W(c}4(zJ??|yHfH<;oHpWNJ?HBveX`5LQ{x6@T}kbu&_tmOdZLL*k1Rj0&e_ll zuWXn+nNiz=egko@Zl#Ku~2{BHg4wBjc?*vKOAu7FTTX$FP;Ymr6Pn#vvoH zX`@1S1^QnT&X!J%B{PKAdT2M9#q)+*3>z*b5>v&p!?_gnPYpWxnL{t>OwR89I9l-(2}BDg&y&b8louo%v_M=s$DZ599HiZvN8WeUA-h|I2d719!)}QvBLU z_ccSp6enwA_&3DXQ)3wBO%K#kcF$7KKh9@<6(;;f{__VMk4!^SPNK*y^~554b%pQ= z*yEDh+EkS1`J&i`>45z6g{Gf0qPe8cd$;+e<#*=$5E|op2`sM3v3YXVNl&r@Ev=Jq zky2MY=gn=eWqdud<5 zKw~+E9JML7EZfOHx`zBUhnFTMHgS;aa0c1wMly%V+t>}A@x(rMP}7rZd;j!H8MgOI z8KSEs`eEohw7G-|9(HRj<{5@<40nWFV}=~QZ%5wF=^j1&K(VSpMdE%Zr9owt#mpjp zHwk&pZR^>Gj_@Uo&huE_)o_$ADan5{cXf5ozUv==5@l;Ot$jBgpCM!k=n zLr3Iu;dgQ}-w>4q6ugXp zjufL8?NB47-w@Jo87Rs;a|L`&^{(SA9xrM>ku=Qz4KdtEWa;ilwgYh27YD1=*8wXw zzXm^@{{6`9;7D(3G?7%m$@+#Egs47jOvN@#6F99~gIK3|%J>1$$T6D0+L4YLDTBn` zGPGL(XT5ET0VG4pyu+6`*0x|&R-npiBUp0Z5D`q4_zVh00(euEE2^j}Vl)HGCAqhB& zI&hLInZz_Na8lJ&A{4q0{Prq)m~jz&npe2J^AtQ-4OP#GK%7Hm+NrD#g1531Ux6*0 z!RQZHXjSg}ag{3bl!V4_hz$$h5DS_J=-?81u?{>wq%=>71@Fr#1W+%illLk&9UkS& zVQm7W$ixvaPTPGIu=ckC2GnansHJ1zh<-w}dW2?`{)a=sLyeOG*=)g0v}ZCS*c#v3 zp!MG1BuP|_(hg%!N)nkXP4cACvzM38?UHsauBw{iEbwn(2(}zu#OeC?*eop4Fq$l=@-*+6dAtt$l?6{QcFC;dqK zG_Jf>*g@hre`bN-jddCH1GuLxJcBAujx5%zTS&)jax!Tf(Yknx))7N85QnfxoHF~G z_}}TfE&T8fe#QfkBrrpZ=b3!@ONK;!)JQIB!7Uk(o?^gRs+Ga-0nd2 zAT?*sfWkdLiV5}v%>e9z4m8hQ=$uGWxtcUjNoDjO!`DVl04f$ue9S;@0-T3Ju`eadl9!FrDR}R7Dt`zdgKggaO2aGmd(oydaD44hgGG0o^7HPh0;M z=r-{f;xHEx4C>VegL?f~iKhGIZvIU`WW%K3Db`JT$AMkXx#Ae#KC*ot+NML|+2QtG zClq+^W;{F}>%P=n3CIkh%NI-^W>jQY%+p7lDznR}-?rqL-?~0xyBQnl-rcLFM{yc# zV6$Qf9cceP>g7dAMdm+3f zE$*ik)^6K9dcPp`%IKD->m&ql)Ag2tYsbJekhcHeyB1H;z+QDL)`eKiKgSMHxRLA* zE`HZlaxbnD+Q+m!3}`PFYIfg6a)hKCs4P7v{)B`9{m01mu0gKsfu_3&Gi$%Q=urPX z=YV8Aa48rInDUKP?tHmdK=d?kg4qaS-vh>0<}2dJQ(~VV8rq1uz|-riGUM$TFy?~7 z#lIDLqPB}a!Bn5W7&~(dn^E&DiftJ`R{`#SF$(AG`zIKGr7EF*ICachJ*jkawEaxj zp@El9Mbx{Jwr6-{7vCG}eHBhWi#fmXZ5E3t5vdWk;*V=0L>3?O`DIB1Hw#6?frIU0jtAxwDF10>$JYxc$dyz z{BA#)PlGQe9)2)YhBDlLQ#=hut%F7ih&I$HN|286stY=Ul2fYop_YH9%2{rhhg#J9+(iY6NBxyryM^)wl(Z&x% zu~8KBW){#A;`Spkzeb9q7h33-zwAq%a|bUWS_&ID`|X`3n`B)Y^~WZ~#1tQY5p>e` zcr8!;=7$f{YD}K??hu4>n@o=vwfoT1&?i$o=2XW5p2SyvHkC@cX!vfO1*8g5EZjW~ zjuGb`J!hQi-hU(C-U4HvGI>3o)s_2U^X0v=A8Q;cZC}PeTrg%g?S?3)TlzpLkSj%j zgcM+GeGdty!lq(P{ zHvC>vB)X~%E};$VxQPI4J0y(QZ00w_Q3|uI<9W*IgzhrD#)XDcH)Zld-?Q&Y&yHfA zC!~QkF_v8F0MA_{M!aF?YhuP53^?!Ds}{C%T=I;my1>#J;W)d&B^Fm9^$H4%MHavH z^%K6ild&^yluk`PS2P3LTx&K{?JnR5f4HuOmo183N#uk^!Yg+rjgufzESpxFY|Kzf z9;s@e$xo)$BJs{y<8qt#U$5@#nHSw~)>t&ftH6c!Ani%CFT}Ag6pk+_NVl?mViR_L zcipk6jk=fFZ*~W3O#jPm^8f(x0Jg9V_vy0!`J_t8PKKy95cW0~B~Z79t$JVyJ1kWn zS%)2+y0r=7KhEE}ra}p4A)`kzYl_K4SO=`>_*?+m$6z!oa_c;6lLEkfZV2X3CUGM) z3Ri+soJ_MD07ymvGE^!K#;J!w(+N%hJ&mDQCK>#LpZ|QlKd;}P>*Zg$Kk{dwK5Z7a zX9lhdDSj4AwjYruY_c%8-!wL8hD%mq-CW%gyce}OVx_b(p%Kb&GE zJV3dtllS$|(6F&~ZR+59hAFRjX}5>F?7LFWrG))gClzG`JCHFr2Z%()ssJBr!U{-r z1%FO~aq}lIw^0-Z2DGGZQv^_^ss{RQBu0r>6PL`?A&M6gl*JzB{JJa(9|UYeErq}o zG9PBhnEr>Y?avp~MEt*X4e1mli+gmG7>g8Vs8N#m!+HYsE0x}gMPDc1^M0oK;iE*I z`~i{z=O?+!F8cbvVNrobT#`H|>rPi;Y)Kt!`g)6{5XRH*u2m8{Vz0esCa-fXVbjK$ zxVm7&l>P48zND&(dPUa2C0Wk zJo8V_re&vcvUq^?Tk+*+4l@i$F-t$80aNn=$w(=yhUn}7A!W%`A^^WO1c3qUuHO)6 zQ;E!Nq5yo&a(?eFmQj+VqLekJvXdkUSw>_@ zBuj)cm86m;`vC~1AwHf zK7@ZVWznyL>4rgT=@xH@El8S`y->JMaHa9hxGvS(OCr{8g6wid&Zkba?3zGjyv#PX z5R(l9jRR;3V1W9K0M3HG7wQNbv*^8oq8aI&*0b`k(YFmu_4zuaBlc>ycaltMBhQpR zqSbHOi4OC?lqf`084d+2G+035rtq(Z&h4$vOIyw5LYxT}2OIV|tgUWAk_}oVs5ER2 z$gW?ZASCJu{v=s@(}N;l~lQE#F~ve0ZTE z977b8xf-`eLF>>I!P5FoAW-I<2a2c7Q0%jZ@Ud;IZ9z}&Dh=}l42-w&J4_T@^C05p zgK7mg1dS84qv|aeybWP!EpUklia}}JM^HkB%^(}@vGy5Z?I4qOG+AI2mo*8-R}IcJ zy|t-YZX82f?*j?dg0}TRTKT|`UetG3=n{&E>R?gz2LWS1rGXF6Uj!-=7nE_d^mAZ7 z$p$%+D1!RmVK_|`!w_wJ3h>Gu!JB-CNoWTdke2~V+!D>W2;TQ5LQ*?uKY58s2X*V= z)68uw`aJ6rattpLg}oNHR6q6|CjJz>4e$c^sz)9h0W-S6W5wceC^Fpv4C)ANcAzm6 ze3GVCNR{=8aq+=beniYCFpU$YR?e_q2OcnvqcqSNYt$2C^)u1S60G;e6(PK&{811c z^zpvgK=;6H9A&S=kPl(vzlsAN&mnD`IPJWZTCVlXH;|JA%B^!eTVBjAry{=Dxdcs! zL(iG($)G>W^n$IK7k$KLB)KB<)-$))!qyi_BR5SPx4CV6rU1L|IA4$6g(DkzyicNr zQbIEK&u=By6@)Xi&%Y@)&s=Va1toN*xaB zM6$>Mow#ZB7wB;hRL&v`f&AmaU#Y9J&0|R}l-yHHP{nh>h%p@6)o8xQ3~Jj3G6Dm` z-ifA6jw+toePKuasg|mcBF>NcM7=U~D#IKL|F7V7$h*p=ktd@HLGH+1pdRUf09$!OmHpiBbUr!HQhw;+fYxZ%0mr z52!C~eemqF`%3VuyzDbZeolb`rX20N{6}I8H{6cZ+R?n9*IwxdW5&9aIFRr6}>9c!CFP`0Wnu;st=+FCz3$-g* zpQbb_m`b<&OpRWX>M#~m4vTdhgQdgbhY_R)bhRKD#Zj7zRyusJ&9^1Pe2BJIzPyKbg#_|ycj>r@jaJ`%U0u{Q?YX#9?{sF*j9U73(YyZN*zQKzUG{yo%)_#x4EEqz`Kr`umn$0qHUFAQj_`Y`MmssZzEJ;E#@YchmQs2XhyK_XM| z1ZC%no!d%qc2%=lZpoU%k4~LkcT{R)fDqeCmZ8rHe_U;+zphT8@aJIe>9tQw*{s4{ z&(Ghu?p?`5X*C2osLwUlU4_4#utZknd>>qnIMf5aFWgwySHt?#$^e9zR{ee9?GpJ@?X@ zNh`TV20I!1RF4fBz!mzDwp>#)jr_VDMH&L#?gH$vhjzz{-3@9NkYgL(d}Z;IQ3bOJ(n=(^cZbg^>|W!RpwHU9Z{>Bn5p zk`m1|6Bf+AEIFP1yQ?lrLN}IR%)sBek>L3MCLithm^IL)e@-CzPso)0!u|m%ep7}! zk}3h9!3!F|Y>6daMqZj;8j6qTm_QVLE4f3HF{!+ofc0|jB+!o}p8G&H7G%Fq@PcLT z9SYSu2==@m@F^f}qPmbdYk1KTUD@lF*zo z{2JEVCdBr=I0lHY66WRnU4T(>wkqvD%?aHZnTHf zJcynK4b?i`qy=3;lbcV=D$0YSV;a`uu84hv7G` zrSv&9TTPVE9jmKTIc`MC7_~Hhlod9d>DvLHPkQB_O3eXZQC^U1QG3yp>2&M*iR0p% z58(SGCtqrI{eLS9b<~p=0qLh%PWyi zBc7>;6I z;aR1dBY(HyRase=vZP%8jj}mX@95iiMfcb8<%F^^cLw<~pH;7gJ^^JNOntnN?L?}; zmcYPut(a$LRst;AM-Olh2e7fxKPb*MzQvJGqr`%Ona>z(CQCf zcGmmW>q+(*x?M~*A3{gz?Vw%l`MPwH+}-h|hPTa!y&`Pakj9}#-Fp%+XnGrV6a^v9 z(G7vW&;oC*qSpF`Tw3-UB`h=E7X zIRWcwc)tpC1IL5LXUsxJAHpXd$3RZto>CD-C?smBmgm0Nq*&%Rrjl{w|=U%-XQ!mlMZeCdf z+s0-F-*lD+`9=hW$lhO8s7KKhy&6%s(7aV9n0G}LHQvt!HH3zp z*n!JHe*Oli7B%~ezsNGLdIcGK;6PfC>t=g(?^NN_z{R1kG$sqizTx*4eAeF-_tsy48h;1}pIfFRcPoZ&3~T~Y_ulyGZy&&;ec?k% zX;0S0+V0SV_-C+{`{)B`5om9aJm7E-520`ADHM`NZt{-Yiuy2rck;FV_3SiUU!2wL zoEXR;gxh~O4#XWx(WboCl7Ef&l&)!B8&KxnQ}DOT86-v9mb zRHO7>3a$<%VCSDRBvKrRow8IAJcIu*fAo_7nR5Bl*Eq#^v|v0NTKl~~g=4+Skev(3 zUV@pnU^=a6P!5=cRvoqtn>20vogUxn%ib*j-}bZ$Fn$AuTP1>wnH70e*2B!p^L6{` zzs9`?NVV+LnM=sO^yvQmZICdKIfn@^Z#N$p9PBq9vuP&3U#lrAtfe;;ip1F{(!`)I zy~yz`-rr$p48!$Tzp&HC9^X%St^|gOr;7_;*KZ#is(HEyM z3meP88qx}>A0MU8E#Ac{Bwo&fy|0g!?t2KI4kzNKz3DsL@qCP(FD6mkRqEb;0ohg` z)#9Gklz&U)aCVqmluPs|0#vr`%mlEc+@ZOE^3okNMSTj;=l&5&Xn$QFu?t-*#DbGA z91hw8_Q3Z@S~`*#gLXu($mxlLg}@QPI2MdFgav`LXM;Ii7PM}+kXxbkV8a;#o64&9 zSYQgV&2uqSO<8hL2cv3{G^KE0U7iJr<*?+U9vcub43Yhy5Z2qdV8-2)Sg#kg7DUiPV_bC5;FaQ5GFFnZS5ch+ez6m7R>?6vg z^RaE9iB=U5>cW0GJbd{@)@K+DJGo<3J_q8Bl|QR@`IGDX-v+4sEL{@>%i1~UBOX@0 z4Qj4H$Ari*^xAa{Uv9EIapTL^&-H4fTHhpvDwPkPx$p9DI6KgVAv^}Kr64AI-2=kE zN5#KZ#VBH1{P*(q@2%oDHnh1~B8rFR^A@n3EG1QV{0)vr9d0keiv?WmNg0=o6k{Y_ zeSBN^@^Kj9Dt7W3c(|88q%uFD=?4G-;_vp&ioj2m2lRusdOia{XAe-|Z|ldIvRqbw zJQ}KF3UNo#HG|gBvgkV|pzX9r zOaE~f_ZR{4j&hoev=UJ4KbL0?tO#tF$cONItKxuFn{_n?=op_}9R;YZ4k}hcVqnYR ztD=DRpNwZ>WgGh_Fr58$9ALC6roiTa5dYt!{Ch<8TlaD~0IYzOK?s^1Icmr>^n#u! z&DNA$T$1%FAu0~%0N&Mx#q`jG|JtiOc>JQpJmx>8b5gm2Sk}w z0%t6pnS)56Ga~@{Uo4l_?5&^$`pa|005Rte&GA{F)$BE$LszE_Q!Qw2L#?8W-DC*+ za?vjc`DY5xZb)jHwPyoiG$;io{vHou)4%tIJ^xN4RxNY?PIms6 z62}&JM35q}6%;~i&RnS4PUPLUN%l}@(7BNT8{3Ds#T@30`(w0DH)*k7@5eo#1iZwt z-KeF#z}A@u!u;=`bEbwc176hnD5@qI6;)*g33`bdV<&kXONN%F*Ka(WZoVbyJw_pP z4N1Id?=Tn1oyhPbw-8(FH)-%n4p~Mlv#+^Y+iBn`!Non zNT&_TnWH&Oegl9`0$}Sv5MTW2_uR!=meVp71iz29i3o;(HJb8b895wT4$9XROw2)Q zCXyT#CfBq1J=D}$WTHqixCDvBY>ST0Y;cYZY~yMKfe!<45g%_ule50VW_?kh+Ds^y zIJI)OXkTkh--T&=L&1-HRSJn(P0MNLJ;ZA7?B%iZnN%FY#UV=YHCW*@$SAZ6s;{!9 z8etMS_XyXupeV^SY?0a?(=9V68$>CNC3LY!Woh`LvBp;Zet%4(4Nnk~Tt7Xr^UFsD z|HQnST-Z?Wu?XS2G0e9>c}oY7udQCxbnTlHa|}_NG&Fl5dc}Yr55budCPfyzls6DlQ|RuRu09 zuVZ^oFw<~^ED^M*()ok=-O3ACp@An3=PTB3{A#qtklxY4QBPj)c}Fm1?`A|9u4oo? zz=`Xa4**5%k*#BoJ)$cpywsn5@Q%)RJ}fIBaSA!6y@lFznQr+Wp{p~bee-8 z_55gn6MJTE$!>pWOW5AlZan!EHa1;4o~~r=?{l!rArp1e(ju(0`9j*l)aBH~Pd9wy zWjN)01+KTH`!NJ%i39!$)bj2X2PMz1E*NKqn7gli9zWUDbw0wTm{kVF4 zu5l+oq(wA0hE{K6STey(arNS7?4WtRlrMSFdoBCJ0qTtF=DED5p3M^KNgV8N-%9yT zpzHzj3?QOBaVuh1Oc_#k!1@Xl@mWBdqlD&OdR$a{fU=TlhakmGSCWj_WZ98j=_S>Jyg?bT{lYVFkP&u*3SMags``X@REa!ebd+vR9{KPw)c)9U z;^*P(i9K~au?Vs;a^PZ*P-zK;@21d+&=)sf+UI-Pp1a21)sZcLVL-NhyQjjpFfsz;wl zHshKDgW|i$e1#mF2Zcyg%wMT@njyW2`DIXuWa&S=?`9qc6~>lv5VjaY1LqNi@QveP zn=3WY7wkL-VjU1J4XXVGg3!7#x_v{ulpiE{Dj7y_HxGR)6>8L1{>>TpJRN}S&V!Zo zhwFySL)=_+XLtTaUMQbPPmY>Vs>o}JAiec)h<}HB#)9UpKI<3xW{>z~1}L&_aA8pc zp&VBLX5&EDY(-z(2$PlrgIyG_1b?}f13Fh6(aa)Wt)87GHf_r7#}BmRo_Wu<$d=ct z&NO7rZBdp?Kd2^ST3H3w1~hc&IjG!*?F;Gc>6EVr?X<|Dl-=TX>f5`MZ{OheyyLNw z22ge>85o)@jualK{l(s0Pb$-GuS=47ne)ZLqzlH+ki?Jp;zkDK4@S2Z1bcJpX!9a) zukj9GG@;CbD~)bfzS!3G+#cT0HkqoLsLO48t{%kVwNG&uRaV5Wcrm2DfU|Vbrr@l3 zPVfz-10M9aQ#+3Ar%$-iUTXJb}n*{9ZSHa)ZBXM zxJ&PI%@nGC=W1!mP~=0Eh$Itla^pL}Pb9el*OkCMs`6=Z6#~XOiKgauEY#tT8P3DM zbi`53h-q_*2@yQCvjeSl)#|jgyc!}?KwQUQ{g*QrKg*>!ufuI~NYMq4!W`}nrmvNt z0I+p#Uj1W@xOI$U+I>bZRHC1hR7N|IE)won333v4S%^8kAIy&2zZZsR+nNnh2n}Xb zQPs`rt@V5O+T1FV`xp+ziYvXCNfg-vWc$^PNT*)kV@8Pi+9PFWfDQ|B6M7ebO+tLM zDS}F!CyE$VXCmm7J0ajH=UD5#-baeOmp$mD%8qN)&AT=Sc*4*pY*HB_XzHoL<~~<@ zCEUdZZAmqATGO?h1#A~QDntcr{o#aT?2QIT%iELcM9fe|F8&+O)NXpm&F{4C!wdT1 z!H4&y;hTFY{A9xrNzAQkesrv;sSDv*v0{L)n}F#3jKdj$IjLgG2k(8p8m50i954;7-Tg=rb4l&$G&!l-+cK&zcw$zkkfK7;Q?ZAYl?Pq}rtT}T2JIp^FT z&x|a>u~a^w>MP9WPRs|xG}B9%;&r8ZYZGVs9O*{}@6~@AYMXc85ObLi&YnLN#x{py z2uOl*-EmgA?m1X<^FlDapD+bosKP2VHwjO**BE=-VK-QWcJ@dTD<2fOsC+Zwxc+;B z-=@nUxBiP*n@rSa1T~U2#**={rJ;dXxE77{wdm&RN0Pb_3R>54NZd1)R6~L|M9o*Sr8U zYwZ3J^ZCQi|C=Gq?}R*TzYtsgi60;Xq|>Z~{1gkVe%2Pp+%8OKk~1JdSy3&xBHTj;pjiC zc)2Ew6mnVhW3Y=p;(O;`&^L31RCh3L?oT3XFP42 zWi7tX(!`6G<~I!~(r#VBm3Io5oh=X-Phn?ZtzI8|GEDH^wLL)lLFo@_7_Zezce(GBhxpLqbu@ z2-Rhx)D_`&bjjp~=}+_So}1_S9WQ96Vs;-*95LF!M!>KkolN@$As%hYzNKNy2B!Ve zQSy^kI@89rxu~7-Atn9jZg#Po+zm!E!%ax-#GIAWv=iibBEAOBX&t0XLp`p2G@fF8 zPDno`ArV}X^kl%$$-zlrKgULyMxhgj9-ZJ9 z;ynEqgOxR}{tmJN_HGOj>bCe=M5IshTEiX{A&s?!yo*mluMzI-SrDc;1CR*ED@tR% zN?&gcT1t<5s@;9xE8J80FJGgc370x3TbuS>L<4_qqWn58#)7UQ3C+@oNo)lgF|0tK z;~WYN7M3|@JU1wN@gktuYcNPp2TjH|;&>QF$QvQDhpH%2%Dpc9-zuGQdC!bt0Eav#7^C@u=OR^kFt!a(G zOX_Grg0x&>u8M$LT;g+IX?u}#VdUnrM1@95L-W_g!~isoGn(o4fg5GQkZ{RN8?x7fO( zqyg&EvS{cicKkR9CIgYghLtg{<=wbP`#?xF4FWMq z&_lQK1gR*9zga*WEj-vsAJ>A5hiS5~HP8(Z!jBa~QeLmn?dM9tb`Rol7PMzL*ay1B zi-XpIu_P(32!dEBEgoE)e0XOdq4yn9BD4{hwr_>8XECkY9( z4sg{;Z;c;4k>lPUX+Cns)JK3layg$q!)2c~?z$b*wh1I-@BZ0S9As+&iugBr!yCgPkR;cuR1=&vK1s5>I2Bz+0ce>peJ6oGVMQ zh3do1wir7_QI1qo5jx5(@ikGTpM>f0?RO|6CG8La?42H*C?I2Kn0sJ~(J@saB)y93FR7PqZ+fQnc$MyZhCqZzei^>=(kkrMvIlSu0`a zEJh3+3e}3IId}nqfoKp~A<3p505AixiAW7*5_&wIW)7w`700+(oeY{K;BP-3Uw!h_ z#1YT|!TtN9|5xwn6lm@9ER!+B_;+Y9Dc}pUk$S?NI*K0XCxb2CB55ANFHB@U3GIo# zYp_v^gNmJiSbityghrv`@rE*h2?V60&Tj5SceX#-ue1^4Y^(JY3`FOv!sXwmjal9fGG{*0TgMQb z6CEmzyVA;g>`FEfSb2#GY$g=wmnWp^IBpjo$_61YWjn;j;HR|KuM?IlC-=k$1YSrC za|paQq<27zV^f}Tx0Y-;>O4m%L{}_ONlz?D?_B_)~4)k_h-)i*{VLamI1#&bO(}C%JgO zo;qP7ZEDF$>4P6(?qlAA#Az4;{;di}kTT@~V_#+=@r~|IWrdOex$9}-F)uxLWF;GJ zvD{I`7Z!>@koJ&yP&Zu&-$JxI1{%T${b;9~jTmZ#xGLK3gBZR1lYfIh)(1P1VGPQc zH8^H`FLFh&cX5xtlwW+vW||_=FJgw_L0NBI(x#>;5av-?%3l|0v@SGrBrOzn+4$vb zBt{sL3UVY*`Ii#?qCIs?o)~HOzxkLkdoWYo!gb)01<&LqeXXwu!oz4`q$?^6#o_$AgqA=EoP&RFa=pFlBq@u%(J0|8&rSYQdxM%MDX6{tk~NT zRvY)%aOGVUrip^lnbJB(`~8OrYo8-Ou2+awt8@6&)92}*rF(r&jP1+)mEPIxU-1!E zSA_lqvyuOT>?7-O|G~1$^5H6HFW4{Sl#SrqdGU*})6N_yIUS16U9JH^>lP3dpQ5F* z@?=xyc6BD=m)wGD=$F2{u?sNi-2h5z6T7e=F#hw@^L}Gt{;dV*XUG4AMd_d1VMO#V z7BaspD!m9;=K!ei`!@s@j7ABD4B|@GK#dr_D;VK^nhzvkXO4Q~A$eFDU05TibWGMn zk@M==GMG%LEi%A6o$g%aF&y<4egh=fs2i3wC{Z=_o-5-{bi;B1V~(DA(l)smt2Aq? zwK|T^$KT!tNk|NZi{wN14H`wE{C>_LsDgiYjYh%e-=7d*2uU_ky`%SaaA4Y`x-eUqkfAT59kjF$@d zCs8j@Fat+So0*z^wm45VL>_pnqs7EYceJpvmIJM*Tm2YOwQPlt-{ z8)=QX4t!mH)z|Y_RDpYbq9frfVz;Ax3kWoDoJc2l-x4*j)R1OLXxX`Y+0w&(_xAEG zx%bPZcE_sCBr9?`7R zR?j)UKpy6f*SP#{C1Tf=mgpIhvYD|^TUp{>{_+hkpG0!ya|y9s1o_6F3pV`MP$r+^ zY1`q`*KacrlsMaCJwt7E(S)hdUC{5EZE1E~Gdr3brtb8C!fBm-ewF~wj z4c#7uCS#~oJpemWr7(AklY&+_ODqtZ1}O%@H8xt2f~6%{|E|_z(718 zdp$4;GSeGrU`YUeC(l9t@)WVx`-0%WP*G>Ql$B6^8U&?hSZd`HkiG=oLW&K2o}I2V?k-F+^3LFXQR zOv=j=GN?L1L~o=i*4d=q2(r#;$@bWHaL(4Q|5K%7r0eY-3+cu(!lIPr(i>C?j%E;d~zI?GVl`m@E^Bym59)5d2VM&bhT$A*CAaxg0!8y&bdFR zv!F!xde78t$e+1GXIibR|Vv_=B9&=n_D~EZ>TdbNTiDdZJyYk zA`(~9c*=0~zmGG&9xvTup{cOF8- z2~-J!4N1tu7sl6{7J@nhKVvqsV!G_4%0O$UUF(vH87#c(qtp>(l`H$=@t98?>T$Al z5}wtC5Apk-yDhoOMUM8LP8T;@a7^H~;OdgHi2Hjq@f#KAc`i}_T@y64PBSFSX&`Ll z7G=nQ4)&YkD!Nyn-avezs%NI4n_r!-MrBC#PU-&af&5(qKg8w`JO)%MMUd0H3Q+738v8G98FGcd zEh7@RW!&opmABF{AZt?}PiF`BHcQ|aSsPXaq{NDV&SjV~x2mzxgMVm+{EfN#pZT9_ zJ{&v~ENI_zATj*56uoj%afwdUWF+F&g1VI`=q+`)PLcm2!6R=DBzUfazW2ft%h{zL z8i{RO5C1W0vK%vF2rD)~YP4Xw0-hHCb|3VJm}jl-xlg^SQ#re^Y~4&vvdMZ`(#2sd z-Khn8oGy}(gW;?4qZm<7cj@dTPaUZI1PL z+Dq4~0n#9E&3{b0cD1)h_rU05o5M6)C)-?Fp;Y4fE4P3GagQ=CJ-f0oyatzn=%0mnxG8(6wY`s+7_@Vo!pS0XPyY5|RsB5X^H-{aJQWS5sqW^ZuC1pz zv(J_rf)XVsisDzNA{o+?ZuxeM!dquAPS;B=Ns)}{H}nY_uyKh~9o0Czj8Wp^(H~;f zR2@;Du|&V=G@{t_!Ztx`is#VwuHu2{RFdFcGYKuNt&*2~N_H9R+NG5Pq%?SV;%Qqg zBXBO|7ecl%>;ut)K?hD-dp=wzU27B+wn0A9x%?!u_s%0Y|0|<=_?K7y;~QXY$$w@9 z{*#B73pwxF^8CX(bdjw|Eq5=BmflC+m$~9U;)_xh(rnh8*^(d5EbOH$54DE(i`Icw!nWCH zh<`kGsX{qtIB&mlgS~>S)x{@eT@HsfzgCqw8a+7XdVw$+s)Z1S+K?qV*N57lQoQ1W z%(EMZcIs_1H+92~uJ>{BP7-RrQzv}E8{c&$Zx%EZ3z4K!!h{S3^X?0~d52^7h3=C} z$8jamAKmnt^!buvvq*;)wASK4j!k$DKtF0Ye&|z4h)%~9+Rz(Xuy%$P4 z4nBLa|HPJvP{NMqn};`J$j5NeOVaz8>C;8FbF;pLslJ5=GbZAjB`FHci1{nhOi_QQU!|c;nC=2nfM=17iO+$2WEjGIN}Sb++ZgdPBCK-{3QK_#!mvI$dk(T70B)OZ0-g*9I=8-^)}nm33cyCQk44kAeQZr7!f zYd5Vj{A921e#gZT)+ol24$A`Zg%0`t>isgBH96``46m}uCOz9ggQo*_&Ap-~VbAO}5^ibiD%ZKs_bwC} z^b9;qkfayHgZ&QE*L4B6o&umhfQ1k8o*IzWjB~vN_(f^tvOp*@r++n3_7Z|TfF_?t zudq!aX-4tTL2PP>-XyMMjI`&ao6T%JxFxixQ&sI}13fIFYLohJ#2otiHu|htzzi_= z!PAbh2nIDcDkm_IN#D1?1D#Rx8W4qek{-HJ_lcxE^f=Ks(Prq$gIPDDD71s0ZELRr zjPID+ZwyayISjOQKp!$J7f^NJ;8B04gPxQBafpDqM~z{@=Ku(i@B#!7?7$x&9>KV2 z#L&Pqgi%oFwN{9K7=YA(P(biw+&vYVCyurQ$Am3n7_yDPKod++UXT-VT7HK;lr98M zqd$0DkH8>|>%Vk6|2^J+-`{_Hy#Bv1A4V!|m~X&yky9<%F{9qfZ#}us{(Vtovsl`SfBaP6{4TQW$@Q~EMZ0*3Gro>0jNc0{m zs2Dt}cNp&Ny)UbQv|&f{6WO&Yat@mXT4)NO=J&@8d~u}0my@Vs;K*w+Kv;|$Y|V0$ zh{Hq>=WH}I+7Hw$8h0BAsYpd$zU;>@k$d);;YvKjlSW66(CNiMVX@%1@ObBK+DdiJC_F{IsAA01 zR6ZQCTr*(NadFAW>$#0=eoSD%yUac1kJ1eI4)56Sgq@bCx;4!8*6Riep`#0;!q$4+ z1{?p#kA49vyYH5jI#0Hp* zJV`%WmT17x=y0s}PP)HCGwt?~oxx4078eo;M2258sCMm94af+I`iAF(+g+}!Ch(qCnmmrgKhzqJD=d0VgDAy_jWlIuQJRYU`S~w4 z%f>saI#oxW)T&)*O!+K+(;3)N3I?Gm3bi;c93cmip*=pa$a zo^PEj*m~d{4uvGnAq$X<75@ydBDaB@=~#$~f9hKRpaw;MPV*ie&f#19&&T)SZ!71` z?u<=5C?8!ZkUiw+-AS{OV1gDzlp|D&SvU%kwwg?kTsB={Th68XfZq-yU9kO%RIgr0 zh=DBa8SW|8zQuiBv&7qA?SRL<(cn*oOhZYPeK=5=}P5 zEUUBn%b$s0r+XqI#I^)5gOuBdP;?>ww%+#TnAG$`7@5y&Yrdc_9~24S)Odf=3VlVe zAGZljh{3Fr?<&S1LfUKwogH{yg*zx=R0EzW`dsTn)(PqF>54ix^tC@$7Jf#P!YfXyBPVb+25v)3bDnK!WW(nBok7PtGZe3&OVg{Y0Wvae7 zxO-p+;Pf_kXor5_gKZJjA%TJh@@Ola1Kcg+{{0y$ylxdU-W{MU>3CUJ@_nRNPMYX> zEhu03Zr8qzrz7`Eh4G>h_|h?wHMy5sM!`nv1QM_t@HD?u9n zmhc)aP>j0?G|)hs6eM&TD!=4JlFlL=jP{SZuhg!2p?|U3=#Jor8aq;`aH(b&xCB|t(JRnc! zRPIqa+pQKo>-F;09b+M~|M~4Q3GhQkw&!2M&-2=WL=2Px!Z6=qt+%s5hV&S?tOkpN zhrW=OqZt#zKZW5{SBy%ZswvHte;su0f>-T6nI-M5x6cQ&KLZ4beQOi`mGD8&oLmb@ z_Ch@9?=Zc6sgS@;ki{1LBv!%sj6dA_FU3cpBghS_LKY*2CWs8b!=NBv6^6VZNEJPn z^kUeP8M}V=xmSf(*uj-;eE%1ROmQJDF@%xSH#F2Rb2HR_&6FAysm^t(yri-NT`-e^ zza(JlI-YRRe~z!$(gLB3mgTdHfdVFkiT2 z=-XY*2{o|WH`o(Uv1sy1ctjfw)ReEhp~zR+79%+Zjh@#vXWt(vow;ZpabD1YTk;|E zvs@_eH80?(J2V)g!OZYQM3(x@c2y(Xy*3`Oi+9Bb6na!u)vT=#)+$ahu z1z4_izlhbfLP}tbR-~jdulpjHTNB@`0WF7Ldxvae2VI6$_TeaC917I*kK_O~{WU;M z-xxdo7cWfY#$QBFP4pztpPgrcE@5MO#)XH)rHHrDWS|p^o zTB4)MoMMa(X?gi2L;Fg8=$RvqR{e+MMD>N(nm%1N+T^|9>(52vqVEZkXXaFCKLrU^ z_l>2VYliJrB^pZtw)>8p$U^Rx-f!_unqbCNk08@#rSaxIPpoYu%&PQrxGWHyY1bCxFdrl>6{fH~Q5)v}p@n6vT_`LFKC^uaVXcNO50GFlon&X$-)LDdGkpLuP2OhY?Q{r-}N@x|-=$7NE~ z5nt5jur-+eq+!H#fUDOI?i`R@SUjN8T26tB@=W@Go(U?bQ^O4M9S(#9zv8=a@Qg{{&y)PIB z+x0&V<^ZD3(Fn2yyif40-u5c;v#w$kzsdP}`SOmJH%cC#mfNkAWXyf}$>nAq=??)l z;2SwXn|;m@W2Pyq#q-*?IAHxJNovvZQP-L_m2~dxeEFzC(BdOMq{zwGK~H4hQvfkGWwgCujw^#P^HVglsOPMLaa|8A9=WF{*aO zj@>xsapg%}(6eu1+ulzeHd{B)HCoL8F`jow6MjMI3!2ajp77gd+R~}3uMAp2QSVtY ze)D5PUJTLwU_we z!zh3f&^I6^xI9nMgw{U<)ynjFQ<|UX^aB8I`Pd6>cm*0WBZwad%Kqn$D*{QlC-E@{ z#JFDc@34&%#E4hVJ+WU7N}Xer4_upp-IwA|NQbf&mt8>QVCPnnu1T8rdt$ZS#8&dy z)pf~RCSqe#^`_iGO+vNV)YV}9|uJ{YI?^ggBVJ5ZsBLi;OYHRJ)2SHVR@^8)mc|;6it)39Wy&cd)chD?q>v}eC44c{5 zalkg$U{_s&CjCSK=Gp?_E0dF0NDq(Niqw8}TwiPbu_wnbw>5+#$oH@xxS85N2o?YLZ;_bNh4nl+Q^vNRfh9T!iAZ*Ze63} zor{6u3vXGi0xc^Q`)Xyx+j%?>I!PMjcZYbj2SKk>&)$In$+2MULsch ziG8QkZhES$9Nas;wKTZ!py`w7r*E8bVLQ3%>=nqzEBVTaaYyF&_b5r-oAKP5*gq;N zwFxV3X+9OExM)sLV2H2}<7)v*2QPXd50Z6lCmx;ScX=UmTPsstwdH#2tc+6Q%kVrv z_*Jv4!vhG+Lwq3@iIeK7U0c2F7<-+l82jO#l0G|8U6HNMyFQqn6N>%(?K{Fv7uFmS<5OJV`-$;4cB~ZQ0W}a#fi4}dXLisBOriW>X~|LnNIoW4sxcMD zzTA#pN)syq@k;H5n-UMF?GemJ`Xz1VMAEj4GK6kH0&tRP-xT#)H*)CoeE(GcB52+; z;>k@eE&$BaBAu(03za4tjEp>CBEYmdgKmmpDWsP5LYXQab!tC^vw|Muj80CBsKpKg@#_zC92J)FA)tVqANevrQ{zx zeJ-mu|pvW$*Z#zXR5O>t_EN zGSkhfTz+!I&45Z#zwjS$1NhHis($O7Uz`Ovfd)nLZGSX0f@4Y)|FsYEa^m#Y%Lxvmg^bT z%q)f=*_2Qk`E|^ow^+TXvZwf1`<}YfH#JJ087N=cHTPOK_7vO7><`$Mwf5yNt^ETH zyVRM7Ul(=m94Wu|Z1A0yS>;(jEv_}VR5(pOGXV&C_vhB3_hCXYPyM zXLP-Who@<~*xq4wS#_o`)RjI(bD_G?@~{R9chyrPic=%1FK(qvJyxj<+x=}rm};+V zUWf$Pvm}b{WzvVL%jH}?MCwZPYc{r&m|vaSrZqLUWi#UF?POj=9w^jANH!wb^tSjP z-}a%<-G(nqYR3!l*dZ?0*Km&I>%qFwI=#bihJb0^SJUX~Ej0B3q(^Z+F4l8`>X)dX z{f6<4u;P8&hY4#DGOr_l2s`|Z)7r|(+LUACwQ$tOQih=6H*zn@d`N{i(sei(aiU1| z!=-NnU-WB7mP5Q>kKJB7!wKByAM$K_d_Cfyfd4uDQKY>@H41KeiL$;nHsfos$edNw z#}Cx@G$Q>5Vle@WEt3tOxodLrbxHS?sXS%!#jrVr$o)0VhqJ9NuNw~x*GHt|f!Ukb zaVk!j!RD=T?&d4AqXAtbrt+-=>24}QySkv{%$)b6{%K>V@=H5YX!CwCw2|1HqNzrgWTXu>Lbf@ zC}nhw4u~`map)o=Qq$8ZlnBGzH!tAYp%-m7B6BbE3MFCH$)m0br^yFx1Gr?g%pCj^9Clq5v#V5DlH&I|QevAs0L> z7nnaDGUHlc8VY};l{V-Jl10LOhcJ>JBI$hbj+IX<-=ys%>>p76wjs0jxelKz`;*5w z6}48{4uyA^XjG9xIK;sa?--*9Axn3n>yS8_E-C19@h>%BZ zt8O&wuAExjbfCq!bH>;1h*N1%;my(XL-glT?$-C{FmCnBRi@PB-i&wn!qR#;e?xPD zTeWD;kcXGF%bt%31G%8|D z@4T|=ed$ZrEJs8c{8-~iPoJSz6^%9ihv zMRPe6*DL2_bvE{PfON7*e2o`P4s%QF48wq|)ahicEjOlBrds zT(u%ARkM=momLlEs&mg#>EI&n^VHoNCWmA;>reg2ItKPp?VwuhR^*ddru4gh$y#Jb zb@46K)eG+uB;VAJiK|;|pE-`Azb4S1cBDD5GOFdjH<*_(W!K`GUvR0=sT$2F4ngzn z==LQ-5;PJcL3)rIa)G*7t>30~`&ulPkWqGP;&iEQ$za`U1HDa$op38Bje@co3H@p# z#JR;M@qwdLv0fT-DdvWkKRZ{I*EIc5&EBGtF(P`lzkKzURJ$4Fp7l7WkixsW5I~7n zay*La_d!O`UbZH!-9ykk`1!ujKm5EdFB*T8QXBuEy+_>a{+n_2Oy%X;Kb>EFga2Az zbG`G+r)_n=&mA;}(df(sq7kB3=x#UA zKwfn+ZIO;$uy2-*SlSq|OR>6R8o;U!7ZMA*DM>39c;|w%ktH4-P589lb+<94Z2DUP1iTwK+NYxDGQyZ zsl{lVPaQtjL*zMS*X1))J=b^13K2Jh4s9HoCPyNEGgk_A1x7rU|Vv2WE zv)!^wqi~=c8@3+XV7V=5am*x^c^Y)XJ~dD6^|!M-{@Lb$GhnOQSnOQhCv{T9NJ?p! zl&=B$8^=`>6;G98(o&nOPTw^g-)eHgeVOLU6e^k=ou@0qXH`GPE5ClIa-mS}X2Oh< z(Ns}Biths(Rt6Z6~(~#8?Q;)ZUsdl%Aoc2{fDVltQu$=A^!EXV(l$?CE4z4;R4Lz=~qRc zlbZM4Esw_=wOrWB|J^E*Ia%$ioT!Vx`No-xagx?J8jOt~ZrxMhIXpW9-EXSP3(Zz8#CGJ}x%;lj_r zIAiXFn2_Cc&sBq(%uQS)=$Ab&Ij{XWt);KnV$-1i7t3eLpz4caJ%3t}N6nFvu3YxA$GO!yaY^r!HM_U*FhDhG zz^Y(J_kkbL&~;XbCQKT`!bdc>oW*sp(pA1Z!9^T5tVo=^?Ra;2mv&;tJ}V9s|FmB% zNXb>PkA+c22!h`;w+o=YTw^qQ(<9rnb`NZ*SVJ5@Qxh!T*7y_Ecjw5%F?20jj2Jst zbwHows~LC7J-sB(cY!33bM@nUS*c37@a~WSt!UXLNrWqs5dakb!iuY5HQJ3ihoCOQ z5{7-k(+|osR=aBCgd9P8KFS61nK_7 zqWslnf@A+)K3O%Vcf*u1{=PDLXfLuS2yIr~{0L!c+|x*R!ZSs7rlVB5~y2@Y|Kx>dm+>U9&O=Ns8#&vgTxh+*E z-|0yUoD0yT+oK?reoN*vfa#gTl>=`y6L3AQ+NgcaqkBeVF-H)b3->D=I3+KlJ;uAA6>qbBdoS0vL>v8_~Eg+ za#?HCH-s+q9SoN)8?EU>89=T9jpwX>M5a4kA#>0|wOp0sj>=`Z@J=J88U8J8M-mh5 zR%(X(=E>*}*bNRLTCnYtN3wNm@%1F@y_&5JA5%SVb?%nV$u#jTmp%JoJL&*xyq$CL zRMFpR(&hs9Lo4gV)}~O40__@Szvv0u9pUc^dH<2{{RkqSMh9(8PfwCDOo4L{Eqzp< z6KMd_q;ua#gWJTJvY_u(|84N)ULRk1%C`a4%8E+kh)H*mK=s{nTbj0gQCFVvK*6W! zPg!yLTR>A%7SorEM=^EVHGG83^UU9GV9=Bb+d1O!P%dC66yr@vWzr-p3$6CEk8kuNx z*w!z8nUy;E+{IdR{a&q z_ggs+$^O-^eWHBYlgDD20aLj;|~J`l_J=-_4)2r zJTT%vN_3f}=Ft_DNAR%HwINB42963+Y%%qG+=ZugK{O|=K+b*7+AZZGxDGTQ!_U~n z{+of;lbEl^?e3;r^HQkMZF$(zmozjaEajbap8Khq-nno?g$9vT0h%YmO{S)q)1IYJ z$3~1Z5bbV%>BIev_BL1O6?Plkt+d=y0M|yE+=b2Ay6Vkik!$k?|Ccg4SiP?*?V6kF zR&&}qFksx-OzAkeY0_~H)rvaue5S(Tj;liUtDz$o-NlB}M~z}N+g=_-;rwgevc#+T z;fJS@dg^dOv8we+)v~$X^hd82y_Xio5`ontOnlcw<3Te9y5D=Mj;EkB@rkh_F{1an zj0MK1bHw?U&W9W>MX%{sA2ubKj6V$>CRBL=FX3?S_ofa?L}<@bK1< zJjSLVp0T_-Oy5jh4YpV=*=*EaYi;3xQNt*) zy5W|Sgy9>g-254v0KD^_+?V!7&hDE(Cm)}ca^q?|e?RQtvgVN^H0@}ng>agTWKV&n z+mjvk6vYJpQ#Us4dyBGsk3&N>o;Zz%Xg*kRgfB&3p~u-P@RZR3{iBkPi>;)4#??#p zE}h0oKsEdHcL2!2TM{J+bR{$Wcqfrj3-&{wQ>jOJbd&ZTJu7YI7{=L~M`+=PRWw`c(S-~njsgk0ph$~^ zv!@KM@$8SfpcDV8Aaal$?A)fHRY#Ai#^Sx)Z|8*> zT$Gsz^9`-*HilvW zrb$0dD{%}wOi=a7Y!B@X%At;?N#`CX1ue{agz-=r^sb^ZWqX9M^!f3-FY;w_!m%k>QQ}Aemh+mWggcfG z$90aFP@-ybqO>sW)TQ43zge-bc&(Aw$F*MU7`eR-1d|2e&HvN7!GBX$_5+$~+qU{& zQnM_vEwccY<{;y6kTv}xtoJJ99PNzD-1)h(q9S81dCF4X)(@}zilsyYYWib8vo-P) zN#rc182jn&op30%%sBr3xo;ln@%1nCzo!muPTCn-+oGWVEKFazn)MtS$tIR!ZxMdz z3+PumKeN@+|424Ut?g`wxIYHHdTke(JZX{q@hp4z7?HPnH`o-9EaSHZMGTYk?l65v zrBBxk44*E+pD6Fq|8nxPS9QPj1E3*QD>wqNR1Q zVjqYJy?foOHG;-PCE~c~LGu%3r#fqNId8q*@ciQ2&9F1~&0RiUKY?yHZ{X7nt;e@n zAOEZYP^_AD0EFJJ^jCi+21-Jsu=)?S||GCWLavdWtZY!U&!wmP|EZ z#bC>tYKey?)T!vpJIf<}(2cayd{l-UY_=^V?vfS@PA19b;-Y@5?ff;1%DHF@oTUO< zT*Yt`8pYHpS#&|_I%MWs*p;U|0#%|;ILgNkDEQvJ`uOS=Tyu~lIWwMY8M&rbd$8aM z2uHGy6}wKb1$p(nWYR=b4DdZSz|yoyO$}SbK6m= z!OH1vpe!on)8gJ5#l(nU{^@{-r;s+0YC$=D0`_8@hSSI{ukFkrS18Xkyc%2Tp3FH| zbN(*n=zaF&_(3hMw)qkWv&M^|5}=Xiq$@^(Hm|Bxx)0}lJCR;|YD}_;rrLz!^@|$Y zM10J>X}$U3;$J;dGJz>{2stGV%FiS<;DAlL5c6Lk_Uj543wK+wNO=$_tRDe3m$az^o`ieum@x8txeANO|uy# za4u~g+1y66tcBcrtJ!ATVbXECqVJ>zl@fXB6el^`&%WKMG3~nvAVq{iwuJam7n>kn z|L-AcH2%7kA*r4xj9wCb-e-q2`>re6yXRU`^b85lae!{b5aT}Iw>Z}y7^8b)Z6$_6 z-IQ41{l0j!j>@sm3OD&`cZ&oAe*76Yidsh=ZiwH=+(%QcOO=uD99C?u4p-P*?|98Q z`Xozy^Tn^EasG964gR_|0al(3D@)fz?dH6#N`*td!=j~PQO=n5R(!x>+ndMyMt}0L zgu^^Ep}BtThFN8NjLzr#r*&@*xqCMyD8?U6bXsIIN951$n8ljn`p_jVPpIZ}cb|me zje9h(BI0VVD3r+u=?u$M<>(y`Y&wJP2qbJyW`C#2Br^S2cz}a}`g*TRjt1V`Q*f== z>*Wg@e98B=UEYg@z==%7Jp1iE1$TQG})oxU-n@1pmEaP zfVaCDIdKU9L6CsdLd9zFV3@+JvP5)4o%6nWbj^sR#*03XDzv>>c0o8Q{d|^h8%AFb zMJSwIjbw!bA3qldKzAXW{58beG9u+w)U~0uK$+*ynhAbf<=cxsdMWhybGTk@&f(MA zN0{C}LZZl157EQApkkaLT-{{vZ?Vw-@;>1rda1Yf#M|v{;=7CK3SoM+>)Dcj|50A0 z0AqO@sySN1FF7Lyy;Yq17^q5lKS%6at0v_ANU}eCUb008m+9C7IyIb<6pq!;S$-RD zMuATdN1ocm#4qy@`;zHSm8tY0I(CkQweDIm+w(*ve&P|va^FO{mSd5bkiw+^#UBe! zYgq`_0B}txT>xuB(eE1LY~(Q|;N%?|DtdcLKZ?c(--)oClA zXru&M@YELtR8wD?exfY;@+!x)ks{T1v`Mm%Er4?I&l=+hp!XWm+IP6 zVtb!?(OK5Wo#)F;q-8hTrNZbPhTC1AnJyDP;v+GoVw4^#n*171%_alP_+*ShJJLaS ze%G#$evx7hY3BH)&wcX%+Kj}o` zmoK54f|$UGkGTugm|&){0HEZJrM|XCxPr$B;vU0g`l*^Pzpr?T_LKT4-!hE1)yPd8 zbs`5eN3u&f3jdp%|Cc6{zY?VU{|WoE!mxnHMoj94qdMU`^!P4SP3$JsZ&j!2g7fyg zM7_HjlBzYA

q>LLLWw7JJxwlJi$FSU?}h*bkFNwDE|jZ4jP*kz^=^ z{6Vw4xvU(IKG&u3EWP(fuW@vc(|IA@gq*EVp0x7JAFJ$ifKE=ramzEk*~OL^;FS3x0QQGxq~6Q%<>$XT7VfT!Y;0k zVVE)?kN5E`=-XLub1-uSZ4SJk&0!y5VCGM`AseX1i8!^`2*G*)Dv<>0gHa%|-Uwpc ztTxcvGeQTw5r0!00^|z+I{9Ct@vph@x0Uj;aU7mv1Tp1d3OOr!Sga~^G+yzHPQrbX zsJ6|+V+0>Rb;p2@GFsNB48`?omI$-a>-+s~PG*i@-A~ktDhNh;0BgywdgJR|J=}&M zr6fiDiDI=tYnq;Gxb-sGZwLS4qW#TX;|JG;Rc9Z=Sn3z`I^SNO&En(X-s--TM6C2Q zntB#1#FS%Yga}d1M+h;Fk#_CZii@jymv#S0Y)N&b+nzmS>-|H1^Bh zry*(-Q-gNm%}q~CzgU1Vb-PeW**&?fZMURin+4 z3u1$Yfl6{_S!bX7$WN4qji9MBWA~(gbL}ecg7#^N@O0Fw?5Je}_SR3-HeEBdJ2*u+ zVWmF3xLn46{Fz1U@qnWla&CuqZan$b+Kizea`w&$)JWJIVv}XzbxDmED?ebP@^sUK zGa^cIO622wCZ9s}W}5*E>Z?7^3ag_iue}~+r|uTcZk)dI>C=@X8}=HGOU&yFg{af` z8?m?g`9VdL>!Xio;ixkHtxZW+@s;aQ=3C}ojI+nNx=#(HqgWubA2SMDBe&(}5k6Va zRpEgl!_^`nIcWx5#7J=0s}j`);G}<4B>P=k47CP&3DI}R*fqH{B76XZ3yHwG{&MYA zc0}qM!wzbjiiW6;7@%gi-v056tQtGWJHj}%C}=v#0hr`b3}T%9%avb_^TU3D6aaz7 zWCF8Y69?>lyvc6@Wsms@AfNc_;3@y>P=XLVHiZ9AA={xEYU$~CXv{l3#Hvl%}BSL zDD`?8c)2-$B;Nh}aw5F!#zNiX7-tKjg>+&hh;H#d4A0TwSnm>o>K#Kj+fPNMZQG*z z^$K@1ZH)MFapiud=R9DTlP}hnV^uAGY%Ll**Xl@U(eT@-bRpf~zW+g9t}8iyq%O@h zh+zr_Qc<6cd4XYl3$S}Ml+sixvsm2w6jJtQLf2hW1YDn8aAHq78NbPOfnXmPp$m=S zqbp{DvdKT*1KL7|14HtaT~opC^;dayV&Yz2j5(R0SHT z(xw9Uf!D>Pj~~A6I?L6jR9wBO=B+bbA;KLy}x}LP(c0i4Jj;;s;ql~ zx%d;cCh!_4Rz8Q;In`$HUm;K!!84^iu*^gQbnCzF?H}LFzsBwV?fI7SIVi9RIRK~7 znre_;a1_A-##<*_b&h(zKO9(|(Ij2VacV*I@~-Lq4Ui@Fml=!&i=%{8V;O@%VZ42| zgdBp;KdVl8azsD;>KomK!94Yt~)-@BR{Eeg^mXt?Ea*-DC z6)wB<&8C`OUN;{U6M>J=9?~P>CWgW6F!e6emS!|W)Y-SUPL`vjbW1-^P;JB3CQgqC z=vH!T=u3tv1ok#Jp9ri6sY-y8sRHrTl_MR`j@F%s-Z$ha^)`d=ndm*X7B9l*2bWU0 z9K`Q7$vM4L=2d?)62ZSc7DdnMs~atdso_u1=YtDhoNIaBALO#?Jzg4>BA0Twh+^f< zHuWAi1%z(q>-dIwm+?$qV1~I72-(IwNK}9pB$9|#)jrkEx>=ryMJ5RX@O(aBMekg+Ondf83<`u?;%tAN> z8U$M!Egi)G-&k`cgmZxG0IYrFE(q?~yMCmKe}H^E8yq|+}1Hxqp@ zWz!72wTZr;sE%!5QgX9_PNFsDDhvjLgT%_lp_qGntM>#gFP(nkhH+sS6u-Zo_Y?KU zyM%td{EH``>q=9T+r%|#m~bZg74tBS5W{3A_Zat7ndjTMr(Uw^6Mt8ga`Dz7zmmXr zALkA}=KnSfTML{NRv>=l}MfD88F@Cn|YAXgoouZ6d`I1J)jzXIf}kh!Mq`z`z<=%D^l59 zv#w61COdjtotZ2`MW>jaOlmOEWXRHtMYd6D>Htar^ycSi@cBH9Ljx1B;<8U!I@)<2z48?gp2Gi73Bj20M*j&Lt zKcMH!NUc_%`8q&6@YOHQx0h%~B(J^!Ye&R~xeIJV$}ed?BiI_WB(5E>$j|AEhUhW0 z>66(~3$=SD_SRbUYo#oZZ2@nRZ)+%6wtP2OZ!qQH7pAy25a@l(Ux4|?%jJW$ISEYC z(X&MAQ#A860VJ%*ZP-%I6Mp!M7WtnP#CH;~B#RuAE( zh&8E2>Z`z}L~dIn{ar<#mZj2H?5<1R*P2(J+c*Ec>jukn3UEt72sbBs&Wnr{3UX&| z4zUMevdEk^&aE4%8DBlQ8XUk%5AB=Hz~{O!PXl{!5$Vd}L}X@(LLnC_c6C%8^7ejR z-G*~cYvEuo)P0hi+_kfcK=)$Z*4M)I*eJXm<&W}w_RyYt^pGvS%w95JQO;tvm?I=E z@G*Zk1miAamW*tHk=+4|?94B(`sFwg_9IckS#98jjC>@*t!U~LVSt^Hi3Tqy8GtEK zYh4KXAwNsN3QWpd&_V^Uo&RvfRkdG62SmoR^bg1sn3+_kVWvx%J^+>c!-N6>_Fu19 zZ4Umfhx>{iq=Hf9efSd1pt7be&qvsq;;*%slnJYIhdzUA^K6eZIY;?ektO(l>DDzeVKwL~xh?N?!aX zqW&KwFa8IIs+Uy+qy;3zC?~0>rpTHxVS7RjsWvPZ4EIlT3r-l6UTO$X*@e5j{H+0( z^N)gc0Ea`_2>gjzAP1qz)Ok1x)tkqQTFCI+S3P%b$XsK{xn{@S0&ZU4kKeH6SyWRW zAbocTNZ;il(Zy_Q(0ZXC+iJ}e9YD;}!Z34oJ}i}VVy+G1K^!R^Ua+eIKFg&s=(|%l z+vG6yBq3UYSfh3a(1#J-d^mo*kx}X_E;xK}Y}?jZo1-T-T})te+G>V$`RJFh6hZ{x z+-&A$*o=&c?UxZH)%S;JyJ#L^Uc4L1FY)ebXb+E}DAC8~9NgzHX?n;+`p_Y4|yXX1(7JrAGDfh*sF zYW@(ES6HbDC}VI2HN_O4GQ-Rj46x%(-VrLmn0*YeU z(RXaS2~#K;IhIX!Wd+%R_pl5LFeJY5qk!GADE3sfyCpMRUxB2Ko{mt+lx#7w%NkYR z%ic``u~^u)5l~WOW5(h7Y~;)4z)rZ4w^y@`Yj;W68`8Bl!2u;^xLo;yPwv^Kqe9{+ z_ON#%EYtOIOOm7sAq}lqfI*lbOll+C=dbARK!z(wQA-M9r(aH2=v;RV@Rk{pdW;#e zPIH&OCHvZQ?dm#&T26aZk##4Xh$I5ps)#J=QOi|jj38!?3;2(@JBc|;hz~S!D{H|D zCkK5M1a9yj05yJwZCyuhV-bvbYLpiwHunCe{_*FUEkU@|ZeLxOgL}Npq2h-M;xlK$ zJB&iX{>3p47x=%#)eOnd6e?Q%6eB1l%DcHPr$$;_ml5Gu8rW1X)8fDbuK0$g@W`E^ zAEaD+s)$8ALsNG+niF^b@~a<$R_uf=zT4V}?*qWQfi`)7t({}6y6Bvp#b|z&iF^gV zYSdD_!&HVM&_u97F@d8J;S!{Y)4>hUGu3D>r4Nr8ng2wIe8EmG5>n3yG}kYtynA=v z5Lt*t0ImWQzalg|d;3Ag=(&5Dppt$WIhb+o$X1?@(hUV)#JcCAzHnZ!^gwd+H%LD8 z3~ssZwP)=7<45P^OxRL7qQ%e2fSxAhRYh1nkG~Ia*ZJoJY1e)E?wA9QhHgN84%AFp zns?j=ot`+xUp`4Jm_d$FCM|D6RI6=j8i)6cU<5mzg1$@yU27Sk?97}k+R!KEn2UnW z-}^T}A*9;hTaq0FxZPI*32^km5WGtmDQjO)Zu z&*7tnHZrWE*TEFph){vMwjw>HIXF~7QOen*Me_^)iE0~xPtCQ&gZlE!L)4}Q0<83` zk~+|MrD{#_R>&!OnwfCCaHefdkNtJt%^Wfhq#_Ow&a!fV5C0P-9Ku5<2;$oM)0m1j z>fX@YKvL>jvFX9utW?PizTA*9Jy9(UwpAjn1rSP;8OF2dexhi|Nxzpyl4)gyr^mnA zx`vMrnB3|PZ(QI>T+eWRtlIHfFoA))=J2KlL=#E`i6(%o5%;wQ&twNsz zmg<3Sm*(*Y|E{$;^jH^#O}O*7_5%)RHMYY{+_! zaN98zx|r;KTi&o?oYoFvKzT=a9#Qu6j)s&ONHQ4~|0@cx~4gxPedLCqT zK}@?&?9YVwK-{pS5`X9`rmjIrAd=&74E`nif zzQ?wFbC_xuzvYsn5B0{xxHP+`+Y4OI`9gH!D-c*;#{SK}#2_oyb?9xFku(ri)u=^R ztbiubAsA;bS@-7Ein4pKZt^|XbB*<$P8<_4R#Mi$GmZ|yK%;Gf?hYcIrJ-f#D$yU_}w6D1_OT1A&vT_BSUVc!I$%G1g!dhmBsiTUWR;cSnl#las*5 zsfn&W_jCR5bL0%yXP%R2*pEO7X3c{k8~M1N*f`Y7#mEID>VAHn zUpgu9{F2z-@Z;Vf-bus`~zveh+J-_2Mcp ztAXWFxZN<)`cTL`+i>S=er4%qUcau0mQ|ML8vEm7NV|GJ`YV=t)&)JwY!m6r`Y}ei z(y+{^b{$LTBMf^mx#4Kz=fId-ZK_+T)IIym751eEZC})<>;Gt68;hf1tBe0zed-%= z8(`%IeQ6lvlQy16Zg9}zXg;7zGoZcgqw?4>)DKAyh6Tv;`gq)yi_g9%Q^gObtkKCRE>ryn& z8_-|B>yT_n>dHBQ1O0gQ%5xzJ=5-QNj{sYqucS`JUTZ0kw0h$w-TG)w&BvD~A>gBM zxV>awn8wy~)J(8KHWOp7`3PM{yCco-%c>U2E-&i0?fm#QK+>D??hBvZ7LWu1ODseg zEU}~&u*3ww64Qh*`Z9o9!<_IvQ>yS*7m2#TLuC7FY(HuQ84RNEu(cU$$CC@Cz-pe%DikEC)-W^-M;R~REXP$ zb`fu!8`C9y#{L$s6#37?au~{aax2eC8sC#Z@>|c`0So_K7Kbk(cAM=>BQL98;oIcK z-V7+dF$0notb~yBqjt)qn#-RE4_yYfXHP%21Pw8Vo-_fPoH<@bW0_~b)*uF|HSaZNkbLJb@ z(ZcJ&2IsN|63RaUs3x8LPyCN1!2h@bJ`qRKB0u98Qb!d3iE`e;Z$#s7;97d_H!Bk~QQtXt3BEpRN+#KBe|$Q#t)x49yK~s>+k#O) z+KXckC3b=mA3p0Xe+{-DYiPkgN{Y$u&MCFBHJ`p6vDRL8ns;`QEkiKAd|PyO$wl z4E2c%s;vyfuHjSHtf(J&U9^4YwMOlihr2zX@ztPFkm zAhlYIK!K*Tr)|3rMTL^oPqT5Kh)UTi7IjAOxVotb*dq8YuT_)zQY+4C?wO|$i>}r3p(*Ln7p;%5 zrtenOA?59yOl?=Dh=>nyj8S_7N0?Hh9bw%8kx6-zQ$-L7Lpkn=MYwyYq%-g# z#+e{!Ebr#ZIlR~rpM0Y$kuna191Y(}wF`%NuJ8YL2Yq84Gx7jk49!h2_Il`R(g=-P z4*wXgm#DW|!)2ECp0ZmF@MyG+usLJNE$_`t+$iiOeU+{MmBpS|EQvRjSvg1bXk!naw3Y@36zock7&u35Ow{%LI@J1PhJ^Q%Je=UPSnyi*cDT!d{%mPJ zQ=p}I7MlXxaN?>4C@Rc3c0o&a%ZvD00=&zbDa_u87OIjb16F&{x5XXHk8B#h(1Vzm z0)zJc!Q+Xo+f!}xOzAsR-*>+lliRu_6zu*R?*OxK1dlZ8X@iP+n}GyGHN~v%f2hXT zG6Ss%T1$dc?!w_C6iK>gAsqbN&P4H6mC;L^wu@eOozJ~m(I9P`*}6ft?J6uyTg5U| zLR7)S#|5>Jg#(A*8ka+PK{B>2_j^=J?gVcX+m1hQ^SYpcAO1dHuyq$^3veuwu*Lih zK9W$aGvhowQs<-4b=9#t*{at&;>y;+V-mYa(o)4a*ZKxgp}t^wU7`*ZW+N@>LvQmN zJ8WjF6)NAo{gz=SY}MOK*-5WS)jpOm?K$Bi5~2m}A_w`3-CAWNGMNieNj1HJD_Zmp z`$5YmUvJ4KMr@4!X3_dpiA}IVXP@qYZR{(uc#$*kl++3ps!^4}&5KkiNyk_BxjJmGS0(bUllql<8>Y-AEeHNXW6cgD~7j-b8zQ*)8p#`|Z6 zrNZ{2w#+mboI){QupR?+JW#zBYs5qhsZp&8sS2U|bvmp&18w;gSkuyG)19Z44W+QV zQE?+K?^lq`xF0wf=E=C^5`PQb@UKD~4=@QEQQlAuFx1{zQXx`38b48CSsVS2!)7;xYl)EHG6j?DVR_2M+Lj?=bu zAEs~7nRlp;w7W2@^W$punX9FS19CGaCjEUP%`np0G>T`6bSx1CkIEM%H`28_8`Qof zscINWJdy6HM>)RJWSu<0NfTn2G40?Unr-6^|FWiuyFtVnn;S4LpgeMJC9aetbWFdy zihb995fx!?lu)YtLxx)?FmU2v0onoguH6XQho#!+9kMK9Jr6nL@9G6qP~hXAJY$Y> zcbNiKea^e#sTU=7ZnzqdFO4F6CxKZ0E~uON;(wwHh1qWWd)vbQZeie`s5{rIf#-*6 zu-?n9IQkZCi#N1j3tvCKopl`hd((U= z*|DZa>e>YZ+5F~lfkcN#e`0cE5S1Wyih~@?l@HYhKQ#e9;M>VQpO>|f8M{_5mVfvW zC6y^EYk&rpTbk%$nE=`}3Rt-`#|ANpN5$R_!zDVOgX7>osP7!RG)9GBrI z$x@r{T(cBx5qPBWaMrUl{&?pom(}6NnKvno9tq`7JaOVZuZz)w?kf^jc}|mu?%2M| z4f>n3jT#3J!r0&MPCJG!ua*!Bu z+X_U%e^yvQjHb2$cos_CM5-%4sLv2?6Ukr&l&rAKhhcRH*#{!DVLAw~p9h1G4LC84 zyVxmNFitESnojY+!uw-D{=5Uk-L;^zRj>fO!cAA!3%u|{L~5Rag}Q;O^|!DHssn@J zIlq=hq=J$x^HP|8-~XZ8={-ZF+Clucw;iEIe9z{fs21!@I^@K21pk@T1W~7;eg={| z&lJZ1)FP<0{Nr7+Ekhu9!>sKD&&3U5%#-Wo8$5FB38+VadHKEvpt-HT`-A}tdM~R) zu^!6~2afW`Otf!G8x7UzI#Vo!ZBAlk(nT^=XSVOKb&QM4kXd4*=l$0gT_|H7Y@>rH zAhTKutmQZ&6A#Aymm>z~GwZ=ASXobZK~&%m&ijMc_Z6U~V$sZ9pvv_Ps|?j;fI%j% z&XU`h+ggC9zx4YNwq?uzFWpTYGKi+0&;+OsWpWlTA*}5thAjO)eZEyEZ-6NuhPbot73)dXqlL4~J?#S~&26DK_PUfwiel#2YG7dOOZLNfZ6Kn z6c>|-Yqx`1HgtRp|G4=I7okf6kU(R^t3^YEJ5@D@V}}&pQd^_IjBtstZBmkZ=pOn4 zCHLrs*;p)U0env*bPLbe%hZ1Z7d-hE6kNY%oKfM`nmam|tm&s|X}7KobHR?I z-ii4hYwzm=N>e*N4NPXb@JM;4px*97@h%)!o>BUfLg%+e2kY@Brr=w86nv1O7s3lC zk7Dj*DwecgczCXqp}Xf;nzfN?`8n0GEh;yiHk`duE_n+mWn(C->uLowN902%G+j@q zvHWUI7oe(9VyIA)L_%JHJ?zsU!I#S`D)uHnb1>(P&u+a+uc69lLla~1@x%sHFq$H} z?l{p_Vv!y)p>I8MxM_0C>m!Shui`gzRqa>&)*)Z;H3{NYfbUgPTBq2-AN9hGpChS~3AR)SxV849G6X9sw^{&ll{0^}zh-tlA<{EsXO}yKQY2 zycfK*aUBf#GqEI%wXte9IEjaV9C^(D0wS+UpjiU5c&3cvPt*=Dy8O-Uz|Q0Cpb;8o zm9H8EIx*_&z*TkaxXK5z6l&`TPeUJWXrvab{=3uc&4>)s`VKt-JajzlGdft| z-XT%yS(}K{q3sx|J9s@!VAqT4qk1AS-`IiU zkxFD<4b#*5Z=HtOPW-KN+|rTFb%&Y2cY2xzvT9O&5UroLZ+6N0?r(=N^bsKXrane$yH!W7gE zmI&w#Z6~6L)48xK$mMBanE`m%4I74CI>p=%=D|ZX` z4S18;h=kaP7)zK4mhZ@93t-Z3=OmEGc@*GiZPbIi7>AJd{`s1=7=7vXJsPYHlCtuR z&$sl{cbNxziF7EB4}QphA{+GydpSRUJC;S?6rGPJEX}e6d%=w{O%#g($wM+^f)LjL&f2 z>9IdN(A0Q%w9D+_v4ziXW-nKqF8Y|EfpvF2k5W2vS&kqXHdu~gD8f<+zPlE^;w&$j%M!cS{exVygE8G+Vk{&{*?mh?vog$t9*kpO_A6{&~OZl zc^cgZE~JVzfVmvPHo99A5;A(;Z>(E+zPe>cVdd%lCReRnuGI>@%6Z9m=B^iI}r7INO2K@J==IIphr#x6zr`e>N`><+0DLtXej?p+7EG znoR9)$iH=FX;E>}F>D_)a`)ZLnGIY&^m&kPAnrmp@$=aQ-8oE)BE%pcwG%a;5j7;7 zI}1;IxS4wDbq=~Ju5NpT;bZ4nO0HG>9*eM*%9D z`k>P0AJyLEL2JC*dd0Pm^Bj?(@L2cAlZK4Yr;8RQcVym`WM1F*69w3yaL+p&1RmqM zp|^pzm+pA^8N*M=<(Gf7>xlJuJ-Nc~nqYliAjaRH_fWX)Z;L%l4;KsPjm|TL_tVW_ zt>=7YW0m#+h|yqaI+sKa*nzx86t=ZJPBN0a$|f?ZF9v2x@_$XkC0avVJEQyI@qhp<>lSB zf9pAKN!-b#uc)!r)b*_-&Bpd45Hn>|J?uPHbGYWQvp?l?Fyk*mz(BoG9Y*fOBKViTE7O2nC5!qHF~dee zb3A#_cMbn(5#V2O5yw+tZRA5mAd+lyifIIAQCG-{(UUZ&p18Fy$j3A?()YSjqj=%} zV(-o4p>F^G(J4fs$-bMiBs(P}jFK(Yv>{8TNJyweW1F#r>`Nh(ktLOwWXqDtE=#gb z*0Cgo$&6(Tv)t!$f0pn4>AuhTe1GSB&v~5R;V1-xoxo_*9(Y0HXbW#3H?zs)c%BCbS5}coOy$p$Z(lP zK0uZ!G+`xBt<1I4-gbRX)pa#V2mG(!8GRhF&FnLT(@4_@X8vcxi~l?B&f9of>O^i? zc=(EU^K_i+{ovwqHB591EWwLD$|4d=+ZInxsYl(zi|HeBku?6vag&<_v{0B#WMJ5F zt(u@jNuCVO+p(Q@5UR-1!7UZhEv=M?+5?!y!(jjY?r8)INk+H8MC-1SzcQ}VWD6aF zjLF7h4)(8y{8-k7VwIl7bMB*GA1b_9w!50}8Y;A0Q84!)XkUr^c-*?3TIutXejgs}o-DxHHA**QDATo*DS}ND<>RrtOmimAmbz}g9_45xloT?ZRj))ygse}^p1=)tF~bTv5_~~!dGfth?Dbys5x0IC^kR@=eHApgp!RO( zO_*DhLXM(L=*Qbeu(ty?%9BJ%+M((F5uAXsS7M8bvH;7KjqlalQeATE5;Jl;q#C~7 z6QZeu+sY4Sc=QcvodGp428iL5pZ24H!;e?NJr!u0y zU~4}nfo1rAU7+OS8k`|e1>{BwSv@1cg{{yy;bcd~$@j6o#YLVes=F0MIQGO9T|W54 zVsE?AsoPJy@4+XZ;>l;m;Z$@#$xtKcN_ouswAG@0v8VmbWVhSq?+)B|vHS68+G5A? z?apyCf%t{X|Ab%&eZzNdN0sJswH>s`L~KRTQdX3>)rV=8weH`ooo-*qZaijET4M4JWbYw za=AP0rMpmFW(n3ifShO4lBS&@`9dLO$3c%_AQubTeh5JHu!e81Gp3}2xv9#4O{;UY zrav?;R8Mscbdx%N5w(r4wf*%qoA_h$Co31ja$s*a)hhC6i56u920U7jvL5>OtG1!3pmQ+qNL^ z-I1+t4KCezb5{E2fP6tXAE7hT`5RpSno0SZwX9GhO@YOIm27V_;8ac_g!7AilFnHje`}j+ijBx;nOt zmPkc`8Ij1)XXc`hTxLK&>nj2nNm9W=oBtiM))|$=?rOeRwW;>jCF2FhCb}59lMmgF z38$W`G~g)XEm!cPVD49Q&&eJFghYCqQrwOSJtWjQs5jtal)p z8VZ5G%shaLLTh3gOz_v1!UERXBJ?e%mq@3*K6c#`4mbe~`m9GjZ*gHyQFzn7hXL7Q zCXnD3lc2+7SJo~~`T@o*iYhKVNOL+vtJ{wmW;b$UsOP$uG}-Zyd6FbPg98aIz%L& zk8O97^&Ibs&J9YO;?{o>5Y!HIbspu$&|uU@V3^`rpRi@=Bjpxr2-+!q`yI2awmvBr zw|rQW?`^CS53DY~Jp4u6XHKG^%}OZQhP6wPZiMcKRhl&+dFO3jPI>RRe52(|>~sE? z2mKEUiCn-=oH^F8mz21dPxLSo_Lm+OTW%e5e{<_#v-k~?$Nv{j9EU+==9j`2aNz{# zcFQfI^q1@Xz>+i`f+qe$dB&y#Q)xp|!?-}hs$fFP=yw5{WQC;a03F}G9 zPhY)g+ji&74%0h^uW=n%nhKJUDT$$K{=jdl1wd$^gcC7?+myHSX7K=o+00=Qb|-wY z0t?kQE*|~9#n$Kcol>@sy2$E#QzIUynWjtB+JE5MMaTTY%!wF}F_69`3#XN_+JwVU z=TDDJJl{RbDSOAa8wI+?b5s`^(t{j8oFPWsJ$f11ylGwF?HQ>0#KNm;s46QdqgRkK zRql(m@rvhIesYx?C&@G%=T?89_a@MO@BHp-pnf4jP1UP-%#!DGyN+yLiLo+Ul8J}S zSKyh4Zyh0k>XeJ8vC-gM-~&L{kf@a~n3FkbdnN3v>3gz#nlY(%`}?A1Joy|n9KDD6 z+zWpl$>&~^nvi#8`sh<9WSQCXW2{JZJ2St>y7A7s*A*LjM8Aa?Zsy{}7?;6wR!9@d zMO6k%G&w?tf{b%VPde*m>T6Hjx)3g>VSZ20z^2RXOTw#)&4OuYtl>5?rs*ohqcx!1 zjx&EtWc+;8ph*Bol7Zhn=i)HsVY6TMZh|wOER91i2-V)mm33Ar zxm6nb;%S(s!R!gg(dfNrIe)AP+O&e+Vh?GU%t^I=7j8-mec>?gC{uk*$!+tZ#cfon z&Unh@CCPJ&r1wo4vsNLx5V}kZ!?qXP#7YdiTE#PuVj2|rreBO2`(k3qe>-;CEvUKm zXoR2ckB2SjQid|EcoG(m=IpPRVH_Hq87h}LrIV_+} zNl+C1G;0*U9qyu=Q%;L|(H)GNF#9(8edMNx!Ai5mb~B!jlIK!1#LX^$Im&h=l&tm? z40#RBMD6*TdHinQ&dd65cDQ+|K7Xw4GpiYIL0y51LztDhb?KA%HdrLQ92!>42qxp~ zVlVaYZ|umM%^tdWt?4%3MqY9-*Vn6$Quk&bq9!q4p`qwbY2hkBhCF@Je0G2*_G;mX z*_`7cr!0IG z;l_IFP;i69FsvA3Ck0eXTT>~Bsg|xPz5T`CxEQp!LuPVn&M8Os%mG0kNq(r6+#Ykz zqH{=gRzCpYdxyZ^QtN{RrgyxjD8$TM^k3~(((M%L+mK8CcoL$s zekZtu0S4F5+KcifaT?s7-2Ld&9>+Z6V96_3ywX04?V$vx_TX(f(BT**Zu_FZg-hO^ za{LXqV&%Nvi%GheQ}+^@@a0?T2GjuNV_zFGHriXQ&UpIWfL7+2q-d>5dg}$!LGvGi z_Mx2a&ilDMAMZ_q*0|xCpQ^LCeaSJNCQd{3k_%~R@*#(g-S)hF0EtwLGd3Lk4I<98 zqzTZCgZ4A?^#NFMC$7=c^RT_{V-p{4#hCd%r$e)2$B74Zp3T~`=V^QkF(`@IxJSh? zi}Z!)vM7;L6k_~pvGJwK!3x3tt5Ik8#Mp}kAA0#{C5Jq7k_|g!J|iYOivc^MHF9`; zif~_Fz1Y3RD{orLr$szSJe_)Czv%3n$flQku~Zi8ex6@1MKBRn|5p| zSU53L>Z6nC(@Ub24%J8pmjoy7TH7DU3Jx-ajeNFyI6& zyaJq{n-KKvphWP)97kNbPh3990PT@0myd$=HV^j!Fl^@-sfb7{m814INMAA%PyCle z%}6@*kNS(qBdh^@+4ETX`5=XNGz4ybU0kZ5WxQgzy7EMd(NNbVHNp(1KA|`DFH;&@ zRwr`n&8iHADjFi^#=7`y!Ta&0prwR4wKM zpkbI84y%mBQ7~-*Lhq|0yhiQ$#!Z4f5AmTEKR@J$KW5{>>^Ajh%Vc)^NpR-BGy?xF zK=ZF|4*18gjjnX&Ck*{KzHt7(cJQyz(Urg3Q00T#c{CD9_;o)FG29C2_W?x*6&k2t zUd=)<_aCMQBOk!YfDb2+1xdo*g(>9U4zHdWihE|tV zea%B8Y}TjDQ!n0nsn;n!x6h{=Z5?z3T%H5t67va==~;@x@G<0H+0*(zWL__oB#5wi zK7Z!2(^{gi%1>G5rORb46^OxI!`bZ9BTBEzi6&f z;Pz4Z6I%!Z`RvD*76I9`*y=u3Xw^*arTF1yXErxklG}#CojZ@EbPd4tIIbgg^9Y~njhj(yLKe<%I=Tb_U5--&5l4uu@DUppuNvUJ!+!aDo6jakE*^`c`7M#WL63)|47wfZaK`n-KOG6FW^V6A z_7E6H?@_A&uAU0Ok>QqD)*-V$9WioXg#ttE1(sAT-5$Vo0zqN4p`ZtVK=S^4C`S!r zRBp2ZV8gq|nZ=+?V^DyZ>;tu}E%3*$hdztIbL{@xqs5K=tSZD{Vw$Qj9+*w2JCdeV zy+!r^(*X`JtS@iaP)Dqv`o40MdSlzXVvZ6uA1ISHm*TvmOosjWvT+I&2Uz*iqQP|J zXJ4kn`>L4L&V0AL>XfC^wFwv9b}H{ZH)qr(D)lVV=o4XYsL%WoO@@`~^P1H^p-wqH zvXR@_Bx0Zc)oJ=CmB2Z>&EM)_IBoYq`14K1Ht;Y?ql{T6@ri}VO%C_8MG01Ix2_B5 z#vert*etEF@40L@>NslY8o0~B{?qFeGnpteQ&AU}BcP$hSsYXdQur@6#O`>^x&lP#{=*0$ioGZ6LJILldc4%scj0K%+GOs;w{ z1`e|!lSm6LY$I;+1s&RO1o(0j2*$F4(Eyy~;Ip#zAUdc9PoHV=T^N~;NW`zo&|H__ zgC1rKgHBtZpQGVth;5eYiH@>;H}5hX+ZH7}J@tuYuth3Q#)=g7TU+-^FsMR~@(M9} zF-;^x(4PJrJB954%U{X?FpEZlf-$FJS-dhmP$|`=|7kDFE{@` zqxS#pv!+}~He(38&ZI{Nx#ytn&_A#w6Gp8DR6XV0SYM>h`&(-(omymrpxR{(Kr_yC z0Q4q+m_3*_AG3lkfk;CIyD4r1d%a_gG3+DSnFyAXMh~K@Ce?1HV$0&@ay4G&a_*L{Lptq#R;R;x|!mao@P9 zpsfG!ISpL3Gd2ppI>UiM;K>&;8=8n{>^cYN>$Rj}G>~uaF`nu);OUYKgWn*9q6gUM zL0h384h#1hw8&iFfEya(g(;T zWm8!%`rLd2m1A`*4wa@&YsVEiKh4+PeG~}05UQL5wpoSaWTIEVSfvGs-ZGQf(~peMoN?-}K6 zmfoNE?OfrFk;n_P4hSE9w#bcA`cF>BjPHQug=@ml0PDTZDXvl9;5kk^3dxHqV=3?r%CWJ26woeH4!b2a>8yl;^LpJ>-m znc1$f5%Yz13w`~O`G`2XLIqqrvQme(NTp?4&VDh!r@zPq@w8(XNe zFp!vh)TUhpIf%y+Q{g0pX3W|NVj6{5Pzt`Hir6qi4i;k+kz{k|hQ2+LzU>C4LyuML z_QzD|AHV+(nwS1b?_iLrL}zGf~q^55$Km3LM@1B3nc(GbePv-fwr= zxKCY?YIr;uwgXJVK|EA8!8=M?EYbxk@k9G$N7LT$?c2G4f5BJy%v==1o6gS~G&Cev zg?Q!nIcv@fSMu{rt$N-W9~6`(NuDS17%rCnlq|+LjTokTKKmi8rBk z+DX1UsE;4M)hjVy;jci$krnvpRg4&F;p||!C(^7!TgFq#9>t)0YqY;SkiGNmxr`3J zIIkl;q7YgSi>nEU9Pr^Hv*l>?*Y>3Ml%ToO{kBoEh_@J`9@fI^V1B>{MUY$BCvM2X zpzy^j>7Y;{ha)6L5}|d6$$202IGemVkAHTXkSL4n7<7nC%2$qG)_@$U?mT+MC0h4e zof)-=nZkmx2!{N9`mz+Z^8>3dv@H9Bz11@06!#^y_6lM26feHCUfk9$IkPw9Hs)7J z1i*xuh`FW#6M^F8$A(4cIbZtO@Xl?zHMDWwE+@(?^jI zRI$uC%+*(#VP8%{c&O_LdSP^eU*zq*F#o^|xEqwFxW3p)PHapsqbW3})x#;4t&MGn zz9&7~9)$^&s6Fzx5Y-)H6XT-43P_ZsfRzP9^&#z_m($YzzY_hN9 zuH0s=z7C!zhQcS~h-L7|HPH%~sE6Hp?0FgJ@bzrvqpoe2rzZ|)aWXTkL*TlY$tTF| zz)%{29(c>@bukn$hzyv?Z@St5FUzkQDta<;A zRBp*G$RBO+R-Jidd(A$i#Rjs>d6<4=218TCEnb~`s^+mqnfF-Ne5pD7L%V|(*TA4)50b=OyDUC` zH2x%p-=p{0$H?__(R|FuBle^CuZ$etp1-xt;>6?NAy*p`o> ze!W?XDL?HBSab^Hyvci@DlsKWJ3SSs&g)JpZ&!6@E-uz7BMzmdU589+CA3K`47jmv z@YB{Nk-*%>5btRoiGa zMd)HO1<{l?^1dxWg(3Umtzvb5#Yc`uL(9&o{1y*fq~j&t$cie-B6O*a$+SnG0> z+>t%zLsd7jJ6c6YrL$sT=iS(fm?iMZmv|0CX_|-?dFjJ@KXozG%(>9XitP47mwu3B zNc0ow?Ci-&lDQ}BRdy`r;To}I)1J_@2@P6RNl?I0327F|(MV0RZgug<3&x0ol( zVERVufmq1yrR5aoo5A!+I8C-~@#)kBt@;BG9Vl^SCwe35-`k^{uCx0m7eBdWIQ1m{ z=R)@2Mir%yx|8_|?$jd%Zy^q+Ul^z$?dpD-^S&?C^L!uMiYU&L&pcBfnvp8KpG_^g z@iXTt-6RNLGuh{v&+FyT1F#*wNP(aWZz?N?R(b7Cs}@#7kotZ&hbii}#YbBu9ZH(w za@-*lU_%_(@?O}s6^(wnhW`l)Eh9oqxXlXqs3R#LF6{x;PGDqjB)t@e zW*8|Pr0Q8@DK1!5f9@-B^d1wm9FvXo7v-M8{ZFG6e(ysc2k;vgiii;f|ABNBJOQ=5 z)}Ow4NVHmdo*&_wGj)=f4VzR7vyisWj+GE+#eDx;sTBYJg5m6eT>R4>nu=L-PmG}i zjWmheNmFmpdmis>81gXILt?ntIWPmK@Sv7!e4EpY*tGAQphKGricp{^9(5bcHwyqC z1trA@!#q%iT3CR6!BG!0%jMi4K2aP8eggy*%9isqhMapxYME*ok2 zvgfu+j2Y+6YgfOst=q-0tN?(b{s^ra1LpHKa%D6H+8pryTh5BdoB1T9W4C7DgS!iF z7Y>%aaY}ja|4Hna6bHuYF610C1y4Q?jm1_}KAi612!b-xZ~C&oqXZYUvryudB?Q#n$f+w3~Mo%UWw1f!Ykm6i}xnkpM2Oh0z8eE7{_D0I3A6xjn(8?#H9+9 zUx#XdzEFHMOU1_s)?XmRXhJt&tKcA_i-1XKVM2)F6`sa+z}gO7S>*1A;#{? znV>WaW=$!kL9XOmFWwy?gz}?t6=0jv-E5Y0o_Ys*q;gm|T&Y%g_Mu^o72~nd!_0s( zl)wtds+w#YE*$PIp{1KCN}v}y;BG0yob0Kt343!U=foP=ZC347N{|~&!F%p+;Rs0B{-xD_K9B--F z7&u+Mb42@MQ)E>xE=pMVlYZ*N${)% zW1ayjt6|53jA_mk=rQ15{N5$Vb*dO{RaTwZxRF;mSez4c*KZ%oT;!PaMAkzKfR_8clcMx{Cxsg{6$lzL$Zr2-W+N8;aFfhQlDIa3O`x*xnfkef>-BUq-9{1d+)VKH<$B?k`F!T_U1N{WX4@th*Gi)>6n#xDft_9@0b%zoC1u0j>JqBGiG5YDq)2%uNe!EuGN2^%7 z*yu;T1hG*Po3IdCZd)r$o|2H|li#%C@$~!z*OIJ8r#^v8Tz)=Y7`XG69%P47VN(xH)_d{elz}zD zhJr3GK{bzRIJ^7kTUVZ+$w^k7wtdiZRfF?c`*euD0E_D>IlV>PB`Ohp=tkvRyYDr9 z_U?8b?>^Lc_)A-+pIS2?b=uL&EpqnGYvoxe?EsFE9|PT7n!+)+?_yyYN2c&ln(^^c za$CgRg7Oa6YU>g52q0xwuwAMh z8Sc56mLdgri46}+12-D4hHmlSA%eNIs50pr-fa*s^ z1zrkuXi_>^Gev60j-kWs#!24lhlJj0uvTvsy`es~S->o2(1nAvn2DZs2x!x|z4air z#oLX5cxB19D17|tp15N?x_~-NFMC3XbH?gY2n+$A$;4*Nh$_%?5n2$RohaS^%;m+$ zLNU^(?Bz@O7J@gQk9E%S@&7~|lDoE@6T%ip(*%Q2&rH)I-e>Wb)AA-o*DxaS+QVxe z8S1!c)j%1}3_p`w54ImU@a|k=+)C!^OCVl=e6x_Zm*%`nnB7Y}tuQ%zM(47}k~j#X zHJJ~6nHQ+);OPrDn?&$``ut@-FqkwjHQNxjx350o*Pq?M6Si|16XJ(k(8C`EEHT>t z#1B0qX49Y+AyRkxwMV9KRiIFvjb*`UPst01Up<(4v@x{&=AqLoehE>lnIj6ThH}d@ z=IbI~Af1vA6TWVKW?NV;WO0jt5?2(A83(LlW3%yAMRUXvB=t5KNwOq&gG}ouXhfaq zuOH@E?k?0n_o>WCRQ+I^QtG~TS+8KO9B3kbG6}Qns*R*d41ivh9Bx35j2y0|qR8Hh z4dh1Rluf9&cLlFDcgD}I&Db_Ml4=9TTc-rMT^f*W0lso8j%!BJ!IT@k0Si+$ailt> zuFQcEwcIq73?8-2kz+d@e4f%;kKn$kvv$I#0k9CglbN%oYCLmzFf#pxVW#tEoUf@l zDW`75rfa*!t>ja^wi)yFy^m?Aqy^OiszdoIw<(4tiW3Xh#jYr$l(vsH@4Pl6>X z^HX57y-(NBiY)26f}H91{I8^s`$LYX(+;$eipe6!Xx!7d`)EmuP+@G4)r^F~o0w6c z)BNq|iFAo?)tLuBidxZ2=K@nSw$DbzV;Z3Nuh=b((>K{q0Z&IBmx~s_G|1zx!y~CR zMIOa3P@U%oRZ4DJ&z*PEwk-y(+m@c&TV|slX&zYi&ywIUlBs{ld<1s}4b`{85jZy_ zbnWs5>%?XoO@}dymmj&`K0cePPD;^=X%C8c7KLfWv#+^OaFMTl z?T<#^{260^qw=$v-XUqfrdvYCrz>DSzTU!+lpbm# zZhD2`A1JYim}^TLhq2)p>V8xpupj($j!;PNe^zVCa9d^tW{IhB_3Q~QIdH!? z8BA015Dtky<0rB+^0pLB=C z3Q72Ubg!KwE~hZWvx0Zmw*<>!xA*0XrC@wmYuIXrRX|BUJU*Buu; z{BI2xaRgKJPfiIr(9vqygi9le7xi?Nwknhj?9w@eZ{AzLOIOyi zV3cfU8ecn=CqVtDTEsL{!JcgTbcO^fN&V`p01MiO>C~0Ke>!c3c5$VJ;{?m!Ym_B` zpeYqMRKQ1It`z(5vUvRPtt6%zJ`xZhonWSx;i93 zpJjAz1U>QEWx7JmMJOjpaFzmigZP#ga0Et^$N_hiz>VZlb0{K>9;*KON3_<(E2hUS zyu56f;!Y<8p<-X9s~O6hw{cD0wEf83G?c#r-f73KGlz{NI1Z z<$s6gpf#QJj4nBkFD1kT9i4iLXeQJ0So{)--u#z5?Fb#q`O6(b?sA_{d*{e&tXdFU zeN-lUGehq#G503`cJpKvcKISp9P>pShY5#dkMNXAN3l8vR=$D7o5*jkcayn6*p*47O zwdj_e#{=u5%@H;!%goeS%^cQBNd4-zeT4Z}N+<|4nY zT&bNuYw3FQ_4n#%?bj)jN7_FkTSJb3YFb!1Ys9ti!Rp|qyET6B#tv}gEqY|GFy^Mn zkNn}^AeR}f?>z`w0{050_VX4C{i}f)iK*KNt*QQMUVPGitimBx( zyE!}nI;b2cm^E3r=6;&nrY_6cYLkX1o4|)r0C^#dYG$fyP5lOWx3cwB?Ws0WphL2M zlju(DKa&G(+qz@f=rIyu`YtvywE#NndRB)a*iW7D!V7$3XpL)Nlut6#ku(|N;*0k| z#?#bIBDg)4yPjeu#`))QQa3bSey*CK&!Y&rw$7|HRbDJqcO zjiVJE(ZYUg8}TTvz9FVcriMn|D z2Cr$ypxw{)p2W_l4=@j%Rze##KMT z-&S(VSubwqdImpKd8lzaqTm)x$QMbI>j~3efJfC!QjpCOU)#;XSRULmO=XBXMe2?b zSFREb4sKg6LyzMt1$=20IF`r>%sz+2IQR|Fu{bm>9Ip_i?>|FAG}~Ql7AYT)vwtC) zaKBCI?N>kD_7Z`%dKI!T{+vGC>aF_W7?;j}R}BUoCt7=;jDCtURd)HCYN|t z&hi&5YSD;J573Ykx)}yN$w~$q%fvTJp)6^d@(E^sosF-{G)4rKFyP?s=`G_h;x4=# zR^aR_Z@0Mo5YBjeG}keq#{x4M^BZJ4iNW#mhvGEp#zG50d)CwU$SFm4?^}gR)7=6p z_v1^7m8J4YE;so$R_OZZ&%Z%{p%9pj629tTMDe?$v5A}x4$AZ50h-A|az`&*>>IBv z+F|wemi(@BjpiY(+n8s29q?gw9#e=16M}t=+mpfPrCg5>UA2$3to0k0EUl7m2Klfu zq~k*o=*##*CUxab8QX7n555eg7*w3`ula%p9@3@{}zP`YPFe0AjNY+QRIOzd;^_ zT*eT1F`I<7LYh5$F~{5D@u5j^}GWWfmb21~wx8&S5VZKG?C8&6qG5Sp`f-mzNB z-o;m1cnBt!jg9RS-Ipf+C5}sJSs7Z3+=*$>-hu=Xce3Py_E*qWD8y6V{O|__Sh~XL z6nB}*$dQQ)eA!n8WG&=UWJ%Uep&EWE!;W|scZ55N#YwXY3Sp*6nO#UQPSF3j@BVW{ z2_J<<_yj8h!Ms2VQX0L3Waw*+u3YiQhsON{N z(1_=%JWd6{btO{9>Pc{;Sh0oCU4m^Xo^_M`s)xPOJpFZ-1?y&pS0k)4U z7?|WOL~ec8e8w@d6zF(d$3N?y8y?#@cY4+$thTN?b;fJw)T0=>Jbv;8tjcnYmPw3E z&Ig-1QXky~c=@Z)IJ`6iO0H_r5z($eL{}Vm{nMrFuI2sshL7?27JGE0dZQ{Z$9IC~ z{Sq@Z9T%V+o7jVp$K; z%|YM+I^8A#3)!mP#mw~8QlYC`AeFaHYlHe1iI`djFSM>Awnu=g1CSsoF=db&4+(ao7ph2$^r&c^y_eB%;vjI(WvDFe89hM<;Rs8F%5h1&Z!Ho zB|3x)-rvJQ;M-tS8ncv$YkBHTPUT!aSwgJ3M~Y;kRXvCipWsQj@0ys&WO(VTwZ65; z&G{6Jkx~by0n|dAIToJH7q1PvvhD+bC0HY>6i7i#v>NI`u(|_A={2#EK)M6|rlpey zuERbSZEAomyK2$;eUQ$JDuQ(lD`D$m6fjOM3sXnGZO#+$eINnv7Kwr1z>XkVYj~F; z7?&wJ&3g68FHN1Q%c}=m({|=59K9@7`KJB52^wsV4S=K%I$>ZMzv(VyJro~d76ZT! z{;g|c;&9b8XETDGX8o<_^5?sR>hh;+M~A~sN*T#ZpO0_Q-(%wR=eGkW?ms8jugqPh z^|Ul^2ewTU-0A-64jyZ8anhB+XD5!J#%(9WS|~^FzHd*%KSP zW*UFySM%Tty$4-vmh)2dm3aD6yTq9{F0; z+pv>w+J2$Y$tGP0Squ(^5~Rwcy5p%4W4H*H7YSW+ke7s=`f9stT6PZY}3u6a?s#AwLFhGda z3WiZ_`x%I|vo`1wjqNQ;3Tec?<+)?Ay=v zMqwLj&Hdrdj!a$8lCNJg#K+H8!~NfmFOYkv8Z--a(gGdnk*!4QRmn_v95&ai*QfG2 zax%kOS5+uP zFa?j_lIvzaOEFD-&bUjdionU6;|>>dxK!{qnR}}3X^)=w`}i*5-T43zv{p)MF{3co zdL(J*5nMOm+~vyL*W>cgdjrZdycdD=Sgy`|Bh z>N6{Pt3~m(6KrECnntkBw#81@N$e<``jJpC#~KJ=MrP%jSBa0$Rxp=sTkf8li6kHr zETkklqHRsLeN0fT<|sqfO&leQFa-LiMM6vJVe`vD@bdoa5xEhIYIkdo#vVs6rY$#i z^wwmfbPYs>8o3uUIM`FkZC0M`pr<&+g}uU!?Rd$)@;UP18y_G}m(A zrfhm+$gTX<9e5-CK?lE^essQp4@Un?8g6#s&w{o4!s~`nXBqfGv>(vFMMNMlB ztp1P|=Job9O#IgE?aO!mScjQf-%buW1C&+|O?inLfg^=a)e^-k(_+7@MK;YwzsM*LITPYYum* zd;PSQKxHJR%Wg~i@afhW%{3TlC-`3bn1w;|u>18Y?y#Gdrmk<_ht_3{DoR3n*uNaT zcg4$^CqA_L%npct*_3uDefL&Qz)f0Rigc79V;=-*srHID9+kLy`}(dpmCvaH#}2So zu9mIr&==nnm=b9KT?yj$I;aPEZYuKe-T3Vtw%=mLx>aT!r%h9TgZLjhip_Y}Dao^o z1>%Yu>NHS`YQcP=gU;JJ*|L=&o)(d#X90S2?whSoQglro-c_x<$YXTxc-$eZy4!VF zF_L^5Is!7sa>9qs&YlN-s)QJQ1@zs~tQIRo<%!5LhH} z_ZrZna^wN4cMR>J6|+R;{EB&ObLN-#I&aFUrH&u9@UGdYOF9`4C$5gvT~_3aHmk_# z#NjgPF=j!Kgr_h3KEq(zKz!ENZr)BeMhqg_mg?_x?!%8%X!-h%ckq!y)=mmpo%*sC zBj}ObGldiL_YfVrOUktfgS=3CcIEu790;^??^!|JvmEH%jLWoR^t0f(P_LK$&H{2j zah5;C#}aNXEyl_htPfM_Cs?h=JKrg@RaBMib`I|F3&EViN}=~Qk!2^<;|%xxNbAKj zWW=>rhDWOn?Jao%J!hN=K=C$kW&-~dLEGQ9*p!bZU`2zjQk-WJGjQKouncvzsZrBYFJG4P~*ZuT`k85sw4z`$9HFq=Adx zWr_qLy~cG+~jaWmJI@6%_k$f3Jpdv!8n&aQn~tn&hL=r+rRlcTk;SR&cF z{jm40Q))L|C|&DG-6D!A;#Z(&?@ha$4T>!s zrGVIiA|sgbpl!VHD zBjfG0wJ`(ubh(b&ZMzm5Am>6}Mo3apJcitILqJR9_ZR%hzhl570kr5}4WaOu`hThN zh8S%BqfGmjdUY7^bD%}hkFmN5#N8~BVUrggZ|ALb6nVyo~X)HidE@T{5PIGVALx!tKe>l zaj;5L9NA-fN3Hh@np3ZDvZS)JCws`S`3Z%tXrXAxmAUXA$ZD&qHn0IDmt0m~WZ4{sbXhZxOP8 z?Jp1h>3Y$7!AkaT5o!c5Am9O(-*PlNbPtFOG;UGhLVl;h34rOO!4ajL74a#7=6g~(Y_nm>OVrO;F0Is%;fY`y5Q8QgHfddpe+{LJ()waEde;QWtNe+T=q z+$7T-1Y3zC>}LJCvR&}W$H+a;DXxoaI`_S=nPMAi9#NA+KH!JFN_`}BKVgO}lyDHw zvz*3>y@-TtA)t#rGg+0ukQVV-5SPyL5FIaDPqBX&_mdRaFef{}*MbJ=dxN~bnU?e+ zTIdwNLrZYsFmYRQsojwE0}4S!LCW9=DWs(Erz#k{#y>6sfAu?F2P}YvSl&E}_Xp9S z+uo2AdV@_b0#6(2TT7d8fho#~@u)QIy?Q6W32&|pdJS%g+zm7EP-!@M;Jf66TRQ8K z(rfnK#=hfUybiuqN$+Z1p?UDh zt!EmwcWyN#N))aA#A9K!eL$4^z0EW}WbR$h$iBV`Wt8RJWjmCsuY(Wv;GQ=9E4O=K zpM7^+H4&OS4~#EdMVc*<_wnQuTZq5F zZ$IH7knS;L9KoH-Ih@X7Yhww~n!o~QyoKbXsI!{1v74tnsGA{R)uWxLQ~nKNs zOx&^txNixZiFdpO5jQAsG}i1oz=r+xVB){LM>!|=KcDk2gZ7{C`QPjL!Dk?82eO|Q z{=kyY1#joo8&STLQK#MFWD;*FCg{2g@(J<{|Kb(v3T84Nww%0WNIq5t^jQF%t8gAn zXu#|IH;ic{24KIIuM>rVS*8Lj@~;p+TOYJ*>l9g_;hNmD2Q&5&oWc%B==}NM-_QH& z;_yLU~u!GG_EirW)ko`*xAsFk_g=tr217?-t zg6tAm1hJCBX|gw~4GzWpXx#;25fA*GOwF>cJDY^=07;ycWo_+`OevW;K=tU;kBOB^ zk3O5`dIU4xK+Qd~^!UNbawQj29t3qygDeo+Se38XTJE!G) zmgo1J=Xw2}-|PJ0HQZ+Ix#ybezOK*mUextbqcT-4dnGk2-&)snUadRfyFl1&4cp@$ zvhB$;Bt^!2<|fA9lK3u_G%@lxc_93@zqFfV8OCM9_>Bx%zFo&6)V#|se$HEsP!rdH zU}{m3peZca1dmDnw5^yD{}e3#4I?63E?j)D;lkAiV{tL(9FIufo0xYIHBrV5B7tUM z!)mn*^^h^)O_Nu8v+&)OlVP>6%VkX^zCzL|hMNpnLJrn)qx79uXNka?3rElg%jCvM z#zrD8#FHyPMyE95UH`^Pxv}~OhixZ#-hFtZYIf1^a~}feLtLc10s0VO!K*eX(i&$4 zwNE#Y6V>pcMhW169%KcYBWZ!LlpBK0$TfatXd4L8V}aJm^+{k<@$=EnU(dB+V%|o8 zvgg;s@R1p$5Ngt-11^JMz_riPzdXdU{(MlMb|`XGfzBz+KIR4RvXflsSDxnh++U7@ zaec zOxJ!EqfGvA$B&~4+~-NnC~G(#jpDvUK~Q(n)QPjRRn|^|WNEW1^HW8BmFcA&mi+iI zgU<%9cWfzVk8>vKkWmykDUrb`O+9Yyf09&_Ru;JFMP+%4=__lKvOV?=bt#DQMJCHf zaXjJf*B(T^+`9oiJ+SR!`JElXTMuTNJbel=HQ5K;w;_|eC5w&$)Y$jX8@3~G82y*~ zoJX1wbqpvF{1Q$NgR1mD(|`p2aRES0?0-Zq5_tV={R&|imN!7DTu9WEy7`<+Ajrh( z+(4k4?hkU$7o{tV#77W+5Yn-L=r^&c6dc={OUuSg_hRp2C=%?nOgRias1!c79c5PX zV~+Cuccyg-tY4Ix^?y}Wh7rxtz}Q|N7~3NmVLhn2AAO8F*r(`!7HI#SV;dntwatEy zzAstlQ4>nY6%gCZ0%F8BJ+Aw$sE-q1>5y3q{F|QH{{&G_W;GwIfNv23<+F#0bK~Cq zqUW}kd22;fMz=~YV5UyhDAFHcFDQWVDYcI%L2aX*2r%2OuUm3JF++W7hvAMzVUr^E zh$F{}%Q%y!Ju>$}(oi&hI9aY|o;A9>$wpDab+N_64K^xKxC=_1u6aU?jn;&bn00P8 zGFW?Os&#VCCWu#c&xuzCw=HilNKFxc`A+{j4MXywZAl?t>{U+16EjN*P^goxBKP>X z$ZJ!oa$GBKajZ#^bdUMxlzOB9%^Z9HFGhbm@hilvr}_=op%a6M-hwW@tyW>Feudc; z{Rzys?JTpU$Msf4qNlat&((>jTj*lxT5NDyazO!2k(l&6@#&XDqeDYe3hq8TQ{NPE zJhu0YmA>6^Yd&i690??IvcrXF+9NOuGZ#s*jD%?c`tiPncB;@y~FoAzy^!YQ8&c_ z-NM~ciKK+rhO)xh4xgk9-X|H}5VZDtc*;mrt+Z7(%m4HFyou#vaw(O82r&qL?a$#~yHD@EM08dUoOl}AzyA#Vu@;Z@^e2`=Yfxj~ zYhL-{_657D(z{GvPB}!&kZ18X-zPdWFCyQz#4t2K*UY{F!2@(NGpHb^fa5K5w)S4r z9ebN7zhrtJeXoQQCwhMLV#3QTM|9pW(GU~9Xzd_&kn-T#b6y64M{4jOr*6^eVrGS! zQuF>2rVu9Sou~^ks|^$tu*|TLCTi!yt;`#c>=gJ>XP$j(`;W^XUwk8zDv@kvd1mWn zP51hs>>!yQ)KB~8VZ;B^d*lB{1BqoQxt9`x-UPpl6m`l1{6eQ>;~eX#1ODwsbkP?w z(T~J^;&`>&Rt^FPA^r}M`%0fZrY)dbU%rrxDxLHPo<4J{Z8~%R=$j@h)_AlAIROVmRfVd+SYH5HQtf({1ln3NJTOMo zbr3%NQ;p0ZxGX^W3eQIbiu?Ogbg26$d0mW;?UpA*CGc9$UL?)zPync)M!E71gHNSz z0Njvh1m_OK!C54M?qf=G<^8n2kS?x*qsS5m>0L*W>g}^#sv!oS&$m6p)lx^otZy(l zQ++&$x#R)LPSQAIz)(5hz_8K1yJuGu9MYbpoPIbF^66|RDU(q12~5yvng`{=4JtP! z4RA~tl7;X2S`;k?9RN5bLqhe&+=JvJ!6&Z9Wf$A|zpSeRip}`kFu-ppkTIu6(rl>K zl>El>13qN+x7RPHY`Fi2?J0U}*nvI|R?BWK4#HdtNRGqTc{=nW$W<+$R zEa{mQYjTl=g;AdVfd@BylRr0%YuOyoy*qHLN|7t>fp_wnblospwf-n5^@tzsbU5JV zl{e_5nz<{4pSp1PLY}4%%cijlh8CL&N|;&&IszL2$sZar%oWJufZoX1-++4Lq{SCK zTi`k?7u_T1-7c=_w{fuR3{PpWO%9~4=vWX`gr2Gr3lO*n)S$*;db-xGY1kDaCfc>Y z#p;t##?orv>BQHiJ#x2#kGmE>Zw)k+ka_SmgHj>ND9Y- zFJmbl0p!%H&$@sg$MD_qmmDRd8xFI?#@FvugNQQxC~)$0M@cM2`du!u>&t1XLZi_( zWP`d_H6#ajK`GDrRUIejD&)jo)mUlH-TQ7E>iG5XJ2}27x;>g|1TMgERxmA3?wT9$ zg7T9WAH-M;A-JDEGYy{JBtz^YAZS})4QVNNB(ap!q+Kq2y6LjAUPe0uWMxlWP1$0^ z_IZL+_)r_vzIIp9+RyO8ilfARR_yafY`rfx&eYR-G z>}{2}WtZt+17Vh11JwYoVtgMRTFK7fkwa0&0)0>oG1aEPE5{Y&yetc-1!LGREEny$ zKtP_WpIP6Nb>={1chd;TP}8T$<8C6Cv?gH`We~xPCan(rO6LIDoxhx#>FuAL3K=_0 zJnlDA0Y1)6C!YgdvV}(Mx3_QKcm4y0#p3`<{TE72A8wd0)jST;UrVO$sm38wUI-Oz+ zIx3htESa?%yOgHBZUh$sRDDmctNNaI#?to)P^mvT0eLb_uD3k?tyV(kg0*32Yzd0S zXHRycKL~tZAe9749p(`eZ%|H?uj4r+;7uOR!QB0t#TO*pR;Q510^5Pa_k19m|7K!J zy|Io-tQoptYk`w1L5KIaW}nyjD8sY)+)?+%CJJtN{YHZPBNeT&0^JF+#U2|3V0 zj-m9AH<1(wZ23@l|r=L;^9 zJ+o*7f|)H)NL^wlSu?ntM?_%iEqF`=wZQDek#k=>3Uah#qT~wRKA#R4i&~3$zp76? z%J>8?v7c4T!G5+KN{&Q8pvXR4xYC|(z za9z&W)K@#;`O;0px?;yFZ`c{JMuURByJ{mQ`^7*D| zP(@5*WbBFybzfeTXj4o!7vE;~>1M^bTIK~I)6bWVvZlsBL=zAfkXHVC=%L^wQJKaq zHyPWf_f0LHkDN8sOLsB>s_(6ywu^#{ig2wEtPcvGpv6LzT$#Z=rs=#SHYfO>r01gC z=y_#Zr>!!V}!d>fAhM|-0M|8`LmXwy8#5Ztm9M#<}V5$v)YVREaTt%c*U>~Xouzm)HN51$vH$JWO zx$QaTr|fcBndi{m88tIep&fBSlDFeFLPDNDZTO-N7{ur3nQ#YGG?obJrqPutb#Bq> zE=?^rpUVp@^BmgtIwE2Flc$cZgQhq4E*#_y-8~|qZOh2?7Z^oy`s)!Nr;NrjPse>x z-&)N1(aL79Y;r96C3$BstEB~-8LI^w3p4|Oz54k^fFLt*xmtx_YNOGpeyuv~K(R1e zW}CNY{wD4?18v?$0~W|(bJGhH@&6r!yZ;E)B(O(}iJqv*oJ-TAMCVRlk4ev)c6U74 z@<^Av`_6cnRFu<`b}#-48X< zg~%pdU0J;hj#?qS%@IQyyHpL>GQ!_8-oTe+d~v`D&vTw((G6~s(^Q$tBsWgVYarQC z*!@DE!R^3|!{TonjE>OD*X{cF*?w>r_Np^!4oZ=O&1Ac5ix{leQoO@X zd=!?Jy&xP77FK9JJsCA=MnW?nQ^T~vENI^}ZCwH+hSAW6EH%F+s|jR*+ad|D(2Ksa z31(Yb^7;$qd!xlKoP6KvV0qi*ZCfMzry5WV{pCbJ@YkFQ0{@PJQ8QW~UUga#jL$TF%3N9X*sf897#0~m*@q=M>yogK=K$9r(*`fxDKJkNSU%ceQ?QCsTr zz*;ar6JARS3|uc|xbdpg9b)8&sL7WUq=dqa0Qu|`@u0FYV(pwq_sab>^BMD*Cj?w% zraFabL}4qY04EN;b*t%jH`NPrpO#5M!Uv1KB8R2tBI$ zppgF|P$+9$v$zl<=b5pWJ65NNX14NYw@Du6HH;E~oGAufgYqcQP+ieLCmAoO|zd`U|wFsCc`MwEwr3c?3b0suaGsVajXIS zw^ysIf_G+V?NJ^E72vwV@R8l+i#x;S4h1s$B?d5&w-Pz=TZK4wO6-|dk5hv%p2%pJ zbtz|{j4L|526wB-X_2@+2G2GZ>+DrE_5CPt=};YuSv0T>e-FqP`uGE(q?URB0TbY; z7RaM`oQ^a}H!c_RPWRkAmbB}d@KB7eVtKNLKJsD7$)AL?{zoe3-_&Q3X-J~!Ftcwr zh}rY$k$7V^@HJhJvYHKTuy*Y28hW?$XsRXcWm39Rve3a=oXVOtpvb#}k&)Hp8&!ZX zTFZoCF(oqdo2HHPr}uN&=nWVd(sC~ijPEf$D{~$n|N65#GW#z0G$=Ks0E5f_LJu7O ztXE9^1@SgepxsBS1M@SH z(I||aH6A%6-HhZOy~*GMdPUFT_4(huh}LV?sUGwz2sMvhAPMdav}cL#hvoK|urEvz ztH@yX0~4z@vF$fU4ow7MN{uy+v2pG8qVuke6^u^C*SRw{o&(*uWZN5gu)VY_Ci!%# zM&L)l+>4Dk5GYF-aq}Bd*kmbR=l8_Y(AM7Gwb9L3tIZfAn;_8Y8l}A`x$PvfM}?oM zg~DE2BTTW!M?*&@Zn@|_jvS#Lyi#y+`lv~si3O=5eSW`~(3y*c=MZ ztU_-w6wb=lNWWLnXt$bHyy}|9?9I1w%{bXSakpg~Z`D0~xoqG^P^|?i`9L_3Bzy=m zHdqXwe*it<%6#c|(NBhXmhw?GuGu+n4)_mxIAxXee%`-#R-bKuk?9K7YBU(6?dv`ce-39>5FS~@ZoRe4bu3$>yOYU+arDfJ_6iwNkI|IPH$0>;f`+AY> zh$b}mvS*RA5tEA)=5hUtfhN&b$r6cn2mdi|`LILO;E#qo0!KQvVQ+JrumdR!PA;k~ zMI8}RiKusumO0*=c@Xn1B?o!be&a}^hmB2D6N~lTt*s~S*$)`NF=!%UxZm1}o>apO z53N4!&POS4^l^LYbnYatzOzhYyzZ6f2jh-DNxo3$9gy>!gIW^-VYn)z26PyliBySc z!9ZThosmO2EtQ_9UK(XsHGeS+4ZCK=bF4Gz-oDSW2?sjd3lRbns4bqugQE;i#3&N# zm?=)=2ky>uac|H2$+{{J?d`PRTl7)0&#rpG<>W&4{`3B~l|1OWo!VljiPRBjXf+2B zsY4MX$>(QHsmof)zEOuL?6lBKO*ZN9~1kc5UL`(Go@40F>o z@EWhg0`&GfyL)6!s^3z5Td=aIjM-Zk-=My+Eh-Mkk<-C+8w9TamJ`L+=lsu7Ub+mo z2sdSwmKNT=qnG5ib7;5U;k}Ekp=SLH4?`Y&(6C)D17w(c;3c&*Ez(3IJSdR2o5nQK zuHRU07AO(n_n=!&syg{#P0O`RTZz3hd>*VH^^~;V8DJix@R>D&B!!T5eey7dFexy} z>!B0bd!;5IV@AQvT`(2pzRN8_`}w2zq@W#F4`Yh+{?^Z0FRD%a#a2f0G_aKk26xp` zj`{pbz1yl}AP-1L*FCm!I{Wbd-0~<3s{La<1oIY3k-^bY%K*IKLIJ3#={V8hm`s2@ z0*)vRsG^nnsX!Uu;^S5COD5%KXT`V_6mUaV4a_!}>F(HqdwY4)Fhe-#f!zJ=1pjTjSf`U3zv{42sx;8$oC2J_`tpCfJ9?^P#y4F*&{e^%8; zmak(#{a1l&T4xQEo?NpPM>^eHW}cRmW^>qHpfn=eSawBdV88gTr$NsQrL*vSAfrFr zi3EmOR8$CR2|?5O3JDkh-tuIHP$bbD2?X$@3fXQFq~2@txO?ns5nZd^lD_NRRaJzW zM2!W`EdHe6lk-a2?9;{qtOf5&mrGR3PD?hus+Wy#6=B!c7cw&w-F*19c*x%Ce>HLX zr?qBGJ?ajuEO!<*0;-LBTvBRzq?OeRP#F+i`~t zdV)=64lkZ)bQ7Z-8*x#=KqrNxXYam^+#N&MxPA0m3};y{WF(mV z<*uMZl@YXc2AXML4_pLHLt_zwCR$iiyJly4L-*9hw!`+$4xI&EA5-}wEgqV&H0HRn z#Q0%)EC2dYWY&AszuGkYiFXr(fzJD-;{H1j=ns_6&*-2Z-LJk6+`7~6dhO>QIyzzA zxl!ciUY*z6PQ=xZ*$X~qED#mT?^@rxbwr7^4k__^+v2us(;F|FJj-Dm@x4iYN6+#? z-wK3LgmNh{#>{n@up(=L;_m%dRveTphK}d9C%Q}Td8r@epE@1{e}otSOk7>+K?cZ@ zoI2s9@R9u_JCPKsw7zFbTO5j~9jB;svq9bpT&ZSz;8}kF)dgh04s=QR>`fxB!Mwb6 z1i?1+#h@VD(e7!xSj$e==g@ikm9x+6G08F{sVYP`YO|&(F~50D70}&m3Sz3V&dRmt zvoP;@*4;3sew%$mpymUz+@2;k;1uMIuKCMZyDu+M_3q)q_XL1j^t_WjgUbsFpI%S7t43)US|f*9!sZHLjUv9+~canW&sKh6~&-2+(zQ zXIABYV9X6ujT$>#zEDf{C!M{Y$rS7!6*Sv+wz#~0wrTe=Hc=pS1X83OAx$_QFnpG(=kou&>cK@F!xZ`r z(wpW?LHfP9xmlX2tZ)hYj7N)$pWLm}VW~S)BV6KconBB~ z4D}bUy1gbP-Ae27^d-!VWJC*-E-|g}dL;H0u-Q)m!v`?0Y$>axo3rP{);$fj4mO;c zLoDs#PJ8y+mp(Sm-<0_>*d*q#w*Z)Bb@H0-BcK=3SC+Yl5=`K-+0|vNaC*_L)F^Lu z)bY{b=*1IlTot&}3C@eIAWZs<9mxUFC1P!FCV*4bje$uimdy}GkUfC_+LI9kO>BKy zO*2FsvY>NDzyAaN`^zySAeQ*giPuN+?+*nAYz3|gu`uRpL#iYV(NV2I;%i8I{^&-I z=2hS=O!mOZr0A52$CzE#s^vqzZs%vB!+vW8{rye(?^U<|-yi>`PR_am$QZ=i0uCqK*~U)@Oc2pdJ)YBpW^}^ zI+1%B=Ydi5s=6CcKUAaK0Gg!xOxVe*--9lTm2=xW6mp&Ohm?8=_RSLs{N4w*2}#)D zxy_pRnQ)-9BsqdyW41)$HBlq_KSSF)PHF=L+fT>%Q9nN4hV?(E_&#>)V`Er98n5b>T;CtJ={iKo5O3GFaFY~%4o#k?W9<@0ymD5A z-EJ=%iEv5p){)P+AnI=HI~ww&NKIO;VQ*aS_%=-Ok{e*}L_7kH;IPO8X1x)<&Nh1u zKVO(CGrAqZxQsPJU2^+?!eeGqFUj?VxIBKZi3os%tXS8a?^9|vxE!N=R-aopNW;Vq z?`FT32NDo@c8MWR%y_n7ZR2ca(#|K~dE;JEQaHm9K=Havj>z7vd%d6|kN%xz5@ zShIeKjfY8P&F0oDzxizyNsm4Ngn)z8Le&dvr-|q6z&xp(tkJVH2!t!soqWvQb?iZx z6&gAC&QHL$VbDqB#De?H_!z%-)G2ju>;tC-O{)`+T&|O%-@Z7WAh2EONMbM&6bFc- zu(bm8s>@f%N?XkH~NN3W+ex^1qVebkR_-(F(5+s##Ve%oziPeI8`&we>;BjFHiyljzd1js(hWzeduSBLgzx+BF= z4N<7+GSpc4SI97sX?Ph)XsI`_HMl@Z4i|L~dYYd%2+6t%VWMn!#l|sG5vGyuxYgp3 z$>gMtj0|_kMtq1Gi#cAEcCdw!Z_lts(HAfup`qtkV@FU8u#Gg8u;o$-B2bTeB-fo4 zox9aUV#AT3xT-RH*Ax4rGE10-e`xzNeFJgG15CTthAs%3s-0?MM1oa{*P#6U) z4;xcLQIk9%(+VO1y9-Z%Fo*wo5ccCs%n$$PEPo8vZ{u>hsJn!&B+i!MBJk|^U_@i2 zaPkiBxGZk`Ca19<1atq-nKG>7PW~M~KW|sxJR#rx&79{?3vld2cI4pvIu%K(7*LTK zae#_c$;sG}PYqb-F!ioiFh8UM4wLF(ZGbfVejEJp$WHiH)HDP@V=DI$fRFFPV8IOs z2NvJ@D(m~wF#O0&^Sa~Zln?A)!gjbFLU z=YkNkz`Mh77jy z>J`MK-iuo6TZ%?XF$Rz&Ln$+lh^i+kab=yuUwlqWWn>Su8N^txzE3@#e%1eFyqG>D zy3RY~Q7}Yog*ZGiAEw1mVLnW8##24q9{PH}laaBz0>G^Jl3Ng=k7F5yi;B{({GCZ6 z431P_S$T@Y+86P>diR>vgByy;V)%9dga+f!}ay_>LPlFu11#-n|G%JJ)A1cEq_jWD%JseN9zD7ESz{K zmL`FPzM*4gDuW8|m0T-1Se}%2sIgM?QgRU7K?O;X9M8rN4-AfA8i#{T9IX@@&>UwS zRa25z%u|$*)ns!ym4+Zv>#H1-2Zr%_zV*S#Qd1x}!5laY-xl6P0pj8V*|-#oH2oJZ zu-MxTd4t3C99vVWp{ni7TV6K=(%9@baWdvR*4upSzk0}j^Gx3fme(IF&{@yY3MOF~<0NCeW(khf8{S{_*)~vg#GtULY6uD7opzktUs?GXFQB~ zsuAv2zwn<7c{_9DY2N1q1cZ|^N+_rCvy%-e@Cj>5B!MS6H?JX{EGL?AB4G5XV%e~$ z%dGe#NI;5iAoDkL*vF$aUMY3Z@Jq2;HY>NKYsu{y*&yk|8?2d+hc~%Vb|Ys!NH^Y< z^=YDfvZuQ30wRw#r*1s`o@;P6wP(#xyn}r?o}@tzric?Sv!xSaqn9w74%*an<>v^V zIAmLX3cwdQg=xK%0x%Z$0So}CjvKKLUq*@!J@z)DGxnU13S0-f^Nn(CiOib!UhU$dY-3!A}y$APnOg!F9Nw zn3l@1JH)EmO3dw@-hRy6_^jD!T`)G5msd0fR$Qo zOmkU@RoO^9p7yGB>qyI~vfg@r8OGZh`<}i@w_BQAcy##MsmVZ=4iJQGa(oj7diI4@ zI`PAH$MVts=B5B0xta>Tt1I25JO~g_5^JF(zGJIgHxUz&cx+Zq;&CE}bqBC#q)1&f?iJstr>B1v?2j`DZBGFI?6i z&V91$Ej`q_>+Xe=Q%X?r9phNzFXOm=9ddLx2wYLOwqm!I%x*$Lb=`eD99$$bT!pOq z%jS6ZX6+1SpNsNb!Q@Q-2q1=F68}pWu;mQUsl){0U?uS8VTu&0dB_M~-kfoOFk06q z$hj7l8D*zyxT)v8a=FXXx+FhQjw5%kChSoB3PEUbk}#oK;=ZSiNG-w323{FlmZ7e4 z^0R}U*P1`x*jyX(>>0DnhG)zh*x#W`hiC2(SIFg*>z{;4d!J_zwyT{p7O{+GMqJX^ zWE#$9AhCnvqQnNsj{K)DXj}g47&!hoKZLOeTskjgA#i@%X02TC=4VlW4ZR<*q5Bw_3J#!C zAAXimk6JnoYE#9}Pzx?F;urJ>1mg~xVdfRfFvegfv!Frnkv{kfG@TDld-erJOV_8V zPmuXQ1s9}hNg1{vn0KCE+s%xJQoL|8(qAD*L3v6>y#>TAonWq3&_TR03FhXQJt+2m z9-uNJAb`lQ-Gd5v4V~ap+TIm{5h+16JV#xCuSua+Eqmds2Wr6igVq_`|9`wdBtSVG zr!yOZR2cstCC&7fnCbJV%$+fvs{8$%#VBX&60El*Xk83IX!b43A!A@gNT7o@s$c6i z&~k<{PoO(DtfaBdlWShfy1IU@hPhqq9Fr`G&|+FG+}U!agMB98hqBf8^2ztde`^`* zU)iDmmFKYk$)>wP{cRWjZ4dsvng7Qw{yj$d^K#HTRH;Ic7MuSu3c`IPc0k;!yu)8? z#k;H3%OkJlQM0`G#f!+2W2T3t51w7rhDrU*#qFD-`%h5ie3|j~4ai$$#x{D!a(=*# zx+xzISs$~3sOveG33_a%)T6q0TZ=>Fa?1nl*n+mG6?-dO=! z{BI4z-@1<`?8{ZUK-&1%jv?UV|J-|IqPun?TLph>YXWJ2pPP@6>!61HH;e%aWIMzT zG(UF47XUKBwL5|z4%{HNq^6JTTa7K59jtmiI5xVyj8Iejzj1je>(bxQ4in+O4&Yft zBZ{(nW!9F2tR28LU_^Y=dKZuMWgY0d`Z(Q@OY>-mwr7;m0{e~*_GutB4`mqIH)5&W z1E?uQ%;5C5_V_QsJ2gO?HSi4BoG*gav||()D#aRuh18>Oy7bQucd&z{w@wagKOYR= z0${8%$SuH2*l*-3gx9IdSvbnrFK;Tj&zw!!jW7 zi)xxm_M29!g!^F+VYX>AcnLpJ`++`$lWIPSnB79OC13B{)mNiHdED1P{5ZFjq_yh$ zoO5}>gJQg=(!--hQgN>_!?adqEC+b-l>w?&sF5k(Kb9$>ACs#W&GjK zyY}8X!So?0{$(2D27}M{v<=uwvlxTwy;Y-M1~cFdTUrieyYrud9_ZFD^9t*2+fWy8 zliH!d6mq=ie7Om-U&b1^IR%-Y%J&CqMRlnjFz;{o|gI(0S5 zM06vZ`-AVB+5zB0^w7!uj?42|J6_#gO9#atu!o%v$=R&EBZ6WM`sYUI4=YXQ*eZ#+ z;WSZAg*-pmvb0AT?Qez-D-f$zyIvP_)5qdOqwr7aV4$pRh`aqsk)lsM>6=f!*f+uJ zxP~R)QW~6>I_PDyt!8Q$JC&64lmdCN5%F;YFv^8(_tyqi8n8`geX9s!RrVu8!hIWz zs-_Y)-h@?V6^djOF66yY#BXq3TtR|?C}T8R$x$sWozotd+#WnmzHV2qa80Z8bD`LX z-)!14V4W$DRlDI=jK%AP*@_AIu{;*OP7LeT-cLjJH}b}!^iCR8K66!Es4!=i9#ea+ z-+&|HNv73O7}PlbaH3Wpn3s_1S{l(#aRX) z<@t8~lO;Bbj*Hm?$X!0}EN#ywm6&{2c1yENM64Zgd{7oTC%^q-grw|Ts*ZjXD-{>-(Zqp#ebiC&1c#}1sk5v@9+f2ViF zy!bXwE5n03x%|%Nokq9EOtv#!;_gf~0TL~`fk(hSEep3t^s|q`8@69wAxNcMB4BUt zLmo@nD%gF{*?M#S=jsE~(?X)jL-O0gR)-u%#*m0=)JM^CoEje1rP) z(2+>kwrG7s(C{?EN3DkHtf$qHpnNLd`gM%KLeGw~o1mwd);NgaqvrMVv0A)HQGZDi zi`dT2w_$?Gn?0=b|hE!=09nSGM_aYI9tygGx~1fvp9nZX-DIw zKMVvI;&ZU{M)XEaBOs;dR40FtllXB^gF9HI`e}lc-$0Xh*`_Xcq`WVvcX?2^`y%Sh z7+ZV6s&EoD#!~rS;km2Xv>*IVUuzQY7IbA{Xl=8RXd6uQX?MKs0@QuV{@)T8>u zL>8aT*;;;Hol`b8tbQ^(#~Eto(DquOEWU#GxP1^4UB&+#r3$YQd=3J-Dl{Xki)zD~ zWvG4X&V-ShXOF#TW^tEMlbvH{W`0uJis4NmPtj9c0ykUwyVsC%%rXUg#xn;x+F3x9 zbg0-R;a~x$-`tr|$72T%Z~u(lW&B?8LbM7WW` zMwmOyuQc6z&1z61F*vn(DGFxE2|Xn<4Fx72;dQ9~zJ?A;EMfRgR_BAN*QG&hnbVVb zt(y;Tb`<4~dLd#i?!nzRVo&3rCv#AAh|m%89a1xiCVO{S&vb9BGfP)teO9`H z@Ky#)F-02U-^jeifounjPAGXRqHUJJswN?JtwN4qWq- z?>rjSxL^oWx%spdhOlj*hs)V)?S7 zs3b;loOeU?Tct))FV?mOM0s_)u$K`VDZ<0Xl+xVx(O7=mHdUYDD^%WU6>|}0r)vT&8O}eZ)*aE*_XZI3n$aZ+sFq>}*g9A%-npzRtgsrIT_VkeM9p(fj55MN1*$(41Pj&+#HT~KCPbpsVp z)vu5^g=U2{N(U%3at+Y?-!Df?H>}z8mSfJzd&X z&h@fkhzKwU6&w|$`#Yf4SZ6&@L`%c~^xP|wLOA@=FlXy7%X=5zmtNr8?3$a=FFSj0 z)3Ij$wlkmiYuZ2MgitX}SR&w;eZbWZRHx};Z_h6;a$WIT8q%_qt7B)slCXNXma*V^ z`a{~9J{|_BcU#n4CoLL?Eq?&r0@q&&WS$CE}f$9l^TTnl!0I!o)aPV(I5E&F;V=7dr$?7fXZ z8-=gwz$bO0et5OZc+Q<-iQnON!;+feAyP*jYwn~V(@cV?Ijdtui)&Xtf{T)VyUCnzR#8>ca;TiO3W>^ zcwll{se|`w*0mE3%xkL;vGFL%b?j^%KzX)f85~Q)41JLFVW?M89gAS@h6I4g8$Zds zHuGAwiK+E@@A!F6lnfg^eM znfrteeW70d39$d?N*L#$_*jahekemL@XaSCN+GeTF?Iwa{6+W}!pk|&6>U_N?ji59 zd;j2mTiG_rbNdPvq+X5wD_R+e=on-%9Ar5CZ!CrNn{b>%4xMkuZHzKfz^I?8lNB@b2IP| zMSm1mH<%yGVLUQxsNm{pn_i$ba#d#|cl7vR;YaKFRv6P(WzirgH^^-z*D3Q4zBCSJ zqluNT>jD6HTc?Yo23P+;g|`loj@lfoIIDWL5Uj^PXRM$%&SX=)01K|1uD6m15( zDu%Vs%Yrt;DA$m+3TtB3#;g6Pg+nmnXKWC1>5y;wBES%TzD~~sG@)&%kyaE0J9(X? zI&lfCPfMtV1O^kTW0~Og?hTkMU&z-1AQOdh0Olb+b+}ttW||vGqoGE*@3|L@Ysy?b zO3{YPJVdX|W=vFK_lNTvUtb9Ec(>09RVV7^k7Mya)?Egihp$KuW!79#l2@D^TgIu8 zG5x~>&LF|R6-mBIdGdhe(+UV_9vm)yyYo|<$t`Jbo)E~y4%@VE9R`(Y#<@o%^CLbu z^rqNr5y;%TeFmSUvG*#(A43l5F=ekD!=@uXOn!xc1)}b7=@62BLc!nAt9Tyu%+cJ{ zcMAJl!*d*)bLUdT#!Sci`l9EH_lxdbZ_8-Qtd@dX!Y-#qnx03~B{<*JNYw37;5xXt zF@Z2QboK1$_%6MPClFF=)?<{m@=0ChUlk~S|NB|H+nI2kPRw$C@VyeMpR1_H%EcoS zS<+cc@M92~K6pFJXO+XVU!FO2hTb{VU{mQSk+9V;{=m%`110DQrZw{=6;e6bl_Enu z>VW=$-Aa>lF7R_Q<0`5#oh|cwtExDg#OAkG#2HqdoFwR*+cZS zv$#+ntYCEJ*{_iKHdH9=$3L7pwS|yB9{4~1Uz9qMc*-pnTVfxAbzarlPLm>ymp>Rl zT1*+aUATmjtY#Y@^S)Y@#9w7A9h`~qI4ez)K~Q8a?bnD-f4 zg4SP=k zNVc6w2a;_ufdEL=9VI~50*lc}utYX#NufRpf;RG2Q(9eM>+pi-FhG^eco0_@_s?fR zXBU1Eaa#{x|EoX$Ug@s&vHoWwUw{`uru~&0;Rs*4XkH5=9)$z52LR2S+3yHaV<2SW z_MPY_1Ik-mC?MK2I1)||#bB2*WLAW+n^5?2&^9)%*@pUH1x{($6{NKp1Q6P&RsvwG zec(FHIN7)CjBGQf@iH?ITl*NbR%qaOmv8k~(&2X+;QHgb4@d~WqP3K$MojkxE;;73-{`*o zrK$37rZO`VhAF#KXsPy1NGe+sz}%YjEu18(GK}mS=Be8}(R7&zYyoIp{o|(u!7J1L znT5kY`#%`dNch4r+9d!_v;h&YVGc$c1E&c$zvFM_b;j)vLd@%{ z+7Hw4Cu!a9i3B`I9Vq7m-jE6!@P?#(Zvgk=4s#Ogp8!Q6D}cxN(@Ef)!5xcUFk1J8 zzdR2HIy3-e=wMudH;yzYyDOvC6u*GwQEr0vk5nijYYZ$G9RKpvr&wm%o-n1r=7E^U zUEHK`*?qSuYl4^|bqb8ZDMwYHq?21Ok~OuV)B4p*D3 z(%*)eT)^^!2@16KTcWhK`mQMqFq7jlMX>y}!U3ws`RLD$oGXfJrsv%6`t|U=5LXWw zeB;U$JJ9xRQT*NR*rj=MxuB<5jTOzgE2{` z@j^O+_&%zyXjj1*sblIR0q-1Tiwqk^jkvTXjFz zcKKe%GjgRxpEsP`=u`Xgq_F;MCJHZDivU&l&>DIB_@vEC9ofRWtXRV3M5Fc(u{N{YHwS;>F z_JNH?aSBU0PzlO{Kv1Xx>kJfwwk-|QM1YS$8Xc6pgFnz9|LSvt3MpXZHjUFU0Q6+r zjUu{$N>H=I?=M#Y!>7w+O9V)$b}ABVCM}Nv7--Gb{{8iR^K^dJ*jgyXM|BqL+B#zCKz-5~!`N>Hv0i(%E^PcOx~K!J^407F%&ak0)1S}sG{Gl!sX%yovytU0 zfk2HlGSFUo(!iFQzo23d{&ptPG#SAGA9=V|uJaK%qKS3!#-wHy{ ze~xI@It2RY?R!dbVw7+Mar}NodvmszK#t}1n{90zWYNBbY@m|JiJ_kJKekw)9DTj7 zcFnUw>_oWht`xT`AKzZOulgjqds5weV*XCGB>^q$RODQv*z4vsbj%G4Os|06;&Ou7 zWq1Yzbp{SDs{mDG-+)ucH3+USG<(F`ZgsgtSck(tRa7YMM3Nu!{skK>2gIiNR(pY1 zpUNYO)cb|4ev40$3J-X8#C*JtuN_7rai~yu5n@)HRNR^}Qr<9Ye!3VfqF@(3lPm!< z%1;!P?qg~tV?c-&3Mc9+LCKf<7I25Y3Dw;Q)RIG^StDOR!FIm4 zh6<9BFq+25A$d8z`bwMU(Yc2ky?agc-8P`AL@)e5?7ewBlzsm;K0-=G*&;N`RtaTI zVMw+liXvg!>%On+ z{(i6T^ZPxo-~HEcnmOlmo}cCYejmsCILxsC21rDMYd0Vei|b+Ikto@@X~fvf*@*UM zhZc0Xy^0HG-AR*gC^##o{P*v)W8&1Mhx;(cIt$UFy&3DOa+FF1=2!1CGFi0dPRCXO zp&ngwWB=w#laP}WZsgL+{T5-L))B*l> zP9*A<4%D3T@)y;er?>gX!y6-uuJbj4OpUF(Gj;5%W~Qolnik-3(1X#UvxbD_^~$Tu9IYqdy^ag z)cvT{=aP3?^StcQ;z3ZR&|{_WS`QFY?<66;2RDUFQMx3}Z zy>aaAj3e=#;}-v&pDyR@Q_$XqFxn)xk%isn#5L%7aTQJo!_7=m-$u?JWY4pj=-YbF zGHl?Shr~sRj!T~-4SKu`&t187nT1&GZWfEsEdDFu9DZbk%FoQiqA>c*SM&IAO*KmS z!yKA3iSFO{Jf*#gU>|$^`Kpi^uKS&H8_-SD3|{(xL(McNRH^Mn+UT2EwWTgP{cYcN z^U~LCUuCDd9^Z;&k+}ZUh$U9B*%<-H{+?O=m;n9$^Z#fJuren2i@*PU)(-zGAtUTT z_2W?>d5d9&6%d#kvghFPL(F1=gBuUfuwj`7Jw7I81+88}(>Y&T{Yi>%x-Sjq#q*S| z(jNM(4qsuJC9QN&0e!x2kk1z;rw8#}{{6;;Dx9FRid8PN{GpG}$S0xol5(7RF@29O z3Z4n6)0mi8LVRocjg<*{IxWE@@?jjManMgXLtB-nc^sEgZ(xW;-7e?dqLL%5r$OHFPAV9ze|$#KC7hkAWBTG!nmU3|=K=ccnFv79V{ zSK&hDhM)Vpmg$fDA+U)HNW11gKZeHED>E;3F7q+jL0LAQNP(DsPRJ#l4T!5K2JbAq zc9GmoEupy%qJ(|BYpxbWd!{*)x+uDt%=_YE#gVQ}Bf8sq@2=?&3PNL6fHMH~2$lo^ z{whi*Gpha@!XOvGHGdJ)VVwONp{AjGu#jgJlH5xozlDgMBytSo23=s5E}&g!%3jmq zk)%@8LWpM#Rwh zK&@qn`)6FNOqH4l?%7es7Me6!ybi}-z`c25ffyckam!ahIpfTD@$6j#RIS{b{+rMr zLRUJAey=XAl(Fidg#aX6XQkC3C*ukDA}|CR9)bsRemO{cG=lGNW@LlIHEG3@*K(d3 zm~3)Nzb?AEq0VGb>j9S;y!IHmcpxbnb^eVVCFa_dXs@q@xrTNw^mYDeVrGIYQCVGx z_3P*O7r`0XMVU!M%$RaTn!_M)1UXO*Z|%%b^$U@{~9mWj0IL zoRetlRQrO`n^9}G>8ZG`XI6ufsryn8hW%o!+v5RW zbaa8SNoSBbS=^_=#Ch#$`&7nSqjEOvxYWs;ty5gk$OGRs$jRbdOg*K(Uw#S%;A~%* zk4S(Pwr=e+btL<{*|Do}6`d-EPM)#A@F;shC0lT&)+bORIwU5Rb)uQ(38a~pK%Q+5@C8?wjY$- z5Eyy$8+Cm$VMYmr1RnFYoFVtrbG2d@{4NB%y%%(XvnJTPvla~y;&b4cA*Z$t96&~( zcwm)rO`$vrpP`?tgVz&uGwRl!o|_+I%iIqN8Q5ucAsBBnBj-`H-ENaz;G#rhI5Ahq za`BqQt^;(nD@|6ymVHkB#XL&n9zVymdv`jqMQMI9PhBhi29yyh9lV;?GIq483*}-| zZQN?F>fTq`*jEn?cU#{v)fdUr+U9UlY}K3SFg2p-Mg&I{x&T#y-@L4}Rj3-SDN5_E z#SaVu+Zt0?fW=G zw4RFFxq8hTu_vM@Zik&@xgE@cA6ghPZl2;DY$kG7LVPI4CY68#Hk2MB_F-n_ZtT$} zbsN7u^`hlQRgtEX`oeFvb$bNVF98x|a1g_o`LKK=W^)_HkFM7B`fGOP#L20rkG+jR z=EOtT?zzo3GJg%DWuwoL{UNb$Rkt7*)wFIfz_2u2zvwx)t=PAza{jaSQyR~3UTV)f zT;IGG=%aAfigVSzlNN25Xa{+k2Ei>Rx)Zs@6|!q4YMz>Rxz})Mx2(MU(9HY+(SseQ zq6@o4b9WUdHYBY+woT|<=I69D@G18~LCpR$6zM9<5el0}f`Zj}?uN8R=JYNjKZ$5< zKm=>1A1otLSJ6}{Nz^zd{vNf8@-=cvKU{WtTKS2b6-RMVS=mcN@!)l`J#TI)u6AG# zG$%H_3e!SeMma%UHwKcX%=%gTQP;_b!-zT_XCc5W%(Q->-LVkG|QU1>B-dpykwjXN@buwR{|1?XA za1e6C5hth*nYT5SX+AYuIX|5LSkB+;ND0W5yVz(|ltzuHdD{7cMMB|vS#IPz(~y)X zAB#660U-HY(p-ZMDwS1ppi_4nSfjG9 zh#~#KeNu*8alF_LTi(txd5YW)Xh4DNTkPj1+j2I>wD4t5_Ra{bd4lMg*d68vZf6OJ zVMQ2zG*2H2FU+Yt*c_HrQc0eJr78JD-7F=0or`V`NQYQ4-v;_v)PAa1=Y$)BM}~Tu z7E1Q4f@NsA6jh^p2VYYniNvvwn~xQ!dJs z1}x*y#!hM)O(w&7yvc(P+3j6&HNUUhx@*^C)fW=bfrrtX&}Vvuj7PTJ$y%MnP`fuM z&D>0nVekWZw~;doM?RE-Htlf5tD8`sOb_U3%QFmn-nZ;*F!z(U-YLFMV&mHP+=mCT zU>dq$+y)3--HvDhSX6h=*ttRdiPt)#>B%oCZGm=0E`BGhL8U zO=YITYBa?vG-tU`WxDVFfOBs(E}2ITnI~>g`*Q5^OrL9*;K8a>L3=}9BG!&-X@ZR^ zIA=Y!8dlkwv>yfE7sJbfBMMMMCW!W_%Hzl-LkC;1b+KEXbo&%iN7vr>s&#GsS!SAB zlOLL~DM1Lj&};r&Of|A_mBRj*ysoI28dHu_;Qb)Kk5>lMWKuC5luYWL0!Y~axu}?p zq62>F63+=dO(riNTbKOA)WGJvk+f)dvYyGHGl9XgK@>WFgXgW5tDBj)cveq{IE9Y| ztSpsw0JmwKBhj~90fjQDQoJs>P7|mhj;0S2yxf>_iDk3!x?ML(c_&lfKE6iLOEqMD zK-c)xz%bw+C6EuZ*)bz9VwRgqW>1Q1)OTkLn{o_y*6Eppv(#HEZscG{l?x7kLv#x7 zf*(vQ$}tPeMSMG#yLbKTZ4tBpj+8;;<6k>+&MWvyehx7Va~IGF%sJ%XB(Z)-f=j<; zD+f5wOY8?Uo6=fXfZ;*As;)EGwAT1(FR8~Uy)Rp=idn?FKGpN)WXATPI3u1K)&%f@ z_VE(0ktyi`V}*~paT6xwvQ8#r^GSm%HC0=cS?^^d z5j~YVH85`qn7N%jTNp^@9Zinq5_@nUwq0i^srI$zY22FVg9n(-uw~RqGkupBY2!kU zaJF5CP8jYKICalMnD7wYEeoE|1=5Z2z!!jvV`0(n`3)h5*Fq*hWF2U67317D1U{M@ zxFx^FM-cj-mYB!EJxlMlJO&>sXwR8*jaL}fUw68mcb||D=i8C2Q9YL_0&QE#o)6`F zUU4|R`cSd2KQ`$uGEDbsJ%j%=RfZB?e_#W0$f3$cZ64yKUFyj3+Mu$jWN_AmtF?~2 zgFaLB^&6rTk)O?YwTnMcJsRqdMs7^e0b9Nr+l5W(&A}-0*;5N7@*xtxbF)wfyzgGK z{I(-RrMLu+XT6;xLVy~)2@JW*v^27Q)kzd5yqC6-_7vjMOWbRt+~$;g&9ven&w)FA zYY)YV3Hyjf>8-nTG5;$6GQS;1y+`YOW<}GdI?+rvyk^XZ?K-s7+rT!jX?41u2RZ0) zl2gHltAK@GOKz;7;?|c71FNmqp^bgrzGj9mS|6pF@_#lGL1UlKBb02uq=s~Hk;{~_nt#%-|7JyhaOh0CYn z8*jx}DAU;K24E(rqa3HZ%6eV4WJH5rDg98fM(E<`cw4o|dD^pc_4nL@qH8-FIfBG3 zsFK=F|l-z4BT|GsR0z#5GCZMKHvJzglE_YqDNR_-5{N=;_RlW_< z{PU{I+(@>!%=_giX0~-}0gyP=Dznf$P1iSwJ}z_t_@_9`)JmD`ZutDBK0BG-Sh*xj zJ6~lb9rpN+wPyTzSH}yij-G5rLTA+Kyhnr{aKD+RU4|ydK*cgM5-@<~W46Kmi? zLDp*0#`2xmrt+PX=Fl?Hr-lzFHm~(~fVS=~YP_RcNK^`2`{?rd_$$JyGRTB`90x2R z4==`TXY5)seu@>0fAvmvg zK@x#UrB*AYTa_6d|xCX2oI(5j8XPd-2Ex$T`I!rNdiyOfs2yQrDcu&bS|njr85 zMwu;|+kwlHkT2@ghh?A4bFYe1MGsz)XB!vwysp?LBpPF@iv>@j(zIMqdkEikbD`Gs z^yB2DV7dPGa{-SVsD9R`dmhE=t?oP}tj4C_wom*Ws{ampV+O@xa$fbC+XX#bWyx*o zt-uOG*QFPp!u#Gs9}Rcm5w4~jI)T~9bs#$|2Ex=!+)ikk(yQNU`ad5~X0B?{nuJhj z@)U_kfNhpSW%r8d>53F{qLpFCZ z4){=ufyX=*EZQMPI9O5tmm}y8vY*c?dr$0LuQe z^I#J6GIPQ9sF4o!w~qV+BUo{$b8Z~~6c(32I(!NK2*&N!Rsz`xtk~b-hZTohme?N! zzF;t+zjU`OVJu;w8=?{kWrd6P!q&a5bB^3U$su z5$LYlV%)Pk)~3PptfvU&^{X=yh_zE)Ay0@xz9TNNP{w`+2@IrEj7m%`0dba6Teiw* znjNjH37+L?qmQTkZet$DUf7erJ}9(dERy9Bp0RPEvQgj$#bS`HC0n@(7+zG+j}C8? zXnW6kWn0;&2vNP3)JKG?*}$`GO-uHuDj_f9N&1*~t}B#5OPoW$P}|JY$X;5%qyLQG zY4JM4cNgZwEGBH&e(#t5XRph0biXVUe{JpfXQ%wPj<2je$V@E#QaLo`_!T}RxMfo~ zR@yL6%>#F>3NR84P@lz@8T7Bfh}X-7Ouj1>tvHGRdV4??30wfcj{$ZJYG==Pn{)b5BNLF+tFWvi8ZzU z_1l(ZqyEfV(2cjI4-4Lisl>f)fbw?Qt= ztPFcvQ-81FP`KsI=@T87zUDf_Mu)05s2q7;pK)~USN~)Ib`-B7 zaTQGS`TFSdlvTAho+=7sk+{=WDKLR3TJ6z3`?1vI7dH6st0;daL%;}z)WISe@(H!L zE9O7%{DCw7f7M+Y&K^{q1e-Lw4DGd-T>wR=s$@Fy(v@M~sn6}oG6%vVM1tSuEU<-2 zIEX5uJGiR@G#3nMJ_D{Y$3t_!PR?Zu+DDhq7>caP$?Vyv{j4?eJ4cCD40B2X%=Q$; z0AmPrq+!JK~VNObn|{+PJbTjpMMSGJJP=B$N8KNp8HocUY0@}G(JpGD?B3+&3K zr2j80uohGYW-R1ttV%5-qw3tE`Zlzu?(@3-b>8OW*YS^jYd)F1i@)%0^6G{D)R9;C zv8R7b5N4&Z;-?mM-Ab+Pq_v~?%u#lxw#wWVa7*igq0F%D1npNRw>Y%nb1Vm-bEkNzD+7Nz^n>(St6NEsGNLWMv|I|BA{3f=oZcN2T zLL1aYzc|q4^-2j{sW`V+n;e8+Hg{c}5~??g;_fw}Iib4R;K)qCWAi)z4N;%HSPin& z$G)Xi1i^2Jizx2XzS6e&6sZ*EjdUj$kBlB2^}{p0JqPsi^-is0RXJ){=YI8EAPQ5? zOY?+5-$rQa269py6_hTMRZ`A>Hn==Sb-8fEW$UnOD>w_D0z^d7oMHyRLG*KrYE>COo!T#4i{| zez`!J#k|~V%YBKArI{OXIBEb{G96GGNM<(t5iRu=^H7T} z-^L%+nfbqz9BYYEGZW(c*ksoi&PbLouAG&B(8ZPXn&&TeZ}R9^~3iq zGA(LRV)I_MaoUgSnjUfke{v**z(`udfWJ)20b|OVS2I&zseY8g+;>IKgwHE5 zX?KFT#YMUKC+9zJ-QKCm4>u7w<-O?l4nUl~;#H z#&}vIE09D|zv`5#RIV!-s*yG?{qRw>v^bgQVi>1pxORKa=)JaG+A=AbEH4t?aI{{r zNM)GQp7f)F%h}t@B`}9O%=AAfCFrP6;s8#8TXB%4}}@$9xe=C|&W&?l~z#xtfIt(~Y%SmY^MGK#&= z7iSbwqmW?UcHwo9+o}7POvU6RB!k#6WjA%bPcWh|$7xJ5zJ@?Pep;R=tk;nWtiz#=e0G~6S`R$3^&F_V^tKJ8Ex zwELNZadEuT#-zj@U7N2&>Pw3Y?GwaH(_VwvZWl8iUQc^JQlTkE+3-F)du&D!zxi;X z%gbCl5v9vU2?r-y64vpnon!%Cy9=-Rhl)r@nA?ba?GG z$D;XX(G?JJni{}-TtjP~y-ma}w8!W#KYA1`? zL*Y{peLwbn@i8Zy0LIQO-8sJgi+*OcNO4jqcdva|mxOP}D`G3VSx87NhS}dt+m$rw zFeK`%v4L^I^MT=k$(ybx>ssxXT>;JTqyi znkCy(FMCxJ{?N(I#}6q+mDEg{JKWUzQYJmLFy&9z#H?y7c&RDvn|@KaHqordDXfn; zxG1;n#IY|Ng*SCMeV(-%Y`7m4=&}eP+~BjoUkyw+dthspDnW^8@_HHUEktzoqRt$_x366qQVbS4QecP1SG z4Kr!@ieI?v1dqQSkm zP0OT)GK>4kp>xnD9CzWA=7LGfmCo)jvOoc=-qTa;{n%#%n^hq%RIgdzFNJlnpO8f+ zHEzVAI^ge0q^l$h*zE0b+OqqSmGA|1zqqYz#ax<cyC~?rz6WYy2{K1y0Z!Og zb8)W8P3KkQ%=^!)@^#gB3kkb~ovaiIW7|~8=Anh`vwxU1wq$&pn3(O!pORjZ6z7z3 zKAO*K_d6}to@DqOq5nBjpj-${V0?c9&27-1oA+5ndXKiBRLXwq>h*Zd{xHjigP%k~ z&)=lso8+$p4Ki7V?Q1$z^9>>BTrL%>GJJaCl~knSH^h-a`qsaSr>f|B$;qz=7FS#VP7jmmz{Xq68S;ng?W^(2t|<(tCx-hn@09TZ zVz}0;4s{4~XLqrgu<-Wkl1k^3CY71)0XtY1ecF_A(dhPbZAva`^TPCuZV>@WL7PE+ z-lJ9Tu)*bH-%s-n)XaA+s_yJ>8_>|a2&MZ5YiEXXeC!D?VLZf-FAOYH$%hiwPNMQP zg=i*5ILldvhX@~jC^6=+h z+r*F>^pGjRIS!$Y7B1P4GpRe7iST+qMcU?pzPqAxf-(XALhbUb+=HjSMm@U*>iOnZ z*ITp~{LEYZWv2$pf)+U7p`#BxbkJ~E2k_95!V+=$8HM!Y7_JhU$pdHVl4x)3eEO$X zuk_>g6uq;2);=PwZp!f<`o(=m3R?>;e^QpgRY)_zk5iXE_r9L*?Rr(De^a4&x1e0- zh`31S$18s-L;t(<>mR)WIoL^nF0pR|J8}p!t&qEm4)c?!9G_4Hgh4wdo(e#=u1O16 z18xvd9YRfTk*)$))|KtX|D?6}_ol%Ai5A@d3QDl@GqC&j{=XB;{eJ?}Al%W#enViS zK;_1wpG03ZMp`l!#R90K@=wfO5OY7Z05JeVH~0ZYn^Ipb-(|Z{i{n~ ze9j)|e#fJCX5!VF1D0N*GIN@Q{iyyYwrePvHOMXi=~bt)d{E}*8E?*ZYL!jB>mvKP z@{srLsvRdt&I|i0rnNQ$@E6*I@uBEbPqZ=2D4T6cDFV;EZ1i}2{C6gMid^0hY4d8^ zWslU5n+Z#s(J4r>5t_}nZD3&%cXpr|>9{34tnfldq0p9Qv%;DODatdzd4Lg=)!`3}0c&D)lOFwWX;Hr0{)6XhW@=uf z41FCaztn{vEg?t&Q7+n!;%H42wtY)dg~s`+E!c$2Pav^zEerOuwTGK8XD7HNeW_Po zYW{h*fq+o%N$y1r{ANHTO1uTw1A98>OMm5sE3qs=AYL>AkBEH;PZx@TZ=D9@=4B3a zB@aNpf_x|7y7S=UC+F!JKYhhtG|T$MErNiLMg5HSAbv&WT&PO~&`Ww90==Z5$oI*< z4eX84SduBru|__#x|Soshc0nGe}gi0KD>kFR=voM7YBnj>8;ADVEx1yz-P+t>niKo z)z`-t@mTHLQ@$4q^;Y)-E3(&zRIW~pL=11I1@!N5LlvAQf4nRD(Q`!h{`Mo^5M{im zT(3Fc$mW?nz`a)Lsd9VEZj0j_+ec3yl5S>R=`Njr78!{MGjj1uRkM$<8C27=V|y~s zHunS+)zxm`*k~%V$KkHh4;Q5g@23kia*S01d=g}FbtPw31wP%wf#9^-!4ju0(o%3d z>x%n{gfMN6EU|h1_hYH+$=Dm-ol7dAXU|^#hPZm#-||r`^AsPijqLZX8J5OzEC<=P z*pCVNLm(4E$s0}A+MMGJw<;D>44Ezu3JVHf6dM`sINrCWPGDdA_8oJa`}ogAA}-vj z-=ze;>uhk2!kRg;R-N@Kl$<@=>$l9EK(NfMdi*F*JaI>6^rO3M*L(QS#_GRRdZEj= zH^=lr^k#6q2^8Kd=tm1g_*m z`s*?mzGFMg53!v1ZdbsOZlER35$ojmV2xN&QK5<4M~Tq2@7qn&q|4b&bOG;fLUSV- zcYXL&na(52@i;LSt8wDaH>|NNUi)&5QAgUY+&JryZlz;)DRcWiamx?L1#=dQe>io6 zc6-`R4#s28YGB?4_Sn&m-+J&@9DKk%_P-#;e-w+C`}Y5W2nA54_ytA$uQi8v0$w=i zmk&pB0Kr)K&{SC(Zt#HohOxblZ`3Cm4*7pNka9~}UOe?p&>bpB0hs5*&7 z!V~|2TK~RC?_roc(;bM};M^yXDwQfL)#_gqZLofV&2^ipiO@E8nZ;nRejgo8+Pa(h z7-Kfnbk5@CtF|jHcasn5*^a}5%5ptXpE2+EM%||b;YLqW3qA^KGn25g<1Z3Voh4@! zi0@+Mk26v`#w8rJ`pi*(Zw(oW47{c-1Hk2$A0=Yoz~_34fF@XX%LBK-Ww7M)z>(=j z8yi5=j%u#+ovs}4(R}f6T;)Y{H_A`4^!od+pDu2_tf4JpF^FXFnALb{523Da-6Z+W zf*6oO782&qwR4OSyc*_085CQ=6Q@h>&c_iP^3Zu%2ezH!xVKX}xb~RG+a!L*bAY!$#wuf5wvecu zwFJIi7u(YU!5@kf>|9g#ZYc0Sr*m^1dr$TBUiHpra3!vQBM9VhF+|R`!I+vI@|TpZ zE77}88f!`GRzy>aQ3x{9Erqi(yH)8GDidXxo;CqYf{@x#zFw@ zXE%^k_ISWCJxpU=!Gik0F|~ZAf5yT>0(oIDB>%N05Poa{<^;8Esht!>xSz1(n+FWA zzC!>R%MieM9Do3Z*b$K1?Z3fT?iUIe5bs#geFnilJ&}L*KmYxvAn0(wkNt*Nlp?K1 zP09ic`tcb0We3d@Bwqv&&bpGT;i^v3(orxZ!dL18!N&>xvB(+UA_8?)JFpY;Vp{#r z4A_+!?Z5duM=gPTOnyd_Z=EmQ399ba>xe@Yw7|G3*1Uj#}29Ty4m7xXy5srrNXRes7e~FVlvs`H< z^rLH_C=~0QvCqJllx4AwHM)~aq0F2^_c987y_I6?SZ3ABE_rcy_i63hx7Y@+1yo{o zL9*1n0rXnzJG4M6A9J@015sSbp9gp z1t}BqNg(pNh~JEDhE+|v%ExEWoV8hX%2O!)+qzGB_J$0L$#tdQus5&ejjKs{Q_Th) zMi-C##+hU_{Dw&5`Hud9QpOM7eo8EiFXIB6mu%*iyq~zqrhm>a@YGRIx?DHz{&u{A z|IF23gHVbE67txN~Yms+2PYj;IuEv6p z1GEy{L~)jP49bUqn^*OQJn4^X2vX0r+d;45{qn0w07evsG&>VL$An)&lfw|ZEV`vhJ zDgZ{J79FC`;0E-6{6z~{QOnPq{rN-Czk3EB)Ou(*-PZ)Jq9T_}6PTSpo`IhM3Z|8| zxw2(@>E~!J$7kXWIFP?2dGoL%rfyQFgkx@Bl2RmPsLdhZYZHraLD=~kYG|Ojz_l_c zraS>Z`jK!SqU{99JP@sI`A{HF_&ipkk=XSrW}y>2sgYskw795eeb#A|KPb$s{t>oX z{+XZV3T7wB0&a&ngSPO-8q|^yz9*TX0=~!|m}KcFLrb3rk<9fUb|Tx2`HpG-v@@gm zz_sgw&FO9VF6fgcH&Z6K{3C!6`yVYwO=y19B$5M8sMJEfADOhHUt*Xr6Vx>ZO}H_d zWIKH1qfWU8B`G7D*MBH*Jf=0XM(}KWb9z9o8yNDT%*T&Y_W@<%wLW`ES{Tu2i{{hmbF3GQVv%3vleO&hS#8vL+ zi&GV?-YM2OlQj4mSq}_MfGx;5(utO93*{n)>#2A5ZOIJ(O5!

YiaqHjS@HR*o0; z9$9L_S0HJ!7PT?q$x+yKG(KrDk`P=xRfIbBmF+S1K>qX$VMM#Ed(# zRubF@M4#i#V(cDYRVs#wlZZb>ZZ&qLx(40y5)9fVyA-h=3-6Lm(h=6ucbO1xzm*->B}}q*jT=AS+0NS z65vHgc2+@4QdNYkT?p6)9JEj^f{Ip0_+>OoYTW?oU(F~bw9OB%<2e^;6BIjsz=WxplWZrEa`)h z7Ui^1f}1p>94A%`nz!}e2PMM;{x2otzad2Q9&lGqe=?z-FJ+ctReehb#}C`$3{-*8m| z@cqk+OJ2Rc{!;Lm2YXK*hlx$J@Ivn=2FTvoQ0)a>>Z}xfgD8A=fBW3xu#j++LiP6g zdh6BEi};=;1Y5XSc^wbS)&FR|vCZSRohwOe&- z>(#`?&M!55goe36ENKn0@(;$45|AI_K@Fs1&H!+(O577DMN!RJ=p^8RWwux;rPaSucUTvzz+DHV8Z#Qe`p>v5 zo0*gAr!)e9N|VK_mNH$6Ph~QEZ=gxoB)D5QaI8}%a=96ht6yT}!&ePULmBAZN?M-PnoH2t=o(>sqhKF@WBL*KmL&vn(5 z9YvGGM>Cu#98{C%6l6>xQ>UgRf1DWQ>i4AA$j|+l=)ot6_CCbdVeSVkMqgunp-Ooe z={q!#G>NwxKwZDOVLDv-j+g5}y#hke)R(p5JN1bDqB;AwCM+TUdvQ4S%5v#G+{Xrk zDSt&<%ST^%{IYoGM#b@=izwgwSt4G?>XDD;zqkK@6czOUh=Kkeu)OFM z{-e`pWvuf1?|)zZ`M2c{?*TXrF#Jq_;jilb4a1-Q1;Yms1<;=}0s3?7rN8OVk$%#j z)7l#J``B52Meo9Pp$jfO1luq_Zt!Eu!`eY1*Xdl>4Uh8+OY(XWZK`xF#GW9SZK{(zbS>r~NZjQhn=o`hYno{z`O4oHxxl^q~dqeAvT`z5_Nqlig&+}GU*tlXXHt6S6 z`d85DpNSD+aest5p`%dbLxe>!+x@_a5Cjin{I9XiKg~;yS&(1R*D&GEf--nf;cx30|LnBO_^}~7G^mfNWHG8dla|-w{YSg zv15zn>t_dZ0@#{E@Mi*ZOCb~W>%jsRChEtJfuC9aa!VcPs9SlxEHtq2v%h``K!*V? znu8N{2kONJ<`6UMXx-4%K{53-jzm8F{4YFf*Glg?=DrTnEGdpq*T(Cdb63N0Ox>44ebI#fPsN|i2v zk%1y?g>^r^58a}hJ2q@Jehcw6n@)aw_~}0L6uW58*jHbGc;3@Eq4eSe=^&!o`7>|d z8rYq=yhc1$d(>s#?K_rX2qZQPsrXG8DL2}wRN)xUMQ;y3tD3c)+zyGel-LK|`K2Q9 zJe-U2R3)kqh7Py@0Q z3+8G!h`il|k_PjT_@NK>3*Qi{KX&5bMq8 z;-H4P4G7(>17m5>k(CC*jGw+f=BJBup@ZT_*iI%$C`V&y4&bT9LKGlS$_Kvzy}bq) zMX(sWLD7;CuzUsu6(P6XSGwcbaHQ_8w{34u+>Y9)y-)1sBZ4=f|EBi`eD(D8&d8U^ zHpkzpTueQ=Pb91U=sMMrIeY;qK$vu~Qk{U_U@1klMw6H3`e5{GUHl^(TdO{+Pr4%H zhKuEnJH}bqrn>LXX)IX)YM#C@NwE^iQ!sUBMp<~Rj|2YR`WsRHegU5E1F}R+x{_g6 z)P+E$X4;O_(#%irR}$T*=7Z}5y+AR+`hF_sP0v)#LZ)QrtP@=n3FIJlsFY@ZLp-US zj*u$~n4XW`HPtMtI$wY3{s%J@1bEB8WXiwqK0m%fdvOTJ{u(nK_`)JLVFU|0xq{@B z{idGvAq(X1I&9mK_h852m;DK1Hy?<&?)!!)KusQ_AB5j^Dl}2nRmXTd$aamfYAd#J z!p8@dm_B?cF?T9aMI%V0^AjKxC?}m}zV?c?+Vf(UrAzVAYjFn~6r?_QG_2Mht3Vxw zAIjyr;$73foBNEMG!(z3la{t#A@6?hn*NZFA&N9kydlku96qQD>I<9o`%$6EiW9`i z=Z)?~4ZBU_`NZYAGp-zEfvEv9OXy{DsCXO#-V=}woIp{7-M)!7&2Xq-{NcOJsIvUG z*Yn%lQ3D;%`ZQqx!E zJRGYngjfP0bP(>(FP0861nyhQb9AJ+hD4JC3&i(3`eu2O&W;SaF*bww-=M_8n3u0Y zUMI1k9Fc2i+=Ho6OI)P_PtvWG^BehUbd{_4TtAd%ib4KhYP{;CFzL2%D~YRI4clpD zQP0g@k4Y8HgX{Z>-5*<^M&-6x>*ehm)kNRJIWyD1^ZVu7mR{>(O~GRs1}oy?EJYPw z_u#kZHv?8g#v%>eX!Z}#j4=X>KZd@IA}=DrqK9FCha5bLT7G_*nB_%J6DtKC%M~Oe z9wZd)7N5Bpn)=HN8xF+_^_Kvz;xrg3zPzk8<01%T8xHWCfsOGz|Dw)cyet2?&i~xc z{|m<_y^#vh7W++zCId}`TJCTo?_`bI8fvB6xsC57ie2#uCp~Un+gKli-ND)|voeH5 zqk&54&vpDjO$3;E+kPtKE7JqeKw@TvDF?wV>z{b9e4d6m%{<(|dc@AxBKP&P<(W{- zz8Ql_{!Ic}r|!leSQ@0$5s>L^i&&`o;Ie<*qa+ZWh;%^|B5L3vmIZ1t0-*`lMFMRK z;R|*Y5Lo?7E0(2Z>L9I+>{)}$v#8y`U0r_QgO{=1w0^?bqQVYYht!i?kFP~b?5;=L zXZZlcgjfC}Ke&smq$w`{4sgmpM^ORWifj%o?lp*_U?_{F?w_i2E zzY`n*H6$*F@y{=!mriVeB$2Gpa1jQSnqGv1?56c)_7X^%{`6g0tn?@U-J$V|Ps!Y} zMa&92lLy=Y!vOhdf=L8KWd|X5YdrclnR+n0cYU|qSqJ8Ax)dR8{12^h{%D5$-N%6T z7d?O(PZ<7)-2f<^cL%1Ks@mup{$JtP9jv6$743vFGdk2S;Fh1Gc@7QFFpOwBsq&n( z-ebhCAL_Qf=(sL)9JZ`cWuts?-L<$##(FG~KrpC4cmakcA8K&(YT6Awyd}3ObD8Iq(|vEo^K*+c9Tj%lERN zwAh_u%o`PU9JQTWYw9Iiu2&)d<)(B4o|_|9LkNazi* ztvjWD;Ua0LWbBjo&}s}fMUe_ZsbC9+BYG9>>*kKKoj!LnT3f%IY*pTIp~B_0roNW= zT>}ImF}K3dOt6Au-bL3JJ;|i&VPu#Yn01?D%K68Qy_^O0y`1)6$6s`id0Fkb(Vny7 z%C2|O9YW#CUvK}O;IsTv=Lkft5A=m1v_#T%c%&$u7az1LvhmIDZDDR}oS`84P5gBSn;j zYHpFhFQWQK%G)9;KIPge98nt=3I*b&@}gX*N8=0tIRw`*6q4%Dc~hCuIp-`&$SN<~ ze8m0iNRvL!AM>6*KW^?9v#!X%T3T(JJIg$GrO75xkVlg}EBbaRN>)1)x8A!lEm)P5 zOjNnZMcS+GCAtC2%5X15gI-Ir`5L7L2s-aDXtTBY?d50XN0Va%9_|wl^ZhL0AnqaR zi*Lcxq}}S}H;}OP(NcJC0dy#Zr2i!2V1f57za$UCyGY?95?SX?T#nkeE_u0sGC05~ zIi|L`o5O}yne{{`Gb20qh?Q0oHTOi{#%lvcartk2TvNo?`8oE_XV!jNLG$$R2q^Y zk!pR|rbx$wFHVHZJu5xtxiZxlWKMzDdQ@?9By_%+i*}>B97#^WT~DL%x2u{+cGSe) zdAIjA=R=k^YR$@26NV;z(1gjmLHZkl^%;s52}c9}*1$3(-L2A}&SgwL>r1L0GV(oB z+MUr|`X=7G-j}M9m8THQN)E_cAl`@Y(?~suo>@oKnn%+XzBB&R0?4^W0N1 zcO}uiwi8@F-KQU=s%Lw#yj>r9B2q!65HfDn7s+Zl{IDeq(7DBXwp`q!<0=@|ryQv~ z`gBzgjT6Dx3T}55wxKEOQ#ZC_D@R^aQ(UdxpuwQ6?WSu*uf=>~u3u1RWGGhXDO2?s zyzL)MK3tnnA~0uB0dJmQ3r9&iJQ7`V5e$rG!rB_ zk8e3{v3r&rcfX|jU~HJ?V=*9O_{9O{T;R*4JJ9S-tX1&!KfEpwd!E|;upY0>M49st zyN$OSEA14oxg7O&t=Xi>;nYCXI;j5zA>Sc};jO-t(pFJ&WU03}eDZUfvZL#Hzs%qs zgSKjq&krPO(jW8eq2E33D-k2Mj7_npbgD~}(+tS=h2&l@0{@tR`zIZLQLvA@UOuZW za=8H!nqnEaZo7)$h?ABUvL*VTa_2!&i}YVmPzE}1Bp?%cy^pgK!#a5V$ddl`oM%e( z0uC3!OXv5Nw{-DVzE7^eM5LW zRY{anY8;S!ka)qNb-rnzfsprG-1ujkr=QpidEG10K+V^H21491x-Q1CEsf?!79Lol zdDd*nTht7QZ_F8&cwR}DI44-Ock-}-`IT+YtCzIP+a{HbEMBxvJt{3q3Xi;SYSWA6 zT@@jxO5{Ts$^(U0$Y0}~(sOVat4a};@=`wgKy@>!*@yBSj_CZ%U?p_)Y)_wW>w(p9 zyURpOVNX(`h1aPJ$>Nf_x4CiddI}!N@OcLBAZd9CJAx+?EJ`}mRovjWNy#p!dymAqpbb!K+;?|U<9GOA(z z;`xg<+4C>jT!#vZuAki=!RDCuVOJpg>7U3)tH1)sJ+kh*oPeasKqfQj87Lt>wwRu- za$A&K-tV8wr2q~`OK1O|w?IDN*7|p&CeTxI2}u;MMdvM4+>43q<8I3-JWUTc*xL4n zz$19rTHqwA#P`#l3i12?7C?YA2#jpMWe}ttnp6fAy-`;!q-ONyQ?j9CNup!c8lF=2)36{sHSt8DwiKu#;b7 z3Ivw40-yLjBKfbpvMlekf7U6+U;*0$K+?^>Bk8A?k#stfd=EB6hd`6-GNCc0D=ecw z)^Yh~aZ^Uw@cSiXmH!RUzZ!PHPkp`IRHC-0tEUf=XoQqr@O~n3_r1G!zYjj`k1QDj zHS?K?g-{pCnr7k#(dG5skU`PCZ|h`ppSghRM$LZO71f~=}m{F6Upef}_#=#XWU=gavgPZYZ_KBmc6H63= z0JfRLfpHyTatuBr4EXC)lER*ezsl&g+Hv`C;3x5}6_NS! zYR%2eC_vK;2oBtl$s3KQp#fKCK4JAL(aH2R2Yt?b663nOS`80mOT=yQpU2VsSEMnE z&SdrQY_4tm%#qE zRAgRERq(wghWQY(!(GSa2gHOZa|`cqWSq@N@8^ulb-J>ufNJI--V}cIt62Z=&_nz) zO;M;)A)S`eZ^J$K|FHMw;ZU~yAMgk%+q5C;R7hnjOR{80l3PUyA*PZgLXs%cTq0y0 zRJ6!cDwUYZ9vS&CU@jica#JH|&uIs$c z^Lu`m&-e4;2`QAbseV41{kGfip?b|$+Vp|8TqZ61Xs8pHnJOKEMVyO?3#dy-mKZ=> z(>|hmE@Q<2cgP$t)y2<<04qhTHf3CbZjy$dfdeQY^qk*N)V&N_jveD~ak=Uf6C)VkkiHK~Ti_`PWBEIFLmt8Dpq} z$YBCZ=O7adtZBqjst>}<=nkV8YT(a(cjmW)0t=od!rqXIVF`oDE^~D02WBE@aIedr zBzft>vTlPSfDt1C4!0n}d?-+Z*tYBgK0+Eq&DRxx8kmm&hcHNxM1$|+%b>Kfk?(n6 z0Y!*0fcp(9O=cMaMw}hs@7$mQP`zRKC*1Wdq$NWFzHZ5&kz~<5jWDokPws#*3#&Yk8KRbb!r?tjTskg)Z4w0JEvAot z9U0V;p|k2wVQHbuFf+%%sGb06j98;3zw4d+ovD=dh!I6eGuLk}(C_)+QF$`E@ll*x zkb{q8mEO(zkX<4j2hF2ZhXruxWuq=M6Apf3MI_9H5~B3*wIj2Fnh9+Z{P8 z#BBZ*cKoxeg8ArKW*Q0$F;cUORbtI#Pz zZ0jEYihrU)%y_eC< zkfKU}gZY3;C;3MPy9vbG%4}W{5<>%|Is38}(14`m0#up;ilmr@4ayMJ6XaAXUBJ39 zh091Y+w6f6_{v-~V#92JTnAgSAb}kQ4V!%cbL<1S14*!~(ad!KX!zJ4tYQC2SR0c( zW5zSY^WXdY?|ADAI2_$De(oHg7rg)m_$sqH$Io4-Md3oj=#y);K z=Il5W?G+o&MEwQT_9cCScV^j3v4#I5X*JjitnQaUZh}CkhL#@vq}XZsu2GTehQ}&r zcH0>h)u{Fgw}nTGydJ;ykGpWTp?fOrcYtjx#}RB>SAG=3y;{Rb!)fghCCL;vCJtW( zRJ(rA(!)r=Or2J48?sKLoq*ADM{XDRdFLTz{`9hF}=LtHJllMYsWQx1#C15vDl} z+K!i2(s!!iNPWPTd@g*4L$b@^*3qoytA($goNo;Vev z{;h6Ik8^?6LXyRSBzv(Fj)Soqz9!cb8tQ{a9rhSF-00Y(Aj4t0%VbB5Hnxpt_qSbD zashhi(=UjF&gBHEb$y3H*<46vPixR6MFH9R_%xxqXTBxgA=6tTDHEQvdv_uc@k5<` z^(>+3Ufk^G*9ErRxjuOI`qJ2Q!>>F2rVDGtFLS>ea?$BIvWjO`;ApLyB~54gedPvM z>w7nOU#&Ejp&}#?Lp#lJpU6}z&zrG2x|_Zfn8k-b`^}z_zU|DJmAm@WhGasey$-F7 zP13!09D5{;vs7zX%=y>r3BcAuU(w7APU zp53$g#>HL7kWQ750VeFe90)ag6Tw99k!B<$pXk%ZRiE?rNxEBquit@0)E}*Uehz36 zx$iwwmqiP=c67wVlMRj^KaVZ-{`&TGQYhcdPyq6?_9SDQ@0e}Md8Sdh-7eOrzUt^f z+}+WE{cWQaUP;{o7ko)n3uK$?=gl&xHI4;m+S-NjuU2yM+#0lbROyj7lwvdQ0G4;4 zG*%*9!40$4XY}Q6Vy&k@+WQI4L2WzPJ#vrgrJjFSQ3J2~z}#;0+gXA=&8PQHMYZQx zU{Gw;)R@^&&Png4XNb#JQ&>Vn^xaJ?@;g_pmXv|QA2ri|REyR8fPST%>A0S)nuTU@q>Jlw|Sgn*Y$=J(X%4 z)8(x!;Yzxnar^m#^}Y8_Z|wMNCuweddu?cy%gsO?y#x14}9E>>*0^wHF_|5~n6FeuCDIe|~?EL?Mvh zKHVj=otkVUFBZ%lZ*`s{+Bz}rnz2mM499#mNTHWgiR^VIWR?W5WL3=|tSp@-eG3CP zbnwTo9c;{U5TtqoR4Rw7LC+ju0j%c@I!IksY~!Ro8HM@q#DcLF4A1zP8-1?TX$6sc zh7=SUq@Z?GOU49=l8aOG6FpzJ;T-LdSJu;J*;pf;f&HsYQRB!FOb(rt0gcfdR&=yu zm8e+-+aJD^3Of_7a`6J!CjMLZNp4gTihJa!k0W)oG=6&P<$|k*_jgtm7q2~g7G&@! z^(@`#-W2xw!y*8XI^p_ok|jLUClSk$g4Jp`b<0;#q4ijL*QE}kXq{Ib#0v+Kdd#u% zz}M9P(N?if&Ub`(sxXNkCw`(X$I^cNXZOKF<9=g`jUfjFSLq$lFGoPy7%I~gPny}n zaGnA#>S8oSC!nBVLu&m1*bb_N*q{_sBHM#m;IxJ)I#7=hYm2=QkjYR_fH^A{J1lzu zjis(G@Mj}>2p@ri`y)u~zyi1Uu}CTU4zRc~Rx{vIs}h2tVF9rH1<*iPhoXl0S-}A< z7zPr>FyomH2CpoW0T$#w0yw(em6vDazOJB1vjxz;%R zxEcd2d1N_&9IrD`W=R+?&d!%$RJd!tNZ*VyqI@#1$q$LZxFmS+aNCp^Bn~CQLQLRy z@Xu2D;INbshMy4s?Dc-wXcXxc zokyQs2=!-f!hN*4u}>5hWEcwBUOv+>_V_#{+Gfw~*!n|#U?l^QK4CUKOOoKiFz%U| zn#ON!JT)3!XxTN>w^!Rojbna(@hc^n(zzRdu0&3oyYg?oEg z`sVX}sH>XC!F}}R`I8x2(PQc_e%nguxpDAgzL)UJSNn3gk}S2exTaf0+1ohFXMl{D zg6~*_U!!6klstH^@&GOE$iFfPeL6!`Y+qh1KYNv zf1>W{75xR@IxSj~>)>Nx{1Vw=lfsVK)MvBTW8XcOw3)eE;T=4EHpW2$+rDQq$?aAl zFtmGdm2>{ObxE7vwI|MN#XW2Xc2;DhbK_{8-y`HnfUONo#Fg6Ic)n3rt}HS&UuER# zQ~3bBo~M~+rh8QhEx_p0C)jxP)rWRsZ5oc=@pZB#$X-|_p>|zgXQs}rH-gOAf*)&? z(L48bC+*U@iGgQ%O#{brb+~Oe)_F&JGp=4?PqIL9tEYd_Rvc|!v=vGOuGAdK-LGR4 zha`rmGYGa#Bk3Hmy|m1BWiZXWq;-P1GiKEC2&H`ZKE}eccDe-WvB|1?cAZRRdug;J;o@>VN^gQGk2p zx0Esxb43R<0z#REf3Ga|Z-f)*04Leqi4@++aoH#&k}Of&F}I2GtWT zt4Bg3?|Bh(3%~b8Q&1$COLrp+qFU4k;C2jO7_~V$?@|*Wm-S$y_4sp?&arf_yn|J; zwmUwhw1D?FVA6*k8(;#avtm_U>0+99k3C79d}HvgfoEzb=YX+zVuYei>jbu~PdoXH zP-EQDEVKv70qUIW!HhenJiormQ;@Mx*JXSqxod5<`zxL5*Jr!1ZTdPv*@m!eXQ*fP zEZ8nwq!b#{S(?2MRa&h-dp6bN{pbo9G^lIe5~HF;YrHPRqQdQH>S1LRb_0!j2;@cE z#r$8W*(tNtXFB=rB{$C`m{k~`e~L*uc$}YNJhb%!=L=aqbh4{F&5AAz*4#6Oe=m3Z z9V&qy`$gms?bgTmY@5z4m5whadv{$h=i};UJ4wLn z2t8sQq#hwrndQ6G7SLe|9+yTl4YED~eO%MGc^yT&E#3zOY6`UgSbr5yIn*vf|HQsW zWF4gr9cc9FHZRfLe%e!UBA6@3(k3hIb!3Cui2LAO+j!mM`6pLUJoXtEF?PMeb$mx4 z6tYZ|gJI6rOO3CqB@#U4LZ0WwnXfu6mJr_^{iUmnqe#K@DqQ&jylPcg3>01MfRoaa z2aH)VG^L25HUT=J$pzu8dX}vCcSNk)LdKKpU9dU2=Q%`qMxIf01Jv~zKw}J8HUWT3 z;8p^Tk=se&R2?PF3zJf{w($`@IVW0JPjtU2bB2y4ARP_^tuGVh^h&OI`kVgCFg|IMF& zIEFv^%2GgISy3yra7=z~ZfQ#|@QfBE4gL^=(7^NPe_E!ts)sVk(YjpNy$n0$n2&Q=yQ)_&P~9s z#>c0qQT)#CBH~*uwFi?rPPBF9tLYfZXG>g?ahN8Ocd#z7)WO|XvWLle2d}Ky^>&J<_GX1YAp3V1{)63Tc}2JgwG_4uJf=TZ zi+hCZ{Pug+`Y(QlPW-c8T};3a9o8KLpu@`N3sAIMNKoN=s9K)t$-aJRlD#!LVDHcS z_=>{w>{Rf2718t*hQ1Fq;b^NYb;e5Ma_gCiZNfcw(jHuH5?5KVah)!;663mXgrtI| zq|NzSqTkN4n{U^9s<_!WwMK>Sta-CXIyS~j4`qg{CS^ns$B5K$)QuLKX-4!$We-o` zZzf*n$s;oBf;p@9NT)-*3-%;cG$rn8^y8Os<<+wbQJ->-rN4do;+);nQ%?-{aEI-w z*0CCS2+#+ue@7qO0jt_;^ge)*P|euT;Rj~9aSh5hZ}0n@hj;z$a@37JPEO8j`2|;E z395}iS7kqi*TNm|E6YhYaY(qD`mmKwPVi?xKCL^Meb5Hgt?wn1d|_4}kZ*VYiZYg8 z4n82Ib-*BYe~p^*UsN+EDoO+R)#*n_P-R()K1ij7!mAQM?8sM%p{t_UTXKl)zg{2I z<%d)-bHEWB=ly#vKd~C>8UkH-@;_5^F3$n`k;hk4pJSrY+*k}XUcmU$6|CtQdc@^M z(H3Euoemq%DyZzlogN+6Lq(CO2kOH}0tEup&R#`)RDtzV25zv=GBa*l79Z2!=n_*? z{Em~4?8Q}8edEa|3Tfr@kJuV*gHDTD3@U}t{qx|1X8b+z}?$}J@i>-z^z~U`g3(If!=F;=9sd1SRFAYaw~rG;i(YS#l?<172%vuk!kf42&2`yU;iyqn% zW01wTzU_HqUBuA@B}Fcsu^C;EopsontGa+CH330+S|0QqEELF zMaF9#Wu#Wsulz8*#Ll@rq7qnZZOiRNXh))YJsCFHR_Jz-?71W>`th*AYe|N;oF+4O zcXxUvN?a8~u;G7Hm;V0$4{Y7PfK3C?w51iJ|5Y)s#ngfPyap8txrD+4fV-r1rmZ0! zY(qEd&0AzRd@B{S&Q6^am9btUnRBsBBU#DlLFPj1tbQK^5>F-4ln+q(gQKLkmzb$t z2RqH!_qQhScP9+=(}h@IZ5MATP*W#G7b?@9Eo7CwKX9}0b+n04(n62!MX$N=1AQ_x zazS4>jYgDN8GP(%sFsh8d_hwtB5wIRm$L;ekEL}EzsFfQ~N!Vk|{>y2pw zQ`QNyc38M&gnRH#uUo>ynEp>n?(UHn_tY7jK6Gu@jg8fHXw2w*H}LqR7%$xmFf^;8 zi7{KVw@vCBTe)5cU##W7AwgMBzg;i7M}mfF6k2)OQ2pLflRVy6D#jFq_WgNc>32j{ zYY}E6%P9zmIO&*(gy|fkp6oizPSkBYcRsr&!P-NAs)pVvdJ)Ml9H1M&WN!bMu!{w8 zZ&^SsLx3CpM#4V-DS=%FG4}(eew$yEP2cNgOnSk;|2Obo^Sog9)n&@{4QrUuQ5M~qZ7w+4| zrqV^GQ<)6FbQ%JfPVcsk&*Cw+FuY@v>R{V8V`d@;Lzt{BtThE{Tzh2MhoCbo+ zL%u%lOi{O;ou6n%KB?!tKD{X4e{<{fid@tcgD2TYQM0SpF}2F2L4C@>gOz=f^`)t% z!hWuXK&GfQvwiZCv83IZ7}W!fk7GpxDY^0=j&Ph+XK(+N4(cedej$s^4gX+x(nGDn zf=L)y=_0i%>l#o=lYQ>eatzzga60?;Fva_U^Np7^>%0d8d7MTu7=}5_w~MJTYR1uf zSgs5wnRlGO*LIK6X$4bx@nUUlhjbe?Vm=GQ+>Wc2(&_pt&#>#M86kK$!=G}(`&Q2x z5!NZ>8$BQMx2l!e48GGGE_*-Nl0kKB3EN8dbyKut4t29+o$sHukPO_>H&Tx9k&*z1 z-G$2vWlL;X#vv8v&C0Jm(=N|`n<|pl?1}{GM9M*ke%bIy}qUHZk}|F2(<_7 zVHuP&DJNl@SBUsYRzdXDi}Ja|!69H60=Ka2BYf%TvTMAff_l^A6OTT6hiD6*)G$qm zZB5do41hxp;5rJ|nSB+d2?uE&u)ck+<3ktErz%k?_e{P>FBlX|cY)MlTm!lRgtt1*Acy$X z8D4i+S&N2O9slf*(p@jQFkd<*vVRJSfY;0Q6J9v04nG=&y^6ElK{y5;R@T`NT)PMg#7Dyo%dn?6p+mZuE6iT)4qu1R1 zGF`D@WWoqFjvBE<_so-L_XE%s2=*u45xDP&ql>vE*5twyX#OHJm;H7TL9oEAt97K} zTDBe8{rY}qRcZ9al3I=-b}ZY4b3ynCDG9~EDFd`v9H_zfn!*P_aXbQK6=dQuj6JtN z(itd6Nj)Jwd_ZIBSSjFXGfQKbyuyB#p<@WSJm(b0S}Y3UbL(qF`G!BRC$ zFOafMhA8)e(T)St|Jsk~?+39q5G}#X!QFdwNXd~ILM}zhkd!}NPCY=O3gXNwaG4}( z8=7j_?aN+&pb>i@L#_^2P074Nj}|m&tJ>Wk?bRx=5CEBG{+L{R1ei~j2=Z#ebqU7Jrnu2Lmtm3sYWTyOppILN6g(11GAr)w}Z{d!>7G6FBl~9^IyFx6$OserLmakp3RJu=hn_nlkY~F z>>SXL_&|_06#9IE>B2cXY0u6lfevB$-H&N*`gf&;&<=S_ua7Y)E54Y--YS+@TSZxK zuiT4f>>!VsJ_fbHGqm8Zc{035Y9x2eDHX+7i1ltWOPcS_jE=%=b-_>#CJ~+W;(iZ$ zlTx5nhqT&98ah;Ub*D1px)6Erx~I%Jq8oLHwKju(xUplky8q<5(OU79u^yg1;hK98 z>56GjPUZEVoPtugifUw@>At*`rH!=SUJ;i;y|g)I#JoY^A22&9Gb

<*v^ zus)QoNOfv`%W!sl@mv18g~V2^ka75>w|r$0<@5oqSsB6$7K)nDgD{r6uy#BUztZlIhumke{Yf9`U2hNMHz{dOC9-=TJwMW@bNX%0 zf+0hMsmIz5x;=!D(eV0F<5$*_FTFUCN)sQ?sGQEb!D`GMlC&u5iX1jrgWUGVRh%DU zDfNF7OIZo_tpl>lA-XP{^reDaOW(q>-xsa*);XZ{Rfd{VfrgtfP2*sespn@de`?m@b?vE-+e=zU< z`NW0Z?3a-9VG;UMhN%~ofN7pk(>967cgyfTA0~4>?d}^pVuj$9KWJekEuL%(dRVCh zz2*t^8WaEg2XDPEhDijcpV(#hvSQ_)zn#2VU@!rD= zUw&Vvb(y+=Bi7*q&{;-f*J!qV19PSIAyTX13fgQAJ*Wn|p!hIV)%L9|VdjPPQ-%J- zTXT8ULovw4tAFsS{)15Re~$OR@=pHX7$i_Z`B9lRU+j(tV7bNa__<}>@qbP(yresx z2*{R_p_MpU>=pKXWgLpN4J4Oy)aSFb`GT{Vh99!e1h?g1Nl-CFA!fR7wOMt~(wxVq zY15TZhV$6GH_f2ubBFF*&zIJ=+#hb%azyvs!|Zq+x_8y)P~9%Gl_qed;v|DA_=BI7h1?JIeBH~TDLt_I*c9q1s}ZA4rdk??qmnz(2PEMJUi)ip&ZsA z{G+6k-9CC_@C{RbS;a0cZOMM^Hy5r{W?dubp}sf#tq@6*rfag_!)UB4`^~Ip;8)H3 zrU7a*fnFaUWBpncdaQXOW0OoGM|~QuvvrXwmkZ>#0ab1gJZQ=mYi6T=Ap_?4zr#2D zbKm(N{Vc=+o})f=4#hZ!>Cm=qo$)7T`k9vAYn<7eva8zd{CMGjS26R@vwq7WPlbxK zIc^X-K*gquOr#7mLPpfY^~ZKsq!buogTo8Brm!tp+jPknKdhSSJuUVwrdp=)%`j;H zeuN}XGNk3krN1LKliMT8S~1g2CmJ*eDV^>K=o{aX;rm(OvR2ekYHF|1O&G~Cq7Ic! zt~MEP{VM(;d`coth?Oy2#`qSuwco4zL^w>=6~Z2*4pksSr6d>+d$wb{<;voZ+8f02 z$=%iUBS=bVbVX~c#4cgF{ue(#E!R2{C^Ee>rHg*OC-TWZmIkJhi7#K0NLGSmsvO$%Ai7*U1-JcUM^)?Yb4t0yWCOuitVUipl&l zrepcBuAY-hD<9u2{CIXP*KUTO<2r;dtMSGToaLE~$~~Uph3YH|8Zjsn6`CYXQygdw zW~r{Q9a%VWt0p7ZTt*}AuGK9W!meVRS0YfTpRr~h1mU4QEWlD{T7~MavVlrUTl=~Q z=ertQsJVT*o(naRQjLg6EfWj?^<;orwli&rew&?LxWNu5UV!Cgi|CV9X-BHz0&mYkk`y~)L z(+46n>;@|ui1r22m?xn|dqCUCKLntCAn_T}!cIpI6a5~5#z=3_-h7;coqiNkj$-Z3 z8TSH3IDf!Bs!9L`AQ=|7rRRd6`2(OH6|Mh{C^zGw=NUp9AZiCWtCB4=379+UAb1C! z;SMCi;qW;?-06(|ju6sAbqolxKz6Fs4cCgAu_^%3OAx5pGhU&4vPf?D1^KUA!3ZUS znwkXyvO#CjUn)g0cd+b9LB973H?%k$dwulIv&^`iMbY}lIzyXsY*^Odg{hzGbT3qJ zt$Ts3Y&>Ove|VX%!YlED;K1+|>7+oCvW`B}7T1)d&@jnT?ez?UH#Mt0JSLL!J}Z|L zf6ysJKdZ4ZjQil8O|7FKlm2;IXesgLEv^+t&F&krb&1C z*axqz??ho5aD(T4{1pKk|AdD$)&U=L zm6XwrYbT6dJ)TP6b5=54`AuUwl54;01wTjoEEWwb&5)$=^bsqG8(Ff9!>LBet_kN- zV{V-^;nJ{dY4UWlw%=KnI=)emH{PNfU!W7zOD-ZuU{+xFVRzA-OGT0-3S|p2F5C;e z@WEzvL#DYh^JoS~!?gZA5aVY;k7#?SoV;9h z=i|^F!x39hayv|pAPeGF;hHY%>M&AXKM#6r_>A0{Ra34|WTJwI$ll=FkDuskDdbHL zEtOhJ!$f6SG!%$F`8v9AEBT<}CFVJ$lx>L_t9mWZ3l41r!jAxMRa!$j4X7}72!pw% z{Dm|x29i|1V_-@e?gJYM_X3Dkq^+jj0$_bR(-1g_@-z&*3iJZkqNufmObmM+NeJYJ zqM6?jkqtbv!0aNG*r8wV zu^eJZ7Xl;)pw`}vYlLQgV_AWDt33&;>Y?qHk(3=137*R=2P`k#v&A+8 z*6U^WVzm5zIoRg^pNt+0O;wJ@E~>inRz71)7K@C&a|0?4O@{U|P7EtWd_iH;y?4J`Cc#noneZhS+v!n#hadcn`b z$1Kc?95qFG_FipPjwWOpl_D1jfZj*#KSB3f2-@%tJ%Gq zC@L+zNvi#5Ex*-To%K(uH-uZ<3E1)Rz^YADeM!SBP%$A2g@kn(Kn>WcRDMKVU?tu7 zakTZR_;yy6#M6MMuhL2dlY`n?Uzza$WeBP>cDFn`{NHNeCoKC=XBhI^k!1TfU2p0LIV6tx}$0Co5MoIy;|f?>ISz=b2GkPK%c zz{?%hQ3c1jDZ}c(vbF-e+}5N;yqtOh(2^ixo&@M@8KqmZ?@p$;rIK&xB6IK2^7um3>-TzwbfmxOl$H+{b6GhqC z7`_5UPhMwNqbDbxL?$M3tm2dzEW|QE>4tj~J5G1kXm+7+LxKIJD5nL(@Qzli?6{h` z*ec#IRZfn}GTfM*s_O#;t~KjP=`ZU@Tl6QdE}|IsHvtsmR!~mqvls=$7PmTB+zfFC zt!`_q7ImPY`XNy&)+thD=Y{>&qC7%i1Lg(dm+aN=m0`X#5~KSn_Xdv2HmnYHWdez* zTB_xk@*oO6e0$eKXQHz!f1jMtTeqI>6%{`3-!&=zXlLx%e>yfOa3$zwJpcbLvb!h* zgaZ078-f5-y$7(M#)FO4we@eJG0Q&G;}+`2Y}D0ZZDy?QneN3z*l)|A&L*$iB~$;t zmqN4%?A9=mzk0(wI%NfkcQJxfAVy>XpoE_r-L(;00KI2$x4LU0Z&^ne0$u@kKRjHnp8^_eIA?V9CvW-J$#40x+LH`Vx*_0Ek69PX?nieFL9U<;tUk)J>%G#)r%Yy4w#O;A&|?-q(upt@8az2Vn;oQzd;ii zKWGBm^;ZDLuT2UeHtyz5ObX zqi(h6c){GBVs@dsB*^amFNczwRC^`P*8 z#?=WE1aaF&MxMLHxcasVZ166>_jM1feq~%nI8J!v~EgDR|)C-nWo;2iph7+%SGl@VjFmdKP=H{ z{#otq&)d_!iHsXjfgY|XrhHh^3a&qO8exWJtW$BUNth&jAvFHpd~QW5hB1LgxHA#(0~TS z!UO&;H##s5j}s06KcE(bp3(rmun4U9@1g%IBd`UXi5lC>JPi0pkY^cpXxioP&)S^I z3#r?d=41QP!$WCn-Ip-24P3#NmuvWUyd4ejHw{5ikDxm``*=#(e7(-s3+$5ZU2vPT zZBw1y&y1g3S#{QDO0Zq}!qHYa{-@4>r6(Pbsqn#)>WhGe8USb*1ZE?47{u>~ab1`h zm0>bNGauv!t}a@t{~q}t93?Q$i?-TA7uagyzuD^V2<)%M8rluqb?QOWPL1R3?3CAj zxXF)o_1YGBw>-w2#nYlyuiuC7V!4yQGW?6aN;ar(dNO&a>HPR}Uf*qT8!g0na;x^J zakTv0DQGpw1ISeXXAnqYzG(rD{NpSfp|uXmQmz1*2}I%gF026~@9{~$h|-4cPTLR9 zveG_ug;=-U?ELsXZX5OKMc=cRR|e>UzRh3or$i9?*VAnn&D6@i$r69)CbVis9xuqa z>X!fDQf6LOPG*b=Cx`SE%XcPH8iEy{;hbjsLnV}X6mvoGA8}0&c*gZ zEebjoPOamzvxUrFZ`~xMO%q>x(mFhpPWgaY5?OrqaNd06 zy>IdZ`2URNBnJaTtlZk{Ae=Pz1WgUfv?}(@bh|&ERUq6E5+!=nYrvLs!1U~a&!0$@ zQYJ~fXIWX*a2bJW!I$y{U!vSWXRCGAoVRUfg!n~rnTS+1xc;_s?G(Dz{blcqtL z^93&zJs)|!JhwAr=el;y%}!a0)7E)`RC~a#65vrYg|2$YZV3N`txZ}SKtNcG%^S2 z9Ox$R$%#C(-cPwwqeIPoD=`JHA|J-7l4{qUx3=bbbF#}*hp5kMcG;j7OZ>6~P~unh z1x)89Amhsp0Jguja6O1|#L(=GGaRgj#iYC`u|u2g%Cvk-941g>8Ar(8kcUgpgu6qe z_xpj}VeW5sdQKd%Nl^*8G%GFFUx}c6w5)Kte{6-d+fFl*J)SsPDg?EZdXs= zj*t3?K;%d{sm1z3p8ouQhO9;H6y>YwFZ0k^X1MC`!`jNd0D(?WOH_@ z9C%xGV&d|J-7!c|4qVDt3G&KInJS5m3)?t>0QXY9O5oEhVbmlU4L?HGA(!!LqLwmh zvQbY0(0kB5Ao%;4R}-{5ru@pN={5thDl8ZF3pC|E=w2=*w)gd-@L0*7noa@F(hl`* zHYgc&_#V4)zneee_MLXoBRlmaS62wGqGA}*!Tm%^eb{9#q>`k;k=v-=ZiQnn>vO7r z7VbEHobRz|FFpXXj-~1*Ub~yv^U$ewosOqd?z{M%sYx2^+C(QY*CFbGZr4>hCN!hc zv@NTN?rN#e3?2@+pUpiWoz3x3e@^=A_2jug(gSo`6;Qeaj-8%*x8>--gK_&Uc_kO} z`>$+>lt?^Xr&)i*Z9$(@nAxP@p)NnaUpPCq;Byt(M>2=^*uo|G+5{%jB6c9d zq1FAvSFsZx;*V~UnffL_NMgu|{HQKL9%<-OlrVPJm7bcuWYeJj_{Yz`$^rvG^YlGsCYU_*;NO#> zwyGfHIFIku7<)z)o36aS_np08ghk$Axr4?#&zMyA#p$c1Hpj_CkvS)KTI>;sWmr%n z=;zs>G)j4N)qSphd-r3Nv^|bc0 zsW(9~2UAv(o-X{@wui8Gz4b^xWJyO(6xP=7Juc$0O|8HG#gBEHbFTAM4VbA)Y?I?% zy%({yfJlMHm8k~wn2cs(As^!&)==;U&FHFOT66NgrjT6&9*@Iro{9>56fNLQ&h}*8 zcPPlSziffl&O5xCi({Ub~C%>w;ofLvH0gclfI%GNp( z<)%J9P}=``H+}eQ{DC4l-f-h0x$rh35BnuBjctX>P#d8a$=0aX?=i%gh5{#9 zqO{}oGFq!|*xv~3?S9Mz;nKD{x#3$3uDF96lkXm#p9Xg#2JXbPPnqr20L{pMgM+p$ zqKsB(J`evYl6x=EXkEP1-W4yZ)u%&?Jv-*;m)MV41}_*^l*!2TWm@TXJUk{g3H!!B zJ0IUz8>@rB`@|4DTCf2*I$()T)?dtfSla}S2SJhGe|w|PF~7p3W)ia z$x{EGrXTtqq1~Z`lp}KUlE6jJ_I=kbD{hs4(RxhmNU~&{c;LOm-=-?_vM+6tlK+Nh zSOUgJ<#`s303%On&8z*7p`XuhS; zT3rQ(_K8-dG0)dxy)b>@bFY`XyB0}tDB)B@qTrD|lEIMG@nx8d-^(EK0PcrB)7Fz; z0Ex{&FqGBdSl~71D#J+%sq=x0B4Q(j0$^Jm0wQ92iuF*P1AmYJ_&FKyM?e30q5#C~ zzv?Rfj#36_<-G#>09h1^pm< znrwaO`qBh*JLyinTlWrCtx4sPMmXr0@l1JAfd&tr>)M*G|Mn&kLQwSK>vDKw+RB{B2OuCVTTwIMu2-RUvzo&7(M_J zU;q&3MXS(^_XvjV(uV-5wf`NO|J2Z+-d$Zt^MQnJ*ngzD!1ymh{-S-`l z{vHiKba&w#A~R&`%h0Q^a_k^=^35cjFY6CAWGZ?e&&_+t+)?SV;sxO4-S;Hg`sUGU zq4~+DBN$nj0X}>tvO?%)Eo*ZO5y{O zL>h6cPgcJ2^EJzk2HW*fy=isL6f^s8=3lfO#E}bcxctPu^rYG;rks8NeHt`_=kP-v z!T;8fseLIMh&o|kie0@`qbiZHOZHxvlT4gjCA=GMA!O8#jjIV`M6izT$*v;CgNA3r z)VupO?-4weZ7g$KZj&k^g@-4|4m~bFGsCW7NcWkB_jY>>Qd=JE{jw&n#KbG=jr7_o z%R9FvyrUCar$Lg2Yo2|-Pzr8_H`5ZvxtYh^g6zW7*LzAn?4Hl|XU;_)R%_)?kDK>W zh`rgkal1^=RC)`7{fuGC#KF}BF&$%f@Vs1pQ|r_1-Hb)VmOWm%1GBj$-FGvQ9kWH9gWMa<_a92<>C$)JH+kR+$Z8R7I?vE*l0|%?+WBf zZi7rexCP% zl66?*=D6C+c)lkw^&Zr!(L4hqFA38S(Ng!~Vnn~h)$E+tBlZ%FM3zB%w`}}_XPq@Z zK3Gjs`DUnCt-w4DWHYme!Xan0Xia}*FHe5MAkRQlmgKIlCdLQ+ z4+dJ`TWe2RjgVMEP#=o5<2$0=$eh}I1GY@1tr%1k+ms@xc*H6HJanln$sB40rBFNU zdiL|em2hLw3C|#@ZFms)K+z#=~$0t=}4ArVNV zJOa?EcLh){5@b`}4V8h$_G8xWvr*ay@3}jNvd^yT61whW-c~;7$&=mH+N;c3mqIrN zPNDvYy&9-QgxACC9jjK%iwzQx9SR=q8Xjw1Ei0irow}zw#P2?pS%E;Gf~CQuZ2OLIoD>?>vI^5# zea@F6U~(uwaDbHY>|9#9u$-U;CD(b?!HojqTeV`k<%dxnmGn4?P!>{hhiEbmi>@|jJ|)?o|h&$ zt+3P8Ow)bKpwp%uq`QaWUzhdxW~981C?bM##Cgy5APS*Ud4=m&b;iA(=~r&L$VPQz zwVE66^Ky2+34SJeLdGzL$8gi0)2(kc5iB=g*v#Q((rS365lvS)8s6aJEe+Py#S0AESD zy(euPR`#oFH7Iy*sv^JhUAJAql_DskQ|8?bxlOywo(9$P zNNH0%8QYb|9Q)WFWqZyO)4Pn;d~sJA+j7|#k-U1{9%;X-6CZDV&~vodfvaL?0DmR{ zwL+ng&o83#JA&iVxdP>0Q{MKP7Y|$}(v`+1)5gsT8dXkKOUoqL#2?tFQp$Z~ECAV2 z?&u?UL+Esj(WW4bP6ynA%BUrmOJXQxIJGQ%g&3FbI&ypM2I{kd%dztr8C}(4oxN)F z59SgP*934_s8~W*#}D7DT!LE1+DbDl{aQ_(@;5)NlWuSN#&yfF7;L-jsW_1i?~?bA z5Zj0!ksTn96;-GL6&vdKQwcpYA&QuC+?L#TJfC(|Y}zLCa6rB~d{69CueyEI)c`y) zY+Hh&ngR2Htc2E{$BFn^RqmTR*{l@*v0hjs!qe;Gc$P!-^D0^eb+X`y8#yc8chhNXB_ zjlC|}{6mcteyS4GYk@w#x%IGclWp1@^V_OBL>6V@CqjtVbm zS#Wx+hQJ#OcZEoe9v2-qz{HkClZv=dF5KHZ$L+(c-jb-}p`sG~ zIOKrQ77|(q1?I*m`#N?nWe;|9T6-HK^r*{}nydq4EAm^^YP{EVvCTEE7-f>Yay zToLg7rEbY$sTw|srlg3aMFaPko4d)U_K1N{(nFrm3FixZLg6BUL!2@c&LUCb$;Hgd zyF#T_xVeYeo@xk)SuU36$kD&P2veVqe()%}rWE-$=Ho$lgZd!gs66-?!`N04wBePc zgx`6U*ckoWXV>^!jZ`l725>!L6JQl~Y8gt<92Uw68|Y%rFoH&IX%5Y0TU#8O`6M8v zb;m&uRg)44^v94lahvjJnCRKmTIfW9X8gp0Z<5oDcw*d!%SVeO5ZnL_&^j&z+H4yD zq+M;=&TqQ3*1xC!{MMZ39`Bp(uD|}FB=km7jml%W=SJ#NiU{2OVGPxh&;eE|;BRJY zUE{Yo7f-V(B^~MddUD&^i`cp&i0U-VQa@b0m`pXS5B(}fjSlU#E|mQiW1M=T(=OA^ zfn$|(*x4PcaeCe1z_D8rI_RPJpi&B87MP!W1lBNE$JrndKwf79n>tho#yqed?!;UI z7}#F|9{iHvp@FA_Ol6=tk+At$QJ(My6Iz>xN9RK-EfVd?VFOTQzM z7C>7vn(2=E^`cpGrprQ|z5N3oNt7*Qrzw;Q z38}0@vL&RlL`;Q*BuU7av4qM_DAFVqC5G(l*w+f#*RdwcWX3WLv-E!ToagzSmgk(O z^PJ~-f4}!Vf1S>_=f3Z`@9%P5pU>y|Tqstmxs!C8DNM?{M;z|KLd~Y~rz~%n-La z`YK35UE_SwPCuCAZ|N%jmB0OEI{f&@N{R(kFm63k2qD&0W|~1&ZtpmP612%mNv~n$ zVzl0am)GSVioAZc`Q8cdN{ODAzb$+FA^7oKc_cQipKgz^2TMT|Ei+&C*4AxABmd{* z8v^so!YxVO{3nVcG({FlWeqL`m=;3;bZU6&OT9fuBCQDf!M+_!a0diP;}WXo8TPX5 zs^B$OstAY*7SsU9`nSsbzmELtDgFOtgxp|Wt42GD2GwYc)}R_q4}-(q#4Q}90wn#c zwaRCRek$gDGXV8AZlqk8NrL4qM{*9WobH=)DwKB_3LE#xslTEC@8I;y^TBa?3D=Zt zc(TQMI?*LLugYkWTWeG5*-wJZ8~l`;f1@1shk~>-=^vy=;I>( zaNB1{aV)4J&qo<|k&7wf$V#*boe9~~kEzEA8mG37mUed^_PsNszgG@1*UlqcBT%or z82vCAl$b~S2AcmZy8q+<&~o~p#ERt8(lizR6`K?cTl{6Wo~Hh6Hd^;-xf=5h z7(vIlU$?oa2X#S~#~P9AomlT*jtyfV4cND)uk$dOmh!|@Nwt(_9{%Vnd<$f{Ob8f$CuTXvxde?18 zU-SbbrSfsv;;3wGD?8PW>^O?Oqs|@s`dyiUJ@24Kl%@PQ_Yv|q!bs?#=zTqm*3bJl zV81b8%UGgUT%$cl@>7M$RfLD*RZvlnd;$=T@4bwFdQfLoR5kRLMV`~n2U}X)7ox5z zL|Q^tWWNRTu*S1f#(mDC2diZRp&Rj3L4uZ*>t?T0)`NrE*Ic(@H=MTcezCLA^f05W z(W42T=_PC6e*Eqi-Aw8T=NyIt1u4LaFVT3EIUHM{>g+~)v}mz2-{HoK!b=Fi&Uhuf z*%UU&jgy7M;+AaIOqzhk=pXh1-N3`{>usCkW(S)dXAl`tP88i-4-e7s2u42%PcdeQb!vW z2cFF0@)M}svExkj_kA2_DWitxFXdg$C32>T|*KX-*z1KOqa*G%RT`k9` zke?G$E1^PEu5_TYz5hfz^P**pzl6N~X=`tNagQ2Zv8|?^0cuwaA>L_oU6hlEowVl& zUg(_)kr_wQ2QFA%-u+~4QnFNJskHA=o~W4gUb2dj)*c!Z-eQd3NRgld8a&`S!X$@0 zN#Hl>R7A1#7RQU4o?G17eBf-RV}`_0aXGzBY4=RUqaHvagD^d)UeK<4`-p6R!qJfG zVL2b!ja2bLD7PbhJ+gt` zHLMUi3iP!DMDaTLL*XOsdq;xqQ?vKV^31g%`b1V*GFO?dveY$<}_)O8RcH>?kw@&Y6GF!;F|TFzd;_{?4#b zGPkS$&f`}x`?Fp$Od_7KZP51Br))y(DzosAL zBi}4{_IZoVn)U1aD$;tEOtcw)xWWEcR1P%m>B#%h>q>_&@yaVEJyq0UofaR14cFpX)6@Q|yUpF>_n z2h#}UuW~zIPIcr9h?nG-gg;IcJf(V7XWUsl!%jY1VzUklrmnT$md-LeA}f4mn4C-U z>WZx@c{w?$tTy6!?YU@it^6+efptFTPl+&{+QLB5w)M4t-dB}AUC1@me+CqqwH3uE zaRqBh?cVM$62QUkKCDE<1k**l4q6iD!}iuigg&w~^zoDv>|@(=I-6O#LBLcbXB&qh zN)pXRSEd7=)JQhep;BEcc!vEz_mLO?C?vdE>GvJV%Zyb^m68`f#BA4wE;~?kN#1-G zY%jdbV?&X*QD?0c>MGca+#geQ%iGy5)!3FN*Se&OWw$o>1$oMzXW7p{xSmV#r4OO= zakUiO24r)A!bQuA=3dhK$uc&RE~iEeeNfjo9jT3y$8fJZ9NrM)4BrYy%yV2$+;YMl zbww@jOfMUKQ+w;PL0cCPgnQpVtVuC;6Br^0GxrHestZYS7^u*D4pR$|_xN!ql`3Tb4kjW7@z zwECB@bRnFHeDj8Pno$x%P0W4q9s^A2%x~`ZpUb^&dgW^PLs%MSU^h3cPjYO@E!`Bp zX{Hjg55kA>V?wU+4P2^RqlN}%fS)Ux|xc{I= zLRqAD$8o)Ztq zjI5{?T7Z(J(n3GxV+X{>k94-QrjgzK_44$k+4Qmr15`Kv7(B2*2p z79^?%IjAiUHbg~Pr9}7MNPaZpscOC;N4ZY#8Rny9Q%y%vd>5shL;RSfc6n#3B3S|0l^ zbFef8#rc0?e&?so-UGWFIe;NGkuh!~*uMLhmslC!8#q35Jk0VeqD%PVLlXUx3v3@7 zqH%ms!*nRX386)MiW+%@$~6zB8UhtqT1S>gIdene&a@{rd!L-WmpW`&w$&pm)w$g6 zVoQdq(fk7Orc4P|6E+fN!_dBD+}C15T)OhStIRmdkS+J@Ws2fWw#cl98IeEwHiq!Sii?NPRH+ z*-;ac!~}ZL2iPj_vJWSfH<0v*JWqj<5#c>=#3%;_E3NWB7WlnmKuLYWL2DTzvu?0wOR8D~yvF$kXJZBZn$CPMZ zZM>e0wobwe#pk3<7bcv*pIVeGDu~b*cJfP=t{s6epSwc^K}SF?{T*Bw=FkEQnkKl{ zE|8t++#3~R5p6Z=GY|GLp<^~CDs2i{LArif*pfSrnNE8rd7`KInz~X394M9#syAyTn20+i}Bu6c`hJb7D-)q8LGmw zfxg_j7e_+(HwLyQK^|$L!sA; z4$NJ-e985Z@CB(M(V?rGpfVNbXj}tc=wm%pUT9dEj8{SlGBqufB6W{$7cY2z@qDy< zr+kz)N*Y?|ik~0l91yNn7i^bSWjq+X3}?p~xQWj7+`eX5M%)t5%gVi3wEKYMp%vz> z=n^Qwcy`EaEzTsc^}m+aypm{?%Kg&s`93XAcM&#*Dd!{U%{Mk{ih#C3oE5@g(3-)V zV60y5b=~UQdf;^xVm;yofjaRs-JM#`8c$QFYp>J?>+UhN4fi+1DXinSr8!fiXHbMw zdg|(AW;1f-!(FjeTg6J>K9NlcHn7Y0>bsq|S;6tGTwJ(+C8H`deoctSN);vYkWz}f z%B<~V2dYX`ODfR9BejJYZtJ<4n>E>NxpO5`gD$_2TFv+_$Ng_=`_v&|%4`QyK$;9v z#B)i;w4I6+UpR>QK{4$PG^hifQ*Z>+q8{t|FXw!WBb?+tFzUUu+Ez*cfTG5lfm

?2P6 z!XHfXevP60xqZ4BCpdq%qc=!nNz^rSY;q1rgdAZ@=gol*@tbD~n&+f{qY?T&4&w)C z+IPCn$lp1Qia?>*#)de8)^&{ zD7LkhNsbs7=l69CQC9GQ)}31&^s2a0w^}#|2uf^%P)lf0D0^29WH*dpq^4VC{?Iz@ z*3vxJOu`MmN8?G(RqWWtiFJtkej9g9g&kswcJ>G3?=r~4#A%7h-EevnbZglgNuAP8 zyS#kLxdxJY7To#BYa^!e7Ek8$jard$LmA6JNRdW3Ip$Ca?<=Kmjtmu&Ys+u)NE)cR z*3z$%U)OWy4{$M}-oCe}L6UnL{25(({R!FvdAmoUTiUk|Ol zx^ahHWNYBF)4^o=nMp1^=1w%h1TVB`L%U7i-P@Zh9SsOR?s7z_O7C6?=;7#W^{%xt zm-VZ(Ksqb32ggCx$9Of=-KdBg1LB^5egicnd;qJ`Bi(=tJX+%R6Fhs}BV(Vr!;7Ig z!y&{TQ(EE}*_U#L?0c>|Q@cKlI5?>3wHhCF?ns0|e1QIzQmHbT_`#}!h0^-e$30lk zbh@B$E}CdI-+;~ue%1dz`Wn=!!OfYec_6;Ape(=q98=>lYF6oO@hcVs>-^L~$RDzk z-ilgM^-85vz|fnpr11f~@B+AFdJB{-b`<6Jao%;#aN^UuK_`w#)!4VZ7HAo9jTDnp zkaA`vi9~8(2@xdoNn_#JgidA9dilyR)#7>aNP~_CvWJJAOOI9*18dwdtS#sSfm@Hz zCA$%{svokFi^n#A=J87d-WyWW%WDe__rEff@bm8Q2T}z2dO<%4*<;67$C_T;u(3BK$`euE0WVA9^?NQe3HK#-Q8^P1Xi7g5;(I zz2fY5ZBA>3Dfe%`yx*q6`>fXd zM&Cv%9=33`CyX-;1}Wc?Bs5~$vE_-TF|mxS}nLQI1Ewn<)xg=o@msXMgbY^?z`A!+3=J&!a@{}=tzQQB>ucisAK+FcY`t)1z zJ#%%Vk>UNH=BA+`uuJO=&Fd`ZNjEK%^Kci=Ub>vzb@r|$1E<^p79&4D@AVK)09}sU zsgP>V=jQZk?(&t>NBLQ?Pw~knI=g~rCOzW@r>}-!g$F6 z_5sZ`hdWvMKfEREchga`)E6|-y{dt!cHM%kgcV@sf`})^8*n+@&(*O{jk2N*U3R>% zep>o~_K@^;ZpECzt7cD;1oMFKSr)1lL505QImNy;%}a9pC@h4!W6QKeJ)zH$Wb%sm zzRA^*LJ77xy1@*xwEICDvnN;@lnKkTPB6YQv#OYOv3Jr@s6Vp*9i@&P7r*fiQw_Dhb=qYC?D(I!bXW$se0RU&`BauB5{uKx{jY;DTMW z`BodW4g@OQ;2GJW?j+e2?|ISGrpGu&*hl5I?Os9M zc(LeuO~Dzd)u))ZvgBR-bK9Bm9x0AUaY7ueB6Pf!U0d2L_e{!yZ03V`zT6V~D5=TR zE17w@0Y#V6nf*caqwCy~j&6kNaZTXj9uK32+Qbr2^o=qE=*Xv990%ff2f}zv^aQ1b zKN@FcHDutl&2dyERo?sh%%o{{FPprssvR_pD(VJ?RsvZ(rH=~hpu+ZIpY`cZoNjWb zCA|=M3@>47e)HbNr26)~k+Tp8#Mgwf6QKz9zQ|dW=(W~RR6gvEM2D(O+Q3lihr=YM zh{?2|)DRl0(VN1LeS!0LSNzgv^tUlC|GJ*=?-33sXm+y+-1=@2z-!Q2P^*g0!0Pr# zA3kD93t&x`0NB-qVEbJkM&w^*SD7&fu#5!46Hs}PRPIm`irhlNY(^*;C#JZn zN-8>WI{lvr4So$*{_MXO|1t>wE?fBfDEL47-ybn{z;haup(>1-SNlY=A=`zzDgsA7 zl@`PsWF+gT?Ui#A=HaG2KJj9sQuS0KWJTded>yRMxf05#E<}|qBs9#;XIjk;a~hQ1 zw=F5?wCxf<$h}V<(=R6dBrq(?3zPb#sOxt*tj9m#wbnkveCb>I8uTsQiv)d3tBr3V z_tCov$r#YL)SiYuw*i;aXa*gwaurFtxa|IL1@tYY+oML36Sw9+13Jm&m1R-&vF@F& z+`Amldv)eSM=+e-aD@|?N-F-k>g|vG+Z^;Q-ZC}fiVvb3LmYpHP$P6ynp;Ntm5IC1 zLl2eb+?JKi3dmcWiywuuq@1^&Vc9crWtozz4MA-JB`@@i0Pf#7b_>BzbUlpXbT;RE z?ju_IhOI7h(lZ=-ja#oQ?~>dDV#m{KXeR6h;|Nrqs%05&Lz#@^Z63{iGSd-b{hShm z^KuS&?uO)HE85_=$aokHpuJ2$up5hMhs~YH1{`He|Edq2#rZR28#WBRJOEa*HrI4U z;Ayd=p$6BF8bJk>=@L-mX@ObSj5&38b>>+W@6D9do4k9oxS*CtuTp7X&lP_4bCS{u zt98Hane;`=^k}P|xn<-&uss8qV8Y*PmR=9Cf+uFn6t%A|vF3vI-he&@qou$*mVaFi3;Of? z@Rj`5Q2&ub0%nGUy$h)CR_a5ery$>Ukp&f2`4HdF0Nwb~6XG;cmR@rJy?V@YL92hiPC%IsMXg)F8?E?#0PF+8i?MYMhz15d8(lp{4t=B_L^kIPzu|pSzpz--X8%#T@hbiLy>|cYdT%*nJoSnO!*sY% zDKonm2@NCwRRyGh${T~{j#wr@ZmgVpw|q*|*>OxisVQRYBHd)`BPi>Z%rNi1O>b&m z#1_#fwf38HNf4BQW1-+_J>XcVx&Df#C_B;&Ify10a+3Fq@9p&f8gV}lTD;8_*Yl^WfUNzOjFv8p9aaks(_|@)^j3Ybq%krc2+0yj| z%|xr1EVi2ZpI}&TuCMVQ=9C%W639N-D~BMMGdifDsz1MhWK;fBW$0o_lr`=`v3CWrorc3k-@yxdXlz1II>%n@%GSug4*c&vo{C5Sag0O` z?|g^cW{bPj~3##8~C{Na#*jH917cO@ys$PxqZ&h&PEs`R@AdND4@B z{)zepcr1O3+O3$6&*NhW#-xkHkRVkk(dpx&kt^Sm1uY-moQLoFw%Hx-VG@3Q{54rv z9C@>#J-`=J1LdU)c=hPOHoH~5St`v_kgw4M{nkM2x-w(i@p)0_lhngRzBu_a7Apl~&DH%E z-%iBNRqI>oa}}Rh;ALh$dzKyfe8_SDZe?ii!)@$(C0r!BF(taaUhuT>`ps7~fsMpX zEJ4?`-?s9Gt6x64Z?;}2zaTnuV9o1QuO#Y7VfLB&#}`vCRNoi#YH?}2VBQ?Sdy96A z_6X)2=?}{_3s`Na#c&;w)uYwN*)_R6A4KcnYxXi{G@3A9Umr%lga9dUfSi|99eq~` z@n97vU*rZ$8#&K+vNDMuyCqf8V&}vfZ+{5GL|mynrwb#b(pm10zLB*o>{6GfYQ&z& z4hlSw{G>lg^MF$SrxPkgT5UJOtwcqQw=pgykWz>{(c`e*(6NJI`-7q{dEnOL-~+eK z?E<7S_~IDTCE4?Xb&S}N7XTj%_E1)0D7|7ojJA%1wMFhkh`*zH5@5A(#wG6_jgI_e z4quLby|Kmkw}RXD>6pp4l<5?$c(VD2MdUB?&+Vpgy`cy?KHQgEObj5ug7X>L?>tf4 zC(UY?3sFiuKS>>bc&`CLU|2%qw}To~B2kxK=i&;4Ul%a7t>o>k)G#bRYPJKIAhaiV zP}+(7)lx*p(NDLj>p$t#A@rXPPX~`k-aE%1rK*1RSQfr?F)VCckQZc};L6O#6hWL< z&G8cAWAZCwUS0dmr#jw5`CsZcYuf2un6P*QQM7jH6|gNcp$G?xM_~PFhsilE8!4Q( z^9J-s9z@3CORT(4XfL1IoI|=nBhk>v15`0uYQj5DqipyzV@9gvRrVK~9*H!90Oto%L+K(8sOSo>Op}jI<7&i5;PSH0>QEaGE zWCem%9hUVeHpGHh7`Bu#w3B_$u&JXv8TN|W%{SQ&&nm>IHU z=QHiB)ZnO%q9t0~edt%jy;v^ zFTo_3Yw&RIHs#CWQKtexV4@B2>Y39e&W_I4LHj(blTllfYWy0^z%(|uLc zF=Da9G49r`@Z8rrmpAQG!qlM1d*<<{K}}&05%kNX@(`-sG8~Z*p@26TI(>zu#TEOQ z4UpFUaS+hlmTDiFy})}=l2U?RM~ z?U*XuD%Gsun>ch;b3YbG>D%-&K9I>(oNstd;=DI z$*(c?c)YMLy%7`a{$UHZ&!!JnR_$d5BjRH)jv-y&Co?4u{c|6;E8PJL%c58`r-$9P zzz@!ZPoR47NTAz`ohihVW3`0^D;q4XwXo;vsf@iJJG~D5QU+c)EI>GLdjuEKZ(IZC zWDe15TPoTgYKVWY7vJ5+>QyL5dCjF`?5+WQtNV>4Ada@b=l#JsE$|{|l1OUga15D_ z+A4bo*gh{6bch#P@9(>0{&4r}u=it^&_fJ)91~Mv=F@M!6T2hN{!ZgwK+AmXHt0PM zDx$}5)o5m9lQJ@NY+mp~$<{|D6;IyxQ%nj9?unP+ZAG#&1AE64AvFK*I#&BIQ}@dP zC-U7$5K>pY0q;T-a~Jf8l?{h6tLjsBAda_DRmRJiPjy~i^5zqEwm#AAJ$ZIUPj5Ub zS7hB4=A&zn&Yvv;{;PBK{Lgi+{%2^{H=qAYO)2oOk#smG;v5l5(jxNRA4xu7ZAm#+ z`l|D3p`Yu$0V^d1`E8STb`NSyT|Cr!l=^@cfEs!Fa6RolQlN(54$2CaS<97W4v*m# z4dwNoM#;1_nHSux`y{IS`qhIBXu=*wKa^podG*$fV82R0)>Rw~Uk~Nl-kxPOulZu& zOy8E4I6wzc1JUIRjlIk=5gMzJHCQ1ehgZd}DFQlTNz_T{bevGjsr+IyJHfMTC-zBj zq~=3z*#Cyv4>TSa7tTV%AMPVVA0wp5N~5AywbNt1dg?D%tzRbCm}cHOZF|S`7Wd=R z_eJE!KOOPJxPY6xfmny7J%n9=a(cmQuS*&e59BS(3Y8f_VpNh|uD{)I=;9;2=wL5< zC(4aGUs+}To-kn7 zT5K(jS5?u8H1nFU_qmJW+sy~u)*CJ4tmr|u@2D_&%Y=v=$C9NI<~&CQo(XY0e?hJu z+qp};WB;_c*;(b4glB!%r^0L0HbTPe7#9|Qz&uPi*P(e3<`f_L0G*c@;)f7EkmT** z@w9=&TSRe)b4fcYWN`l(*AcFDh}O93C8pbg^x}(u7<~O^IDU5?42+z+W}ICQCoy%R zmJ*LHt+i13rqAdj&PwQ-31*z}Z4`+q4}g-f@b+cYp&xn5|Kw}=+1L2bkEcUsDZXo- zRNY9BEcD(7o>T`l;kCRdSL`cKYGP{@mcpZ|1}Ll#yBKjFeo|QS`{V1M-`IcqyHY>s zg9A!*CX5GsNY{yycrC)##Zlv%%PNMFCJ$HN;w8%$Rl^HD8tj&n%9hxwezJ8+0}7r~ z)pqQ}o4k1nyAyg)h9+WcF|^Z zs)AWmZHFtczl|RJo@NQF2y=jucfAj#XwcgZ0z;`3f?|D$=lPOg*F@T0@e3U|XHID? zK?Z`J()c^<(b+G}d_l6-tg8y(ad%02IH=_R4u<$6?}2^>HIj>4@K}|k-7$l%p7wzs z2e~L}f)}&^DZL0=WnV@BRVC9y5J_gYqvlTj0@#PpS-^6(4IPGBUNhrb1MEK){NAtr z(cddT$`EiDVL=WdSdnbYh&|ObCu>IQK-27?y^k%1Zn`T-JwLrxDx+KUzJh{z1JsEe zA(>twB(2WEo?W7VUa8e(VR>u+^Wn9#8xjyH)T=09z zV2zivf%%n()PTiPmLKeM=idpLlvIhmXU+|L!hU;>Hs-Y=rSH;@AJx?U(a*lr;=Z-R zegwcoKQ9etzSgEy38>zRTR%fCg6@}V9S?ux0{QQL#{2_Ic5QkSj&RJ^lax-@awT4< zX&?=Y-WgkX-vM0J=C4)`>^m{gZ3jJY!(|L>{d-;6Zox(yZ1w=x!a{qnw0Of-c1Duj zdhTF9WcHy(9vaj2F&{ucNY)~8(;__Jy|dNBy9sa8#=H>rfpw`8C!m(4S(R7izw`!78wH{k29$Zd6STV9B2%fs1E!q{jU+EUs{Xx{rYT;PJL` ztQwIf8QAlF{BLJGmjW)|gzR2j|J-?ua}(Ld>;gBJTN+$*ci4k-*M}c51_m5od{|WH zK>`iW(P3`2I;n(N&4-4dWY79YsauD$YKT&+5>;wdCs80v%u7-`X7`p1DTwLau+5qy z0m^sA=aIrxmCmO;Gd6thvL;RYg{%Zb0|JZ~>?8#xfeHBbSK#^+vnI_(sf6ppUIm04 z=SZH*jT5=2_2j~ck@Y1laPnStKZ2AE6@YZ@S#J71a-YSJ9C`-$LF|SxGRG!#(qw?auy)9f^hI7}m10sS9+#L3j?GupQzF6#*WLlTe}5019r_t^`=t_UBm(wII;QNl+n7b%y{ORiT2oE64YTL3 zsMy8MTx4w94qkT_z4q$SXlz*4nAus@f@oB}rkYW&e&x)SjY}H}VWS^#*=X99R%)s< zlp18CbdbuGLDG+r440*v@W@Arda(L7>&+arjE?TKO8tsHDMADSoOhlyX#{684{II2^(!jziOqit1AG5*tK4Iah<3ioNHPI zDBDqUYUxKV1dauj>?s&DNSircpurpz)1@!6To1D>24%3@X}3`yo}WgI?Dz!Q0QEE_ zU|hrmg`nl_gUXn5l=@TOIafHKtsN|-bT zJq&}<0B;<*o(%G&8OSFU9C=U3oTB6A7|vVr!*Z7{h3a-OG6Xg)${e>XJhRx533l3= zUf6l~s_;CDVB!m^s4`llu_K|lS8-3C=`YXo+w4AbDW2ZAOGzYhZykq^M}wNOSsGQT zl%)F^f^G%R2f=njeUA1yr==bU(%+x)yS>Rb_i7k z2c7dWyo4@hUk93fOLcDB`qOda-Y}9df8nk^DSGAOl2Y%&ZWySsu^*pvHL*3>#9(qhZ;%1 zX6MO2X(BWk=v4W+*qVDZDot7v0@O6uZ(U+osKpVi;Qs5tNjYmV;G|Y_PET_xr6tdA zxbZ;bhS0=3cBDw1A<}{5Q2oYeTnRVjw67-OUVZPAx8AMCd`J`^(3OPF8M8f?M@Z+$ z2ybA>a<#t7;cfQd%LSIZEs)i?98xlN4#YL8>huc=QcP0w=0lAl$C($$(@|BIuorjpnLH&!t4VWjD7o@BuX)|u1BP^q z#=`>279+Y^0jI^}Oe<79mO_P%4skP;+FKY#M0)NUWI3bze$cQehGpBwi6}7Vp2^5o z+witgF>{7OEyfE6y-^N@5i&`+rI+gj-Ws{}>@3*?0qz^4udpL8D|Um;)BOl>oi`c_ z&-PN3q_s~+$~Rw1d%e$FP5rVr3}n6|uWXdTs&~CraKMEwE5n)~tMwZGhX>&{)^C+Jr`hLjGl_kt)p$ri2{YXd;s6u^tk&yQ65gCvbs*Xw zId@XdaaDiN3U-LQ#mx^g0+pUp+Gfi<1Dq>SRZy56cHPTjcR_3zsIg zm+A(GMlPMZqNT*_j>;`uHUfQ4fmaKhNu7%-*kKe0Hd7&opwVl3lg`3+cO0vIJ5(#Z zbNA7g!WxFl5C7}&?sq8R4`Sl4j`<5k{0fWsBS_Q#w(m$S*3dWk6PX*JY*>3A93Mip z2ZK<0S(D*N%pAHo@48Q4FfeOk8~$A9HB8$o6;v+-JoOTifmDIRi&>*n)gVJX zGfl7HhW*RJ1SYQDT$U7b-;BX=#&XCRaQHbuN8$jRY;H_6uC=~CUu#*_$G!LGwQZd@r#%- z4Li90Oiw=iuU?IG#vTJ4>JSfn|c8rs9AGYNDCneg<3!`9>1lu)$7P0?;79;-# zEk>?Ei@+?Q&DD7lOO~FZCI?4?BJGE0P*Uxg51M~&9-wOgP8c{HRRn=18pCbO^S?#d z{Lqwejo?94QEZU!poC`hhL1=Yptl4u*GApGd6!G{IubU=0WWQiN}6a5FPhFpo4}2G zzDEKezM7Qnba09;)CPRTFQ;I^_RC&=xpsxWOBJh!-s+7!*PcFAs8qY10AH(t_iZ-JzHXM9i zYqKsv_#CBnQe*r@#3360*WmOIT%nv-hcFYs5UA=iL^6L2IMFY>gDnLML1-{==^fwN zLSY3^Bezit8f5@VvIT4WJE0q^~bXWYaL zoV4K`0I00Ymwn*P{A)zDvE&Kx*bV?z`EkD=s3Hc3ZvS%&n16F}y|@vPoHn!S`!+Vi zO54ZQho|aGkMJD5r`DnZy!Q4I@===18?r#U{Zo(ho$j;*Cn1M9Qg@C44wu7Lz0fU(-&wQF0#_DXp z$*TH(e{{OGpwb~m{}^`-zW?-cDz(NrJZM4tN&T&$=G^bhp5t#oceD)_5 zzkm2i`ism+-vpMIox6z&^bzc;*DKIPlLV$`>@zZ;XKc|S*rjnMlrfg@d#z-4F;WoU zuD5^S|Nrb7^A*P5FGmM?3=IndjpU?3Ut(jLKpb%O`nNkQ{|JTuzwp`OcN{TB(N;&W7=*-8q|C-@E7*tIjB3REI_ZGa9&maYQbFhCrp)p^&Ffd9N0n} zFkmor00s=VA8rK(4A*$6n&#j4BKK4K&Klsqc%%R75BkH$Rh*)|!d@KGB3Y27V0D=) zTA{V_pq)!UFt>5`urHP_y?G}~^8^DWO-*ffrsM}+3y(OT!E7V@m1hCwFX>3U@JZ8{ z!%z)Blw{d(<+3?Vu#!XMi9^&4TO#K9EFh^jvac=YX{cW3Qy`G(#v1>daa(h1Uj}_+ zLng8nmp7bBB)N(V%*H3Yd2zn(($-;ZYt7Kzi)`=Ko!rJ1;p@C?giVI_a*lh}M+zcw zLQ{cd@bS-(&CZGIVT7nl9hiKp(j>(NJ26#F=}duM^^g3Mc#y+|rvK!^`J-)!ZiOAm zMK4@ehSBaMqUdqM=AFRl$907fasD#|6VCb>qEU{C2fQ;19HaY>V*s234q8Y3auDa* zL610xU;L59Vx@!hG=bz9IjUBT>=YX$3mKH!$Ks`~+VZP}YQ0I9-`#jfm7^hxv)yBD zll#(c^*|?TPVp(&bX_!{mdF+p%>PvxOJ_eBIb@)s+3G|MoRa`StVEfOahZ zhR*DtrY--%ccAy6YwoHFF1qv@OE^8a6ZMtk1%#O241HbdH)#nNLG3?RQpo^<23{!r z81QGIRiT!M_VCyIb0Ec&G$gpv8(^HM>H1A{G-@OrIJKWg^T0lW)gUc&5B)6GZhl4C zH4sHoBKv%X80|aGOj`L9$lO1wpO}#r#DDyyN5;oaXNWTYA#7Ay2Me)hZ2 zO409yc2eX34F1!XMB(&WqHx{;Bnm@J5TC!h=l<+f{!bi_n4+8jFm7%|2;m_B4@fBy z!^VuPrS?&02F?tDE>sOow4oRJJKZ0M7KzS&X#Qaw^%C^9%7l|Vd(?RbaP*e;frb%V z&#O71&dy1C^mhW@xY826iMMXCh0OWBf^e{q>A8Z1Oa&}tY4qUyRn)&X4p6Gt?2x%d zV(aKUekzlUs~=w9y&ukL)W6!0ex|Q6{-r~7m_>r{mGO@cMo7R|L8uO-=ya#)mst=4| z_E63c9%gzzo{@$&`sDOZpt{kXcmjs1c=!bGnE@t_izspJs16KR()T$tH%Infa7u$mIoiy z_c%}&azD|`76X4(?Vh&nIZCq=qNHlesV!+6xUdF>t7Bh%Y3d*M;+4YqRQCN~o{mx39v*f3}JK08YbZ^wW=HM=-Dj zZ{-Ah|x=$$ufB`BPU%)OK7RP=-2> zq*1px)T4RJRGeLX(#a0B&prz54ZJ5|AFgT6(%8H0VVDv%W0m;Q2gC7!zaziv<9%m0 zg>m<$O)4U6Y%%gzHtb?tDFFm*KLTw0Df~6CxOt#AHv!aj9+~nPa#%JPC55BwVLp7K zbIEQ3d7w}Wpuj*bwh1=MO~B9?USbJbVT4DVQ>=8TBk>dcCY+6Kj-^KHQq?=g=V>ez zf>aD@MGTvYBOfV~0u$OmCM-ciI>?jEhMSajFv`&IJtZ56X2CoiK*0#`5 zbTb$VF;c5)-(EQK>-%C{ss8J6{LgFp>v4QFYX5p1e?5+0En5Fu*N(fgz{!8pX`(SH zg?!qbkXFByuRSk=H8jHqt6qZ_109Bnb?5D)k3NXAd9JDHoSnAR7{{V zHqd#W1M~7#I~{iAekqIlg}k@<6s0u;w`&MiinK8`|72|T#j`gRtj)jKtRCJm|GSAe zgpT`B-1)Pg{h8!~y2fvF5Ev{lrELX`MM9yZ4Y8oF$PNUtVc?Zp?g zO(uY%!)ir*F1K?CA7pLa6c94dwTi-eD@TcfK%`SO3~CcztAn(!XcTlMc^1Zrm3FJn8C;A$^XYR`Y~{k{{PS zZ;&pS+$#{g<6J!dpj<<11;Ana`TPz)pd>H zah<7Wx%XAN!5?`L8|EY_hshnJhh!#4;>m_?OW)v3^%U=m?T*ncyhR4r8T3v+z5hvM zadEtD6J;wPIoke~UK9W3RTo6PNM2m?s#~WDe@sTUYPIzg;yII zp$EhT@pph%9ngww1grRN5h63~HF{MsV-@)|H~N=g>2a!!~mw6MSHPjcVJO3#hi*RzS4@<@l%7|Kg4PJ3s0VeV437 z-*k}RFlOF>?tA=c6l`_jP>MS|>11}1m1~~Wz=d7bp@sL|B}@pMhJ9*5zBo7^5(k5- zdQuJ`TN7Yg)J0`dXZcp+;&bWrKHYAJqm1C_n$_< z$QY1xotq`hcz$AcA%*%p&{lY^vncZMxA)dK@Slitq}gzc;1{b%AQw zsj13&vLvbo>W3AbN2+;Q)h5=Fo+ElFcv-_ztDsJSTtC}82byE6T03t*kp4eO?f&S7 zfpaFR>H=(HZV5}_{eRed^LVJ+zHfXasR)r>O<6-IvSk@9wj`Az>nJ-(lBF;sYxb=u zVw6PLvdcDhB3UD2-*;xv_%ch+;k>T~^ZK3l@42t%>7QPj*LRNj&N0Vv zd_T*3c>n-pVikE=fP)YWUxPulKVZl+BGktZks@F!!BxwrJ4o!HgHI8y$je6TTqH#X z)$+U?w!gM19R8Wb`gi(EI}X7B0|FJ|!^WC{hBP<1KvkdX zF83ARw*E|oqmN+T0KT<5XonNsgI$t+kDhZxW_(}I0Hn?HW1~&X0Ys`i#Qs_;n0(6B z!A6}fOUB)AO`@K&#e{Dgc0Hi^(e;<8Xa}pid=+@U8UW8%{Q7vXA3DF% z$BB}+rd6$zYZu#$nH(C?ZH%x&LtXr{!z#f3M~1uK%w5`(J3>p9)x@G4l_d zn_!gc|5u{u-{Dz*N+bPGiB$jD`}j|!k!BINgh@jX7a;eos2(GmcIGOpkI&s*&Kc>( z^^L5YKe2SdY1&Ntkmof;r2-{dHMsC^3fe#&X_h46?EqO4Hpf-44Tyk8?9IX6Dzd}v z<;JGCL7bM*W8av{EF+;PMm>6Wy?f$wt$&#J@wZ?47y7~PSups1;%Ukgp!E^u4=)MI zBcPifu~(G_-W@bKIIQmDBy-SnsWXlh@tl?s&7|>=7RVE}{t3I8cFPVFQ$XAH6V4t> z6t@SZB7F}epfocfrpzHw%Rm}SQ6GCCg+Z8O4a!2xr$_%I&iP9;B>9&>hZ6)k6#$QM z@lOnNR*6XvJARcI2AN6KONDF~kA%G*8Xg?tUdr&pq2-3+)tOxmee(aJ@p?tbrGz>)M=yum=E*8F@Z(VPZuoZ0l9oixl^*CX(l?bE)BslA8A&=k~*X zWp8gtIz@)>*xP%JH%Gc1aBSQ}x*AR9w6pA8mEch5>ohu_juZSM9xb5VSz;~?tW4_HA)ptYBe z<3w{%TB4AF-9&F{**!kOv%r_L)u&#%_9PN~Ko1U0VzyiG@YXc>*rxYYenenJx6A4-BW@z5`fRKT6acp|xd^{$in_dY|eS#rHD&N%OZA?he`Z?qAo{ zqcT{Ml+`OpytS&L;+rD6v>%BOo_rzo-1yPRVIbK!@nHA+!Q9!Gc)EknWCsO3Q2e@O9`gV zk1*rI4?0a-m-GtV9#9T4nQJ-~c?#Nwp)a2{T%*X7+4G1)z5EZCDpg*eXK#&x_dG7=NwGgetUc^Z4Y)gcV*iqjD@6dkUAs>tCa|rN_P9 zxg(N4)yU78IV@9 zGE;AWcana2Cjgv`6(HAVCxP_L*&nc$;)4W3GROG_>1<(Uo`x>*lLD86K4}Tk#;>I@ zKC)odGxbPiUf3T>wLnkaM|@WV2PA;L@PH}&uFW()AX_lR+0bgdP3cD)G=selT1Y$olt&;M^%>OmI}V+F?a z+qRfVMGxvK9CyXsZB`YTEPTdi$)!j6uXBqae1vbm1o#^|_#n7{30jIw8{uWe512<( z!P$rXFSk&p&*wMun=h^JfWO(vwcya*nGJLYCLH{J>hwQg`~aMN0MG#FVnGjCbLZ6n z@NJ(xprNRC0Yo)fj+x9}dhQ48Lf-@A!J>}!sA5Qh-zVd-Z~UoMPw}e@~91dj~e08BkO!eFxdyq zNLAlGt&};vRekHD!F5`{Q)`i@zGul{@y4S#7ZHOhe3bumfbIQe`%I1I$dgT!npsL5 z#g2B{?KL)u7Y<1@H$4GRA$zhip1jr6A#76S(yury_@aYrJ_h3e2qvP+mUF$1w;-01V;IplQq0e789A^SaIb9<@d&H19TBkF3X=PotBU{s@ zGj|&cZSOZz-?k~oiG&C zCtb+GF?+E$Z8r_@j{EHE&TZj3_*M;}HkX%h8ZqchR3|VN)ZYS5@>frbaYu}#vBySV zRyFu&R}Z%q?5~(pRl&=LzR<6u^7ps}xiP-}vc?O<#w|4C+oRP4;cB`tx2OLk4t%HW zO{WPe7Kfl0KVbKe6lEjBbvlv=`cscLFCcbw)Q@hv_+c}E3d%q5;sr)i)qh)C`X~SX zbD{A-jPoC0=#gmZC}z!OlmoGJK!h585l}dPtr^`(zx?0DtoT!bh<^s~@!=0xBW6no zB@I@tBq&q9={8I?#E^}^2?-3%NxK#h+qyz96bwXOJN%kob^NooK%wpz4)w2CjDH+a z`kx_9|KRKNc>2Fn$)+C7+9<67&nzr1ZoL07SqV?7C)m;J;G2@|({bF{f)7z0*86KY zh-|*?`Fk$!(Q|-N3+M8tovCnclGzVfhZGZftQ<+zO8e~#;NcV-N(Bbu;{;6cD0vM( zOJx8|u&GeQZ(o2>-yQ!s;?H^f+vW6g_55GEXYxQCa2qI{*)AU@tP)?~k#)$!ny)2; z^JMWK<>K`YL8bX+WxA0z7OI{J%15M+({9uJ3B`s#)PT?X%=}xocM2je8R38o`~llT zJ?Sk{hSzL1bMi)vhb2|m>vmNK>+w%;NbrPY&`A%G-|&S`_W7#nfb;)6V654X5_!HN zHe~pySIHf|9HCmw@*D3 z5490pGP`_`;7@G%N_80+VC2CL}t16QBQ}T+D5!KQdO-#K< zPIdr^y*JkM_SV2_Vfmm_;cl}6Drfqa52IM94L@Kdw#y;}SK`+`RaWowgkWN6z%sgo zTp_hY+ABwiJokZ0Wbm9;(#(3f*^XNMb&TY^GVKU$!J&hu}ml$PwPa%w^pE zeh4|ta2Rp9#7a9imz(FZQO74%G1j$PCDR%V(*BTs7KlK>apUZaV=#?VhZx33m!6M) zzPVhf@xK9KvGbZiM>4j`6fJV0byhgDYPpf({& z&4cli^*>;I2q3h&pexhhuATXM?#!p2)vH=Q^QT#rcWLM!k!$K+WlY_rBj)?a6j^{J zv%5~d(W1(Jbz4+9Lpb-WukxHk>b{ffygn~oWit8p=rf(Io18xOh?xo8CV+3-Io-Zq zLBK>*s-(Y|wp0vRS@p9KA2lJ`7rt!I zhO!#Zgfds~3>no_?N!RR`|Ncv$o%CLpJ5DyrZR+#e|$~bh}@UEEOdj^XI$j5|1)g% zPCWkE?%D7Xg{j#ni=A}07+xqex%WGo0m{2?BAT{;&sia7+NqS2EX3JdmB#gUdExQ~gsT0#& zTbk*=eDuzPKOpo+>OLIV$>{AVq2Kp;S72Ekj2%|4R!?e|Th|}aTj9_TT=48jm0jS` zBp-4miu4~+NoZ8!s=YQkFV|lZ3|m$YbJh{RdC#xXgQmQySTi`U`v}7CZ#y&caE|RK|aBZbFLJ8x^=R1a{DHx-I z1k}*VV1fQii`l~`?PxQ5RJq78_%Rm6>Is}PgOy>6@}r;uHj=7 z>7YN{wqU7>%r0Q@VRzvW#?_*{e^Nq?I=qa^X&@vZ5pF%%_T z!QB7lcAWXGY;-Lr!g}zlM7n2KaZ^R+Sm}3k3M3b&b)!b7g<&3L2=Y%sP(b1$-d=+??88gM#qAby}Um=N<)CnXi7pm0zR3c0oVs{6n}=*e*PYi?S}C7HPte#3JyFU5H6GeV}#7gF>Y-C$kz zVC(M!6B34MFMswnnNX-*UR6-?{8(@U?o_c%#+P)<%8;Y>@^{>|J+$_jlIQk6a~=dS zk3RwPh^*N4UBRg>+Zkw_Zw6hO3s1VwV}Gm6inL%Nd2;XE8~=_)biNdzaZcWiLQh~L z&OBSfi3Q&a`CkVk7x}*5m_W2qm+5ekZwa>hBu3h zyp<-W!6*+a$g=Zz{Tfaog$$u{6D{)1-_5V4dPCr2yKS3D3%JBa00b%Sg zl7o+U%1}d_UV(j{iG?U|{A9--bl1&Z1pWpm5H-phegyEfX;i+4V;$S#Ji*lcuYbTM zH_-J_%G^Ns+wuxHORgiyD6HFDDsvsMz?k(%%w;>kA+a#vK?!0NCS|-Wg+1atI7_ag z3EPx@pk@r3NzEyYKn^lL6Fv4`ii*%AE3DUIHVkAzlLY7rG%s$N15y8S4yCNZX~M#i+X1}GF-4m-;@ChGO;rytC-}~wRGgq6do!obB>x1%`9mFLvr%zJ-dv#v z-QHh`b?x}mxsdizJIqg0o}*}tK^XXb1%u8yn`v%VRBjd9cK6!&<10LNuYBtH$j>)~ ztDAq#srsWhRqUA^nsKdqB`1LdIC(WH@H(ELXJ|{xtMavn6P_j{mCzD)lyefF;at`4 zDSdyEs2RLXip*h1y^O1;Hu@j{ZB;UBF>1%FJPZlE%2zc2d(gve4MFD6|I0NIf4v6t z%QXma&H69bT&E$`|G_5IFA^%qz35sEs*Lw93;=Vo8gw-qy+Z(S=U8+ldhHMZKDe!| zSONmTVyQqrAONh(04InH2Oi9l$Gh4L6CF3PMIKA`5FU;Fg12H1v4zL9!ybu{+2SVU z8}gzMh3W|n%%R)O#2Kty2KsJ4!nU;J^Of=dq$ExuZR|+JeXm4PHC7ri7R!-ffXM$I zn6o71EbVBVG8J?2WZOG#`j}`}3cLBDQtMTE z$mF#q4Fx)1?iyvojjzZ=Od$s3*y@y2v>oUb`(A-~CeK&oo1K?$MkuYmtQky-z8$Sf3q+5HzR;A-H%DYWef0{enSXnQpq@EhepP`EZ|zEkc$ z+VrhFF!9!wpc(vN_*U}8{K~?hv1b;C{Aa|iSYQI3G-lR^Xx2rZAa1V?$TY0I`0gB# z!owOr)b5;d-tO~&V7%NS_iY`%S;a{eI?w8SklB|}f{Wdy^Xi-OpFL;o!{w%P9c8q< z_0fvJD@{IEbWfiDenaL(MN z{(ChayV5=?xwN#Jo&BWjA4$)p;^_YsB;4j=!R^!rsMOjcq8f=i{ZL}_!ElB3=Rub* zr(P0zl+ovVfgOmY5BLh2P}h-;rH&^kI0l%WsCzJ-vE;RkQ=C;fwL17TiMsbA>H4?U zO=IU!vVed|zUReJhl8z+OxL3}7{&P*y&-z2qJLvu9kz3nE_Mt8lqfI*5_yuH)bRsm zL{vddNL6fltZJ?IPI9mIRN7MaZGkO*y3^Wd!|Cqwbc~Tvj`FqJ)~5bJQm3u_ycjoT zi_urEgL|X<0b>L;7`g{|&@Oo_o5v=Pp%yNoWFvnW4;!U&c?W3s5r`4-QW=C&(QncA z&==6&tY{_+v;2Z>A_n{wtJDF|U%%X69(Jt206C9wAS6A6S2h!SS!bOU)~MZs+de~l z8?|Xvd4V4=eGF9%D8U0sSqn5^`~cJxf_%x{^NXGc_+jX8+E?fd>|b- zvF{WH0oDnF0C~VzC^JeN>amkKQ<>2pAGb{j9I>=liEpIei?E`U{awPtOb3m zUtn_11n2qHs5|Qb4F+u8ILXpzGW$CC*)H_CVGd=B?!duDMD_2DSqQ&6HthEhAk9Ec zs}_mBe%OP;cC){jE>P8lh-H7VbIFOd!{0<+KIl-z0wL00aIy7y2xkhKdLG4JfE1u0 z2W3eisLhdqe8J|kFDFaQ+dvv-L&3r2O1ztKYStF$N=Dg&-ES=^pJzh>8ihSwPu`GLtX6r0iZMD3h z;)rM^naze2Bc$9{iyTf$l4p4 zQ{(v~FyNnfj#t) zgv~z5*2Qu=MPX&3h3IC*r_cl*AEyr3qAjK4QmOW(?ysyhaEeC{JQ!dFfHdr zLOpd~Ab?MF;F~EF6TnKBe@a8~`x9u!zsf-V5kA$h27|vA6$WRaGN3+tq@00}IXbD1 z9~??p(tCAyKSbJCyNQdHM{8zo^7W6pZGqMj-L1~=c`VeHM?uM{_M1G<2Kr9-u?${V z*ScX=;GQZ%`>_$`j=}5Zg<%h(#DW>m`~RC3<@CXds%K3Ut5CIoCgP87@i_%uI z7CUy5^-!RHgIHYG+{x%cwqE;BW$UL=eaIrWyrn#RsB@TnIRg~QCFPWj>r;AsXLMfB zcf3_gLEtTdXWJYLbo+tGsSPfQ5(Gp}4`k9VbPSv)M)Uw1c60y-mZNBy3CXPP;^AuN zB$RxY+jyX4g;gx|*kk&Y@CTcDZ~nfw1MNRHa0)<-CF2a6+m;Q-1h>7&SDlxci%#&w zr@6>~X<)lD*i`9Ja0UT%EJl7Qx9Q*P3O!e z&}0D3g98jeT{RRLg2lUffwq7hE$QWJY!CvnLjsM|Yh<8G<)1W4RRYWxDIlUMiUJU` z+IG;cI-JV9azAwE>>GcD0$QkDIs|ztlQSAxhT}W2KrA&KO8Nn#m&cObeGwm9#UZc< zf&zEMU)7}=Mj=HI;!hU+ywRUC`g4u`+)O`@r=REO|CuL~2X?F!1Dr~D(BEvpQ%!rP z{zReyFyl5^jF9M93_-xKv@Ei2L2H#q$mycy08!~d zq$p!#K%n~<+G&@ySWjl$C~*oout_w99^%8s)A8_wm3rg4BA*k5w~E5JPfY473aoS- zZwYX_`hw|XQn?-^EV6WTbl-z8G>&bO4wUZ#7EJ=G6KwB&^bzZJn=zujU#kCZ7IE6n z{iVzqU*>PRXB^uiRB@A6N;wj)HAI{~*5D_~eb#COcA#j=is{kU)*6CgXrPMWu20YyIv}u4wBQAr$&+p4_s~$H@-p1Qoy^_TA`vjMp6ZnGMf&dQpNj z9ahQu64G~}IC8}Hy2@F1gC=?Y6d3&qAUax;#L>0-R1Q?9`NOZHyl$M06s%7;lPUrV@DDvCOh z7P!oKD{W?9=_%Gdv(ATQzQ7P_cDdr?*r?NpPz10fCGghYMd}bF0!v;w$1;eSYVZkI zrb)VgSgHwyR*8d@N^d0+Xx`RI-EtXEAh_pEI2W}_SXObI8yOkBy_Gv78Zxq4yl2F{ zRDjRdXIYpkYiJ6qE+IFZR!i#Ol?aU`q-~4uZ%89)<_yl4{FzZI!u3Y z={RjG^)%TA&s+l!HqK~d4v}wa)9C(spM^f_PAI??S`RmD>Oc279pc23ePP@Ao{FCF24H1(_M^wIcq@_+wizBumDqsT{%^a6!}ocSxP3^6SL1-du*O8I`PhFHibTD%xvJYM1uEwofG#jknW$0 z9vhpktwFewEh(ueCThpF4JO;$f^b5Wx9F>lorL+#nWt_1k~8~1yosne8mpnQcmu^i zJ{~o30m*d?{}w|Z!5?Je^1g9b^MkI>Cnq#6Jq(DHR}f_MRTIX(0F`7>&gGm8YBw$d zQ!kF#+cDk``Q$jbs5ETqocf3x1@-Og{CX#-tAZSNY`G^-(@-u#t_^3T!n{?|p}cwz zd-R({gAXGELz>BPCZ?&eqMQl}4=gk^&zs0XG^EoYST;Vb%T|xwPo*iX7q1A(mN<=I zOq%EW)#vk!>$D zOL4TU@)0FTqH7JRS;#(w8yWShymo736@qXE|5*!>kDT%HaV>)e>}3ZVET4tYuy=~H z6)E1Pfl>5dR|ZecHa=0MBOfLhHnt_p7C#}Mvbh_Pc=4!Abn7ecy1Y`M@@0h$*)7}m z;5*t<}1O=DC;IArGe zA#NcCZS)4E&J)a*M+1Tbos~sp-&_+j_bE{9BUoEof4(ZszV_s(AJbX-Og{9oBdABn zK<7!)js*~dX3rF%KSCl>GF!ryqmOtYJh1GesIZ_SecOfm^rM4KY6$JQyt*iGXx7@t zcQkOg!#S#1%sss>R*E$(^liOOQFtDe>@-^%a1o|Bp-yBc>5VbNA;d(2|AV|IM`(H{ z`c5BkfU9;doCu7OzV64h$H0Kj=;j^+I<4IY)eZ;#ElFqR?|;7JM_GaL2pQvgHKPYg z!&D_aQ`+j!IDZy!e24H-I^HH!xtGUih&DM!Ku)nqd`p-TT`h*{24~(K6hD}cLe1bL z!(Qtir^((t{z7VJS&!51?79i@JJHSewvs1fPhZhF-g0YgHnRFCG&$iBBB_idFq>Bq zjug8!BzW{xt}X9=re>O)`LXWx`1O0KOvh=76~q|O&S`hVVQ`f)q*Hdt0gl=GXy?U) zZ%Q>w79CGiUR{&UQB1gxNfdMdy)#IdxdG1SeVJ0>RQc~?r#al#shA&m=pI0Kg;T^q z@?K`y4Zh>Fq2!n>j#3W;nVhdrJDjC$9h(}ttSrVNHO@A|(5R<|cu?io=NyWilL1-i zV^(wo93Ky5QT^qy&_NNXlYaA~I(j$yZ(jqs>#yDbLwo%k=CAYhPtDEu3b4t*&+xw| z&sUl+HW*WOkn`_`e}T?lTFIanIQo+BU?MRJ_^Dn_0)DEIC{O_H3I9o=|BuGNZ%>VX zmH+?qryDiue;SGYKMKq25Gcvhl~q)fK(_dHi5bsl-@UqPxiW>E`nj7;ttNaCPPun; zvR^qSeTv*wR)O#=(Uzl|yA9X%@WDhi3Mg!ky-MNesTL&L z5T>fx@YWG;3{dbOM|;Wmo6A?u6`Zx6J(tBjxPE!XDy9H+Yn?f1BPs-0)Iny`nam4S zJ!F@d=pe!8ME~s#Yx`SshBAMgc_WPg>pLswgqJ}fA>66Vi=kd^N`|50FD64~o$|9j zTw9(?P0{<1B}<;dp=(co8Z9$Upw0uqqpIMoNn&o0un}-`N?*A5$@M5BgC_guMgE+( zo()&eO&nFLIhwi*;8XA-&4xwikZM6nysFTsnK|woZfIzpV2*epne1VCGIC+SD8wR? zujKPHgY!iPK5Nl$(A~(Dy0gq9Tb`nDso8yB$2=1*VDfg_ZzyRJGQ|cCm?J+|q?wrN3xA6MK;Nf+8kYImf9*(v~4#8P{l6*Pg zd>>hF*XZD&DNZ}fjO#8eR=zw@=Xq$Q@5*sn_LdlfT5IJkCb7IZ zFx23{g0cH+zWL3oW_!Qzwz~ObbS{hR$?adGkN}&dR$1AL#OkTqL-d982uD`0glSgz ziNCuQQq;C6ob0E5`o6kM@cCocZ>>ceq3|a(SD2Rs2yNz+S7;lleSi*Xc_I-qT9W1F zr2SG>5c7Kd2@KYqYQzPD*&JoO?WLJOjUdi!?KhA{&y|!q!61XupsAa8NjJVPbw#Z1J>JgeoxAH6V?>AJ1R4?u zZqf@rbH;OUEUNL=JE%?f$OHF9XW~W5bbaT9qLG zuu#zh>9JK_gA{ZOL-r=y|A4vHV>_4qXF(`jGL#T zPFV_j>^Z%au3*Y?0EV`THXj&jJ0aK8d?NaAf!Il#b6zLu{l9}{Ur6o2;xFfg!SkK{ zoK#gjnI{Ajr;KkjmX8h@nY5)3f76zE-zY7uCkcH{zg{wB@fj<+`x&K$;!8y=juxSp zx>7w?8@~t>4nHlNfAq5U#_mnIfZGuzcc0ga#Ct_0_LDVVvp8g!N*JZVJm2tMl70G*ibUSX87Ht_E$?$URToOd2&t!t~;ch!0VVi!^Wt^uIrW2S=wun zdM*$ODqeCh227Wpq8O*{@>Anhra{6{qdgr?Bwn&XB2FXp?3sP5QA4KJ_lfO3Nu$eQ%;3_e~oN!#@z=J7COaJ>#iZ-~*6EqZC zW26x>O~=yZaGj`_u{;ya;j>3m9CX6MytIrb7H$Lya@)G^PxXqc$(tG_22zq$<4IowDusw`d?($Fxx#-1FnK3pT_IL}N^hEcM>ar(}X z6Tww2=~+Q#(F>+pz72CISu-_Z4R}WqN=Rn0y0-t5t6a^mNJ8lxDea#M&mbOQKLm-rDQ@xwohJ=H! z^fsY9uh}^g%_7q6Eq}NNH-I_XF{6Eiq0!p0TEW8a?57^-+Nip`_GNE&YN?%7aj#R(k=W!(_pe5=ktAWVb|AO#)_Ru8Di%W5|JKqAnCe685HrqQ(G`&GQ@7)^ujdqqO|=`XD8q!HZx$ZTwFA zZL*e(Sc1(l7hjD%db6{^*s(ZF5xi+uRi5&`T0+-3XP6aOrRXTl*(H@f^qk2KtKaG+jOmGD-90MveW7BoMeY*KqfI>D&3 zIGr=*5VO7WyYo)`VM$7ZW^0ma)?1J7&>No1J&Eld!Au;M2J}9I5%F~ci8B*jQr^u9 zHv%VF?}~JaemYH4D41W*t0NO|f%`-=jQPG}BifFkO$osH2Pd2w!wEwdyieu$5OBZht~`e~yj2i?MGHtPoASsu;J&{R$h62{6)GtSWi;)?800 zCwWSyopV%c%D{TtR~sII%$q4CJ^_cVoBUn|got;`P-BS3x zMnWwA2CnMRyQ%dac+m~=J=74Qz$T9*2=&xgb9^P+jGXqlU%P7HUhJs9*T`5U$@t;= zOIte zE*r@FIa+MaJoIjiKaM~vta;~2Ft3!_3hBubG-p)A^r9i2W#RgNQ2_yMA&|UWaXD6hb1yhFAE8nOG+*dokhh}j{^-}hq z!Ib>wJ@jkfOxeLH$CVZU*puaD00!8+@qLrbvPe~T4#NPP%MaKN(1jkLsR7W%1=Lh; ze@XrUOIvYV1XZMC{p3(AnBc`tP<;3W=~DHZPxz0#?LPti>|Bpjq4uDTP*TWO@#SU$ z#Fz2i@ZH)Tce`bmmvG*2`)ijJQ}*#&xV}6v#@4KC*!gS+ZM;*vi<9d`FUf)+YtBgx zj--C3a#JwgqWFTriLU-jKa&a+uMN4wc(!-#U}k=! zDg%^a`u^&&3_jrZV$ExXL~sE5xhx_JH<#rz$v#QVa@*7C>kiNJP3TKw46od7^{zkd z`?O`ZCB~Sd|7*h1zn%%yOjIZ)e{|_9p|sIyGAdMcPjP>7bC8mId935}=ZzfhqAabU zE^%LER*^aYH_U>rFU8(QZVQzlAxby$`9FO)e1VhMorz}9K0N*S=3_T> zz>$Og^&W}2?_BA6quX6X;{|=C-?3?pS*+KvZl>v~>Kv=&A zy_oO>u_3-6Mqb9t$2z6>dr!At-n-&-_m%}gm$CJ*pavhi^-=$MWFi`lusR?` z{6Gmu!3XBAHq%E|%8%zr&z`HeBe^MemHWiBWw&VEw*o z^$YjDPR};|M3Y{9f0gRzF=Y|h0XAK>n1w7;`Xxh>1F8;DC{+{13@MQ=FZ4GW#&?-{ z`c0os3Xb z2CdeNAMR*+DJf68*56fLVT937m68vcOhzqD6QU*(0;p1`_EOi{3Am*5^1JRYW-1Yl zMKM*+T25q|Tb8SJGnNQkN})qFphn;aW+>@c3oqp1!P;5J?IQ^pQ28K3@KKX}Qf!g; zoTg$*9=C}RYMMJ$#VASGTT;P&#OtH4U?S*(z>fzpmtu=5FCF!2^pNv_HZ1{v~bX#b@D0C-QaSk z^C{2cZ=FedpwyGcy{5N^tkwrcPJ)++00l)R!E5aq^_E%oefcpJ?U57k61(mX%J7lB zw53Wn-Ym8toIq891I1!G6a`0hr5x2roY_A6>YJ{JBsu5n#tY@-&lbE@=fl6JGWQ^H z^29!flO(pVEgy^$HASjF%rU8IuXL9S8ryr@>4oLTbJGQ|*;jcfcd%o*qxs*Lr#P+- zFNSWip0Fr+{*CSnjY9NG*Kcgdd)}8ITD-}m#Mg+;Mm@Uul;dSRS0_v2w}-7gW6v>& z<ON?+egp=+n|R_a{|-tK z6)y04<5oPtyIa^KqWZd`;8;s*C0|UFPwuE$=gNl51h1LKh?D4RnXsZ4JVOn8bhc8L z1|vOeK}D^R`I`WBzt03Br22G^DjllpYK`pjRcc%MqKifexO>{^(u%-)VJ8BdG{cWe z1c=6!fqt~#h>1z#7(9rzkyYT+dk5a<$LX*xaQ%^LHmEt%^ZW#mA!N8*O-Ttbj4!Pg z@8VHPf*q*rkI$hsA8ycxH3>#)A30k;p8n}T+-5(SW4q4-H13+d8=h>;8fj{u>o$3J z6$)@}^+jApSPiSzpxQ?Ftk4ymLqg{(l?REE%cBFGe6!r`C^>ZP9aX`_(iC|Bal70! zjx9{fce?9n^#c~)FVCj!NGtzNJyb0vX|UJwJK(9x6EysDP@j-{WyKl3S(OIY^Qic% z5{gW5CYp{XF8qK!wAQ%f*1-;eFG!b**C*kX5~+JC?pPCEyqn0s=@Q*bz`W!{GHeR-?yG_no3C>zO| zXS>G@_op7;g=ImET5SvCC=&4wFcA}8R^j9 z^1CR0zaf#*3(*qlYk^K*-7wpPtzX`&k6~9!pA6UX2F13nTgaGPs=6_GTT@_$%epmM zZ#<98AwOvwgc9)VsR812%5?Vs#57&@PG)Yn?eJ2pT)P4Oxc^N^VRwwB%cP2gvwLq@y+=%9waZAQrDvWisBE z&IO1t^&<>DODkWstU0fq*yVk&c8^&ze`OSPFUZ@4>nBD@e3Cz48>6?>;BSw}q8Z4? zAoL{WNjX!D^;^A+d5`j(p=gulePxck4}}b#r(T2hYut>bLraZ*&}`}$n5{wbko7@3 zC#zQkOV}swnyxMMG=cP;;iI`pR@2^W4m*4W*W?}z@s{1_Ygmp-VX6vrV-m5e!B3OG z5tcds{*CXxijr6sX;1Fq5p1F!B(xk{v#(mpvmOx6M8}=@*A2h+ji?AZ5<9UmXv-7% z9X2VRTf#~0T;}pzC6wh9DsT{>{mT^yE>Uk{dVpVzn>9=&w1JLn5i(yE@5=j}W$5 zddwzsv1a9F^1*cPN*&95nF^|MuebX8oc+^ijR4r3v^^mt3zXLwF1 zWCZD6?9#1FI5m_zpxT|kto@A;B4`k80js^gmmUONjQS+-NbEX>@1Qb($ZEn45MM+o zF`&Q{4S~`ZT{M~?y8)^g>-6MPdP`A&pdMV0fKK@TfQ6({xiW~L3ikT^IEKt|_6Mw? zzh~nL{bxA75mA7ku)-O^6S07qDjWsf#tguzAbH{gD8jO!iL|`6Q+y^^Qfs==a}9aEh_NSV0Bi?UvTrtk z!>1hqLyZEK=%4FI@<)jB|Ekyivq#*vYToSMan1fe4+%pZM%OxHrxM900mGqclNy(f zn&=-NC39@uB+Edz#}20%R_!K!Dm6um!6>RbC)dv`lX>1@9U9%i_LT#C)c6kuz=-n$+}Kc`5aNA^6A3`XOxKA&=iL^bPK z-fKU=H+9k@p%+>T-)DRgZ$hb#Sjz@2#8@tZ)2&>~$ zaJ-@OlNaaL7Dr?)o7L{#J*k)9LY|{#pfV`L!f}||1MgROuHvhnx6Y>aN+~a)i;MEc zL#fZlZyIp*4IO`Sd@m>Oxyf+LaLaTQ*h94jU?b~slD+Vk{Rv9r?n#pQ6PP_AugX67 zzsM~bz1`U2jz4(F?6B& z*t8Gr7YLDj9XnzXM2C^lAfx6F?Wat@$d}7XA_H`o(k}Rq{ zCc!yav?UU0m%EgE_@dU?2nNv=Be-LDCVeTj|HwgbmZX{h1vRmLwB0DDeFBO`??bYl zUa6VFMGy0ul5teKUBucD*SF=T_n7WWGta@vtn_H|sn4qB^60xjoB9%Ul{@bX23Jp9 zfJ#XYWGRA_pDKg5YGpTy&n{Z?oo76C>`ix(T&!SMcC5>x{RLuW>bCLc(sm!WMx+7t zqVKmzhX+|t@|yLwY;uQRp`TeLAB;a9ysSTYHG68I3o_s2-Etbmaoo1-AlL#(-L^yC z^g1yVx_yF1*0QY${XbYBS*3q!GfX z+MJPyCHi+@*`Ii}gBaWm9;;E92*iq~$JT=EHX1+WjZ7d4+CYo5h$xVcyx)#~sUfm1 z80j8jb<;1?<)B*Yp0i_X2T&GZ(J_9l`~iz#_Ahk~4)U@+69_{+14?SILMMsGp|^38 z(;OTtG}JY7wN~^ln!M-Ov9LSsBr(q?Y|+&xv9#ZO{Azx{WRTa;wHKaYh$o=;hgi{t zx0N3;WNMQGy-UHpnHV4k^c&P4(^TF#CHA((!oqFrsiE>Sy_cBsasp;?m5|x3q#-9+Ud6Y=H23*_9 ztF0*MfLGy0@k2PYb>O1Lsv{&b#wr#ERxOh(O^q95VY*)f*EG(*@fxVixWjlyY~*&d zs(%!fA$VMB2`Q%+usMn&p?f0?qzXBj$;37J$Czy^JKBEi*h|C*k0$hh<^+8xSi(xz zyRD1*9yEOL8`Mp446@%nFB*GZ@D$ztLzzw5Lys&m{WwURcnItcIaE)%R9NLLf?iD* zBy*wUW3le;X1dQdE8HCC6{UydMAllOHe+5(eb$gxAOS-UMX=QU(4Ss-8-imG#8zEh z{Vj03y402&9CRPQjeGony(R4@fnM-#5G3Yf8t-7CpA>tvLFW`rDx*bm5k3ZCS){SuNikEMnD=UJJxwV=^2}VqHqQDOOR$&Po z>kMlVcoTI?JX=jTO%37XZsJVqMD->m_*oqB90aOj2z=zX6wk5d7&SOZc<=_NwXkIK5mIjccuXhGi^#y8aXp?x=TofD z!kzS^MYZ3oJ9a7LhWaw2!11uB^!LgsL77?H3e%i|fwC{1Y-4A^+)QeC0(;kgobEx|a|9;rz+c=l)k zrq1~wnURuG$%e-?mVR*HX%4=F<9(5%bDD0%HsQ{lh3p%&0lPE5$UN@G9sm!sIP{ti z;t*nqCI{^Ii6-o;)!^-n5Lqk@ZWUJYlbjGk&Z!|*%%lmz_f>w{)QCyx% z#LwW@jYfT=9{V~&d4BqUQXQ42d-_yyZ4mB)FXD5EO5dw0p`^R}ZrO)E;8N+rZ7S<0BV$dWxm8$*f8GG#5w*d=AjmffHT zlNrkxv%LLYI``h+t!gO zxDvH$63frn@>1D7yTU%Xd^+HQmqPRw{G&4(+mqrSthn(etV~f%XQqxNg748qKb;Q1 znNbb;>dH7tE{Tn zoOQx8s-|4kjq{ijLf3FGj3=C7SwY#>#YRr`{qA1^R}G>YQeJIHa$>D;&=ueDZW6QX zlP3>H$42HR;2wB}d6!Blp-uFsQmnD$p0MNVskj7ThNJxj*A&KuRnDfdXUx=4wJs3` z&#&w1%UpL~wCY3!>2Z&LyG;U-s#~hBma*i{`{}Y+k%{s)uKuWZE1ISU1O*UyC9p{a zqIc_}#pF+r+E;5Yzg@OxB#ZS)1I zDczMSRTq$B@LDW)B>B0npw1(SWV0{VE|_ASH?NLc^pWKkpz)L^ZEjek0RdoWinP@^ z3+S&D=^%se8^%6VkJrr~tzs#8UQ~6gh^$pnBdWwaiQ)T zVGn{XxI-dMK(t?Ixr?*52Eirn1@RVaP(stCgJ{v$&jFod10qoQ7Pdj!h<l*Kq3M}J-W0aFp<+67Bk_8J7#hgAFiNrBpk`)Hj;JUIVQg0An4>bc; z%riQeW`*!Ny?G`PnEUUNG_<_g{hosPNk{EkWa^ZgqZa= zyf+r8*L;{hAPMnrFx}>N<2W~PCKVVE&9v8)a`yq;+O8JFBn6%>Xvp1|DJE^b1A8eZ z0F~9571254D|}i$tRuYag%$Dy&6F)%RRyh@bc6zWvXP~nEk>&u{zi2xvv3$vK+%gfRO*;F3fvlNDrbKQzQ;j!G?o^??^x6 z$}eivzM11y}JSIjc#;DkNi#D4T#TLrqU%IojR(cXICnXBu`x))U;Qvu!c_Z_ z0B6L!%|X4b8-0yF#!{DX?X_3?aul7dQyqHj3%{=D_SdO0r(F!fU!aX!Kk={yCE(p1 z0Q>Z`1^A0e8F<3CWHrC_@5aot$D%L@#0iib&x^%CY zV`(_j27oMYx$-X)ZQcKh37o?Pe*cAzMKM$&*zpyqoDh8vIN}BIDbyQ?8DGb?o<$J5 zbkkqF|(fLk!@wVhT**;1`L=uJ&K~^zWbc!}l{< z^d?Tc4zWGD*R8eGo7*zK{>vBj47G}PZwznv=bfr>Pu0`wE6OI;MljY{hOzCbEV?;~ zhVpUaY%4pO<_ig#nl5-oxE+E)Xop= zve0^6sj@5V`Gx7dE?#cJ9D#4MPk%nLq(5W6=#OO;{=P>L?}hJzo~D!XE+z0M^i(>M z6{$g_1LDO!ye*7kvp!GvmE)VHohjka)^mx=EQ=FXVHSGIm31%O<>y(jI3DQEycyLD zv9eJ8c(KF7saEqs;~LTiv?K6ZdHhNC=GP0-eGbD6506C1YViMIPNP)H|H5jjpvnryQwBME!^PBqt1UTu;mRnah=(VH6TT-LZBMt33Hi9SGsNKvu8CkT_jl9WTL zK&`YF54=Wh>1(1)GK%V3ot#b}p5I*?K~i_U#5-eL+=-pLq+fr9CVcz^);BhAa=( zmAi7ZDU#~+nBUQR!;XjX8OcWw z?X3cmyF1*+W?oB$;Y-<03>0V~@~1B9DZ>Wq+tFihT&h_o>?+wZy<|zar@UnH_2<3g zOBAd$qmUV(KvDN88yuPYXco8AkJV7e`j{Q|`4W-*X}q$}t*jYRy@c~G-3<1jz63J| z>A;lw9>FPb;@M`7G-TzSG*7_%ddDldMR1R^>iP7u+hSssbYCAojCMCYwbJKJaPbOe zj4{G2BRkGFNd={n=(9wi$U9abBq)HlA+sF-&dy6Hjcc*=i+OiY;zEDpF#USrjknHrms9K z?5crW-L4#b8~e#^yF0)5llzOpwh}jTNA2C?Eny7kY`^{bZ^=}nWbpr3#k{p=7V%Gu zBHCOmZ{T(P!S$2h#Jc&*VlnB; zl>0si#42I_j*jYed*pY%H=VY6r1D~N(-Q|J#3PIDt~HL=u3K2Hvg6}k@#V^$JHZHL zq5aglfGMv>#3`qB@9CeAQy1$NUet#Q-twaSof82k661YIZkX>ezBJ_2@Q72d$v{VF!$^u%93XUmXRoyijAlB@I{B`5>5F2lj7X zviZc}fGxd#fO9+GrQZEI)t0b_N3Hf;h?eUpk=#9SFTMhyNI2R7Jn;YtFRv38-3mb( z)qF`8z4xwjP~u8|GQ7X|%t>FQjNhK%RZjgYHiw9-8}CD)d>$($E_m^g8gZ?0zkcbt zyj}Qmjprf_yw|08y$iD$t=oQiJ#y;`Zo2S2E#ZmW@`xND;aR@V{8h|CiX^4Nx}TtCNP#nzPJ_<{rK!Q z2aATTj@RoO-uP}m`6Ydc&Ox(!ZPf*Gi~UnN>>Khl^Xik=U1(g55Y7MxZ2aRPrb#}Q zzUgcwKSPV4)~67oX+Uu?KG(2WnBt^*-l;z)%U`uO=%ifOR@sQn4-m+QcEiV|sBpII zu5|m@{5lk0V`)?jQ-nwOnK5FELPpLr70vPpaa)I&MJDTT_u3hPO;$rK`#xB&eI4AbbDfKR2w~w5{_qV{>i{b`t1`iI zC2sjCMXAqm7P0v)WA;Eia*IrvscfK$wEx~MX)E_FcE0_`#rdIXztznsmqexNv5H;Bh}B$IEyQ?y1y0)+N!M%Jr#x?%tg#kz*1OT` zFKah8A&ekY<|!pEHj{KUY1o<6P$_10db!tfNA=xQ*zbN$#DmXyEdm>KZ>`=DVw*zo zv7=>NQCgtX>$}DySjQytW3UfD>R@=T>Msp5e|GZXx2W8~QE=u9SW=w_$%<@w3^jyv zpWZS-A)RJ98@;W_yL|kTSXV+JV-i3s@Xk=lVYYdn`4W7!W&7Z@DaTi$`$Mc6538L| zjcRwZbYHsghVY3cy`keu+(SS;gN_CHu|o`9Cu%e-sLt4FROjV(nckD-9fGz$(P2o` z$2nEmu$g>JJZU4kHwLdlvkV0V#4z0>w18}{OfW7-hy9%UaNn-Sc`AMmV==e$fZAv8 zS(54R?Em`21M`w*5o;xBJe;=#rcN3YPpqrE+OvYxO8`C9V_x@h{5cI|20;RoTh?~Y z!(CL&M0QM9@My&fv09UHiAQxSxqwc{FKghZvL}DJmuDOdV^JhX_`Cp2-p?F?Z`RPN zLs$IT;)M>uOmUK-p|m@UF_<3(I=oy#h4@rUJOC?XI*wu(t9#hN7!lG43zS|PEP+|h z|Cu3S5(w0Tb%DnX7!Lq+Q(A=KmSn>SvC_|sU!Ri|^iD5aZy9~b%g7gERa@$Ia`=gn zXlGQrTm>d;y7+EHa5u6sJ=dD$z4d+LYm(rqoySBh&;_0!))&j&^V-J~2i!V!80Fbd zjvFPOA)|w?2#f{l+;Vf;YLY{`O8LYqx7MV+S1Mg!I);hD$l}YeJ7 z%29j$(COCLh(&R?2b}UXt*D)nGP{0+|fN z-z(cH&&YyLdndx-ge$@j=wbH-Y`(Z~IBHBWl99ku#Cp*&$_k>ilcnj5RFhdzc@$lI&< z@0XXB3$<6m8*49oraRqR2 zE*UH*Egdd8?%^GMB#Ypzac|~gp2m%th@Xx)iM=4N%EOQtyLq(F5I*tzvr~tPiqc#O~l-x?|0zcOvy$5mhkjLF(Dlt}H<=Y>fCT&E zQFYF;i{5_la~tPz(ilk>0BamQiq3H}@%Kr$)EO8w^@u7r2g%Z^yg0`B1d)-x6U0>$ z=(OR=F{isvMTfitCCPlR4$%cRAyXdrc? zT{u)U*4>shZa5`Yd!cG?FRGCu%&xYAN*qCl^r~s_ABJwJrC4R%nr>Lqt=8js^*vhn zt!YaB8DKT?86A?z!vHR~$h4c}b^064k#^_nZjYr%}>HDOFyBLO=QBic-aW*P9c13)RbRO+lf2c;b3pkuk zgZ`5bKxcIjY8IMB)PTEQ<#4e$#(0q9DVrb-U!O&M1m|aWJDNPxGX>Hp;{&sZF((9m z7O@}W_Y}PS3okdE+J-Kb$hrBbGcCRHjfA96WrLa6kc-yPI&%hI-5(l?0;{CA4-K1u zZ1u1*H4qGwjA6>hkr*36<>N-$GSE?TeyinjoUmPxM*F)ynRH7-!3TR!EYlCYcjXR( z!@N&IK;JwcMP-jM??W>eq3JvyH1vo#m^TgcM`xHR>3fuPc*=IXsmV0VC{&EBIkT!# zqY^Ei3TQwpXqFtoUFvK}U`$^(adD9QF|~UDA|3!uch$Z-LoG84779yKm?{t5IHdiytoW8-8 z>Zx8DxvMZXR$}H*CX&lyT}B~Re2g_Er%BX`LZZSEKA^0y7&vEwRu10qRjMwoHUui~ z;~bCv*5V%imp2%9`_u!q0|UoG-W{*)OnsOANpR=$ndgHfDaV;^fL8X^f}0CIHoT;X z2IJHnYCm|yGUQD4D_?(*M+K67|{n>+r&2SQeE>|%x~Ea)=}$F^=(neM4gGnAQQC^&G2x3gEOa@ z9&TEx)TuJbP^msI8F@OYA8~KFUh!)nn_kH9fR~K1WQMY;)+FP$5uYe8h%1PaeP&yN zKJDB2(d)$3VF83^Y8aqd#_+39FtBi4S(u&03qwVEMn&E|KN(!rYsXR-*Kt~;ekhIB z6G7^I)B^}I_|$M*RFBeFrs2^iuU_Oe3FkE?-2N%P{~@f?N~4xF*zoL+sAn3&@PhHSL3Ows(pPk zYxf44UgSF$E+cC_85O%v`*>Y=5%`LR|Gm@(?kWop0j6pzP<7NMFg27xFCc)(N7Y@2 z=wdmb7tj{;0{-JnT0IYC3X~cC(2U#fi8TH+_kQ2y5Hs{c_H{GL2Tk3Mo>@l_1g)6- z;COnthV)<DRKse|v6`YX+F-e?}NX1eTnks>r^R8b}&- z0mk{g1e5Io4$AL@juFs_obU4A|MoQE3~+xJX);ds&muxdSs$nU@Zv)r(yLpNn=j48 zwtQF|Staz$y+r2ZmQSS3BmEH_*>rze>S!62Z>%rX=X?%}b5vk!ZNsUi{#fgaaZ9%e zKCR*ZvZ{Zu>X9~A2{2w-0OSSAZ=$0wYV{vw&tPG_Hg@0-|A#B7h_u+sE^z`6mXj2!8@9 zISviH%SEvS>>;LvCD5OqX6<40_nMaV_`hd_0JS7l9n50Yu9pYOyi(QD5YI2-VmFLN zo%Y=bVnh1KW|S=SgMu{>9MEF4_UcB!BG#S}beE@DjW`K~=5qT*0e3kue4QpAZgTFK zE?>Hr_ryzPYyC>D*@j;d3KI`{kySk(?5fZx^_NI0rsN9#22Bt}K^ifUNNcOz-P=~U z-F;qffZcR?rm*$bB&orh{HBN`%)SAYSfK~#FEM~k=%^_iS%zt$fF3KV1#)H+Q54K_ z>Bj2Me`(09+~&y68Zx|JAmwH)5 zdix$e_F8+P4Smnfxi5Jb#4WEPV~23HM3WPWxS$4Y(6z^#PGU4)0Udgvb9*to{6leG z-f^D#4DxQuj>(!s`=g_MxLz0;xJHU{i-{C}qpN(|v_} zO5AES+Zrxa?`FCau#U=V?;Ez?cy?e*$vUpe&3(B)19$ol++(zvLqY>>YZd5&xH)H4 z3>6> zfbAqtti;e4H9?bic23UG(uJVMcd|hy<%EqXXyZ!;aDrcvww=qO{OD?alPv!G#PNUA zzuDJ;bNG*erauNBXg#a{*aMP z4e6~a^@6DMMwu%JTpZYdfB-OhLnCQa z=Lt0Ah)JEANFB|kZR_o+L|&uAx(wm_>Va$xR}a_axgzU74;TS7NN znU)QgeplpSIiU=O(BG=_7^EqZnDE^VNf#6nF)Nh(AG%66-K0qHRxV!5bKY{Y8FLN& z3_WVcGGohea@`_Apyg~7Cz@$96SzAlJd(KDZpbLEa%uL9`ep0l}sKTZ{z{iJ_ z2~?}G8G}dYQR_z1q*H}=3>YwQA#b4j>NwD6G;>{Fkq&qjpi#cM3pg>Sy61{l&dQ^t zx#HFJ9uf2i2-D`OSD-QM+0Qr`m|LKQkuam>WD1uZ#lQj0ObDX~H<7YaLRbfIiqGe8 zrqkk54=cKI0nT)L)aI7AR%)&NJcji=)dcDeEV$DE+?fgowP zE&lQ0tFCt+UcA^IzO_|lRc2E^x1?pmO|IGZe{mHjdOg@eByvPCCVZMC{Zh4W&necS zB1aBAE~P7@ba?5JVvP!kGiJSm!<*2XN9GDWKTsDw`i9JsI1mE!fuKb(w{I4~9Dk3V z3=oI&9e*rH`Jm0gvbYY7Z+_`Ei?G!I0iE&{P6DwKoMPYBqrQ2-S2g&*D>e~9G1OH7 zkd71Jdg5p14^ZT9_SJuSAJ0GUH@%N}FXMmHLz(w7e#ZOx@AERo=yw^y)G>Mp!$!SK z{W>$ZJLG<6S4XGVXsT~Jw4pQB7vaK~A;8P}(aivP%{r=RF)AA>t-%J6 z1au6#Fx$e1`@R^2%p^U(o!~xG$?{Cvni#rN&Ge$<5Z~`|NR?@FbUjW1jK(=|o=#zX zyY&hiKT?|s^t#x{Bk5XDlAcwzjQl;Wx(s|r>jXC#FQ_oz4EOYdKbdpHL9Gn^2jDHa z*r461#1>Ls%Uzm<%FQ0(7_k^IPK#~_gVS*V|FuxeEj+aF_A$L6B(C5T# zQg0w(e9hNneE{%3?x%u6;!ic<{JPi3aE19KsOUZvg@>h!!5J_xXGD@RYaUy5CTrG7 z738?+sR!PC;>%UbD@J*TxD5LEzi(J-EkjY~P}k53$Zo8yw2@p_A+<^S-9>@DO!ED3 z&&)#=2Fh|g{wSWmZmuEh$au&WD}PE;s#HTu!RpB39D7 zk0%V6+|{IkcR6D$0S5w8ycHB)%z%AoGlW1XuC0;omd$Xat&{N7dE}-%5^HULjW*-@ zbnS%CVi!GOde=8Kj^BE)3D3+>4d`PW0$va2@rt3=y!gh-qf*0)ZxA3XqKpU4@c|*<{VUlL=Ws@ z5CIdIfI28{nPWhersNdJp^nJ`2gMbX?i!+7+tJi*bu)rRAW`#S8%V7-Xw~z4$@rPk z;6MPsXgQ;eCQ8S1?y13YIJG9B-K{cBqfRYHWP(5LKA5fi*eoV4L9fz8e~S3xSH(3p zxx&03X=T6oDpu_6j3lqGz6!?Luwl!budmpu6{1+p16+w_a; z@vFItw7J?CAkZy=P(?-<9CrhI^&@*rI=WqgsFNb%PD+sq=q%g7Nb9{zS*)s#X$qs<-Q#VjyX2vzM2u5y%K-xox=ufV`kB}f z0teVg%p!PnA$G(spsNz7EQRE(&J??ZWlMGejN2yE{Dt$=GruV2x5E5M0YItq=i|Ip zm^YeV@!9iRVSX#jZ-x1-Fdr+-M@aJ#(tLz8A0hqUh>-Go{wb-A4sD=^YG&SXu}{$# zT0RtIEE{vi>C|Z2+Ep@sSl=bX%2SRoZ$3+jeSmY*P_$ot8E(I~k#)SszWl;`CnNXx z4W=u?{6xbwSE0_I0Afh*=L(Sj6pqoP-X1}Y3&Lx{bqV?m3|UXLuf@Kv=s@2N-U?;XQ5jnge z7(PKFH)8TdIZF$4;bqRCzd$;IajBJ3F^kxy2GYHLcS*1*XiTKfUpYTZ^DAV2cg&v* z^QYpxMVPmS^VaZ>vWB4Vps$V{MG}KPi0%idYtE>bVo|JM_{nw0^~xw z)Qx?lwgnJ`x~CkJ+27z1E9j1$K)}Vqzb^u!z4*`?=8$Y4z$@pJ<#z zpo{fiAXrDhCFkx3|1JdOZw0<8laUOio=WU>3>WEaNKX)oPbXq@6JBa>X_qs;4(l3} z;=ZiN=IzC**HZ2S?mE-n__I)nwKiqjK~DJ8M(q)M=Z7}7yH7on+flMTRCBBEg($#0 z4)|AmFXGNJVrYZ(b6Je5G>jG8+4`wzjM6?hN~gC4i|z~g*kQU7$KC2|?uzkf{bb4( z-1>gFif;5Y_mw~8(;mlzol$E|#N6a$#x<8YPImt$@c9po0_-6MH%l3=>OmR-sK7zS zmVPWHLc9|?XV}7((>S|dB<@8X5$fPze@+Hz7vzg6s`z29#I$$=78~pNIUYSc- zHgd_}CrSL_Nrmsbq4)8$&LXnWloe!5dm#Bteg&52b8Eq2|BB{{)Ejs?A)b7vbm-Ff z+Jlj%dURr<@&q@`D=?>7mkNDwJOm`DTS(?Z+_TO95Y_?uSV~YI3`)^bV>o;2wsQSV zJ>~XC*>(13`gEGq5b@Uw7T0&~cq1HvR1>xUv$GDc$a5;Z7W`<* zi7l)LQkzCT>wj%D*ajGe?0e|q&FD!T(46|a{10VX86BVxeXrooF$7pv!vrZdCc_gy zvH2C;+O{a{U1yvh<;zjSAK#VBqganKt|tr{K6upa3~?kc=(Taede+bpLmxL%XkE~}F47n9TNkw$Y}XylZRJABKwo{x z9BM82X3_k+NlSmLlRJO?e>SL?pYQ*-=lf6GNjU?0CaKe-(|r;C??0n*2J#{qvbzf1 zZxFF(8+%zhqfWl1nh9~=>%}4xfbP__Up2vi`pAx2H_9TL`^UChj@IbhgqN@mm*3Fw zXBUK`&zVNWQt=ast@0g)rj?&7hP(-Dl%b zF1?l-@a%#8#r&WcwXsXA5HELo`BOe-vg@LXPcNNlCnc|A7jy1{HvVKZwI0fGPhz2= z(n7eH#Y2x{?( zznEh63JirJ2*ki9GNJC`L5D< zs!&TwXS{kuaE7f!oL+ianmsL}>)IH0y_uL_=70?4WYeWY$!i z_GdL?DO3D~B%|WjC)c_hzKNfbJ5jb!T4M}@X&p>u3nBZ|!(k=(O#u{)mmSH+%eE{_ zdsON?c1CNjZTOaFyYl7L_G;B%cu(32qUgU0jeX>R<#P@jj^D^B)FB1JnzZg`84PFP z))*Catb9V3@ylf6YZo^wE#?|wwodB&gzDV4%!QWYpp{_}U~`IYr$L;{_9*5y0mwS{ zM_w_oK0w-QJ84GdxIXApxdk*=gz35uuZHI|RQ4bL% z7>(rVQ3BM5MvO2619DF{=M1$>>@J9{?UgHBykA8nBDXNKcRCSyngh(E-5y6ekU4Xt z5-`n_shUEsk-}`(^AVC~oUPW|5Nl2Ji0LB}4`5!vVJVK=!igR=H0`HA`)F$E%)AE@ z#~nNlbRS!3y6?%E+wpYb$$QJ1nd-leI}vkm3XV=h2?`9!a|GuR@^i|1m}$qFO;S27v7B)L$$(kZ=JOeB0kya102y zOHPsos1k4m2iWwjcZxuln=cA_?N2)hmtMib*JFLqu0uajB2fmD&KJ`g@paP2*m}KE z+`mkEylmU&*sQrpKlN7D--b?f4P7?HRL88tw_$RelUOP^VM1R^d{RZ9nA2E>lR$EI zZb4ROtWBx0rOaalr!gU6lB*A*?gBcDU)RY=avkcN_}6ukD<)odFn4a96vWuBd;`{r zl;mV9HeIw^VCdnbf*PS4Oy|j`#(AeG8mYeRe_#@PC4?%uoa+xOuEzX={U-+xC zh%Ma)Oht%iWB)W|%VQjOi{a|j&quu@=qVkq6=ObON2k}lu$aN0cg3Vn%_Vh8!2Erb zOOTmGyod%Qez|~V{hI@S-irVBChcz~@LxPF=kJ-H=l_dq;7@PGXFh`{#E{|EPYOn7 zWhRlafk%aVf^z+yd~W35r4-tDt3Pt=L`XINjeD0QCyDh1Gu(7C=f2@#mO762IU}Q& zqhE8Fl;5PPY8XOWeeGe%8CzCrnBUR`dR4iI%Mcszxs#{Ejl}S8=*NW^fr;G=*CT-z zP4bSr7}DJ*qBL(hWpcHbC4I&uO^qv#Zewj{d<`J&Xx0^X&U_6ZFQ20xxjYv@a#`^n z0RbdhV{n|9`TIZ-KUAgX^updE+ek1kz~U~QVVOFha*%8la|%$;1fnfac|J>Sv=E)M zDxZz>&OA0+i*63e@GVnA$SG;AP_8uSfZ0X96kpY1TXOhS#)Ftn#KFMhq>k` z0|FX3AawTyNfGtWq*@1--!GT&{N)naqX| zhthV%z>~ey`HO{ylas4Gs(l;MVA{h>@^-DJdoP{%K#BWrOx>VtU5;fa)?LRNbPo|0 zP_#Yqij}Pt7y41DF8|8?L51(iW1np|alP>}U|6dHfAvTDs^;(MvH$%{1@oos->;lE zKZo;k`0rW+fA&)LCmy!&AYlIn;KQdaaUgMC0Cxh~Dd!_-q< z9Th6N1s8{(@&_tb+d5aT*&!Fe=VI=h!!@q>D?D~}b2y|hJhvw>(`WJlEu9@dzp8FY zmd!DfNGp3%+rEr)r|afz^Xp zpArxF2o;EdQ14U1xThu{3F6)ZUu^A!sM|nij$+SB(&*Y$Xz2Pm?rF`RDQbUTrB0iK zi_UY>z>85u(^DB++^BK;TNS92mZI*hr@knOkYh6Lx;IK_E?6aKM6JR&AZE;e#EOl4 zifGfNU&!Uez$>;<@6;TJR+KHGyzA>Sb5^~}k@ww~c$sIRTOAp*tDfg?Jje&>0D0+q zVTgo4p98gm2&PB4v(9zw-2=4*Rp0GT3j9QhZ=aRFDesz1*ph(WIWhy78htN}+-U;I z>U|t&&)t2`qw3XzZf<@5BEn&(d)T#pZf@>8Wv0WoVwj{Tb%rtzJE1jX7BNz@1E6bH zIWoI`jWZd+L4zkK0OIKdv{p`N6;SKU7@=Pwt%82!O#b#6Kl38y@0p*&`8Dt#Sp&$| zgnvcG3J-RCi9LFt*>DqJNB0keBYRr=1KC3SusMMo!D8macd$y_tVQ3Q=syHJfAsJF z=Ku=R)@aOmY$Vi)Q*52`3S0KR?@Rk=XNRnrjN@4qQAe6z!a44NI)=P1z@7oSax&u} zz@8obCHgu;oVz&|7V2Ot*6VnMU3}mD<#{rEw^*~PlalZE?Y8iTGI0KE8J=js7o-k zXv+O{qYYkE;53;;a9$AKJ^pY^HNRbzlD$%`qT}t0C?y6@%4kBVkKq{1tN3VGoG|K< zV*aSJ#%{5b`w7NgFaT#q0tri+pVR5OW52GEm*{jelCu)VXP&ZISZy6_Oy@me4`$pROc!AT1cG{dZp(VFZ9bc7Dcw z(Va3~rcg|X<^r=~NoFhM)i->Q|8~u*-%2<+fBpZSqUQYE|Gzc&!ZRPi;kUM-NP7zl zT}RXE#fIo$1xBQKZV-v58r72hC|7KK@+Dd{xwTgv*@AWx+3iNGZ%^~KwR>XKXuT>IQKo%B>2#DHmxal}lBK*0rSJQo1J#4jIp`|$5OMkA5{%pZ?`+GNhlDbk+}r&J z;#Q~(rH9B^p5k8Qj$jIn9OLnOY2Q~DSR|iK3%r`^70-XHvF5H>yrZMBVF3q7>zfWk-s(|Zt9wrowT0PI` zG+W583d#+nD6qD+ZR^7Z8!5b`Vy~@m+jCOB$xg#O_nDJJ&jw_?Qq5XcL$WXF;V=UQ z^H#BRp)!4Zt^RKDX5x*$x>e^}bi8Y>%h^YIC$tqt$f_(p{$Bf<8?i{~-|m6@y|w#w zZv;r(Cy}_})o?wO?KWLT@~7{25Fagw3460mh1QW~${oeHyz`NaWbciH&U&63!0>)g zBCG*&yg)$s7cyH_BT4rD#>P1^+p$b>rNKEe+pMhho=4&mJZnw#FA-n-&is6BgbzB` zbkv(>-T!gAtS&&B7p|mD<IA|2gQGs^)ZsrsZw7s{hoK_MPLKqtcdkY>J;S>J%ms)=qdaJ)aA^Sq2SpW$ z0`CQsi>t{XqjEw@3uMn*yMWFZaz+ue52M0Vc!nei~P=`)O(eo|XD9ruEf zME&f9vdV8t&+zRoKkHNF@|GB5u`S`+@-l+6GUb;}gqL6>|G!XHBGQH9i$?=3)L2eJ zdXXY-TkspHvo>4l4gS~6mnx`sE;NuDFnH>6xhmJ2@G}g;AZJhMVHvS6GRC~9b|uG! zTT~1WpKYZmP$*$HEk35-N-y6f+>{l??!75iHz&*bjobQfN4XO}P16Bt&esQSOW@-? zVqc(okoP{#cfWHsO(FBH?Xn1`%NO^_cn2?3mA|%>d2gAw<29hM*a7YUd6#-5aE(e0 zp>YKuv7Yl)VqE}~gce!OA`+#uXA##x>TF;qnw$qPpHF}+8FvdrjX<%l|BY(c32dgr zvxp&{bl@&Zqh|yo7)~U{IuZvlMPSQ;m_~LV&m_wjT92W2K-9w!lWPRcFpYq>KY%5$ zFduOiv3Zl-EMf@_sMz1+;utkx-iop}^st37u+Nn`ctt#~1VQ+kCt^AJ5O{ALjEVe|&sg^k`o6J5*$t8!3c& zPygYU>yJyYPVyU`9i;^C@Q6|7dqTK%T2QMFAezx%Q`;}q>9-gUGH@#y$bR$SZmHb- zS4}vrQ_f`HAw6Fc%YBDSH8(6#e6S2!#})zoSnW7~olX)hL&uEGBIH4Kto+)(lz&2Ep``SM&8TM8`9jcqp?JO!KMx%E4+1lo>Hrkt z%(+V%)7(ng(5wSffOHS;NnG)`UxL`TB8_Ko_d2$ZT^AeeHuBG7J>w*Gxvc2=^ zC}W$TrO_RQqg&$B?k%wDS207tuat-TmcI6q+e*7aa2K60qDbP{sOw1Uk|?k#r~Y@;dtgO!Wqk0 zcg8=xwY7b3hLh9zF!`)%vhen>Ja-rJ?efANcHBu%=Fq8r->w-wx)1zb$ngrQB z@y^*7i&Jf#v#J+bt~(Wf+h*tP^p~_+O^wzJ@47kAqa|TR%gGcjJBon=nz%*?RCD!k z6lg6uhi)sV>Qk)X|Bx%-ZnKlw_9=J?rTM|t;@!zx4lQ4%iKxRne)%^LKu`8dx)%!5wkg`a$ypgAZtfw}*yghc?g*z^;0 zj^D%kp|u366>7%NAcBSXOkf=-8UgS~8K_#1kpvo-tdX1*3OGez0et|CMfnTRC@XS% z7Llv`rH&!X!;X#!M$tu{&LSF+n^7;m|LA|^d*KNJWfv`t98s)(Z5p*4Ry$`YElZu$ zb#EE7cXGsftd4b;-7ub+9BjB&GQLgf>MwN7d=qn`36DTKjU#9(s&Dr(%I3rt>EJXN zENVheAF6Z{Uw>JV^qnuCx>VJioG9{w#l<_i52r^}1t|IzDn1vUJToWV@EsHy0sRx@ zg}E0;WT9||b1%-E%o9e+GjzPbNu01+lWTY7N&b1zSUc$%_l_^LQ?N%J5#2L+{M!%= z-a+b3M~!xM>e0>Hp~Wa;~yJk zzj+4w7HZ~g9H$myZ6&1+1MKvmCT3a>Y&1TY@aR7ZSpR$Q`ux9tce4QZ0J38OPzCf= zajqtV9d{jrfJ(qIHV}&gd*JE2R5)53B;o07uu(TKo&mJFJ$mw#Wig7n9X+E!0DL{8 zTA+a$M8Z(%J8_v39ZJ zB>MZCe(m$^NMu1Ok;9cvqDrHw58xH*d|;P~*UTcg0H~R7NGWj{^!*Rfe}7A+Z!lsI zpsMFTg>Qad=GV*oKKk2XGJi7s?HHXu8Rk!h|9d9`5zWXZ&)NIZbXpwSES)sK5B&6+ zm(av0+&n4_thxv^5W}c}iwh#)XtT{IAJ9#Qo>}+RLE#4u%EL9J2g9%n)EfkiA2tFZ z;s=q&S;WpgkmO6-$O&?L?86r@OFJ2-dQj_voL^6Vc~&J z5Y7rXW7-6!hH@`*P9rj^?m9#l%K`K^wwqD;e?-LguY^_q$!jjpA}R6iin#u^9E8g63va!_>GG_x7r3bJ-ZZg{}k#=7Kp|rpO*39^}CZ$TjB( zI8MUfgi}9ACH}R-^*{ZeeT;L1bizCwT0nB7SoUK>QyT`UT_cXGjVj#a<5m^9pSvDL z4z-oK{JL&$F3S{i4gCyFxy>>I_vN}pTq7)JpX6ka4KNgvS`EvnRDfg~nT~$t+pV{^ zVR&%U#Aa0CAF+m@wHT@ua>j5Mk;P?Cc(apL2rsz`g3X>jh~uA*p4`@2il%~2$QeWH zzg|K5<%*84S3r~TL)_EiKf|L_bcypAFUpo+lu#4;OHfj7xqxmXK+T!*P}hVhWf&4< zmE*AH-pU%+@yLr>5l5M7-!o?ceJAraN3w3Ix*v*{!WN@$v&Js>JY>_^X?zJQLt1>q z>hhB{`G>c1tM5%55c)i6^XsAhKa_X+36ZXE8FhXorvIbauaJPT#TA^7t^~NA_?b!i zCh-5J5%(MZm`w93X#*;@470SGQB&3aF7xVr`;h{#!WY~3t0z{o-d6eVkF_mcNW_H1n+x<#>xrw#6W}EmY_);yGm2GW>{Br57K|AE!jq%4B z1eo8ycf|foL=G`FgFSvP|#?c(P5(SykTmgNc%Oju4`Vm z6*haS3l(H;KE=Dh$0*}|@C=e3$H|Ae;5XfS<7uLY?o5=6?&s}o$ob%3F;Q-|z+`E) zLQr^9~Xgs-dyGpmL|S+ z7O@!ZK;i2@z_5o@jwalD(#qU3z66`GElT+ImeZHyU1-_rn|C-XaS!V&5u-@zK_Z~*;=RuIKI>g|n=TaB7M&K6Nd8ee@^=uK1%^wO; zIhby*{-&YuL6W6})$%JJwv+nxpL#HsosN3G*QPc784AY>{Gkor7%*POFPn~UZBKXNf& zK&>c}_mm(j`b{BGIMbFB^_K)xw*gx(lmi0U3JOG&3*WbO@9*1&SrpZyyz?^0oG)s)Rh>m zc~(RfNqeks?6@Ye;hn9@xKx^GyPV7eK~GKD)t{eBax-uLg+SA?7@dP+uY5p1%^{2t zak?I9@A~S5J`@_O-M*iAuWTLlfMej=1;U2yv zjnClQ!WLgisC6Sn0`P`ql8OmbYhvJ?c(w&;1uQhw&f$4<8njk1j^IX>%zE&*;&jVSa9wE?d9(i#C$$!!T-jX2!7x zoo(my8bj0I@tq#Ne zH3sLmk`+|*FCZe9@oY>)w(L|QhLRjG9ch52G(qM)+sXAp8spBmy1szKoWm6s#h(#>w0m&H4g%HwgKax2T_lya~mbgHW>*!thaI&yQ8{S>@0tR`OW(K1Izs0 z4mMd1r;gHCyKxGfRJJ&*#W?sl_pN*8v*T^YWCG}KEEYa77qJp37EDin{lUPhx0kzT zE=&83>Ha_Xy>cC_0`{viEsk^54aHd#^=bj&DCw(XN0G#!52B#n|K47So67U`h)_5P zj)<$^h)8ao^v2YFA0!Mhf1m69Ke^=pAA4^e59Po2kB^Y-MfPPBB5P%<$XLpfWGNJB zWDTK0$e7ELeOD-zNfad}`)=%O$ewjD0e=+nR|>)C=&7J{xWmA(Bh1bD79+8bVAg1lmZ>~s9(m7C=+MPrpB%aRiIWjVQB zr^WDZl7dIoEFavmIT2+1X`pBCk`}Z;*-Cr{5GBdZxkMCbIDo0NO)24EG=e!|qst=W zg!tx38Gk2@Gq28M9y(Zt&_}!0^MHG4F81iU2nRjRlfw@SYR^cl1zlY`S}R*Lz>U%9>vHmp~f9RnTIb~7I7AKgm#?zc8Dh^|csz7T}_qN+c^Hf@Cts8C!qmrqiJE zq2w>8JhpPvZt_aMO?+ThwcsoKiY~>UhXBV_f!{)%WJA7+xK=?jbeMgu-)vo;<)txV zZrjaPbd}h-r~yC52JL2>4?T3P<7@}hO8l<^^`GpQC_r$b8N%Z*xL=K!73EYC{lBc- z58%peC`av^yYG_!bjQm6Zo`6ug?~GS;pb>W^$?N(L3Z{+dj#xmyc<>B(Bstuh zEt<|>86O>u>_2J2@M_k(CrIG;`1=kxK46A31jUZzo*+38{YDCQHksZ{eJPrRt$Vvk|`_tw9V<v#ocFxCTV+C2ouc^-Lgq$ z{RaN<)JGdJc&kmYD(@rHt^BInyJ?5Qz}#za&W{?71FP~p`d!|?sVWDDBf?zJg2+KE zq>#o#I=E}{c{q1SZezsus-b-avuDd^dA4hs^O>4g9w);cM!}TCJTVpo>{*dt{4&pz z-~~o++&{aOAAUDvB=kAK*~wQ{raDPKZ&&y$lPj5-)^}T(`W#7TjgTemY@mRA?Y!Hz zrPYop-Ja&NA*OKzQZ3YYJA-rhnk1d{3dr za*oSE-cZ7FRjBlT%lywjIg9_Ry{-NfxX=H!4PJEulVJO_RCXWvhh9THAE^cmVWQL& zB-`39lAFL3n@~`2gDai4`ciHB^tG@sp)0{!A_m8vewoaTfQ+J%qnZ)~*b|sDG)D6T z{$7gqC)a7{kYq*1iIWlxI*sr3gd?JveZ{k^DPaFL^csbiny#tjPLnStO?Y_}lB&No z_`H^(JxVy~5#^xKulK1Q5%QT|WP*>$OCp$a0tkUsU~sMEi~gZdCV%=?cD|?$`qJ~B zSzeOwS{jem1YBxQKgPn!qb~=M;%6AH3nZ>mU(}u^WYdmv)XW9QQ5=YCLCQ@;wcA=X z)nniFTz*iBDm$_tt!HZ_opDdVgKmB(qAMQOhGL`KCwae{B3W7Bhl;PrecHH*Q@CxA zDdw2)K=#Dfy#rF0&MpX#F62^pO=j8KLp9I(!{-Y_iR2-6;I(P<%BZ`aeM-+i$59j7?J=*$gWK*2Bmq{s~KSsHll%fVtFWu3A>3f0Ev%)B*R%w4R|5 zHe#gNjM`jhg};`XHHpw<0ve5LkKwZ$(GO0<&-MwCoLT*Ls556R&7XJ8 zn^$+r_Lh^iti`y+fir2WN#-iHKOv7+P!%Q=?rSr)+oEhFKr%c>mBY?Gp8I0zOjK5* z!~LRzFQrgtlum5a#QU32c(&%nDKaFBnZ&Wk*?GWmW&e9+3XxyaG~+6&Sjff<+Z{kB z!O;)7%FHwXICGLDT^gh{8C<<7_rE%gBHZT!k5<>bnlkYmcdgP|VXVHm8Zyh^>0)(4 z3)>j}hVv~_27J`d`Uw+FqO7E+d3R~2;88^v$8|(hZ7&+6Uwe7oOXZliC+%S|;!BG7 z$qw^eXe_jX4>$ag)&L0CN)iHRRI|Nv?^@^luw9Ys9WcthJ)Wv$t)A%^-rqL0kgl^# zE>H91Bf68A9o~W}UL$2#m|)N*aX2zm!wNLFb90)gQ|6ZMdcpU`126Mg*lXrN4D}+o z@y8Ti0WKH@G}n!A*)UlA8h6|hLNU@{Q8jci1fiXFu#HLs2uw(3LJP)79byD`lDNb z`%G$k$ER0uX=3Yz6*OindkeJ4t0||tck8`DGi);O#S0jFj(z>tH=Icn4>A+2GgB3B zj16@pIb|&q`ndIZHeNWezoynC|L96TEz@!HDu>7m8-BZnB;d8;H;yeh=@J?6cWAN> zn)t$JF;oi+`Yuv5rCogY`DdnQH;9PWQgLXP-%Cfjs^=(j#MburOJjCpnh`1)zs6 z^Z-iW=wB#q_Y=^lg}Dpop;(a!fcJ;H8P|)jtGJvjOiiV|JTA3S^!hRJdr^A^{br9g zEa78Nl1zEogKS#Mv~uXUuE5ZSPrjb`2fg|TJ!sSw=)D-qX=(!C28_GU`i1yiuSBb40n z*yiYW#{&Q-T4(8Lo;FRV(@A3+R}D9$js`U<)M&B-Na7t#pP4zqcv_40`*cvcny!xyIJ$(6>i% zUHu#Vp7yn+wGnNy0rMG5oI(P}uDh@_9R759sVuM+(7@6GGoLX06evLx)TfvAhO<42 z_Z5ksnj05UGaXoTW^t+xo)uN;wbc=q`5v)x6b!}7D>8D)pMCDLBt1NI;gB-qA7PvN zpN7Q#1NHa+->>-(>@RR8!Twq&!X30n_^Vt&2(|tfE3?p~ji!!w`2`30cen2k{}B%A zw3JTQpi5i*;%_0~tN}6M8L`?TfT>IaK`}VggoEYrbDPKSIxvoA z-E{z-WrkYryx}YZ8Ll%0U}y`^d@m=z(_^{Mb>A~o7LV?7*&44pU>1M-T@|Uh}9Y z)4VA^A*T4zMCYo&4i#o|=LDhrJNNK<-GWXB=tC!7W*W9aeBmV^Rn`fwM0_RKMG+lE zS-jNxVZ5MZ-RCah{s7Z$S909b+$_TCtoSQOm*st$1;!42WDQyjtW(vRV7$f)jUT3` zNjfA{s;Ff;mBWH5aoR*!n{!v<>{{P;G+7s6QA#?7*KNb_6vjm07hhax0W^3vZ){>Y6ul=UVIdQz*W{ovL7q($W2dsYCfzcki+)4SgTNoR>ML| z`HhN7zr;D7O}yU^a|6N^@wxC31!Z+Z#R4zmo?^KaAp5lB0Sxp^kpS-g{Rd$!2o);G z|913>(1a#x57LBO&#y5|$S+oT@P6=9Iuh|HFxBqK{>8lP;ux~v3yI`rA!8+mcsgh2 zvk(qLiR}ZyuxD_Rj2Es4ze{{X>dwL<6iRKybqT3)6)sofp7In7apBTCJ0X-k_;AMyv>gBXh|8gpS-f=jNnCjJ>(|}HaGsa^V!$;xTP@Q&@bL1T zO^nz-ChssCuR3})JkiSLM%sn-jR`-fx0XoH(d*axcuLRQnmn5z{j8>F`D3;O>dhO@ zRuoAFSkIkldEb=8SBQDOTN4avce31tn!qok=!%=3{=6k%5(8PDl zH?_;a_$ZABL=vS3Jxe-0mnn zYC1*ou|xkh>#~o{^~TeZlOK!S)Ka9s$BCUmU5uJ%OwfXv(4au0KFrREAfd^ON0u=~ z9;@^X70S*r%yq@tgda3Hsqqd>3@t8+bmZ5cg?@&C@(jKfR7qy8tIH?PG9lOW zMeh9z4W+lJ`UMdG^ahYTEDSViz`V9eh96fwyl3ld%*qB7CnUw{27S3O4z-R8Ou=1+ z4^nar`6akft4yudj2JKt`s8oDp%=k!-#gc;MJB&La5MN?JqT;BhQbN;L**y;}Kc2b;sxC=G^%g(p=w_L{IBK zVklAKGSo9HE(_-?HM-`6UX3HO@7Ln(J18-@DIzu%y56iyf> zS!7eZ!M9wAkisVIpMkFc zoj$EVC`940#FVeNQdbq5)Uq&}!QF|8M%mYS6-ie%%#83(R&?H5Vz)j+s5JzWsM5_t z{ti|I&%#zYWh3?2wb1iX9@(z95=kr>G)r*AO2joJ7ASYF0IQ(5He+3*14}K|Eyun| zbV(W%fQkBK@`2?n?Y^f17UH3>Z0LlzWe^|HlvEeQ>xW`>`x<&a>_UDksu}72Lx^_B z{f=-(ZkzFykVS2dutOIJO7PPZ*duBBoxyfgQ6H|4z|>FyWcd@LXTtel-l`X8pAx(0omCh41-HQjR7Gcwd! zg1meZ{cPvZ7g*Hjjw`$iwGL$>@=w9ZeJoe{Lcs&dpU=VD742bgflf`xIZqA znhx}*Vf}?Ahy1aZm(*26E)DC&&Z(bpR$VoUn#V*5dy86Lx_<_r0*i9k4|f#&XG3A~La_TeuXfPOO!Ju5c<78v<0dcV<_NJ26M>VgGWa%Q z{9sppw(nEP8NCbR7$(?EGMuFhh=qyPx{%HMmwdOYg{~f}dgiTKlAq=tt-XBZe#wSN zT-POte=ddFuX&^%?5Bsuw9Xm{h}wAYnj9Is_`@MkvQ(+S|h;e_HMvCK~d5|0~lPXr-c?L#k-5hR2i&w2r*^G~O8KF1;eb zcSs_>o!r$hGK>Um;uxcU>qv4OgLzg^*8&X*C-prf+NvgB!QIOkK@rb*w zwo$^Fqb4b=EYfTs_<}A`l=_aa@Mt=l|Fxiw3ufP+~SOPzV4t8(>fkU zynf@lE0{VqqC9eV;^LNpg>CVpH&XOft;p0-klC}e4rb77AMl<_b6AQv2LCK5lQ`0c zgS(S?kpr!Md-YdH6A3TR?Hr}sLQ%K^bw_G1kdLkGKf>frx>h*c7<%69&D|_!>DxFy zBP`m^vu_1Ih3o)IBPlBI#u~3!01w8KCPiZGL_Vx22)$lOR$|01?wxXZ^!VQV(CFYc z-OsO73fKUl-=ZgafeSh5C!|@M);}JJqkWfN-aD~cKdJNU2HSu#90@pPj>Wj&AtUeUD!F5RE2s{iZg;9=F&z8l4 z_-}W;gt1C(^8?<<4>fPX8z*@O_S8-_l4E5S9=r%l9EMMDwqOKmMabt!H7#HULE)Mt z&0jS#yqCgyz3zFNLKfL1-E(zKoE11V?|QbkzlLXzc?Lt0PmfiW7hC=b;i2*B=eZN@ z+U1C9>$%w-wf07?Sue8hCBEd)dn~=8B@D$CkRKoi;UyLA1xiFuYVP;CVV|lU)}Z5y z@;9H1C%lmFmeJMlbBZ-SXv@D5aI|&nqP`Hu3Yf_O*B&St^+9%K%9R&~^^Rzq$e&|n zV=E^!I-Cy~N-uZ4PB$DgI=9(m!c1`kf8ue-u?(;Fc$;SZOZe9b@9y{lKFJ+&^0?A+ zo$?2hr$l{QLR1O8iwYK|c7Qo|Bl{4S&8@cLW|?O`b^uW^T2bQH`Pn}9u1fB0MXg@os$oL+*V zT?*UL?-Pjqw|+tdXOB4y1w^mP?)!qz~~1 zqS{7I(Kk)>*c;w(*5>^SQ$cpmnbi3cax!cXT;WK3J1yJ^?)?jWv-CPjIY|0G*Z0H{ z&MvG}pzM^XHRTM8@!_Ff3kwbF6!IG+LfwJa4RT zM%Wsw^uLF0F}=ZiEy*cDa!P?i)AH$^<*tUgU>KOr%shu3)nHXzN6n$5F}yq7@&|Wq z2UXl_ZlZII{Dn;GY9el$8lAM5v_G@X5Er*QvjZmX=9>vbSXp1Y$pH$4#B{%2rmWvN z&?mFAb|P@f%G5_k+_$UNR4;7z&H7%>x$&CB<;`|*c;kfV56PyfkuC#p#xL;73sS|3bIY>SQG{Qilf7kxAOeuVL*>ufZG*7mzjivu$ zWvH5MVcMGTgZ9IBk_^ljxZZUUw4i1PS-ePM6E*^Ql;rUq_R?cENTKhF;ps53(s28& zLpu*^607nwWf2y5wMG*rJkZ3C8^AK1;g1( zfp84xJz96p%8_|*yPgM1-tsM9AX5_Y-J)T^Z=@lB60{_dauj1ueO@nglX8|6h%dQx z=y}@yzRkRXJ74EGW{+xQ|D zwuMCl1tC;0sR;eBoPNF!XEg6xaP5o7nR%lcyYo1kWwWS#CJ8bjQn&WZx{e;|oT+7wxCMl?NVen`utc5>JNr zEm4lh?;Im+wVSnYeiLez_!1bDkFdsCQ|Ty9f6B~j)0B3Emae7 zj^S-rF9Z>5-<^8K(Y#wFVsELI{KU0q}J73KzUsJw<%&T#c-_<053_gTMF%Z`&?- zVBj8Grx@h3>5~IlF%$y6nZ~M4P;Zz%P>lCWM9_Q8)%h8QcvxNc(8lFFYf)vYitKoa zcn&>hW!x@BuS~~8;cdT!r>3`okGy7iT3;0X;qFz%%lH9(#%4LT`Ks?jYUW2dVosB{ z$QSD0Nj1PCQrsc2M0{_YC*A4^RWq+X3abibhH`>*jI0YZ8i99umZnUxDhgC&2ztwi zssA=o9(~8tj~#ZEJ?)Xql+a_jOwWCJ<5w>RY1lkHa>4M&J0)X2AdZ@Bh9E0(^h}Sa9zjZ5blFO%a5|Ij6yuRh~3?13v!Q zC|@Z4onKqaVT+LF>;s=pCs)cU_N(l%k`c5~co`=yi-2yf7sFeIiI(9D*Yso8UA>*H zhs&U)S0yd$qL`08yY&5@rTr6OoI`am>_WX9vH@GRgfrNWYy~?wnTjZ@uRf_tli0h( zoi0J_rSQ{%HOIdcU(8Nt8ORfGu`o48f+s+u3gCg_Mh@TR9<5a*C`bddXYl?N0bcU` zcM?s@Dkt+#MkBAMiX2LkzOMc(Pe2TD1SH0?%6_6;BDv?2SR24{1Sl%Qa9JDWmJRJE zPNeWu>vBKOvhj~gIo$3%2RYkSp}W0H;gTiVkizlaO*C0j0YS~^rA{3oW_)~QgwL#0 z?ZLQ>fXHIT2}tg%jT)Zdd&lfgb!$NdfJQq_o>eFt<}R8I@fcsfrb6(oHCJhIIC#Ov zK)U9~S%xM%+01u0_DA!<=P-u3%fsg2xR>}aef;EV3dJ-=B+x8IwJ((?hB44j6m{<* z6WLUd$_&QK3AUNqvI-d4TjnFH|2ezPNeJ;_=;uI(?#M^9=`&O_1D3qcA2CIRzgO^ zt9azo0H)+m{wYdmH?cUi*LjTosgb7r6H(cAfgjLSCzIQS%|4k)U)18U(1XwiLqx(g z|3KuY0hpP0u!{B3XwOZ+$!lzJ(P*hQoeZ-by5}uE*|Glpwd$a=k*7nk^X(h7m4|9e z>maSAq74;=JqS2QeJ6QoWcc>Q_!?{n*6AF+SYDpesGJhu2^+4BI2E!VY~v}!0Gz`V zkFIk&kk62Qz@tx-#7zhZRcv>ZS3qdw`e%n^iK658o$PcXHQw}MYr(D>$cYHOk8Dgv z_WYXq8lFWsGk&gK(w#Vq$d0rX9k{~n`NUBGn|Q9Bo@)I%>%~vVwfsGh`X4!sumV*j z9&dpeiymYJvI!`Yih{WVPXrTN61n^zLpdhDxvL4%e|BVI(mJfq#vQQyL9xY@8t=I!`zU4So{^tSniC@?diVUo_P-!)AEb~ zqaJt1d|^K6{wqn^f;kU1^<=*I8HQ@zXXe>zLxaouIu+$rtE-yzp48l8?fF#h$Hi}fb8MO|~xZgX>WxjM(qa52m_|7Yz&qw{vppR*+ zV=;YG*-+8@g^E+F#yuy;~a=}~=(Bq>gu z(i}9FCL2pEqQ14y*u9drQYIoZ@)M${ihkU;7*xLmoL~fAfDOD`*wfT+EvRHwI1k#_ zE4Gv1H#ZzZ(W%MY#;)>?$Mq&-yF)D`@8U0Nm($ko*YI^i6zO*!VQH)`1kK)QStFB( z^G6SBeN1jk=VOVAIrZ@)2>C-qr0Mtd-(sX-c36@$DjwsGimC2v#(-M6Zk_T$A6H+M znvx7xP5%wCCwg;QT5KmIgu{MppPTOkJfRcZ#2RWUa98((zk`mk!|auP->h&&HIj>1WttnV@xxV-^?*k0`t$7s%zV%5sl+hF4K5FTvlr}dP23?(H(iTPPnw2&kJK_d zxajP9){b)m1gUSj!;b(nnU7ytMGd$MjyA(DJQZ-M36e0X5dPsfhmi)IBQ)hoi%E$D z^+qlO-uW8?IK_FL0H?vF6sh*Xb$iU)=RRb9e#0 zyE}MdpzPfG_b{E$b>4cq^)s z=@86eWwEF{Wme92>ipcpm>ZW4bG;|FUz%K1TD=4T%Myk#P{nf&SaDSWwWAfx8iH0#Tk(b-95oeYIT{0;3qtC`BwdcX@ao^6b3 zfHBt!X~w#IP<*QH?|!u=Z9O~5!)`Z4fI-^3JNKE^>bD+90Gx7Esx6Y3MXl-!LyA-m zGd^9R9T{AR+B7WWb>6x>ADZv25&LQ;eCm}TExlf0MRlABw={3e(hYMNM4Zf<(9f(n zR+P*K3r6>FlBB#S-0qenA+$E3dA_)Tb~-}MR?#qi#AlH0O@5zp-sKC8<`z1<7a3lE zS-3~NM8YLXHF8EhXnXgfRy`jA>692Z3TA{LK^(1-6ZGrLqL_g%!4d@uo)-$;X{<-^ zk9KFV?JzdvS0pdH3pDnQ!BVDT>f7Pcx-5xF%{2{C%jv!7cpvaZ>u!4!nC6~^TpyRxp>e@OJCFTi(JsOTK)Se3gUiL>s@=x*ARe zFOBSjmB5~8um*T7xciv`Zf-MiFk(2K%OG)+1sAEA97A&jW?sYSUe33#8_mWmnUyIm zi}C^MJ=Y$7XH&iMUiziR5xGTWna5cU-$sd2TF`C`4tyIYHHTq1@j1+QZ+Hc&Vz&)N zG6Ray@dy(aSn?+~@U;3?eTRvYSa*@XB8#?d7VwmFSC#-JvP6Ilz0fyqs~rukdH!hs z#y$T^;r)jXZiBfq<400K-Wv39Ei+!q_$GUKs`GeHleC11FJj&6KnO)oIhv5Tx2k_T z7|KCww_aKg$EO5Y%|+sd@f+k8*H1b=oxTkH3M$eT;=XAH*(s+>MH*S-h}QdCbBgQW zX~}q%lsjV_INKLX<6PwvRExqb7FA!wznjCPpGlRA?;X9hq1Vnvw+^oYid`DFNu1>K z{@k-qk`H;6)}246*dY9p{MJ*UIB?EYKpWNsLBGwT-vj%&q2`q9L!gps0ZX`$7_nwI(KQpc#e=f|8&bBo2$w9w!3e7DX@qaIUms}BwJOrF+Ha8p;KY^(Bo>R z)HF?&K7#wzuX6;p^4*d0h=H5qQ#U=06lRa-CtVLluJoBah7iDJ8ahIds){YG_@ShZ>v|A(%d zp1AQ5^{|JSseO6xtY@ygl}f5@z|-86aboW%7466d|91^xK9;DmpEngh~Uyj9ACH$)N+;#`d&1iwJo#Ne3U1IvnV1n7Q>S$K{BpWt(AI_+s0 z1UdV+EZo0fAqAp)y{IxdOi3-kyunS(;$3POu$_=Yx!OGCubQ+L_0Uefs@*vA=)!$1 zP4Hw3)dhXOT(LyKVc50`dT@VlkZ2uC^h=z*P($l(n62uvK6y^Y84>KRDpH`?xVg~l2jtjbY+5F+z<3meZ}l;_p!wXm6lQw+JptV&At4T7-xRcpp|Uig6GfSra)9 z2dNUSQ-0U+qfNHftGj0X>uDSQg>sAK)b6v14@+a{nhzi94G<;Gg4pCZwuqC;wFakf z48itJ)WyzVaUaMqWZ!ssRv`t7_YoBtQZZmGYv~73g<6@$Cg%XhaW%-*Nf-JHjF2Ga zKiZ(cRcJ%7Vz9CpubJJY9212_uW#I&${kdZMFf3pnoW20>0^sKB@th51@UpRguesZ zii<@syFQXM2A7SVnx^$+N_x0od9lkkp!l|oo36jmhfKWjqU}7V8hy5J;hbVD9B&;Q zX`(4u6A&rteRtm!w)@o#x&r??b(D^j2%3U`VKp?zzXT-J)5K_Br=@HAYsIstgkskO ze1&d^*F-vS%Hq!7`SDUtpIatmG*5t9`ESJ0G6n1LY47e4AII1FnHo0h%P{Hf=Q|3P zU4q=XJI#k5jpP|^9yflUCP8i*nLbM&^hmnLvmXr6J#Lz8cqIC9by`8zjoVYXQyGCb za*~2Xo}osnt)=Pmq6Ot1cC^sjmZserub zziQA3L1KR$57#6T&Sl?yXSzI-FUuCnX{xw2K@QOf|&7Y^O$B}T(O~>z&Go(U9o?gJk(O4bS%4Bn1+PwHSt84~XsF}cRyVEi)J#i$+~&P9o5R_0y6(UW`)i}DT*v_Q zicSoK7GN{yLg6iGIRdzh`6D+Pu?uJ76%9hFHtS~}CFZq(lP10?&FZhCXE~4fm$YHZ zDq=iUj*fVE%oYu7sGLT4EE)=AY9D1RxFci$pHj7{gs@N*-~L}K8h4rMXZHne*oAe7 z4d%V!uom4Z=)J~s}0 z;$%4NFE;edlu7A5NwINUW!_CtGwhCoPy2}*^7b)0K}N^J_D7BXqj@)2FT_x+W-;{s zUSSi-a=O!7IZgRb)|gi8yg)s1ly=P=X)%w4Z2YpIBJpo9h` zxm*cedC)BwoBzRjCfoMPWU<$6F0uRhyTwxwn9)DN#Ns#ZIJ&DZ5WQa;0Y$Hk`|a^TCwGnof<)bJV9YyWxltoF~XhGf6~jujpZ z`+e3A=nqq7z`b)A%qag2#gUfy-!>uP9}U(0_CxrWLpPMw-QBL+gX<#y!V}Xf&0L-f zM@<^75WTl0zrYeCXiF3f8P0nqj27ZpRUt_CMdX>fgSdUte$i zg#1;1S~v##*9y*HSoUuMTMCFJhJtIe^Q9Y%C^38J>|;MS@rwsUzs{V7QGiZxJDfuT$4+SDBA}>>z!_E z3~sX!Ayt8)re{|qHneSgci!jd+J$$)-O{98a(FPGg{Krf{e4^uOd z=O`DbwL2Ig(nLc>TkP!01pLMh>&tUo!*M@O$8ybk3I`3o5}mpP0d%kbotpJ7L)jLz#* zc!h|bKOs-HW^cbHfamL@njQ23AbZv3;BDLEvou1x6mC#;<~&1+o(2nqVn?k8?gC7- zA*kpEQpR7f=_lmsz~UfooxwWhl1M|KuU$)&Q>yAK!j{W7t3j}I#!fWP>f@qkHu*?_ zb0=a8R?|hrFcsAyrFkCjE*iX*)_t*YOl*M=tfxM@K(#Ig&B4&}+v}QqRn^rX3qyLS zq*FZUa(IYPDb!`VFwMO^O6JS{q1!QlI6(|5ot9HafZLKo`bdi@`+z^p&E z>52Yzg(8=AIU(>0Ch?VP7?d{Lzdxyiu#PY6ihm8W%( z75df>g6R?z>YlEyl(^&ISN4m(3+7G?cmgI~1x1p!tjk^#q-6!!q43Eb6`89y?+8h% z-;ZlW-ky8_A1M11Gg2N(G60gs7KpX*pOD+u`n05G_{cgS0moZ%?p;9h{e+CBfvH!{ zsjo~cARYXOBF%s=0fTdf?O*bOmkg7d!S4$;4N@5A(PV){TC&Y3?hi!WKek1{w#?QG zMe-}Kw-*kW4F3uyy^fd-Z}53|sIY%Jal!9J3ofe}<*8ZUj6S;ieO@7&dTdEmCfi4= zowAtR(1|O8IQ}r;$@@Qa>OMuR<7- zWcPxo`53p&7%bk@D`o`FsL68kR_o03Lg4GCs(7@uQV(@+Ir+_AvQSjJl!EJJL)juV zkDKXaN7^?OH{c0~sx?EO@bbmZh=}!gd)$F`%Hzw-H-iFGc+9SIMYYH|JWbaCLDE0< z!Y|M~g6!60r@p9X@Xs|K)N#{1G2|0iKCGfs8SqLA)JAFK973+{NG*CkMmhta`q&lo z1XiM$NO3Qqz=swGaAkF8SOZ;;m8Gru6!p}8xMA@ekt>!QGvDN3;i6!h94@%(G2sYU zm0B$^Q7(iLCBaIu9;}cV`LGnXL{cCV4h%oKL7Sla9fmag=rz+am1hC`Ynm$JJoOEN zm8r!md^~LRu}I6H&8K2*`4VP3kHU8+EIiLH@t<28sk8sn{UWPwHKL5-T=BD>t_wW) zI`>A~;H_Je9iN-o?w;wBNYMEh(kT%v+g8s@8w4OUm9T_nNptR`SrXgEST>nYaD}H9=EmB@_9evi+gheB#a`R2B^S=zN3}sMKX4ZE2Sg zOhU_IjRW=TZNJ{W(VKqQfUa>u!tsL`Uhg3%QwC&t`E9My9GHpWg=WgSZXnDFBdN0! zW86J9HnHJq#!#2TCsrP1wuAW2;M!myTK)%ZH@s z+~;8*zsL`|>HhHN*~t@&OVIv*6gRT-iSe)s6mFOd^IrubVMl$}3Y2bz@_PzO;}krv zx1Qh7<77Fr)hE!=3JFTQguD+le}mb>_k}hAyjjpsNLwD*@cfKy!ECEdgB{DaGck^_R2qKs!(YZ|P$uSr1{|#uF;e*l6V(OdUB)=7Y6N+|RCRKo3AkQfic=&Wl}( z&7j^Q$7`B-=7H1OHLIL_(6hH(?F&gP>_@BYL(UFrJi;&=p@m1c!bvCV&@Cnmwq?X{ zH0jl?@hknca)i>e-z8YOv>6WTuk-{plD%ooyjWfjw5B{lvcy2c^?-olCkMJ~7MFyJ z6ChQP`;QII!c>`O&YS~RyZM@j#0j2}V8=ByZgT9_H)IdCG@jh3 z!68>_>$aWhHK2keK{w~$rp`!x36s@ z2R24%BSLEeDU0LZc@5shhvh`g%XBr1eP9wxIP`|~kJ;Ft{~qv!hiCv#gEk`g>JJhEozjvkCP>bU#3GoDet_@G6LcjeGzj zeF2!;_CSlhaP#hDS<3OqS(UU}ns@~3#LL#|lt~-4eM5#UtsfkR)=*9L>Hm+Q1lsrg zwXK9X1_Mz;a2{*RW+J>yAs5frswvcu^Qqw(eq=kw8eS=%TR$XWC>I++pPZPrLk;{L za+OEFkNn$GhTOosm3>E@AiYJ|8!yTuQb$zew9hg|2{dS!GIq&!?qCz)^azC?`g2}U z^}++9(0WCcK=p?x)xPJ3GV_b`nG|32sUgrP6l{?hP7%?u^}UhB+3}p%Iat7Q5kD8^!{FwiL6Ne;j?RT$kcYQZeg^g zM$M4Ju9+ z{|V_sMPu-`OrRe3E7MjT*#yD%tFxe(pm~yH{Lw2#L}sBbyg8Ay_xa^++Q7nnBRws| zec>}ibQLg@0+jes|t63>AN^YZ6uW|fI4x*fvF zD?O0Ml7Fcp`AeT6J%L6GipiYGA!Mi3_|lUd_c}ha*c*cu3ionT#(k|Gu`wj`*d3aQ z`ysoXfPukg-ox260D>R!Ry{b!#g=2f<@``aA?nQd1Z04J?vcVb2{vhsuLq$ZIj=QS zhJeB)zhBclHR9z9(i7J`F2hRUlZL*ktw|oXKT1^lpN70*8(pR%kwBvok{^U5X8CYc z4l~$jYmxTIvb-Hz9p|ztI>#ejt7e~>`R&f4zn+Ku9Q;G+jaU0W?0siglkJ*q00pG? zEC&ZyB27dDrAdiO7il5_0tro!-UJkcD5xMU(m{~WK>_K#_YMh22&8ykznMK} ze`RKWvuDqlnd{6CFEj)a^FHtM-1l1RUJHAcn2d5xcnYrOD96YwP=)TuCeT?|eJY)m z#SXN$FOR=jxM1^f@r@uj3wC6_7`c@W+$prUe#|PibXe_#9}1X#$qsQKBFh4JU2Aq3 z$M}l;{oR)n64f5)$V{^1!M|<<|DeqtJ*b0z0jx~INJg#ztah;c7QRj=OLh$ZgFYK3 z1>g=W0O%Xyw@GTZ1G;px1H9_5UIP593EE%x;{Rm3!>&6Q|h&Uyq=5(bg$u@f3d!>CG+HV>SLLYu2(DB`3TU=4E3 z9I|F+-9B2(y+R^<*Xs-HjB5qjAlY|9sbiT{MJd?V#0C|8ylLQLbY!@Sz`hHZt|EcS zWo4jm`P`6$!E^GBH40oC^E{QXgg$ua0M3nKQKOuM^@1yQL)Cb-Kd2R|u&$XanA5#6 zE^&H!TZ1K-I+pqAl!0BceP;`ITw~FmwF0v(bXp zsB0*Fxn1H`Iei|e6{1z#lS|FaxXy%dpLj^`4=1qZ0^O7=revgJF1>wnsR~0ZqyA3R z)MS=>tI=FKNBfaV@NI6=zCBp)_sDP)gdX;l{}NzGKlz^2kDy)7-n`Yn3%IJ51t)oy zPUwG9t}{VNJ9o?)e*-xzF15>J>k?nO>;7};9NAhF` zwo`EM;1uBUD(-~Z`P*MyJQ(f2 z1JM`ttn9tV?CH{#n>zj|lq!oX_>!O(*V1wi>^-RqQq~&=cl4l)Q!qT0kIOXHcDs2? zH1cJK+AYgv0j)#x4-gseK4Kc&0YORXo}uS)&tNsi$mc{)yw5jhl!*__FaN3@xq~!; zIj-gnRqk1)*>5!H1wvjfER+4MP#0|p@eEkyt#4nI1^P?4y#fRwK9 z@ZUaiaICHEMtoy+G!D1?Ff~nl-azZ8H3x3iQv>_)Th%_QK>`i=j9%GS{(H3aU!O@Q zb#Nad-)=s`0d4r%1^B2g$!BtObZ%++s`#$m)a%L62f#8}m;1%ubQOT2PN37n4^Zs- z?xO~|7E?1XZp^>sz{oyjo13eeVi3)FE5IDqDN9CZc6Gx3?0>VSY&J)k;msG&g@*g? zg1VQioYw?SHrUdoA5w5lN8Iu_T4K29fjgT{c(oe?-!iKE0UCVz6n>RoELt&;x2T2Y zM4Xt!f(^NpZK?1WL^_+vvYtp-~C0No3kCA|^@CN&--+j}Q@u@tyvyftaCZ_~Ho zLZl|nE^oc6bh1k<&RgixJ~q6%%p|w_af?j3fM~+Ys;A(xM>nVzgyT?iQM0Eqn-pBo zyY0tIw&Ky}UH#=`T*br7$r2OTq_oe9GRHORh`yRH4MQ6dST+}_;??M3BW7T#1KKEo zeif+dNNRzFUR$#0rr(m()o6zPi#=S=K5oReAf6|z_bwn>6X*cHUf%`As0Qg8jtNMl zn^0X=RFhlSDV`_w@up0^jufBa0Gog|l%xbi81U^57tGlWF{`yp8)`yBZ%x$zGlA1* z)r~+E`yW7irWWzx!HjFO*fLz@hd?^wtNH-U%`0Pas*Km|?CovVs(l%%&@z&klWE#Q z{0vgAN1MP!fWg{%bhk^9eq9Uc?Ym4HoR=S``to@Tu_jb)aE@LasOwRQUN$!8Zk%eX zxyXrF=Uhy0&JtaWYzA`#o*!|PADKD-F>UBtu2;m(eJzpGgj#m1)0k+Iy&6lP2@zPU zb7@eX11PQEVT+sp8`oRs36v8lEdT{7=3>MW=`Y=hl+?GlIhu`+zNhx1!W~dhnAeB(s&r|~X`2@Mwhrd@ zy^D2Ai_0bt{l5dZD3f1dGl{j*YM0%W*F=OEgqd4hHzN~+FX)p7Izan zibh61YK-&GS>w@OB3HlapLSSn^7;<7vad+zi6GZvoEK05ZV}%@+%hky0n`wj=^T2v zY=SAAu(*( z+kTeI!E1Vo$0P6HLTdD*Fd2B$C6wzk!kOZYxbunW@`#C#lhSw&WbU`AYo(o1Lbt?A z8@@jjl~|~&h8saE6j~<}!XOlZ-k7UjPt?|H;+Bnnu)C+=D;tu^$vcHtfy)N`d=|Y48Ap&{^%}s$Zy(@W zsS5txBd{c6sN65R^Agc9&wE}n5`s2j3`WorymA9I&}}0XK87^&?ryDzUi!>Foh|vn zEY!k^0{GGC3S$e?Vx_vCsc$~##O-^#dM6QBt}0x)n?J^NUOYo%QIF|MkR2CIMuEuk zbI+5V6k!7zO-CQE0dy`<-uCpQ_C#(?-|}{Yc`+95e<*b=O6^uOcITZw}baxyaH@MVfO@Dv8>;x zaKN;GXv>OsC6Gzu=w50q)e@{71ruFQu=ukD$nO*}{~OWu->As_n<(2qzgNa(R}xKW zj*VYBc8#C!Xh)4}gndKsFUqP;JMO39wxFj73RSXu}!Kiqb}tddT3VU$gAf*6$k z=Mfdj`}2+iwht=vArE)2r5Po*39Q~ZtE6h&B-aOT(t|7uW&=9IS`Q%CESiWVzJ)tN zp3YWM1((!wVi=8#Dz+eZhR?>qCi8lR?ISd0sv)pTk89@Q!$H1)y&g;qN4p3QuIP<| z&{xV1KQc5`5JWPvCZOoPj40WuT;|s}IKjHkPI%**%?)f7bI(y+VsZiEx_E=;PxX_j z#WNmTZuhxj7|(C8hOiH`>w}ha#jd$;Cc|GN$F;B>IAgrE-|Zl)#$Ab*B`){FO!|>D zK?UQl7_(?t&IMO#UIl$qbI?(Crr5mvmo4e9zN-9AAojattI^%VzZGBonsM!yWH;0$ z$XxNEuNkEDfH?fT*W;4wZO75LXsND`ih@arUeNakXuy2PW3IOqnE)x*-faO!%y}T# z6_kA5UC*jn=jy8N8+Du>msa$0=lCC8*DTyOEA!lL{4dCXn7 zM3SzP-MfCti2c|JN)2yPQlleW#nIzsEdwDqWG;}6xryHJJ*0QT#ss|5mieHwFa9=7 zO?|yium)MT2KxIj)n9zULC9|j!4NR~E`pB`*noyK!B4Iq>$d8_D^&1nDynXGW&G*1 zN8A_Hx9bm9S9YN}2R{W0O~9@WtRewm6{P^Hs1gJ>8u~Np9`iOZ4}9(ov@EZ&tMD_p zwL!|z&Z_KDyp4j67ThzoHIzH?8FR{sPxS>Y@t4#gX$V%xT$P&q&<^i)oG^N4pi%Fd z*OaTt7k`3kjeeHht1AQ8vFLV#$Mc~ z>WZRy^ST@PyP6_r`?IftbyBv->4f3l zpC&SZXD}nRfp}$JbP2&gMT|Y=>fmaUzHnCAw~wujKIg3C{e+enlT?6eBB`zk>^uEK|3EpaF&;%8;QG3jJ!_&N~=(2iLLdufKkkqIPY6vE=jZ z$^K{t)mPZfjajrMUSYZP za6l-YWgeB6!WE2+e=P_~Y3X;tEF~&tZk3kurmvjqwAE--M6R)3LGJ=qe9l$;e1qbT z3U85JF-v^Q{M35xWGq=oN<#62KEAyD62L#Mn!tW@fpNE9AL$pb=x`(rUwCKHjOql| zkwMjjRIcZk0)yu;JBq;KjeR}3YK36&%Wdg*`|6%?j9s|#+TM+PD6UpjA@-ryCXu6G z{m7^(j*~<^yRbeWvmhB&B`*fURSr6QqFCOtb(Y!p>g4m@wvEe!H}Mh#IcB##qN+zB z0#u!Y_rfIG^`>QOPWx@{c0ej*Z*V})1BK{ZA{aDa;={=4m*I?QjJ6WV^ewqPjBEGn zShP{CCa{=e?}VO=w$W{^7OU+}hFV2Dc(q?PY##rL44OA|Lm8aK0YMuWjZ5_B$gF1+ zuf^UCpLiau)P8Bk;(7p~hI>0+)QNOD0S|`v6$3kIm_aW|845R6x4=>T_F-LuG8n4{ z&^xzxflTRi1mST5DJQt}2ro%sWyPu|3&*L*1n^1;T;50zc%HrZDBM0ERJD*CI0nq| zBE7WQ>{aFNJ+b=yIRz&8}9uZn_x;YB1&p4jE=>+(DWcRvzFGFvi9zWO@ z2j@MoL&|mBhoE&)VJ17IFPH|D+kUbN#zl?w6-AXu*%R-<$!v&C{%bP`n;aM?Xd5Z9 zyM>6ZmY-D;?Cl>26jr2i8?sbYTHo&FcK}B2Ps7e>ufXjCH3uZT;0DE%jAD{=lC*Ylk1yzwF6dW&U{;3K-?dzYYf%sIrIXM`059b5i zDEDzr<#@;~Okf|UecavA#o@DyN@u~9h9Bh@9(LBM`cr`W9oVQY5ca{7<|LX_bOAsE zhL3caXNIL3Bkea0Aq!u!pnZkkWCmMss9@<0(Rx%PJ$ z8o~xb9wHDhYTy@+2eJ;^QU}8441HUACE9$;l4ngK&O5enh>esO>k1V)B&fe^hqa)E z32c#ADO`w{d>WX=O*W?X7DtRlM3A(d&?3ERw~GhDrYup_^{a|2o{aQGO=(}K0T<-* zZc1&TY~92#?}21&5_{cSVO~FaZi-iADTwFwESOukR_eO#Fx&mBm&t-LYKp( ztKD8x3tgmgOhjPoZb;lblx!ghqeo+a>EdkO`AJ0s9chLo9+g!4qa~ zfobZ!^EQrSI|t~MrY7&7vBR%!rIRatMLwoB=Td4q5vum=kIqA6JjA_sg0me_&e$l5 ziq0l08{9o`mZMeHj=33c>GsuHn(k_gm}HGz0)qeb=-9!}2)>g5`P1QBKks(870Pc z06$;6iGA{`*MPhLA9be_zrpdEsJso*9r)@c=%Es@q-aQp|1wGaf2a!jxnlk^#AfjO zv;Qu!nFZ|6b1ZVlnRqeMFVeDl9@%Nx^SKK)+U%8v#_4|rufpr6zC#=k<4#Sea)mIR z%=h6#fd=_GRUZm2&98X%rgHe}F%gG%4>Pk7CZk^*EmI1P&Y($7G0|<__|h>Fa8Av= zo>;QwCRRQO)K@W-y?u>Gd{XBvrB6tG^Xu$Onsess=>)aGIZh&V!+Amx>gep}js?lB zS~Tmz=(EYTSti06kD`rj7Ik-i%SAgr|1s+5E4DJD+$F3eE-fseOv~{F_O1{G1OZm( zIo_stGnVHG+~!-f#ajCXSuY40ZxjsueioOxR|cTG(U5s80)$%A#waX104tPz65axk zsPOOCw;|~B`@lZa54d`r!2_$eU%BE3s4=sb1Oc#5cN0JxjeMDhja8QU?mai#cA9e$ z+Pq6oFhg$_x20t*xL&D?pnV)gyWt{#zBZD99d_%|_pl=?J zgXj)yx0;P?d|Qy6y59HA^4S+veWW(>9_vbU@`kVWv^CJAzj^u>(wF^b6vs&+Vgu}) zpZ9_&sT*8gxs`S~xrpPq>fJrA4NC0hNwK17|G3aHk4OBD*s*ssRl8J!z(@WZQI+@X z5jDZLI8eQ`9Ak}%332fKF!SYce2bU9d$NLtyq~gV3Zx_6nc~lJ%>q&I z%c!kVy#9U8j0i*k?2!d9w&bwyA)967ulM}=WQN|kK;6o$=kB_(Pdnjj^V4=FJDLTd z`obWxsXmH)O&`0J+0nrKtR;mLNv|^ zOO3O{>k_~?t71Q8Y|<#N&Cr|StcqG&;sHw)|Fx^^jw!3mMsb3d{$aC2Tox&w2B*pZ7VZ&u32&-;BCjxQA+mmNIT3 zi~PcF0ycD!*yQP%X+xD>Y|Oi9p?J%9jyse;KZJE_zwQUl<6qqGLYY2HtiAv!gl#o4 zcwfwd$JZ`E!z}^`?M`vs>j3=S3@(EbS?i1lrvP!=;=wM8p@G|v7eL53n<0#{Z~H5rrmk|V z3nhyrU-J<7`fY0Qy_Yq2gcHrs(b<!a~|YWC8+I%s;U*P8u(8 zdF-zn*SD*Xc2`P^T;Z~$0AYQ~d<=NSKg&A8Mv?8{$#g8wr;ndL;f~NW{Q!-M*v@#L z?)2PmnqlAHk@bX>pm)w5|Rxk^wb2|m}UUIV&oklYWw{Zq@ zl5r&RArfz({vfMqzuc{fwJ8-7##&j)pKw%eyarZ?`xRSC%$P~^{cc?K9Hb+G9M;!x zq8ft=l~5I|FH20Qu(Y-lXDfin-EE22^;9iDWkaGNv<=EYGcVPkJc)~J8D%WM8JIq{ zQf@a9JFt@Sm3udVWS-K2Fu@)j>9gY<(BQ?65QsCB+zL0_#L1sDW2vV*Ayy)t7Hta7 z_&DDoSOY%u!eJ+7C862RuShiP(-^kkV)4OtNv4BW{_NE=Rh4neX*z4qg3G0`Ajf)6 z5>E%#2{;?G=7EtC#z=x|Z=90Bytz^?C04iZxp^-fJpx$t5ZQJafx{vY0?atcC2aNn z0BI`&7!(f7h#Mx_=IxMB20W*i^eS65Z*BwO0w ziQ*CHl96tt0euk;7F{(O-^Bt^eE1u1cgZHAZIC`e3G4l&e8TYl4IBS#WB&X3cLMX) zU)cdMe;5FkZ^66zmF%}oRByeTQ%Jem|EMz9FLd4NLG@k_%k`=PT~YVw%@yCIy}Tvp zD~cWzz%@^7w!y2At=p1VS7KIiiJS*b8?02lsxfj@?qnw`-=bTQpqJs8HHAzf`4+K+ z7zMX4roigs^f{mQFE(JzDol25hM2tMI<3W|lgai3P89jK>pL|{s1rPiA%Mqk@0bPemEwm4?w|`k|GZ^qlw&+Bp}a#*HG5$w1H); z^2wr8WDZpC|^sv)vu2g2LFco%f(@QosUj=!M&1LYOR@~ zDfDmxshVU{+Aua)54arMZiKr4jrC!SM*()yyD3d0oJJ4gxXCS-=4VEYqQbP?PTY++ zq=C7UmwPx6HGvzrB=7=8q0M7le#L`&!Z3f@Pxn^UsC_8o%b5{sr_jfz2)4Jc+bgcp z=plnqa~Z(lasw3xJptoTXkv^@>7BL_19Z- zsgwI44Ll%3Z4z@=qr<@t0g>64!dZQX`3i-@C%5)ef?U|ZGS-&X?d&hEP-1mj3PQhb zc~)YD2`s(XZy5MC{d|_|yi=P&$_GDW6I#oQ^6l8ZN5!FW<)+l)3j)$4axM%k; z>b?=~B6O?A4xP~2z=Ja`)?b9Ox=sZ3JgvBK+q%ikebo3x{rXC6ynrbM2Q5GL9(mU5 z9k3$s#@N+p{i-@{VbVK3n~ZPE(&E8ehDx&`zQWll)|VO9nWI>JHM_qum9Qb_4v{*>} z4Rkf8>_@-rBK;(_aqrM+`;BVi<_H6fPW%XGChvm?(R`I^2Y1go z`CC{;IRXaW^Z>pzPhbU90Tu9L8y~>GieF7x11|^bNr!XqJTk|ecyhAj1^shxT5sWr z)zDEj7k%=l`@X;^r(Nt42zJsh1#3q91nfNsnBtHM9IY16b=HN6xvoCvF!RaO+ zdZnmb`jC6Ft1>E_k+b$MSr5whzyq{S;ckM_jX{+PpoaKQg*Okr0RzM*~jfgE)^4ez}1PaTAdaFi9y_*9i6b9iFaIsmS>00aG_h;J`mK07J4d6TRQA`V`(!JA$xp5)_XA9yg&Ce z)EQV4{3A3P;(}y@x9F1w;Le}W#luZ?)zx+4*tjQ)kFO=Oc2=CQi}imoQKN1c!0l+Y z4e`I}(Y@1rSLyTm16@1?sc}Y#_%84o+yy+LQ^?ItbR-rii9#)=ZSR*69EeF<^8+5+<-Y#&lfLrfFS!)cuTIG8 zEFzO?+1!ZzRLt2B>`&Evo{gXR7n~g4+Sc~Vxr*k2-d1yJ+kl6)S{#KnQP)Sf_ObOZ zA)mP@)yard#-`g6+aB9@0UfJ^TZyrKw8hwEz4%K-MVEetgU>~m{z?YT{~Np0)IV@L zJ;X2c5|UBe@G(wf;5>2>{T;X3NA*4E=N$p%fXza{D!zDmSZ_LnP3^RJ?k2}^YU)R; zvDiMY9q!b5L(_43+ zd0A=|b>-VmNPUc&4}Vt|9BEe{`Iv!G5sNI}+@~O&qW=Lp@0JH2bG`Kg)Hi}M`VTUq zy}co`=<7c~5fCz%@(ke#x;jcOX4~Wlc{pL4`M8=kU)l-O!RD4K9EQmMPNeud##~75 zD0g7qf-|83f}1fKug03ye7bycX#4$?txGIy70s< z-_`ea3&O%rV-^;FReDH%HqKUj7j_U;XT(>>chHW2F=c4~Epv zkY;Dm#)g_f{1X8w^A{Y|+&%;KmIDqp=Cx-aEUxzLjZm_`?p^48QWqF);Bg9fnHS1U z>bEtD^6Aj_`>r=umF3vFdblIyZchP1GKJOLGp_8|guQ&X{z(Gi@pf$_am->uAf8Kn z(IO{~GbcxY=JjLHz0l`I9gr|ES})<$dScs$KtGJ^M9*H0UEW&Bow7!iqI8ya;Y2CY zLbL8Y*YAV#h^<{TWN!PD270pTwtLK3dX$UamFn0xVW7jC9$AfdsHe{~@tKz!Ync5< zx_vb>Tppec*8Ky-N$5&o#ho7*ypaN+I);N}gFA4a#;HCx^P}s7pRJZVd^AxnP@F({ zlbkJu?$TW_8eeQHF6ecU%t*VODgJJOEvUbq)9Ym1TWI>Om;k#NuaF3FF(<@=}^WN)|=@!lhF(2cCh%EjKuhEq7i{Fiz` z424eh=%awhc-^X6!!_IUQ?ZwsG)`+!ai2cLj)UTXlCept9*(}a2aTOABb5V9)SC~o{wSWPw zL8XhhgRW0TxJtEo*&?C7&=@Edf(7F#g2D}t;Z_pl# zRLuc!UII-&kTb(oTUYz+b_~~}J&7}S%A_O6Z;E$xpXzK65?%ZBi}lKH{1`z$hW{49 zvSp5!24KjeAWXn~N4yWcYuh1wg5NFtI2++jLoAIkdUmHO{_e-@ORJ~%p*i1wHIHGw zrw!bOcK~K+0s|-dyMT|i=;2nCzqg4{2j>B0;1$%7fbC_VhzHQbb@tD&Uh_r9-?~Kn zYS4u2Hx3ZyzYTr+)yDm+3-H$$1Z~-czIgmd2Cz>N~EQf-0)$GXtC|$mCyPn*ye=`eCiVObb_2xA6h?V zwKD14nop;rREK~d_h%=n|D@QRX!u)6y2LM%^sc{BuA^pvQGCTf3S9ak5aQa~(c?Y8 z+wMV68{j$;J$g%&rLxN(N-I>+*jlK$P%YBtVE259;r}4)F*oTKi;HW}^cca9hb3w#J9tPsPXKy$QEI zp(VFW7iUkev&t2(%LAN!1?;!kc|w)HQo z>6=eSG5-|ftf}7s)YEAFINv$*CP@n1?&50OEc<#kA8JUFRlQVU@U1s*fZyV$8Yb-& zv1~g7j_qi#<%B0hTR@RImxCyUKJx)@)iFkAJi6(`TQdKtED~|-=1u1XVD7`XOn`IW z0je^;KnSs*p78_0441NCuo)zJZP;NT<=WsC&n}@vExHwb2_crd)UW7cxKvo{9UD4{ z?@@C}@RN^b!$|@dk0#nIBYli;MUMd`>N+4u*Br44Wk3hz9Fu=6Ebt7+g@(~*_g{B z_ah@lxrkKd@pqL)AB56bJIgp-?mbox4T&2Ac|{ijZ{HL*z2K&^*$xLCWLDP?RN$|S zd~Z$a;)Tw<<&P9 zhM;fH9x2V9f{!5p1~8BI7(|LA_+V&ta4zUBD_qvXd-&EY!+OenU)vj$ItG5RvH7P! zb#^qDE&;P>2YyiOPb4H6Hk~awikM?xL+3@x=7j1M+0rVz$|Y7mFSEV&-rvy^C8ZUA ziJ1fCPm&`@d&Gf3DyHZX(eMgH>x9sVr{Vdl+MXEL@|5H`Gxc}DtDw<+1H2M!0L}-W z%Rn%zXrVV3j6#Jnfh!t!K!0g1)8Dt{xmBTPBEN#k+t}cB0Sdaej=_*C5XS@nOli7C zVkh;hvEuIHHOFUgnT@2g$$ojLWG)|S@nL~v4G~l&Z_b?!%C3l`2zBekVY~;FAJzk@ zR%m3LNGF|xkq^HeBpC2HT0XKdJoy=|&F%P%(s{*nDTTGvxa(vm^5d`TT4Wj$Sck`} zq)!l1xa-F4BLHJ3K}z78zvK8n?M-8h zL{97{+q7a}W95HP^K&F96E$u71a=*8t|-k*fIL7%o5Q*5oxv@(lvP7B*d1)MwEd`W zwo`^%1%``UwfJxudSDl9@cnoyyIB9`A$G0TgF>kWKM5XgW9O3=V-^yzD7w$aac?_V zx=UIJVWKbmS>LvH; z!b!7;J;zxN$QHdYDfTLkf2h{G9F7hgFa~4XkZz(N&AL*9;0g3eY(fxabc@3b3!XPOU*3jA>2&g3m-thW_eH(E3Dkjp zXyI*IAXO={okU&p1N6dl%Qwx085<*-P+sf&&iJU1f$yPm2rdkN3f6Jid^=_V$%b>f ziBXvWx`Xg1Uuj#@cuha;2aeM@&{QNX@-cbip7=odV6=pvJeF6WWnB=<9w0!V^2_Fp zF|Z=$ZGB38UZ6E;BXVT2BKKf1U-2h@bRhX_%b*fI4%)F=D@X77wE+3;(k(E05lI73 zDpsF?Jw^piNYN!IZZ@O7o9)8UMqpr3jP4w;ubmeD+ddGC&*N)X2O84<*(vhpNWsN- z#JKl)c3$r{n8mQwUiFvSrwTg+E`AMqCd5;%4GJWO&!s~?z5&!dPXWnAYL*>P__{#m zMOoCX%^{zTwiF|e!0sNE)6*5rlkn#@~;Lv`e?EN<&$G#+!#ef_0$Snv2hjf@_qqRf^~q) zQQP1JafLQGSD?hi%!hYY#N<h_xRYsPPe0i;s~c=pXFx%a*NuI?HzOlHuk zUSAgwuA$jk*7$A$+KxWC{7jX7JMKeo55vEB2V7)1rBhqua&A14 zp|MG(^Y-EKKMBfzN~wPSL3SY~x=5#zUc-jLoQ}BifW=KDgQ03_W#;XUJL#gNSZW5o z9_}Un7X;S>o@fvOKwTqG`Nai zpGS8@@QWKQQ08&C702@{b4}f`o(lN^N?hDnP1!pjq!Nn>0VM=UBC6pG@LDbW=Gc-L zWS44b7F|-5%DWp^LuF!@=oVEUDk2NvcTi)JZQzPgK#=qn@Pmg`*!Y=k%uP>DPN(5@ zK6}-j*FUv%n_RkGrQ6L~;C%hobu!a}Jn>|u`}5CN-eMWYB(by@-EFyr1e^ki=b}BE0o(nLQK-io|~C;Px+Om+VJx@Luqy%k?Mm25X}*X+Ul4Fb|LV zxT5{})}Gse))(nn>Y?RtFiY%9O3=Md5Om;JgyQH4F$?~{C;|16v2_K=zR?}O&)L8* zSbE}hFw<&;cd9r}uZ-7R3t_z{C%M7tTM$a9-6YXrk@Nb7ji6+N5&Vh{H`~ z);c3ZW`gGuRvlg8lbobe#uz0bOJD$qdQ-vT>qmt4=$S}Yrd!L!CGwb8LS11M_0}J$ z4g$0TYw=Q~Q4|$0Ax~FTjNZKA>=VP48)T&W>7?WX%P(Lld|A8Rkwa|RtKdnfh+6Ky z_#!bvaOJ>Ri)@EtPY%JqXI%Xys_{2m5uONyjOs&UQCqb60Nx|}2RJ&=HL3lwKfd|_ zx(|(m(!t0Ag==t;7;nuQ3}@t(-d$C;enjYy6oiss-dx^pI@FI#H zeh)S2jeUk|!rKFV3uS|VYt?Z=ggB=qb90q^pC0}5aLvHZB#T5!20yjuWcbv7gK48` z1*44$~pLU144}fs$;ulEH?&qDd8oU~2L~i5@QqU$%WgL^Eq#$#NvwMyczQl-1 zBO<-)cTzCq$H>105o5fuI>Zn)B?1i*SqY76xP;I}wD=Ymh2z(u; zSrt^hJi2}QfBl00jhpk&_KH*;;3}gnw>im))kazYKw2dbYeOG>+4& z`Qog7_5Rgvr$`muH-0G1QVb+QGH2*}@%3}r6DyRAe5Ey=0trJtRAD`C=KK9<$;G@j z2Z)3!`DR-J%4o=lN zeipIH?+I}Qq8L`}0ud-Pof9JA6)6N2oasVg@#SwDgGN%8E%f!7B@#MQ7t-XF96Pw% zT_Z^2K(Sl}@8fKT;Dlm+fRYN|rx}@e-86J1K9-N5G|_(#J2r-fz$YPSv)*HJTnB*h zB*FLOz`uRKl^ALMiPWGCcQUFaYT#~?w7VFOS0FD-BI+ry7PYu2U%aXx2Rz2ux@nOj zPheoaCDSb$Pbzfm!EVOk@$qe`<t~M{t6R@$2svz59~IVeTRyB0Wz-bqTsUk;MnK8y&-gFGqj?S* zV)Ab5d@B@8P@9j`+n)b+WN+7KN;NcRpU{Jx03fhamQHxRK#LJI89(!mxcMHajg?2` zxXm~)M*OxaRN2RQNng;fN$+{0>E5_NNaIm&^DO#`_|<^U9hd;YQj1(_mI3izrV|s^ z@0jh$wD?9^ia#P)?G6q|(u3nf-~=|YK)w3XnuO+&%dwY>JH1lN24tJnI^521tM9)U zkf-r?N@`a>k9q>}kc7K zL2xH<_-r{O%qSGbx(>cmmDO79E2EULvv;R1@XNBIm{Ult<$EdkD zNQF^Y{TYQA(g|FbX_$Vdb#08<0MCkcw9+c`Gxxi6t&CDRS2}CEK)ChZP0sBPczwXA zA3>tTEBXNSgEeTO1 zi3SX+8N#Lzyd$S%Ujt0Ym2*CUXPx;XKQHujd+1d%qTcwGfGNm4<|jn;?B2-s4oQS? z%_>Vm_@;PC$~_a8ME+aWMrW9pybu@GD1f3TR47np1kr|Kf=OH7u77e(UfS7gKbylU z>RtTGBj`<@Hf9gQ%?;PD@VT`g3u+nKveF;(#5 z%jq!2+oU)Wcc3I1+|+v#kj68@c%9K}PZ(aj)$(AtZW(VmJ(lLrwfC*HRMpV0V;hP; zM^FS>Ju1%ddiR0#Sn<392 zr-~Y6U9!&)Zr97G=Ga(UTia;!ShCdml~}%@UWzL#Dh^A6b^*4!-q=*M$F?dSP(e1E zqe&M4?XK#3?VO3}t2CMOC-%RM(gJ$t*uhlwS!q)`&9sZCN9!oGRxc~utpKQk*tgo_ zk^B0T>1S%~sb0tBjlGS>k1=Y^&Z+^qsXQbeT&^KHEdem(b!tL8o9(sb8~NNN=K6l< z5skoKDJ#OxT?(nZA)>Cf#g5JXJ3<`JMtz%A!9gA`C0bFt&9syWo66uao4E@Vnd}2VHfYi(N}hP z)<3=KnnT8*-)DM0u14hboS3mBaWH-BIuDr>IHm#N3Kt|p19{P+BpUJ8fXj>XuvPdV(U=hQmb(3en|JTqHR`6q3!buVogz`n$5j^tS+#UYzWfCjCxB~jAZNa zQiBraF`{N%|6JL`*myaef70YmYKFs!r%7)cznQWJ3m!21YKV>IH+cCwZ~pHdDf~ieDV&Bw1!M4+gL-GLiXM$(R@-)Q7t))zPvG(vL~)rdpA=Ki zDENN5BGo9AqRduj6-;|)Q5A3osKrwf3^BcJ$nm`{V6f`Ie8lJRkmMZ&Mn137cm-ZE z=8KZto3W!3qGDNG`(JV|j_GT7%Ll!Lio=FxD;0k6>dG=OxsZ=bH`cL{G%#!FUQCd; zewcU$-!r>|RArpV#J)#+Z1v0yez>nK2WHt2>3X+*f6V`t*GgBs$q9&m9dURyZ#jMI z9sWX~8Sv?c1)6=vjbci$E0QPLyh)h>-EL3%v&F=yuEbG&-Kp16NNx>b2gRC`$5t@Jr$3hx$S(x=&Ru{F0D#9WWV` zUHc^=NgU7=_WevqBD=-do$^IVC|SQGw4q8`-;#g8tRFawuHF3yR4sqDy7_yb_YW2T zlI>Cb&ET{kz>xNDkQ3_Q>Y+bl_wWyRjz6jm8QwsSo1GIkny+rnl9sb!H8m`WYiSd} zr)U!G(qz6WE`jf95W0zB5W8MJSSP9i*_uUb?l`O>UgV6{&)I15td>64+r#(TGo_VV z3#paZ)wFo9^N&j{BIx~Oi?cZ40O9NgiGuQ~$&Y=mZo#|xEDe4?BGVnoVwdW^9HXu) zT$HyAO&21KPTamdcDu#yE{&Cy)!7a<*Vx<#(Iky1?I{8bOqeHhY9aDb-jfaWZoeRWkzU{JPqj z=(QOQ7+u-N{ac3a1>uyoSsI7`I!EHJjaCXxg1%nJ`G;SPvbv zWquxGRza>@c`-gIiWmI(ig=BnS0bf4@(Y3&)&j58-C8c0^4p(NL;4imw$vNYl8;et zDG)o8>6|wH_+^7NrCxpGp7@keG{#cj_^`IEZmNXwo$)DD_s_DMZDG zfCu(rDI&_Sdm#D>T7;jmJ~ z=?Cb>tH@zApQSrB@xeguu8a+b?=mb70%N0SQrq z5OaX60IS{z6#d<5#O+Lz(puBn7aHJn3c%h)>4Xzi2-Iy95Xl*%VrN+zC}2IDC?+*F zSMb;q!sV=|pQeb}Gpo$kwoV7uy#RdLFK>5lIaA#HKkU7CToc>cHXH<`i;8q4NLQMG zg3@B62r8f=A|)syAObc76&R4FQ~?12C3dhtq)U;|k=~?*l2D`v5=sc9_$~H6=X}Sq zpS{oXob#67>mUAr<;F_lrn5ti|T#WFw9o%>p;DROgVuma7E?x^<9v+J)n~4CT33L zwJp?b^2P0gf9?J~Gz&^3=|gl4#)fKHB|uc7IOyn88CVIrqfcPpNE@ITG;h`Aa%dh!>V z_uqyfhX3WrDhfQhT)$(W!`DhdfmE`Qb-zUu5;CpUBI&A;K#uB5w zLRXMox;|Hc>{iX-I(nuL%k{@6W#fkjwr|aKb3CEqK8}*TbWNrtbWYC9&#`jD)eM8( zk|Q!t`M1ATP+wW5qkY!Gw&Z@I$32>=uRz+%iw>1>eKuv~BMF-EMt_UkLRL#OgF52d zVcv36tac@iM^DzJeKNT#GGmhTusC5B9)EgZ*X4z z5_T7qM8E!@x{qvQY&8CHpydMa_wTzB0jWfu-xa?m%;aqfRyN8R5dw^t!v=HYLzw@bw z>!f3kifwMcGMV-@Sk<5i36i|ZMeLCrZ*xWtfz81V|mII6Jf z%d|VIooUg1ciz3EYsrGkRlI*hb^ngLEi6g>oBUt->)`-s>l5SCSJ=Y%um?>LIInfP zpvqC`JhZYv(qwP}u=r>)Erk97b50cMO)F=xkV1oB#IDxQb}4DvCSKEVIBm3?G%`@H zeX4$PTomhJOPXZ2s?Ah>wfl&z@f?SjJAtQ+xoPOg-Rf)2 z?L5p`O9eDN!?j>A3c&~Ai+CA_pZ_45)SV1fe!!fcTNQA}5FNchRku92?<;6uPKdK(YZ8{6t!Xi=6>+QeQx-5SHH&U*Sz}m4gOjg*Z}(O*DCsJ$NAs4JJ&&YU>IZv z#T2Q8sz_a6S#N7Ry~FfM68ooRmfL1xX;V(+@Mt&}!a15%e`GEQi**&T-Wzf@?^!C= z6PBx9f!A?Yiw2%OfMRzYtR>q5E3jQ8zQP_vdEaJy!O%q%R(;Vw$t?U3 z1c0}A&;-2jX_au!@*yCFDAGFI;bSozX*}F_vMAv3Dbl$BD-oldFDw30wkOq{#+3r1 zJE0y)s|!GgnNO#=eP`P?v(vFRo~xygzlXuJnZjr^U^ESaAQlDK-)G-q=(7CFz#8XR z!&4GbjIt~MLH5`ON4Dv2Vfx;HNB+cqhe{QJeW^f7qz8Dy>9VmH*bmk`44CsE>&Q>4 zCG-M#QU^Je!sWQP(`cLn{TEwgQ)#rt68XQu~(uBj`$wzBwqN7wMfn7_7Zk16>UDQ%{b$k{Q3 zj7DFl($DZ5m?ymKkWR3^n#j%R-=WVnD>Ob@`*b&1*Jl$zSYau}B*77nnaZ;x(P3H* z)7X)zTfxP)hsDfQ!{kJ_+%?}aBKOD%_tJOnhu88W3>2gs{s<-t1PqD5R9!I|zl)eX z4{C-okR3~Yem)y0QdBl?!;z13k$UXjp@zkjNCjjPL9xW3D7cI-OLHGq^q%m6tnn}P zBe8p*@V5R;T;U%j#{f`?MgTx%G(992L+5!5c#mW~Xvtrq>A{}70AM*``h93(1(bAv zFGsczR1clCJ-Eh%%z1B>4NPV8@j)GmIk96n7}evq&D+Vedqh+trTBX6p`OFGu@1NF zb9}qR)HehL0)IkV3i5^Mpf5IUw3sbg~JvHJ|zO8`q z9MmKEYoTTYspn`sn8YfGZTDP3?l#AT>q-VDTs7=z+n(L8F7a<(_qNbc+ClNui%@b4 zm;pgFR$1Ke?#BaP9%r#Jb|{IhoK^-djdAeVoBT;-Ey+C>*bn|`h{PuZkGq|>1 zO_!hq5<8{c4}WN%tLH7+$--APU@fQT1q%p_K)v_=Q5nG*Pjsz&5MWhx>&itdYfqc|` zg)xW^2zEk_O}D+s7Ey198VR*MyV-o)rla#SS_bD(m0OsWmO}yXx7Vv$?d~qry;aUn z<;EGE!&W&SLl*iii3u&l=!ahXbE~IvUbyDp`3gH!8fca$ot`jgk)YP_+5Lcl;Vqgb z$&j|$#5l$d$x72tUT@a7?X#kQeJw&>ty=N+zOAMSy}F^sBwG;vy+%Dqxwy0&vEDny z;A1R~XG=_ffA0WWc{8p*9=Xq_aiQmee<5Ac9NF{;f+6rPK|<`Mrnq3I=cHT4zBq;q zL(=N3LtP))McLwoMAk^XRII_pN*WTpgxB6XG+ZpJ)ePCUB`q55-pI64r|5P8?=a`| zWiqDy_fDJn){1{H?ie-$!f=M<=4~w67td@-pvaB0mSCjU38r1TYIqCV0pZ`n;gX43 zm>?hZYvPu7B|iq_N0+E|ACE|_6}u&}ZhX;D-;dJ;Lj(yV;c8`1DCCxJ<4rq>&yK$x z*IKW3TkOh&Yi2&Utww8|sR9)Hp!_kfUAHD;LdDJ=4P$Ph9$YmOM0hx(Yg|<*ie7i% z5IMTi)_~i=BOc-|Txy<(+nUu)e2XJe&>@&bQ%{JtRz%IYZhXd;e}&Z%Opg$Xp)6p| z=3VD{-$B&pwUT2?rk9R4Aqo?02-1|Sp%k;&fU|fdcV|*vf~#|i*DL-<4md3z? zFxqwC##Z})I{jz2--7t61%`5J7lvpB=e%cgkXW%a!a`7bUGvT|OJd(;-)z5J))uU> z0>K;9Hi;GMLI0v)l2Q^M6^p806EXNWSdAD+D( zP^|Mvva(!^sb?V$yaMWiKBNU?k^u+^YjWzo6$Ppo6f20`UuJFHaIau{zcC26u-330 z2o-y@dR~su4SM7zrIZZvb<7?(Ixjg{DkjMJZsq_%?Vzfh4(xFAUY2nQ8^(P=K)h69 z2@XC*8lP>TWcD?#Q1U|!sb66TvjnQ&+`y;WfwE>E8F~!R5P>#Ho-T39yttjUbu)t8 za(58F4t4F8bVvTo^%d`lD{B*FGbOavkDVBMGgX1K;qkLflDLQaP`+rd32S3SNjytH zw^(uO$AJt+$8F^?;g!Hzm~^by*4oc}$;%jCFezn9xAsAwI!6X9&8aBT7dO_#BOtnP z+TCwi6e`Af+4Ei`h0^$Q$ewMRolCaAa7~r|aCBJ1+B-w3Ke#@2;qaLJg|e0hM#6jW z%=3v6U;-mIFIn-VRYIYw?hno@#sCfkY{ZNF*5Ff{E-`zKj0sCz3oItU!JY~#LmiP^ z&u%gV5^WzNzT`i@sS`x2E{xKMQrW+(#e1Wr9n{C_5>aJGWQdl5xsjXAQG?ji<=knC zqDlPB;wQyT&rZnmm{#ov6)c<{B*qbE-eGMMg~w<<7xL?;*kDiX8@7akPdio`j@)|P z!fAjDMBpDH#Aw&jO@?{H8QXJPzZBoE+v@^9`B~cSD@@&}?A=YrI-c>p&CE>Ah?^5n z6u~~E0=}e6jONsJ^Bz7&T|GzKjCCWB#}ggjeEMi?y+6xX^ZLVjg;^9Y#RH_j+J{?d z3NNt}*h|9mgiB9My!mu*)CyS$llk-Xnb&qB1Q(DeLCl-$0F%f5Es73DwkKA2tjPHB zfRMWKW1#oj*1cd2CKRwgV(;NqsH`@?ei{tE?w;d#zi#`tIrw4+_@)oKA2@h`Wo_;=Yk=@k!pB`Q4dOvUV-w)Cz!iKcgWbL z42KT_O^jHh4Pq|>?`S!Yp717uPcv2^aZN`kIAca-O-CqoG{$&m!)sx-nERlwr3I&% zr&NPUz}AhZ;6jFU9vg8ayvC`%w|)|=XgN0SNRI#uCgk8 z-Ge3u=;_up`M$f*K6nR#|JQ-PhUcGW-~Szx0)xp-oLL@f!8#2s6;vVwaYr2iFv6u5 zHxPbVr*}B_Tn+10!^ANs)-q{syc5f9w!a1{MvsLvw(bViv#kx!A^aHF>|+2rC-kUa zKXOrR0!P_fye3Z!W1yVZobdnlk<+7?Ej~Lbb%0W_Ll5w~kp!)sx|9In?r2{8V#hv* z508a+Hf|ixVl~w3Z)GDp{7IDH-P#w(D)N3*3r$GE8Q`)sGUW@Kch zd*r#^O_-neR;oJHk^hB9q?Da4AmU2*OBi%vm*rY$XG1bbrcF4uVTjkfe%G}AZp{W- ze7V1l{549yX2@@f@Bgd54pI~K2%`=HdkpH?L^9yZ;JG#Edi2(?wd=LCiW-?#q+ z%lwP(8s>-lBg}^7d=hncgB#Td*wn^(ymyPjUUsDxb)kplA23PY%}8RDhp<0cIcC4D z{!ZV1glZE6LX3hX27|nmqa#Rx7leyg*?X^%91r~4Qe0hR2Dvse*%k`#SwfGSGT5hw z=5m2_vkJ9X`%ZmFiJBC+5v*rG_C{D zd_H%(FiEspT=wqkn7gVkyi&{}=xqT3t@f64zq5|~D;O`o_27LSE7A7Fn_@*P;e0f0 z#RS1yr6|@4o9DDi=#@#X(W5WMhNBMtYHA#NYnOQND*RTIxw65nq|!&zGC)nM@2G)* zd9d^Bkkx~LxN>gQy47v~`sYvEM>VW$Ve)={0m@mjPSNd|B!6Og*sM)Qy)wAcIuhLl zQRKeD?tLM7&A1SsTO`FN^K89Nv}lMdx|D4tUpLDlv56)Z^?75<`-x{^EdSLbt@&-o zc|mAC6f+Bg5^c{gX;_9m2LA#+AJf7M^%s1FJ=X`D_un8$H$$WKG-zTJ>m7YGBT~)1 zp?d>5RiVc}D-kYS*I92g>Ul#<+`pAAt<45r*%F4>;HTv5F^mkclYURqOWe}rAyb38 z9QWdy1h>Q0Ai`y#t!Zp5;z1(QmM1$z=(#V8g9e|??c@+_;+i_4oz-Gv#=5!|P}vCK z3sCdM3=yd3A~0nS2d`Cwty`n21B38h5n2`PE`t-TTUKNs9>mOUV1c}-DE$hf2Ovh9 zNrCh0E5^JVfq$nZo;M8dcaUp|JY|1 zyw@Idd(lcDxrF`#F%+-I(@z7#9OT8=9Gx?W?WKnkpmQzQ)0tmkngF7IrNc=NP>QXE zPfVR}b%$ck>mrVIFH$2lW?V($gxt@0pXY4;pl!n(*fWeVo*v`vie{DXFV$S!dFB@YQh5I;?f>U*fFG6aB6121k9*U5@V2<{ zjTVtlJNtK5GMCRypV?x@_IL`;?#FkrZAA-xC)cEh*pTn0f!!Z66e6Myj z@iWg=4-&_Z*&lcL;BlZf@Lt3y8p>}sOnaBoy#2`!7mtC}67pNtpA`D+o4zS3&R-gfgnT1|d!U04Gn&QOs}0 zT(EkVMl{4`O5b~@^5yec>Osx9(8s%S^k$+?7HsujU+NDuQ``O)@C53pLEtmMTGloY zEb-E-w`t}c1m0Xlk%Z@0G@EM)Ng=~dK|a0Bg3&|Oe&O9~+}SK3at4y&fs8Rzx=1G- z@vYJZvQCi3AILCut^NvI`tkyrMfyo&APA=AXZv|C4{`|5-^_+D}Du*?uwsz{L7t zRhqs{f_~uI4q2;y9^5CmSp@aT*j; zkce^yv&`Y|W)cdRN$USFll;Nz@w=TQf}jg6GA+@+cjx}-?_X{n+8?oo-8cCSyMLhr z7^jgxb@Tje8CVQZloMjqVm9rXV;`U?M^|ZS6kJeVq@SQ#WgM#BP&e#cS|?$7FYCV8 z7h7K8G8p~PGfZk1Fa<|l4QG9}7Qk(+3HVj7hr|0Ia z??9YKg)eVct3Xz)zR*q)lDzG87u{5ITCWW!7j*?01&E6cK5EyuGImV$A?dQ?hJn&P zp`T1%mF=0*8N4|e^_bQ&VoKwOYB0Hi6_9|BMCed(UXY(;$Ez^whn^gA-rFK>W}Cb( zdlGz7dNfL*AAyg;%pdjJ2KCr=g9mS7Hk`4AG?4GXR13OB#&_q!3m<^myQV5sVTzw> zZL?0MftFLW>7)iG1=)*a90#o6J;-V}_1gX;bOtCA6#66Y_;2k@L>%EB)gI^9;N{1k zlSM4?lhl(mDHMKDtWmM;;yEFhq3rOWlbvc*sE$tuDVo8VlIHDoCi)Q2OD^{Hl!)Z% zTpFEKYHWN`a#_({@FYSd`Sf}T!toCdz!EFBfkt-|8yFv#kn|1M(h_Cp7XgdY`5UtBihuO{R4R88N z)wr{dsGg~MNltxZRk^8iOSdbqo_3Iqz`69+xgdB-IGjA5T2U~^JqyE{w-iKXJQ2TM zFR=58u#9uVMm>8W@mD3aWIjp`Rhq^_G4*&&d0u_$l%96CxYOrUYZt7yq?K{?rEEpR zO*8uU{V!YH+gpKWMh;3*ln9(eMt}-gTs5W_IpR?XHo=O*Pe26Yv zJ`-P*#H--p!D+X{DYJ9)eO)$Ye|Z+LQsv?Z9Yh@Y`e)OivpI3qdP?lshtGQKsN~Q7 z(2sI?^mgXLaba`w6d`H(9i|m)Ey`d%{&F+1XAsaC&PJ_#Cs=roWsOV}60FKxM~knZhJnVcUg z7via_+%;{Ktj^V2$GN*OB7%SZQ(uIb1F$D+XDX>qK3epbjJD@4AHU1Z&^(o0lrNf;KZ*4}vAbA6I(}1~w3T%1`fI7K z%LmD6V6Asss}VUzehIKQ;{Jx^SR{?#u+oj#U{2B~7}5`|Ht}}5v@7LyrDPL4A=6zp z{@Hr@&-u8b>K3lHFfu&bl0X_pVflvBXbK*iy^l8>u_7odKP${lJ@V9_IWP9e>?hs* zLW{MgAs~437@iXv?6=!{;WQ~R>RGf|n~#vvLzi}*I>6jjQsu}N5~j*0aIjL<>}qiU zed6_$+CMW8N%kvNJu4b6$WSZ+emlzQfcoELb*44 z+*6J|t|Dzi_&TI0DC=sz^1W@@3}x|FHb7Ww^5f4mBx%3k={x1hp45q#!ckMRRu2rK zK5}wMtGuVZkZtFTRaiz1L**b*1_-UUY2NX<`MeTJ)8Y>zaP5BUz$H`Se?iWMn;##= zzQafY9)!w4-Ccx)pmM`ul1Jz`^;lwtcX*X%em3F!g2abA^G1EHu*w}}cz?rNbQ-B8 zN=J#q15cYloU#Z95Wf*hcQaTH98MrOUD28I2$zPeA3TOQol}rtvL7WObo;%I$7vme|Cv zN!`@Iqh?<(9Nx;Y<5v0;BjgsayjkofoUlzi0%VXTaM-6F$|jTNV}~u^Rc?e3sgZ(* zUtwn!dGA2P6LZ|&IaO`mCL?m8-R12zh}x)9NYKC_U?5L7#h4cdwoja1i0f%P&cT&>#MpxE?)4 zPqA`nZmW+>x7%(6e=T4{mBezKyGanyZL3q)aTcojwCUKj&{_#@m>=y9`=?P6+Q%F) z5DDJbj!<5Xn6CGxR=Pf)DBa>~ERWt+tlmG+vY`(+WE^)AR8~Y+*xTR;r2F>E%6l&!?K6X!7H8hO;|Rrm3#38178>Ccrj7 z?CUb?h%8kiq@~_GaM^%-Xvd3)b^CQV*Y}bWLlO8lsXIt)RXmyoNG7X;&!#NjW8b9I z@w7{~vv8lsN=Y+6y2PA+yPF_$Gd~#6n^}gGAYOS}8xd2J1d^UV#CFN%hM6iG5l@*l zzpJr{tT|vJF#2f0E^c4zpKN+-aW|%wLa?Hl6of#%1*QoW{ap;n?CX1_xZP$?=zWDL zY-;jN0uJTNqSO)cHsTR=jnVv~&XEXvo;RMt{ZU%9MpM1ghv*03n|+0xcTL2R`GK9X zsDW-q^a533OOiF&j$+MLfnuj1^*eJZ%9+Yl%N66zDwp@$`@D|1tXjK;PihtS43n%v z@lC76t#0yBU$Fx~41G9}3W$~$B`HqxAAt=S#({W=RagP}YoJFDLUd0BD-rP^UMFo1 zs?2gJz#cuI5&|jU7)DQcCvSu?oH0XW6dan+C$@aK4KUO|>yw}d9;h{Z2vxtlrfko= zYWo*13u7dLGBk*vnAc6?f9-v>nK9oM8h<_KXu3_c;`>I#-Tmu78;E*|B~u)}y*v^H<#`Yc z0Z<0pH!S;@1g}Xx8WrvPYSL_S`v%n5-DG*E)XU>dUT)QC_Lr~t@0q^OJLO8gMlS*} z4D9i)!S0G%^DcH$F&i76UsRMRvXtD|AU=|BvH6J9se_63mv<`6++wUFD9^rAep>oI z35eN0Uei*1UNdp^-MgI!*Qv3OiqJLb$p}1%ZixGF_y$(`X(clF<~aIR%x`K&hEIF* z)kSZ#@zj=#ydyO#aJA8yxfSt!&^M13>er5fm%ynej9wsX{1*SvUFnjCb>;g8)_OuL z*TWa4T8dFZQd^(a@2-1!C8WKU zc5akSdXa!olIeSJwYl+w+f(DXRgeNFQK-IKiMUPGni z6GulDZo(dWpfLgFD~rAygP*PH>{pn1A4HcqLxlQLvE$_HYip8^7Uj#(N~#d;*y>UG zV3(9vyd2qc<{&*u$$bCv4Ky%a0O4=q7R>Af5+sh|R)uB!##-$x%NFh(WdOR^+( zFe)vf)gubvd+gx@p|2qXJpe?DS_l30Fl*hlF#fcR!S5@~$7-#!5f$pXDa7hgnV;#^ z!Whx%K*3{;(fbFIYSZa)Q)~jVcMuV@%t18^zlL{FCM1~VX4FO4%qsC}Jo%I+fhTH< ze~^I7fK6)~X0$1Hj>>ZpwJOp*Ne7HZO8QjoT4L$|FwRHZeRyOB8Vt!73RS}&mtPp3 zNZK;W9mRH~TW+EV_A@%6|GKU8GDU`J=Cg-_8^(zH=(WjFdBgR@UA3a((kJ8gsP8(Z z>2WFX{vwXMm}!r2WOL0_1RieK_IFy7g$5uw+=>&UU7_A^g?MR>BOe{CiTw)b9jEOE z59=0%-!uwmAN4omX6{}>Ph7-2hWGH|qidqqGjJl~5k5#f??&faEI8u*OP@V*-xj-S zEWD7_F;z4X9@Lu^9U~;e7~yN9`bUqj7Hzl#UUXf`MFd#{lnjIAJrykYNtk)p1O%D? zGh%T!`ssgNA>q73Zer^e^Zon(AjI>phLhNaJU}9gVJzp5nc=?%5(d+&$0^{Wl}49i z$gCZh#bhmp>z_hpTMz2419^gf+|>7M^PQ^@Qq~WP%6XMXJ?^uih+_KWB6)S|K=sng zp29-I*q!-}r!zl~%juZ0HA6%rj}>({Z6N8I#t6y#?aXRu?Mx-whCGTXdWyXIW#Hq| zKyP8bVU${dqL6a;xRh9T7r?f*{~qd{&mjX!)H_$EkvYo*&2*Y?O0CA~mzN6{A7)vz z6~tJYovZh$+~xn^6N>}t4E+Rp+>p#n86Z8cK^D{$;^zb@Utv`uV*$K0Y0+KNgj2I) zO+HEN(Hl(^3^aI)gI8{!!#tb%<}E9%NP_FVhw2F47)D8MG<{nLCIkBuMC=n@u&Yxf zk+l0NTcMstXfbRMEE z)j|E^z~xd{_4~tI&DuNp_xxKK8SVy{<-IhnE;rgaJYAyYrw4 ze~}SM!CjO);2@sR+*KO~WKcdq=a8#AP9Z5Az|8ox$V8^|lh|lFp=7XXJbUT*73QCa zTf?7z-?9(x(T3azeSiKB4Zk}4SI2Nk8c8&7=}C1tzJ-!@PQ-wd%i8YQ=%cGmFxaiw zeOo7(2}KGjhq`wmH%|)qNf5`Ldc2vu6$G>>2VT6^Eg|ixuM>-j%>-fyF9NZ7e~tC) zCrYh!#7`PQq0e_r`#&S!|Ce&1e>HEqmNWf5k^U9-TPFP!$eI4w=#NO7`m0_3Z$3_M zA04xnRab<1L`>;I@kF*Yy`95OKyQZ|=cz93plHsq+rMGH7=54pz`xl>zXwgcwu107z;lsWs|f$LrIoF3@mF`> zjOG7u?0i*-7~o%3!XxcO#g%~mfVB#r%l<}Fr56Da$C*&d&dt*Ii2M5_-`WR8Y&2qf z@z0L-Z}P?8r(l0<{LDm0{PSu5#^0GJfq%2jt=4RF&A;2`ynnOJpM%*Ry*AtbG^x|d z*1h-_vr0cO64?cOPh0eWiNgH15B4>vZ7s`3tsZw>)kBZ7{U-G&a{@>`1^}tY7u}1# ze_ZkVM+u{UQ~Q62oyt%#Yd>F$_T7;PfeYu>y$OgttOH~9vuCses#lL*U{0rDOLU2h z{UoVsRY#o+EwiFa8}`tbW`(Wc13`yBZ3`23W?q~f8^hKQS~t`kXO}oWe^`q3kiM{D zqc?2BVO@n42C(H``*k6*TdG?^V|-+Z!Bp?!nlnl?uT!^Dlld2^vT&IykE=#}idmaA zDnEL$Dwe(C9fK1DsqTzcOzuEc{-7R-*^1(ua{ZQUN>SgjeH$!&&W0b^Kf0@N6kZ%q zeG^rV?Cl;w5O{z~XzwFU{Mhw@^xkUL;?Rv7y>q&aDd(c2_9+!&g7~Uo-j8>@RcAGi z^@e%MmS)jGiKWBbJ$SZk;ZT!cZGlEntant^fWDW*nCMv@)BM}^?K{+WSsBPVvuHi? zM3+GL2dFGlnj(ttAk~Ow@*X#Y+|UnI#bO;-RF;HH)$jW)YBcVdX^(K)RCJ%$gQ7|0 zK?OkXO{?ebiR*RnGbFtW%2(qn52{73o_fq*Y$FZR97xLNxG*bh4y)hXZ%Nyfy>gxi zAgj8&j7|1?zG!q9zj*KXmxCL6Jyh7li0W93CqoU832dGV(%VCPHHsu;aGl6*3@>EE zx}A?!{MbD<>cvkZK;>I?^!W>_katrzORpp+#Jl*cpV@A`9V=J9D zEpPRdF+eX|B3wW$zGSv5UO>*Bvoa1{DZNyX6o`vUU7`z7P`3HRf{;89KD)@SRs9#s zu{`Bh6G*ucn&NC+pNR@9c<_2pxTsAb#B{8OU6WWcoSxKe!Yvq`FDeQs2+<|+t-JLY zcZSnx&KG*SY6nG~!Kq2r z^|7bm(!Dp&4S9t2*+v_Mn?+5#RhP)hK7OJ)YASt`C#%qAL~KM7bD)&Z`3x$U!B5ew z)apOLV`$WNeOmirr9x2hAa>8FZN2d$wV3-xx{>P;yMbdS!ZXkfK!}v&T}p0g7rivH zcQOQugS4EcM|V3iYLVH{Oa`eIuN*p}1XZHbDR<;P-e{{%~8CNGtnTfcnL#M z2bRZZ*is+Tzm$n`lW15Caq0=Wz4Cp-Tu`=m%|3=Y?kqw1ANvAh=ci|t%DX(lgo zPnA<}*IXnL8^$RSerIc`d(ab{7-7UkG*EJ{Qo!9UzzAZT?nx!s8jKU>C*NxI_G1SV z2HUa5)l%4&1$>d3>dIBsf=1;G5Qi3*Z%39b-^&O;{JCNl$|2as)U=BbFU1K>`P4Is zdr-}$1Rybl2h-3TXPhrb=dO<%bq;E$_P_8_WM>HB8d;0Z9-c&k;_*BLEybe>M-|Ld zp>JH851p~5auk$h97lv9HfIi(l43%PGBPB#5ptIvy4_e{Z;abgpJ1~i#9s6KZr{hG zg^E>NWGRRDbcMJ`0o5+!opX0r{{6Yrw7QqU@qE|fE$Sp3?=lx|S%*-E@G+=)P6_}j z*#iG{{S@YO$?fM2!z#?);g!TY=RYs@HCQmec6oYUh&H{j%;2ac3J%wn(N$_)NhSx} zIF?@LE;wiC@a?)Y;IzOyD;?)7evUrlRA5Y3ezJ*1hsu?Ftj`YNqP4n%cUw|*7~9$% z!}<~RXNJ2@KBx_B_Q~2-OWjRf01B=m`oYa_-}DXhqwj!n7=_CK;-X<{irzj}o`TGx$_(*CxHC7Yp18GmQbymA&MT3-iJ@1=@gw zCp(0H4&^>LHONYN?d$o1;%KX1l{nVkye{2PP2c&gQUkZ+K?R<&%?(`vQcXa!Xp@V$ z3RolV8xCW5TgZ=9tVn`)(Mpb3%f>t2D$*b{5}E$XSU%Y_A6KM<(c;M>xcaxunZmC4fCtP|DouSg7!>;mSqs*GfS#9`mP6 zRlSZ-4_uNlJibo}`2v@VnOOp=HHJeagX~?eGz7cmxiYj?EN`FbPv5~|6ZvV%xM=+T z)t%)=>uT!}%+NVp4*<%hRq(P?7`$8`N0VzFd&pFUzedVa&!n5g@5u4zJd*ih1C!Vl zy9`kNGrh(kH*pk18rFM(Qr;M33ebfS1_ll<{hI>Uwa3I_2dw%fywp?`za)ig3&`99f*R-w(_Syeal_Y8k4H6P zm36>x#z&htn>GXwWe5>d^L#vY59+tiyvua2JLfK&A0QQb+U)m#=PF&X~yC&tmSK= z65b{dpV@JA3d?9pFcan!ZP{pb2-YliS6Lc+rEj!F=lreV(u4Ge2z)$4hW=8I(-|pD zD1qFDrrmDNb}1a*MJ}|@PgFaTpq!I7=$sPyB>FQIP2+!VV-1BcScu(^w&13^1EE(9d>61-tXA`U1BXy33Rz2U0Sk6}YUlibz$wtAj>VLEmYZ(Px?18ALs-@}Ad* zQSqBiEJifoVga?=w?OSSk-;$v*3;kAZg;Gy-GYF?otuDa5c70OeiIGs%au_j7GGid z?hU|Nod1oKa0JZ{wOMC_6;%gCi$k_90doKUOAqwB=+_I+&U{LhpdIVb6PePJ46kaR zI-{fYp)jmB!$QC{JRDMdet4U-=+)j9l7rYa`it%_P6S}mv&?mVg}Jt14G&O)GrUt( zHc-?L6PI44>DOEBiB`vxY+yQai(m7%Y_rtnS`1#DQS6&tq&}hivLjf zVW*D8kjDnfNad}>K$>-0^_vN7)#ft;*yiHr$-)L*r{KqeuJTF*A_EZO2zNFn+ zqvoK(G925#$Y64IFME##(jR6%z1L;2xLI-|n{sXPDvrSgPe)GTvO^xO`Zne_sf1l$ zyvCO;fXuPY@5qgaNZ+#rBcmHyfRl6@a>l{D*(yq5L`29g>kO3&5x0y7aZ?)ykv9EY z-ekdq)3e)7o`?wMif0y0*}^2A1m2Vv9gWWHYMwXcb!inCbIIE-qRQy%c@l#m^>$$3*r60AyoO7e=X;@#q>+te;h81X^ZpELgh~tnaclH2)Z61J}5IUXuN3%>GmM{-IvYg3fGKeBrI;tfF!1IQtXcsyT|%I4u?vf+C9)1C`{nNQhB} zn!kYKSmUaNGeVX?tVdu0aJyHUQh98Fj_l60V2`~i2=)X4s39w`!w%kptuoe@x_`GS z?T?iKAk_B|;o1UZ6-3bZftCtWiVw%Hw7ZGJdhG{oe$zEC5ORD81~?%Bv))bakx{*dSv?cTkD|B2Y0DiUwkt$(aUPI zN1~OD&^>d4>gp{;E++0z-Xst`@>pDk(bKlXCHbGJ$*F+c3E+TXL++|XG`=;8mdRFSS{e278@(9datPs4r+(a7d;*p~u)59AFA<7bCIJ8&Rvf)0tQO z{)BAs!kI-XlMsZCjH`$x?PA9|my*EOpiIEc}o zUg!WSBgE-iXba?Md$0nxAy)yj$(7Bbyxp10l?Y6;k!B|15JrjE;_kjfqpHQkv zN~?nR>Qz?x>2)vhZk}p+dr>uSEX`BtOZg#B&R14RxQk@rG6M41(#jB65iu0sk?0U) z_Q%8^#CL(<%g>2n3R2Sx3-R;b=yTb@$AY-%en`YzD^9Hx(7%Tmt@0tpuRA zTS!u&J_3tiFhF?!N6ES^lW2~VmrrPX6wBf0ForCTENUoR*h5@;&&`4LlEE9;c@%>k zj(f8P=$!9ty{Y*?=ZiXYoYFiy;Wg( zq*1vKejmpKIbrT%3g)6pm`0i>Gf=Ol3>##3wr@HGcsg4O}Zr?n0(r1%M@s)@z662DX8(_L6 zqj>Q|m6?>GhB8&p;o|U*7sk0yyC>1^X0^a4i2CB8JI`_|cnfQEJ=iM#LVx~Q zE65+NgBa?-tjA1vfVH3~}y!q(M( zfxc^&2iEv8nh1#eV2~h?MXB~Bk(@>nD_R1I+!D|vQ(M~&-N)mf7C5jh^GH4EFOw;` zs>%FXN_?rEhM)xE=fdpxvPrf6c`}{3M-8Ev%de%+MjUgveB8qD_`4Sy>v^|u){YDQ z*){#r&G*gxFBrLM5}dXf8me~|38$Q>#umFOcx=nDM(3p5_`rF@RrynVzCf9@SU^jj zU)~zi;Yp}m4C5Tl=TfTDjc`N>fS1(KS(x)$*7Iut zg-y_bB^rt{d^@@~k-_N*)Rv9nW&}119rcNg2Q+$Vt3qyl;71y*Y zZaq9tSS%J$fMmj58@h6c?G*e==sKmNALOe7>=J z6&U*8EwAGke3l)|zt2aPlIE_e_OUDSstG*FNKk`fgLjkkxxcqxz14CMo}Cu?iV{lI z@G8cf{@$>5J+KLXyZ8atFI*sUQFHn$ z>@oT)ObFe{DZk?{EShTz*l8L#&TL0o^ z!}jM-AP<-k|GWF8Wdjmc;xODok((oY@)K`K*J1E=&JnIR8P#vzGrQO@%#+dPRPnYi z>0pimjPp|x_6)|?l?23=oA5$}Jj^=UvAl@c(vWfEBKwDC_GfpO+Nda295xUxlE?13 z$l=U*?2KlnCzMLr(az8xF~n$xI+af?n0H+_*{gEFF4nGADC|oSH=mlGp+oJSrrTdw zg8y#J|CEUM$3{RM_M3#_-%tBD{)FE}{6A4%Gfl=*=Qsp8+&H3 zx0U?CC&^t+HAZSn%Z_lQ7EP8SN3tG4-R-WxaM09m4(lHLvcOv8H^%DVztGrsE*OE0 zdEmE;X%;~BAOenh$ge}K--WTW`&i6SlF&@+;>WHo?Q3NzV`?_=)u~3NFBZ2f!+f?< zI7raQ++D*EK)iZxu1e7OdgG#UrS5uDerh5W*w5u6CA*)|6RW@# znseyIH#9J0fkdh53u&2s;phY_U(PBiSk%^Pzt0rSa`6omFHi}>LE+WUAu=K_sPeyz z*v_Ibb>j&vYqmD_rgSol=1NZnPC(=!JF&Lc=P30shyfBGOqqa%HS7Ag_pXV)%-*yW zS(JGA7iDklYQJW0%L+RIqDHXm5VYP!AXz--T(LxMcCZpY0?HAymv1>yD;Hw9>d04Ei!)c+UZ$|UJnxz31~(H? zp*|c#kt*e>o8CtsX}fOi4YawR2>}i78*ie<1AsR7{9)r_wPK% z?f2Pk1HB1z*D?3uv8;4~qOSWJYl~sbV}>GHvvucUZ>;k-oHQl0j8IC&81OCd|WD5oGInz2vWS?TBtcMz1)%eoocDJ zP(yxWbaClb4mOqu&oe7=dh9e_BD%5|eOE)uJS%^}1FP-&Za1MpIBg{evB&US*g&V{ zlQJgD&vd1CDjU6@SL$ufK59~Z=3{E4i}W3tWb&}KGxh)%3lCu350>U0hoyEdAZVJ% zna3bkumQM5OaPZaY4Y3e8sk03;8dlawwtgMEY-Lrt!fK7HE5~tPn*% zwCixD634gbz$H{(|7497Ui}%)I$_;r=CswCZ47{|eHIT~usZ-Q zKN_}fd>TnpZUI(?-UOVoYi+-O09DNV=1I;1mm<&v$=HjGMG+@JEDvfk1_zWo{9Xw2 zeGAjICG=mo<2qByb?7c;;w5+_d={X{mjpiPw+;SpHY^hX*4ZF6>{--2DC;SzE=M7?ce)s=-@Gfdx0qvIsgDZ>k}}vtI-r& z6!2DkTMLm% zLHd@rNSXlR_X41A8vs2yc;X-4;PdZ?jP?N7ZHAO^MwkT}WCK8?X3Z)7wt));9{Sty z00Dq+2h7hHs(en<2Xk--7_YTZ^;)&j_bsd~U6}8N1YGXBLBq%(ewa8vw)~n(zoyc! zsq|}A`sY>p*Q)d%-Qa)`NZF36`p7qIS_9{#IFu(>=NF_+U{BM(WYjmT8$=%K>+sey z)3dppV1L`c_RGlXUdnBf6Ztj~?ys{O?~1%` zF46>k*mZtrka@le052iI6jrdys8k{-djS$+B(Goy7mnkgA>SB2rI^+f$>yckzpHa| zd~I#hTpy@-=!#L}KC!qcR%kEugH_=jT07UHQj5t!1jLO9M2>9fthomy)NdO}XMf01 zkvQ!xb>DbgK~Ni?#%Wq*i+8+*kNFTaq}}s^zGIZj zkiPf0OBOpj=n(0AuMs zRBx!G`xSN-$B5*o>;1-RJa31oSyQ+HDi^^MO+Ro`IvAuwdo78O0*ZBlKX?@KBd z#bc;6kSa&h0Gq0-$HiTA6aAc`k_6bx%-zE0kombYYG=VWEM`rg~iGq+)!=llox z0TgDVD~dzs%ouj7IX@w@jk(V0CdU9z#dawf(GY-KvQlFfVfc2Fxxk%0q;VBth>7KH z7Z^z1!DE%ptvZL2-Ac-cGTyQu3eI2sc&y#()n^+XNl`?F=3)X9$mD!N zdRcdVm%ZXf8(Wt-@{+|d!O;s|=Kk+a+dD8nrR-^S#SzaS_@{)epq$yI`x%c2&eyy0 z5O?d47l{Mv43H%Uh6M-x(UOwjNNuHlK`ALVB{5l7Yiy8emZSbwDk+x0t-0Rf0$YQ+ zSgNmUUe@noGG#P)_7Wc02A#$cfUD6x|zp<_fM>Kl18J?Oau$$~Splp(@iP zQWxWNQa84mY`%I}W8=JU%L!ytCfbpD9oVf08PeXC<Gj}KWn&kNa&9aYLDXdgSQ zL4P%NEo)=F`F_r0V-kuN)dT6Lyqzks$RHoh7nBo+BC6Az^Y2?(Z`-TDn(Hd&pi6wi z3iDG2Jp!P5JZ%?9e|G+zeTd~B>_fmNUDQMa*X2)m9RG%8zX@Q!D4>|326|I|{{OM} z-eFC3>E3W?(mO~e2q+=~B29_}RGO%K z=K`V$BOscXV<%T5VQZ(U0GD+AnkkTrLA){3=y5+A+d8u~#)p@3BqO4JW*dqtKi90-#Uwo)zOrFk3IPB@oV zFgWtJFI@P@E_(eKwp;hlP=imfr`{G`#FnP@+GWYYA3GsRiu<02R>c`f2J@aYl=?1A zbwPc_aN%y@@QJnqKY1V157m6C>R#@h{dR3pee__A>2Q^WPFvTloi4f*4h&C8kS_s@ zl8QOHd6#pfw9EDj2UEMw6!zo#FGxoMf&{kwf4uU4{VD;4RcFlhnIw?5I$&bJ|-J&7U&?u|a|7BqOW)a5biC>0(84^*%Syl62$({DLDHL(t_ zD7Tc8NK3J?x~rg{;L5aB)hG?^rrKmql3f}H`4&>n<#UdX+m6X3h-Jok+`oT2__ZXT z`*zHK0}7It(n4Yl{|Vdc>Pr_RX+GA?Nql=^bIY=vO+L)kQR14EgiJmwvp9sjzNttn zt>&j(Bg=HMMVgYn#tQs`+;*kZtQu3!mQC9%r_~flNX5R&#*O+=t5AcJMertoQQT^> z;KfU9`s1o%>*B{FlvQR9*px)PIYyD1y|=&Ynbecj=RO9PaRywN(#b;jaseS0zgoA) z^^`3v=DuGt#lduZD?k+tu}Gk{?IF80;snKFmULmh?=K zF%pQ;;h$imeAV=5_8zqg<2a>9Th`g{ zuCIr&W(wcLJy#uVn1MHWJpxK~KwTPjm~1iqaFS~QL6R5He2=M2vob9@l+1U`HP`Um zgVRwJb7I)yUl8Y2^5MbI{J`V}SjkE#EnWwY8QszMQqEtlob6COjN~ddbQ){<9QQhv3`!! z%(0aVGUrrYo>`I8k8?D5#jugVJBA1`kch0CWJB^#nPu~<#Amq;zPWteGi;ncmU+!g z{?)7WdjUg0`f9Iw1G^iEX{swKXuI^R%4Y2=RSWjs3tbZ+Di;sMp4*@-BBCaV){Cm@ zbx>{}De{`81i5Smo66Vq%uO4QQv zI_w+)OjWtJexBZL**sBA@m*>m4tk_&{V3~k(D?2`b9mfU1172u5x&e# zI9UCV5TbGCaHqSs$DK^)vWn;{DcW_1P*#>)uY=iE;~1`P1}M!;;$t`4iL6UqUS4mm zjl!PjTcM$;UT^ATA{}jNlcq09w4X5I40XQAvxh-${Wh3RbXBx#gk4=F|Htt!hU5OD z-hG|x59Xw(xQ0>&Zy+RCfLYs5)d#^cbc_Bprsg@gqxaEmZ~QFiRSW2yYnrGV^pVL@ z^dp>GS16qobFwtmbm)HSVs@TeF-E;z;M5~~#UM_3$6$u~k8;l?1$RK8WdU=$6HHu38G1?lT+3Am=(Lo{0EdaiP{RFF#w8~4}M@fup>i2MQN(* zsr}(T11QyuxJ8w-Ei2T2Q0ALd^m6I@r{}>f64?y$cEh2UlEviiUg$(+bszaN&XI}e zOXp~r`mt=F#?8MIRI;!bCZ}yaH&QZII7H1!kPIMi2-xOUC~*zR9MHvU04+2fpD;JR}v>HNY_ zw~cxJIb{z^e@pedcUpVdvbapPIpmI9X{#@6W+>35?!qtoT1lvmbfKgJ2j^!j7~k`- z$(`%JV*0Jdc{+x^I{sm#8cVv*;XXtyiLo@L>=Yjn<0=XPYM26jtNirOHZa8%U z%-Tg+-0zKD$8avHqg_D9`c3cTIbu?^23qj+o@K!+qaFcvpTcL=#&~cGX{nM$*waKn zpz8vdUywoEC{P7(uv`CkS^zNN-75X0Sd9`=sT(EYhUTK$vFUf83K+ zANsi(hASB;{gCX7YV&fNRx)1mNO#$H!Rtnwb-7@y4qKF@=c%ioXgvwXW}~y7pUe(^ z|NbbOKA+J7H(F(qc%^GZqyf!{eE1q^Np`=U;uVVYTsOVz#IYRQap?)`4Za`oWBXL% zV}#>Lq4*4@ohWR5EEyEhsY2n+7%r-ZFHvw4U2|6ZVxQc|2X}*uZ5rzBnW3dQ)U2X? zUlKf=x$4fp+o!OEAs^sv;fBw+EahjT>rdYCEXD4*YMybmeaY7R;(&mKn}mMiH?t?! zi52lN-YIln&}MsxAzMMqTcUlscKbuF_<4K%)GMFLeOl8yvGl>h=f~6cFiD36fjt*k zp$mtoLM<}~LfmI4l+Qkc%gmd&QPe&^FL}!!^@?-ra;@L|AMxnSKV6|SBAsY8Xs1cM zQZ1CtK&|V8R>X9HH@w%eaOh>Jgj+_{CktP9rDivXTKHJasVeX21yf;~99ggh#ZP`Y zrL5j!&pfbi@V@d*;dK1Mo_1aBo4W55r=ABnUB40}^K^c;tgK><-Xmb~{aCzqp2r8> zBV8x-_zeZE)+{KdsBUPf?^E1u;Ag2tgf^lDM&Yo?RSj0VYaBE`gt*W_lOrr$W_Py+WlAkrCB&P+I}#zAnh2rNc4Lqjxi;;4E|nWXcY zvvxYe-~@p6IbTpM8T3e?R#TJrc7A5~+IyuiPKlx&?B{tSW_G+W{&Z%~v+p+6j(n!) zjUnLdgvgVWJM?bY-k~})FG}9;-FcE}go;W!zJ91taK2lR>)utf!mv+a6Z_3(Zi*Z_ zazv*Vo3()p!4R!}K_E75)0~T|(r1P9COX`v9p33M(bEdBo>KHj-2Ng|L2WvGXsV0$ zU=aOt;2~nD6N1*n6ankh$Z1?K)yatf!v}azLwVpwM>_J`7>)AXVsS1nwQ7T1yXQK& zYFx#xf3b2vn-e>Q8_o53ii~B$tS)jC#e;;N&-5gseU7?Uuugg}F?RWe7Z96cF8xF@ zdF1x?mZB&}!2_31yMA&zn}jH{=v-dp`zbs7vY~z`h2B5wAR;v;~r!fBDiH zaP9u78cGK5=sy*9Y48#xyXl*x^9I`X_{=NU;4A&p)gBC10S5@K?Xcs7540-pW^cRa z)IKmpoi{wJjA*$?rUF;rVmEwfa@h5hiCMe}NWLv7MOeuvCzY65WrBfdqIDMT1#AQ` zNVZ9dQ`ixGZBla|eXm0J%qVp9-beQ%CR!hm6LJ;!j8~`knmQz<;2xqFP=g2n|GTE< z3E1K35}ihJ>na-WVt)pn9)oE6Cw=u5zE~?JHz}sis==@ZRx@1V#UHrBu&SKqg-Gl| z<(0(XH<5?VsStUIuSP>KhNb}|U{98v$S(bqBE8R3%br*cX4@{(CE32(1wj(an*aZiC!~WdIPX|BcO0 z9if5E1|hp<4;ePi*~Lg^It|3I8xVctSyj(EzYW`dso<90;OcFCd}H85p1#)@M{L~B zRkD&o(+g~4N_p^4i`2tWO-y5TiuMGv-qRVBCF()pu8BiRvt9t7kxkTI)|sjrzto_$ zJ|B=1({ms*cZ{!#;e}dsg7LOv^yU%nP@PK^R#9WS_IB9)Q=(XD*kG}Da_q#IEDj!{EW1wu2S5j#T zPRX6S@ky@x%URB0F-x;&7_=(!Ee0TtstA+#OPlVWhy`C$6~grq?)f8EHD96)*r(1; zy$<$xw`C6CqWtNcL1rx83tr^?tkk$)5U_(9Zc?Df11$q%lpVON2VvjB$yj=`BG5Cy z<4lnLGhOV|qr~*=8MMR#|9pT}GW4bXn&XTD$J^}vX)i*roVO5w2r=sJD3SE0S*gOw zwz{vnk;=rgMQ)@*{xNrfs*Zz8%@PlcpnaX_0vj?&t63cSp_jw@K!_&Tv0JlN_I$r9 zFGrblc?C!Yxhc7eyWZg!j2-;E8^vLf`mh{wZo`)By2Ze9W>x7+BStd?L()f9W+$^_ za&n(<|7qur{n|xVTn*120PBOKd%x@d&NJ2Fny+>n?!cHxMpoH($ zn;cP?<~o`ne4{Aw`{E01!8a-I=F2YlAeS6;oxJ`T{MOQqOt&-dvl~)Ayt~fcEUG`d zE+;voeQfSk$f)p+3nCgvJVhCCq`jiL<;2xomR3r+qQ9c79Gdk}e?~OQAr{fFc{=1s zj>X|S&Fg2TBXSzPE#d>;*}#CVR0^d0%|fMpMXueV`2KIYp`p|4 zGM~!hox(1Ao|MZx)c%m~uGJ#>IhloGhYX;llnPgnZKnMT?#~9U&s)Lfb#KqrP<-{S z#QH|PPLERBs`_m!{~0MpPToZDAU#S!TgMmM_LyShj4^%LCAN+mwEMOmVPq`j1gJLK zzptxV{ha()XM^|>{?AHrtnKJBt7R3m6K`-iRCr%^_>&LO%cY^8v*DY5Uy>zaw&Il1 z-sZJ1+&j0&+H1%RI`Ibe!@^`fgIZlUXL0^m>g4A>FI0u*46_W+J8Z~O*S z6bhR%(uEQql;?+37`2Y2c^gWL5fka0HpKNt-SL`OQb=vNy+G+FANaBoXJ%f5m_zjX ziJM`@#tWzY3x4F_ZfQt8G$xruI!wcO5GV8J170cKb@%a{wssCPs*sYpCuS`ftV9AV z|8>eys%k0uxEE=)zEbK#_;jGs#qzSxNnhp5?H27iThIJ>%G924TWQ~R*~&$;(egnY zA)2l=SPiEr-4fq})f+ROLhy|bkF4tO<2#r|WHnbG9WH`;3=1*rv>79lfpRN;5=2NR zYnhN~lYHf`75*So&z>(VXrjhDKH6vL7^5N2CvIvutmn&jcY7HMD4tUnaQq#_AzQn{ zQSIfH%~xHpNp~t*Jbp4z9wNk06NnP4C>&_J1c@L7kt{j*1>L%S<--n?d*=H~lTqZruUt_Us=6XQah&eh`t995=X#=AR62xp zT*K6;T692&+Q60hPonDvt8)Ms)e8%%$1S>YE@M z1}7ppzzT1s&+5)S`B>lTh_Z)6{)?yf%gjZia0$M(gJK8Uqf8Hp?x|({aXCTWthcr9 z0)u=ygehN>fM7@8@*1JceE5u>ihW$8Q9?S9pJV9hDf8%ni8a%EXQSEXN6*v`?l+f= zyW2D#R0ic>npATz$sZ%e3jz!#Y`0>m58o{47R0>E?8q^dei2vw?r^f!`kosag(~A8KN1oXjxD${Q8=?| zj?)rh97S<^)LuA0lk``uI@TiPjkp2Gqg}X2 zFg44h5B5}FdGERds~^=ld|PF%Jan%A_4+m0dyifn);oDbOvaIGMj@Xbth9XtTr-CT z&?PCSwV#QUeWJM@28p6hlrXGdh*Y41*>k85O?1)z1rfvc;D#kO$A@6I-+4*275JGz zxuEO@ulNZS4a6vIIt!FF@rqD z(@NjF@a~&+WM#_1A7z!{jpHZWe3Z@9t>-P-iYLe3N~*tAmzBRxv^Txd1xcF5!Sq2% zkEM{TnCISOO-`&$nT>zbK6Uonz0q+E&~Ov1=a9;-8FYvVN@rtT))d|z_yCr~voM!M z-bgwJ`Rr7y7xjFNj)&Uqjlg4ugNJ;~sZT<-6JBRD|AH)D&T#zZahMHTYM?|l?Wy5I zYLZl&c=_s%52gZ2Aa?HQuo7JMMAg$~sd!)Mki)E>ApzXNYrISG6emB{&AMKFnK*~FzhY)Vu3u?XdKm7K|qgfq0}uXkf~hk6ZQq@@miYF79Wb+%J0b?bWw@86U+KOl={HuGmPlBVzR7)a3FL`oZwp9qIh6jwvJOA*Kt~ zka#a-k62X0;a4G?p3FSiVslcUS1LlWq4Lv`eny@(oo+4~+zLpjvT+-XmxvtAl-3rV z5xsKZl~;Vcdh-&=5Nh6ocv3z|w8K5_S&+6&B~IF;1v`QVY6?1g=nWBgUm9}^u& zqo~kF(abP+rH55s;mN*Q;`B5Tj8FE0+7qnY<0EjL2kI^I4_|P%e*0iTd>v$X3`bV9 zibmcbQZ|p1Vk4;R1rvS`ZGr>UWIumQTmIl+YCD_ds2eEEJ-kB@yu-^v)=e^8G|+(#;> z1-e0Z?gm#E)7jcUDJKn-W`>9I(DVD#Ab`~N|JbPgAr1H6AzV1$vK@@gF3nol>b6ch z;Zs^B*j^~MBB_4(!iwC(xc4Wb5*UsV2GdaIzzlN|OWZuuca7Z=AvE8}U&m}W-p#rw zEKGfl>YOBAhU1k8@R9qfg5v<}+CY3SVYOsi^4@bQ>OpzU_Uo%$kRtZ;Ji?~7G#c#{ z%xiU|YxB9ChiXFrpi3uQ3JE1 zrbOV~g(@U;0jPpl34kg%Akg&5U9p9|uqAE?(iHDaf8yQ0PA0i-@6~X9Kpk;x2S!}86odp>2G5K_02krT5v!0Sd z%v5FNv`x`5CkZ*P?Ky-5Rfl|yBsjeqv5y$u0OPgyOSHnov%E4n@ea(=nk84gs^=HR zo?3V55{4jQS1-h^#%+(=5ovN)q*ZHZH(%xtSmsLX?q8|sOn+FnP;Am34M|Dfd`9c*W$6Fy=`y%>@DL~XG4 z^VpG1rzS#NrpCpGR%}O3d~A7jy_iMJ<=A#w(HB5&SOP?W-%`Lo>mpZAP7l>17?57X zy}88E>2ohqK196D88VrgjtgwdL=|?3eV4q)TPU+*>|NwnRo~;61!J~&XWKH(UVs{v zPVbW)H@_{K`BGNFTgFM|BQw*G7_fm5@c~!h94%e?fxD`NU(UgETJICATN`5T)%P&W z(c)>ha8w;tf(VV*{s46ew=Iox$;;rK1yzq0a6?>>oU|8cP&?*B+V<(&VHW~QpCu&? z?oC#{y)R{Y;eJ%O+NdLQRCSuwgYl|IvY+MX>Z7~1Q$SDh6#2!5LIi_`J$g&S&`VQ(67Rs3QL}AEz|dR)+%&| zVt3c-K(Zw-JG~XhyWFd);&FN+(B!6fvU^9weoIhjvw6l86Lc>i?k-FLB&7Bt{WXH9 zflAgU0OrONb$zh2x%IJ~-|(!3MVyHlgnQVUaL-OY>3AWc!d_zNFcm&EuG(06cijD* zZJjUsH)ox0Gn}8|kA^(fehz9E`0gh11y3X^akWKBlsjj^D!|*I?^EQkr4viIDAf5j zQvj1|Z zvTDbbO+KwQCtya<&GZAru-v=Z3xjSq_f?k0HEGy;(5m5uLoDVdPOWo$f(+hf;q=ge zHqVFjCW#O};PS~^_Mb`I%Q-mB>PwSfw*=nH4vy4&#UI);eqOnc$(tSS_M&S8ul1SVko(L*C;*2 z3N1XyG;nTrKmRo@AF3#$Stm`ZwI?4QIllhST9Y+&a z+6lzYa{fo*a%;D4sbTe#^4l+l#-oWgcV~`)gdD>=LB6{p*Sm}~_Qa>!6{`&6C+`vc4OgdUZFfCE_~Hptd&ZjZK#&Z7 z>z*K35r@(?$Y(-9Hp&7@B^h4PXktS6FG!qHDcJ)6vp5$0Nh0-wsQrCN+xrn4b_Z9D zheD^qCYL&O*^wit##BLPU#;)>qxW=yapcS0D>%*juN8a9O#j+P;!k8lRt(h!FfjhP z{R53o2ABYOY%^k$r62t>qm8B|^E;50=I1{!pCzhl(`Eq7VwoPm#o z6ND$5;E)lwt(_9_(6p?k{FcLI-&I7z_TNYn8B=TNiaqbX6z~sR4l;QCXph12g%C&e zO+pF`4*BhLz&{S;_Nyw3D<9RlSG3;FuLm{+sTzTE+)lpd1*M<*#7Rb8=wy+fMI~S!(w5@&RHropj`S; zZx5)D_(JE!cPk_=dMBkr`)|8>{2r&+v8!6dK_mK3!$y!gq~NCsH8A!i+kqYV&8t5QIT*e?oqG$cNfY4WtF(qLKh(C5(=Qx&_|lg z0~%5R%$D*?qiDEpJe=L`^n;#(Nb2;?LH5x zDY#&Kw5saQC;UxPrvoib9X`!=7#4OgpCP_4Yb&|Kbb3S%w0-)Bh~R1k;$r=dZ!nej zjZxN*l*<;}rEiLuV>A%`g$rNIo#m6{0Tgh1mS{y4uG&l^!>6Ah^(Yp8JW7iuSZKqCwhWYbO95X`kJ)I{+>;;Ev6pE;kO6p3QL z9MDRTX~3vlo$+P;VEmA`sVf-4LDwVO!da2l#P8TA_nY^}o#E(U`UOF;$GtCvd9=2h z5xxm1eP11|YS2{;22&p?jy&o$yAww7Jqj2>+ex~uxq+xH&fS$TP^95PBMjvX}n zp61Cz9_|;WZo9*Xw!9GmbnJZtVX916;L*OrX9t>^##^58PR7~E9`=_iszv(^4Iqku zL8jf*YRlwaTi}+Rn1!xqpH4e>Dg=I4r=qYX?dx8?@BEq1V;GfIEcIE(u9&ed0!x*@ zEXksD86rJ=x|0Vqp$!gv48Pp#V`e|GFgV2=*U1&)o}KL|EJFxEeDPiO04|P~LEuCq zFMp(E;#^WGpQvI|!wJ1BN1am5QZB_7PPoY0xfu<|Y+GIWZm~{iqn#QAK%8CWu#-4q zSH$~$0~%=;ManBnKN-oZpOnG>MKi1g$Tt65=t(MW6A4b??hhOU=9>Bo61(FY$d3Xu_`YPt18B z^KO3&rB0QWLi%LvmU(xNUgB-Wg*aeiHNaAIcbCIyD4lBT|A;(JyzWEV0xK-h9kl*h z6mw@{>n;pUv;O7CL8g9J>|vOc0Xq+lZ`YGA#2rr8I|GxVv&P`~0L5}e*#*4Om1!Uk zs{oU(*~5e5M}mq^$U_)i?<@?8VvfP_Q$g?xi3A4>rAx1}M$-?0Jj&fC3y(A;31h-j z8@0LJuMhP;kxloxHevmo_~;2G^M$8ASGp1#y&K99j);&Jhi-`$?UG5zyIX^J@^j)$ z3yedb!G(Yob~BLqR5m!5VEvQwLuEok>kHG-#-+{L6RV95c1bdsN8n(^XsWQ>rVn|3 z^BQN96`J^gdPPfQDd7FPO&!PM88!9I_wGQRo%%+6+s}{;&IwGze+Fzt7@_N~rODHx zHQD8^x8uJi8L`R)F9&n~=CK73fqxB@WjRG-0&VMqf32kVfAfE;G8n@k$NtY%*ASim z(x`~dB?D|&*#6@@l;&`FuNEkyfizVmxe-y~`O@tzSu^+f!~9e@|YGuIT_Q8W2i^bC;%% z1>L-3!7?{k6SQk=@^=D;8N9y+{wWZ4o#0Cq#7)K4B4W01($&h-XKzw2*SOO*lXJ!k z>KC|dYSo$~oXiPx3&#QU_;338*l(Z&S@#Q~P&)k!va<&C#>2$v zv}JId{E}dC%^Oi|mYuR6PIP;W)_P~#Q$fk>n0INPBoFJMoy9MU!(8fcMa8K+yY9x)I3D>}} zT@S(E9XZds7Wxaq%?V8Zo+2V}L~rn>7T4b>Sd?7~R^udD=kxBYq1l6QTaE*kLHo-P zWRbu>k$-=8fPtEcZa6^&f6Pjr_M%FH)-(wZo61BPfVPkYSY3F$(Bx&b6^^*P3JgIa zJd)`wljvO!#J_zGK_5gAyK%tQ@Xf9lE2}n+`XH94Hv!+<==agD-f;@3B8 z5#odJ@pEtQ<=Vx8f6TB-2lL+Z(h=gLmS z7B6I9p3+tlYJ7FNIw99V@)_iY=}+%Jorw!{Wb#jEasYKWLP zGuT`vwxx@#1i*>)Fv>l@hjvkc@vnP++I8vB=hd#A3x;F9<7@1By`Igz0wq5wL;{p( zt-38M%?0Y3Ptf!78#|MT7?9prr_RO#ad(==X~wSiSI;=ic#9TnhDi}?oOzCYb17vh z{(&C8L_e!HHAf2=1A=2$2Q~_jbrg{i!{0NTnS6;xe*0wMsq^zBDk! z`-I7s+%`iyDrG-pT4l$rf}c7S7x>0Opi3iO>q`6K_$OP88A->+-kMec5Sr{Wssiz9 zEmYB`-12ycW=W{gR_(D5z$4_YzneEj>zdtY(Eie)Rfkk5i<0xl6zGiYFboF6%yr_f9wDZ*;)LaEM0i?0kIiCOGiQ}+J5j$M0AL%s_N~H z<0jXP(>LP3pOnAbD7Qz$@QS5co0xp!57y5E%BPiHmMrGo#&mp2P_Q1Qe7GD6dz9g_ z&X{C7wCqE%H^4mfR-jtyetX-yZ)|YqHp`1ATaKvNURgnXsccP}1oCX_P~LE(G2P4Xg-x4_vMx{ZN9>wc~Bh#h;$+af*&H4-H|= zNVpt))I@FPIjE8RjtsN?j?OXi3nGhX=%TP8&_D7P=MiKDQ)-IUW87j*?k~ujzVlM> zmRNjZW4i2af6$$FsX_cc{f5U)RW3KP#_bF5FVUXy|G+Bxivn@5hgM);SC1W`Ou_3` z+3El6IGjF0!Td){A{0HgYDOlJi7&f14(kh_>JY)HkaW% zy%|XtDzfok*xMAC0|SkPxc(zOqsD_VS!yaqWDRmY$Q7ACe(?4^CPQ|}jwBb21trcd z$+ZWTh8jcH*{#~bxTxBF)wRr9DbL;<>k!R;bRm zbVP<05Bm}C=MB*1vS3#Zn|cCsgzw1POa6k?vf=6=1hgOh{%?;yw0w}>@S$2UKh3H^?IDOC z9SS8jLW>k=0;)V{0yARD@PFYe&|>x~Wx-udDFbib-#mEzb44`23FF)LTF7xo1_%G5 zZX@uy4%C&60wr0<@i0(c8Lgb7F{{8|wz@LuKI=Zm2RATR1qbu3 zeu8p5%WX0f@#GX@X=bX6kE%(Oj+hEOH`DasLDALuGCuql;_=Y+lG~)W38C@UQ8R~l zZ&7{743vG;K-wGZAT*RNmK1ozpsm!sderSB-yPqpM;eN@%tG4eoe<=ZWUS!(LN(J9 z93PL##60GosOcH3(Wei=_;60NOZ5htfhR$Ci}OA9cHz5)^CdZhvx;g$YKR)~ivgk| zuMg=k7-C1^yQU(7#7k4*BS>RX)-Dt4i31)q1md}L^;L9gL-bbE`qZVObQ_iOz>F%M zPw|YV2#(F$9)E4AWtmd;T=IyKc$1LXhq^2LhF8w(LXI3{T%`&~01HEaviLX%24-df z3iFR!F1szhfAo))fS`(~()HYF`p&?Ix%bz$O4cW7*#jno+t92QQ5JSX@Pmm#wHU>>croJ(na9 z@UFzW*$m7TJr0^Rwt<(djBXlN>8q__BTG5m!M?p^HFvc-+S>E!;Gm8&VM2G>X8*k! zk~joG-uH98JP4&Ys)c;9aa?n#&%5EESC8dqzyRb1QFcCj^VKWl74kLjJu*xRnD92u zN$m~C(a2&hiq+zC2wHSPf;6`rNMZI{@HjBOfnY?KQRe!v4(s<939m%fCIhP%W1m^@ z%kx%F<3NmY@FE{V2OAi;>!KTAk+w(!Zc-^ley{hbCA@RCo+R6hYI3lpzP#rbQKJ0V z2}~dC34$BRL_#!Z8*r23Gw^} z@D~I=jmyK1{ z?_fmFQ_r(*tQ!jrIvJ~cJCxU^5CqBK{e}hdiZ@(bfb&Jx!^@oem%|b@hUtndukU^X zF_qtaCycyLe+$7vW$^aF=z8(apW9xnD3DWtMWgpaOOvCVc0;Czm*7=0KdwZUBwgyB zIS*C@XEWvpd~MRYuixm`Ee+6~_@PZoiL)U&?$n@e!%OeyF}_U{5LHFzhwP;N0oN?dU1q!L}5W% z6%ll5*x(EAX9$Yz)+MATm_X6={(JVr!^s0w$i`ZlO(xFt4Fm-yw&x<| zC*P-E5L_2s1N~P|G=EzXB~Wd+pQcnr@RJ@Q=-Q@KwelrUkvoO1$B-2OR(<9^SV;f+ zZGU{m|7WfT+``*iBl>V}b<9|jvpgnW6Z)sk(Ph+_v&}DR z@1r*OQy=R%{4b>Ci$mTXHLn$>sDXf7LrXD$ATQIBmZ?@GiCS^klZ~)JO}4Y+%Wu!B z#T-o)@%9dFtNjfl^(B$gvY!3hBX${17Y*4w1!}>@Q(aU@B=z9m@8ckJCH*#7HCp#i zkMw>Z?DMxfyIAr~-Vs51gh)7y$f!!p;GHi1<4gXyTz`zNKjz6FVP!Y<{UbpBRbAk} zv!1+f{dT=jZqd*gQE9egwc$c>qqiZ7VZksbbcFq3Vr;`%PP4)Lor<%iRb@pnI+=00 zkk(p9NpbQ!I=cvwlTh9a4Y!B&*C_g_T9AwfF>gmtdMVS2&h?md?{&I3&Y<7TIwkU7 zElyl{DH*j_KG2n`W6K}l6g_$u^k>~D{_QrO?F&e^|T@tMf)|9}4-DK@D|Iy*y~?|xT;5~k8XGuO68 zkg5r=`+fBRy!D_HZ%uYMS4I40rEWKT;-jg2^V)Z=sCRW9Z?g@8&a-~^aCOrydzZ2B zpmH$$p;lU+jajkX*r-`|C)bmEe<80cEqRSJY%|S#apgcmOr}OSidJIe>3L9tiM~9uJ z?Hk3;e0_g3>hnP+BGU|}8H2}7Pu9X%Vq*r&ZDSw5FBRj7z~0)M!V9 z0RPab`lE^e_Bp+wGJ36VXCA5*%IztBLCr$xiLc5LW^M0Ak(kADx7GMAos7IR>@vU_ z9H=0{=}qIHRCKBoWsk0t18mf5WKgh0*t*aoKVf^Zqn_K~XN|z71MA!Osb+WZ%r>*V zOp>kM@)$Kmrt37Je88R9e>({V@uOT=YgJ!(P$)9TB^y-MGpaP}IFoMj@Vr6pq7ML} z)z|pY-q#$kr^?aNt6gS14j|+%26x-pe4@17^_LxOUZnRspPixMcMjEn{y}g;(_u_e zdSy?rrQWg*Mk;xk<7i-!F{JAYkm?IxTAx~tK?wSw1^qU=Z=s;X$LGIn_P%=4x!h46 zR}?!F#rDnF{&6Vxb9v~*ve%Sp1606Oc`DGmKIPhMLhE_E)`G7VMVEsyLW@@`s@?^1N!?GF#2plAKR zk@J@$$|~cXEKZfJbF^@J@K_y9zxMK@kc&(r*@JRpk-nciMY-W_FIecU|8XJC&&A?# z;lVH+mYSaVnyIv|DslP?rR{|7Svp|T1qZbpsID!6@jD9Y5vxWo->6RP_S{!&sJ^v6 zr41;YSOS1)Cka3S|53^^$@Ep?!Nna1D(Sc3jwUbmUP|gv;?3jAkgTg0^Jvl88K8<0g$`rqP6PlJes4{c$uKapc z13(k{^=iw}D+86r!Po!oTM^)|b?~et7~boCP-x<*kB(hF;|E(eTQyiu{zPvI(2}b; z3A`bA)<#}F*D`!;B=U;;-t7IIU)D)Yp`uk>ZdOl)c`c7)0rZ`d&Ln2YKUy+I08m-HKOkvZtK3?6qg-0qpM z8b-T|F%c!La=Xqr$g>c+Hzn?(E;0-i!QTiBiM5QC z?SJOe)Lf^P$i*N=+^2QhAN9LM4a>f2cJTJ8!MgcgN2>I@q3yC0M7 za9}>`!{y|^GWt>P8wu#N9MQtZWIBfF>E&Oy)REP8V%{O%>e^hX_ysZIjHLjnj~T>g z9MKFrvPzh2f`u5Uli-c3O!M}Nm5~UFa5X+HO8dmY>5{M$@g;eU-YV8iw29+r8{AYh z&T(=_jxGeQ@^S1hh(yL0x)QA1r)!7pDtdD67exHz2Y1Ko3nI+W9Y8XeLEv-$t`6Z7 zkEOSui=K(^&DMvpArB{*dZ;U@#NT4u$Ic8@1q`0)%BPD+euAnv^)yMi{BaHRhj046 zL6xqR<=Z#>zspq#8{Wx=sY@P|OkhFv(*cFeB$j^>>O@jqRx9`gxoMk*rG>t3O&I^U zST?=NErbfaDImbhlM@8fg*oD4p{3{sxEP-7wdqmWU-~d|M7!*fkSf~gZtGaD?HV6C z;L5>v-A}?BaH=5VjAa!Kx4Kfg=u_N3TQ-!wz^s>~I8rRD$WkO+#L%{rLXV{jP`Qe_%l+*w>0 zB1*jURLMSY&<5!H=A5c+qeBZI$Z(1mY9eKIx%@yl(Tac#_0dD~29~Pq#hW$abMpP( zTJEPDFMK9zTJgl#_pq5mG!T4&4<{3p=q-~)V*%QcY3E91Do8w^_<{?snsX#Q)|S~m z(ioq>!>PnyC%ihziFQFe!Z1(R(1rb=7R2!{^WWRNJv?!@?tgut*|75LgFhZ2Jy1b< zXHrFi*5Rc60_t9>_?%_BTpE$Q%C1lvFBE$R4*Eg0^bQQSD)?zxm@?0*X&_{h z>rm;7%x9Teq@t7V4uZ7rhPKDnj!azC64esNWi4B6s?wfm#&W_!R)9p{J>}ywt_?=H zMl9`~*WUEbYT68ts5iD;ST%PQG?*`uEaTr${YjWmuQz5`pK<2nIV)VzZg-+(6*q>z zlYYG3#Lhnrf+Z}gS`48HKx=|{K?%m~huSf}pQdMBS~97}=vktS=xFaOVNoeOee^-l z{-`689JESsiwe+fapX~YO-U^6SREo7QNBDL+k_?`BoZ-U2SfxoX{<&rk+Z)b76up1 zvInyLIiMyN^{ke#RAG1enegVxjg8mn`DPBtS{GSPftvCF%t8ly5QASQ`D}wSA4_a` z!s=#q&+^UY?P|50thq-?l~s&5b2K3ev*w1!(V_#w=?OMNbEZ_`bvj(T0kf&;xDbl*gh6$pBPdMzl?UMhOJWB5`32DfFe-teg?1J{CmX*M?_cnq&+nJl!# zy(5QEP)ONcv;zh-Qp^Wyc=CJRtL*zoGDv1Vb9#7ef8!I9iFM*f3(sqpiNf_Udz5y% zv9s=UP6vRr6DY2=k(Gvf>#P|H``pr(} zrwL)_r)+|5J2IMj15KJI3bjcDbhjq#rx|v3mCly+1LlSnX*L_1;IEMJmASQ7hki2% zmO+!NM?gSV2y|9%1x&=|`1wxdgs3-eZJA4bNW9gBQIXTfRh{`>mNPT(QAaZZ>;w@M zWps}N(S7Ox8!AoKlTk1s-sm+XX8IJvYyvNw71Mzv};V!Kj%{ zE^~%;4^9(3)rMe1{|F>0e0Krvs0W!g`s&bp)l4J>i)1^p@ANhGCf)l-U&vSVV)xZN zI&g){!A-^&{4Vp0-oZRN!qodn9#VKqEEByqAg6+I%+322T~qK@d;>F3_;Fw=_HSvg z6Xh@<%_9axmM|1WKc!6*;(>euoaI*ePNR5P*3PrOg*~cA%z8}OjQM3DnGURR=gj9s znTSesc7Qllgn&`|LHrj4$P?ZLb-`!5VFum(2(-wij^e1ssBTod{eE)iH0Up-i;{Sk znR2vZXR0!NX_>bHX2r>e-RF>@E&mp$VP50IIi_RSEMS=ajnZsuSxoTE`Z{(*u5pqVH+9v5*k{(a zlpWp{w$_f<3p}@%iSf$kE9-be{{tp>8j(}s!a%WrX+diO+{fT&2BFi658wuGrj_Ns zBx3P}$Iio@ePdm>Zu?>v(t}{JVg{^NQJr8CpU1LM#W5GJUAqt1a^7u~SB9O+D?gQ1 zJg(flEBRKEE2x$bAA;ypT?S$HKq$$Z-ZZ*O+v`p=-Aoz0p9YjIzu$P6HDMg+1&u;N zZ}*;WZ9Dxba!C89EMpy0X{S%XMxO3{01M97 zVAuwuiCOm-BpyCJL_-7m{GnN#4eSOSM|M%CLoS4&v2R#O6?>z0hI?}6Rq4F)m=e5oej-C>#$J2M$Ts@ZB_C!e$qz7I;VKpGK;8ljO|TllCN zv9gEOOR|+~hgTf@^=lQ_vK&eyn%gePOcg1uVd-q{zaaZyQMip3kV150-*!=j)i*22 zRlH4Auo1hdJ=xwFjqxK|na86#z5D}x^B_0M1TrtiJ-#o`s9nrAc__#bW~&Jg0#8%H?5RAenxAwD_xcC;B)P^7Av40HX@4p7}rQy=Pcc z?Yb@+q)Cl{^b!OVq$*9N1w@*B2&nWD6cG>s6=_m}AiYFDKtYNEq5`5+si7AU>C%KG zfYK!a41pB)%Qx5D=Ua2mv)8%KI{P|5_WT8@BjbI?c#okcwgAB%q_S~gH9$awB$i5N~{1DYvVf!ZT&LzMo+yD0|5%xwBZO03e zuSF7r7A_qsOM-~C3eggg2NDCOIzq_Fo14G@M~88ckB0XntB`=R1|E|qLvZhaae4{0 z6c!cLoe6y4)-*bBERZpGmhE)wsu=_+3;Qch;Xge<-a`Hp3Ss!4VFnN+Z~ni(W_ELv(26|Fz%>M*H5J|=!8bmC_3NX%7U_Du2o-8IxH0QTi{yvx z?g5vKwufUMBUxwy6k8ICK!1spivv#Cx>m@nx|VP&;DeLYW!3Md;u*FlB_oY)QP+T% zRHvpRmr&M`T(V?f8Q_v2ilLfyBc}L!LW?LK-Fc=fTpAlosTEbHTfi0#DBP5B7=6P(b?OJLA=b zsX}zZ7zTVp7JQ>Wcq(@w-`x68m8`5LYm^%P-R|3w0^I?n{`R-1r{VV|h<0Q@PoYZ> z$=spl!}sN6R5x@6&j(>^Ym=|VNZI@$YZxK>dL0z;55cdSPr~%6A9Q$wMx9Yz{9=Lx zrP!Lo-sK{lb$pk5q;#&?74Atf-)Z#^r#ojj9|2b`-9SHr;VsOk&tMTT^PuzURnC8L zKlgmCYJrEh!K-`LSAG1U{6S8?e4#S{xOjU-xZ?JC z=)&W-@h)Q9hEduq70Dj-zkC6dHnN8{VQot`L&sDQenYf(R`1`cECC3_^mD8x&qSL7X}J}2QNsQhjKVz7e73x7FzAwS4%@`Y;hRT4yO`oK>~OK zNV)tPEXm8dgIqps>UaW_xENfhrMI`d)dpEGob7VtF8CgIAM=_n?A8jPlB#%gZvh;o zyWV?k(b0Yo97UkCVHA`%t1*?|zwiOLyR$-->sXGa;E`~-utqRaTG8)qL}L*qyW6?1 zPb;P)QG2+z?JB<^$j}Cy8kBS#a1Hj9xaHtJeAJ^@+-h)q25K{j7-u(S?IE^1@rFllF$bv!O{h?^t@86QsDh9wUBe* zK%t+!_ORe0LR0h<$d^fSws^(*+Avr`nSe!J4Vot8g?q8kpkf-DBOfUbW}(- z^Kxj;)nOMwh5W?~y1^_}v!~xt+LsO7hy~DT6iLzWjod95^L@+u>j?_?a4oJiQ`3F? za-*_G!3kAA2&BqWbM9}h=T9x*VzRAx%PFS|CHGue9qkO7Qkbp#z9{>pBL>hLNU}Kv zNMhaxYIot%q!+mK7fvey`~mpc^J!gfZ%TWk--)MKoPQosdHd#N-8R^L;cjv!$>hk1W&=R}pb|>Knq=#NwiiM@;HUW{T94h@)=WO@{ zO_JpL@{i2XH67{k&UCaKJD1~t{D`aMUE}dM$K)P*$#WHRzU;IvSQR`*H4J7N)ux^m zD5TP*8aSGDCe56ST`JSf?Pfpw6LM~3^!!7~$#7YU6DbUD(_TsHnO85%cFs%w-hEg} zek(Oq}lsCjh1u{YDhOpuABwKo?^`7Ki8vNOjU>Uj)}F{DnmQM0Ppeb zpt;Qj^hU57J{vG08sV2>H_NMyeWdR^c4+P}BKE&wdt4BsX?|8tXm=pIl%(GVM&_g= zn@&Qbsx$G9U%9#^Q{VkmB|rOldVDF<`ethUk`!SG0)fyMQ}dBt=8q87-c3U!^(_a{ zTjPNP#fV1<8y5blhpNg_mm+(6M10s%^Iw(jRg$V(pwy@6+X`}BaQa_44k)O@p!*wL z`JdLS0>?g{&@44f6|F2{VPfeuqW?x4qfO=W{z63MSM|hG>~Kq4HNhj=I$!?4b}1~x zCA~#93-S5?hIkqV74T>+EcjB6N@8u=n(r;O->I2M_|<>n^f&!ivH}W=S7K7^7TugW zUK|#Q>Tp>SlPmxlOyNi85XI;?ig4*l=DSH1Jt=+zqtauiMIK<&mpOuzC_Guj%Xp(s z>KSWN2SMM=%%}Uy`lM&=8G(?%j3ZjxW`gs&XWkb0W@*MVY)I~yF2t_e;q~Y9AJ&Kp zU%=-*!!-xHxRzj6YCqU%1s*Q>#2qaAVqTi{Q!51>$W8}=8HQ?;60sEtExgr)k_a{1 znq;xvV%!L`rP6T1=WRCG(DEbo{@Y~s2c3s?Dv zB=v3i_`Yo>$e@1O*^1AARDQ72DWn4~ptc#c`^Cm=Lg7UHsdGQ%$lOde?-c~^kk)FD zjZUCb)I-xpu%p*7R-h2O@tSQp3=EjjSWWPOxbPZStg5#9c2{*q^7eq!qy#Jp>ApR9@)4;$3RS0_hq)#|))6N^yD{mS+1RP}3#Vd4BOtz)dR7ASr3V^~l-Q0uE># z5`Q&6vW4$!$uLE<@D||dzbP}3dbFnI9w$qD(dQKH&$pnrQ@r}Zb;4&b%pgocPC4|@ z_(wRpAI`h-<*k(5qN8_|y`}#7$J~YMmEM+VC0$&(JWSk>RpheO{ zJx^kcVkFhxEqeZFeO+yJq+#N{4so|_DgW!Xk>@vNl4^f?frbV(DO{N3+A0*=V;-Y^ zhjhzH6f2aH>(i*9DEaZ3xZSa2!^t9>)O&9aDKBDA?L}yt(pdBXp@(D_jHnEJdr?5m zZci5?5e;o&C0Qc`D2C(drlZ46AKMte=UrBoj8W^?e14z)VVbdpF$wlhpCuqRJ}iWw zQ+`8gzGt<2ZP<&#I`UvGwZq3g=C~g?n6pIwkM+$6cVavQQs*F22)u1tqEyJ*1 z8xvBHY~633eSDkcX7*tz$%%d@J(6!lCEOh|J&NPs5cXBo;b_Y2v9EF3_O!uCG+q_G zpa^6e(ypz6T@W_n8A*EWzy2st47u%-K&dZ-FGr4(=wZzM{7&!JM&=xNSvI{M$ zgH(kW^jXle1p|YzI;v(!g3Y`m8xof+v^X_A$#=|Wlh@iv#an#OQm&$ZcoIu_YZY;u zszJ?y-_B>Hjqb7~}X^QBAUf}77-exhkhSj5aFn9%ZTwC~9E z;ToCU@2Mr#(|sQwgm35_YIz7^g6U8I#d^Wt0nY#}11SrAc=5E{-phY_!SXi*T7A$Q zIE6RlF-CjQhZ6M5#d$3Yazz7;i|?({gScenkAKr@F^KkKVul^>Mzh@3d4F&~fUiYwAs=?rul-9}Rz`vFP-P zlnSS;NuCt#k!Xsdu||PNy9-8#ycu7FH|V!rYdMWJo$$BGVz>^_(!`MhI`L zEaFZZ1T_FX7^S>|+Y}BTj9RGZMjlyJuhZyGRjeppd2lU8Jo5)%_MGXn;*{&Od){Gj zudR&xxs5HLv%V4N>cDpMX@~aRJSCDrY3&==l5aVhsq)-2rtEF{(Kj-#VoMJRxS30y zW8jR0Cu0tuAOTayyg7el*x#5S-a|6`5FEqHyukOqe@; z$(%D_oL_bVSgAH6RTAwm_^V6f3%!e9A*=59fMLElOxEZ8OAUM@KjE`!Q?ALe1v=Gq zRK8&YKq?77#n^Pf_=z8-b=V;H|B&0;c~43vuwAT7yfi|wd}}xD@NNzkM}Np#_arL@?GhbHz6(kMlrtkW-d@%tXn&9Z*9=m}#$?K7M_vpDyo8#cf82$DomYU)mYtnW{{g0-*MSC}(#UMGVCX>Doc$rJ`afBWCo!A#}{7 z(b)CVWO3+TQ64a0t!f3(g*&9}9_fm>8Xm9!en^7V<;||VlVis~KtHhY`47F53Y{rM zDbH275L0nfP&m24^&6s!>OhkiGWLKmW7MEN1KuneS=qdss3St6H^Tp{5-TlfuM4n# zbF}@OXw>U#UjzWT-qOX1PTKMJiR(m-n%O2h=^K8%iRok5Q@rO0s6>>L4+;F@&uH+A z4dD1c;8XcTB&m=rH%Y3RSL&b&ciMl>e)odMX>M9-RA9mXaPd=jRt;7v2ke8=dXh${ z1Rl_FzW5E9_n;?r>nIQ`*UIMd$wBGAAqnro(KT(LL)*JR^MuAA@eUwh;X!oUM28B_ z4~hjPVA}Hz55N1hah>p~!_$@i+oh!zF1oX#gSzLmESOItj6)BRX2?dN+VyCBew*l( zD}W16;V0j=VWfIfbslq4*-v7p^1CCsqcYZdU+{7}{Y222j#LjC?aT*^ksH!w$oA7E zwG*_iQ77p!5&s#rZZUzdb`}#6H;(WDhfsPF9H8XglS0qn)7y!8uv%|!ikXy2!w+nE z^;6HEBGr}oX6tu{T<+?^lHdnIVzPeUS?PcR#3bM7Z^%V6 zq|w;lXCmqLu7Di^dDmY!QW4qtA4(`k2SKb^S=Tz*a&%RPv5CO3er6&ta$at7W8Sx# z_t;^N*_hLPZTmJW2nwGP&5RRH7TCd1`L*w2@E3g_!C5G7WNACE7A3rO{JUF5c4JpI zY^J?yK9jl9Z|$XZ4?LW7Pk>N*N!#QqzMG=g;}{q-1T&>{JMZW|lm8sg5{{W94Q z=)DpVaAG`}1@go|I?ZdH%G}{Bg0Fcz8hX0e-jgq4sjn(hGiZCQ@>JnY>4R4}?qw@k z**XPz-sL;WddkQL=dPAQI|0*be-DK#4|6GpST(lTd^Q`(24Y%mw=?Pq$Sq-Q@Yv%|t&xY%N<=74biwhK$UWtXgnS|tn7)hHiqcT` z1tc>?1OFY9*c4ng?IENPH>)Of`;Jx?%? z*_0ep&2=@bYO1P_`*E&(>0@2mxug5a@4=`B?4aC!1iAPu6tu72cV*GiYeGRp#+X7X zFC~={xredLlI4)_778!*9UPmBrVAGV zdrpH<^hf2ZHx!Q=rQZJe)uF!q;qlMw3bbH1N%@}TQw4M+5TP^55bD+!7GOzpqNEzK za!};-n#CA4SS@DsTIQJ1Yln;F920{gMqi$s`P@I&EkNl^C+p$Rq!YgDEc2UC&pcWD zW|UORNKKB1uY$Ch`<0XO{D-BId-lx@t(m~R$KpWdlw;DNdy3uB=q>>ZpRP(rlx{=u zJBI3#w6O>=C-gIT+>V0E<<$2~&Rbv4J;l0kIut*!awUc64mX`F@ND77Z@jeiwAZdy z56C1e-B2ekOD3odp1o?p{JuN^@^@MgnU`-?HB$16Ja;u}%l&vWlAoAsZGTx4u{03M z3=F)^Yju_TvoVA{8`48a$;WBiv_I(1U+DFg#(yKGD;8yub$Y$kVVZ;6I(h_$tqiws1~S zRXHf-zy`(_K(Q|qXhfr^CJ7CXOi9`-2*BsP2(dJ_D=(J!-P;CZM?3?L*csc_jScK>omg|7Cqo4#TU)}(W7 z$$<+o0l+`%JLCB`N^tp85;XEW{Vfgiw%w-peA5Bdq|_yMYX6kOCw6@IIVqIl8D`15 z-S8_&>@)P1S0C(adLCOA(qDK2esT@GYa2XH< z>*@U?DTcE?0kJUHZXT(A8WCznI9{9LLwBk*(A4jg=t89U)~P|p+70G6ywDy%Gd}Ix zihk50PRe?1SOW}PC;3(Py5nBM( zb__`gXsIRH5e9K*y32LU^l@47-u@pTOdb~1fv2Kcl^s5DUuD3vSU|6PElGlf{!UH5 zl-A%&k(khV@Q!1Nn6Z@C0}Dc(`84<*B{u#Y#)NiJaL2+nr)L+EDn71vV;YPZBEs`5?HXFDiadnwPq#FOG38vl-B|pwKI2-9 z+AsHeyhinE+XI-5RI-vC$gkW72$CkAf!?1lSASTdHFTFf-JMNf3OjA!7k-q0dPMOr z?3o9sOTO%%&rZtLsb47~@Hv;(v(Uxf2^L)|0?Hm+K!Z1wB!?gQVG0!unz=ZjxSymIF)Jg(jNu~NOG1v3}f zI@E;#ZACrY-|QS@v$vmHeQ-N7wTpMPQ8#?}u86#oYtSUlHZ?|MVa2gz-IcK2g{*dO zH_aCd`qKTW!?JcPE&xDvtSwWXgCiST955;>ZC*KSIczmf7K~ z308dqIHM*(vfP@^F+xD8*UOd6ImxDNMc)&~#~f$uvHbU#%&(0R3yfA3c(-?B;J46I zgWdLHI->Z^P7c{6%a3=DT2#-OWE!4j5qu&ZARTcu&HUm%)5B=q+|o56qu8C9WOwJ5 zD6`QGo6re<<+4-V=()hZt$hb=3Rby=080;x7c$SuZG2RA{mWfiwNe%;Y_b6QN~Wxt z-(fh^ql~rwH^@s z{Gy%=J#mnwU@_!FE0}#VQKTABp2**m$vXsGtkob5Rk31L`D}Wie1$1)^sBd(5zJ*Ey3fg4Ok@=s~>)&Q1OvJrz%JX{>qVl|!KmAJ) zf-=yutT^~|CXHu?B0Z!Ql6-Gu?&ptn$8Rrj{Kx-LAph$JVp6|++EB&HKKdza?{gOQ3zB>uU9$QYfB9G6`QQJ4 zuQj|cpRR;7UQQdbpNKj3mEv9AT$TGR??P4!C8(@}!ChXH|YmxyY^I%~==d2%KSUIH;s88yb)ElX}n` zT|Qr>*zNW){szNqEKg&}Ey_UytZ!{td$)a%{le1c7;XEl8#7xQPiBm`I40my&tQKL zD~6GirriK2ok#0KbNU+zyNy=kiW(LKX#D|I=B`iT9dsz$*_3ebO{iOFd@UG-HG|?( zo=IolpH$6dJCgLVzUArX4ta=D;u8Ue*WG8m+assQqGY&7T97Z+ykmg%K8to%?9RvU ztTVCM3a;Cy%qYp%|J|yXTdkGmF^%~}aSXr(^y6pdK4KaVpuLbmG zfJ+%or&htbY`Hi6<2eD?n&6&iezyb!ddQbAY)R}0KfSS_MnaLrY2(=y>m~CW2or*O z#$6dP+L*1LHnV(m@qOf zkW#5(DwkVAvI_Uf$tn-=T^A(e&*%3bV)DcZbzOqi!Ix@O)fGf59nRPGv;X0m=)pKF z^8Q;_&N;?D5hW1hU~`d`pd&tNnbnac^f^_uH#>T9q?0qGJDRJfDn{&O%jn2Xxr+X1 z`A#z57hyKx3tSz(z6X3&&Aec-=-<p* zb=6oV&5WD2dfWwbGs_~s54*&C%|?>OtZPfLp%#XppfU|lA%DtLw z?96^_AYR=&CF|=#T zyu`xKz8_Jt`=#tRN?kXQ(HOiHEM}WzI08$2pjhlVZyMC@Xi}nolvu~!6dd1j^}fQ` z3kJ^5UcczO@}T&-lpVQz_MrZYwO{X-w#9qDE3|XHegCXal17Yx_|{=HLO=;P<0)!J z4I8mJ$^iY|(0lHJ(4#pU?bxSItzc@?f=|a{42dm79?_Afc;amywy-R3d1P}CtK1m- z>SHuZ$!ozUEM+T039Wg~8#4m$=>wk@Tt|DXb?WfYzQ7p6567hA>Y~f5`}_D^=gx&B z)?OJ4-d5>7`b)M*sU{@sPa4dF661lINqIxR`8=qyI>Y4Er))8^bg7ShisM-%1tEJP z+;@U|^z)Y+vCfe|HRd?D9S)Zz7Lr0ts#se~>3ltp1|5qRPi~4d`K3nbrcHh@CG-&Y zhgG2)8p`BS3v*9v@opP}eIbpdAwz$Z{Uu9kz2v}XP;JwNj!Oq$U z?E4-l!T+OxK*_Iv@vJWM|L5FtACZ%Qy6quepXf(@u5}V0*exbZ;CoTJe1bBd_DCId z8xRe()!iWSC{nF$)T)k)y%vU;s&lY_R-Qh`HOB!)RVpLGfnZnCdRyh`wZU z*<}Y$iN{P3AK~|Rl+W^R+fX!?h(`ff)$KPVBi9uqxE}O4SbRR-Ay<+t8-j?!aQX;e z&M;0`bNYB8hHt%ldb07NcGKei#L{3Jd;rjAq5A!0$I`mqiE1jHTXFT=ma_s* z6MO?AsJi#}S|41G&=D9zW2=!>>_Q@0(>AeMmVs}0eHFi$Iv5*L?npgx(s-Lb{^Xo! zKwbu^bOFgt@c;`2h|c7K3qAJ6e|=3D85t32uFZJWdWn718`c=7Vj`8q^ux7APd0dA|9V_e86|-c)n_1KlLRu zRbFqVLZMDN?-lwnCZR=TG#t1@{ZLB+)Go#$5z0m}s~`7~sAIhubh{=>v+UZ`)%vT) z^juWN9u6Sg)Cd)wVBj9J{*d6%ZJ&>#_~VJ1)DYYBFO45j7GvHFBYE$JXB@l4769ZT zy|1qTn(Twd*21gaUNA#NHd)CxXBwBeBFM{5^;9`5avC%?>On@i8T+$EfO2vO`fx|v z)2zI|N7HGw?nB?!Q|k*CY}0fDDkZAw%vC&Pk!}iT>{IyruCkXNEdqt7*7s}6C4NIN zN`WL4 zx11%as6{&GJW<66`)-GhX@l08yTYHNg|=3eRfHL?OC-0rm;|s|sgV0X1A-ETY*Rrl zQo}`lLngf4!K{}}4K;M42g43`K~EvDm`*Q50+>pq2;Cp#wyoaqK8J436+d&j>GFDl z-JFhVYF5Ts<|GlDN!7pGuz&g2W}6;ZAn`wi=8%61&DBuh|NJCqi@^VCWB=iQItsYG z+w{McX)4x`#gi|d9iPx8aOON%{BZg0iuNmAnq*GI z>1C4*$BKiUbVk`wA5-8Z-QRQc$J8{{8Sflz?ssw#Ti}Vxzif1CpG@4t!zl8;le_aN z5vhAs_A5$nFC6mq-oH|gKYZrow|4F(k?rA@6$qV020$G;#Dl5K26!-lpaUpFNZzQ<{@ek%8q|Bd)xPFswhkTeko0W_!_+Dz1X=}Gm<419h>?o za>;Rg5q+agY0(p3Db@*b#yxm!B65SN2Tkk9gQD7iG+M-Zr{TF8IrXvc@~6J|E_x29 zXa@T{L{8B+Gy3aPf(S&wCXEtEa)Wl8KN6EAdCHZUzq8Z2WBeO(PxEPS=Io51ruB{U zcfZh8*n?OpODQ5>P|KezM7h%XPlzuPwF^l0^*2*-x>8Xlw zQ9l}MIsU|ie%f}&K2ziCThS`KW%!vHBlL*LqY-bh)ry^p4=swc0XPHg+|m4bqZsiq zrI%)@bDe|U0dF;+8hqAnq8a@%J3pFu!0j06L;EyswKez6Bd{U6)aS_TEuLV<`wSOc zr9!yq!T^srHU_ovkcG)+$Ln@Rh2!hyk}BL{JW=^u2z%Y3? z$$5L?I86J`s0VCFDX4Twd`i>mZNGVExg_MAYXpmA!l9ndeEdb|-Z2_5jUN)ssgkwt zY-p_McsV9oGU-|`TgKS3YVs_2yqUo+ZmohkszJKW3V={ zKA{Tuep3zW{NQ^Nev;;D9a`&&g4ahiQ@36w9by($HhYjX)ODAqO+!A_xY$nMP+5^n zB$R<`%z_Vo7!D#YlFX$b+lHHq*cOGKgDZ5weT!tN2 zFZVqde!twCmd^cNE9NRYR9{!D9P%tW*ILlplPh@imB-U!R5_N(e6_U7L; z6(>$#e2a5nW=JuTOrVPV?LhEs-)S_Fo%#aAL=`DnaL~ecQ&1<8^+UdntsBO~I8Y^m z?~OR*TxDF0Iv%-Uq%XL&C_o+nn4!o=$Tm|go0sT^_ffH1_N%*xhjM119Ee!UdgLc;F=1&Pvx;GjE>ui;SMX*g5?T65hJL$_#sd0R zRm4}Ml@iiI7rZ(!1{HJratPzl3)<%_v*e6SSO4#9F`hScFWBC^DJaFH^>F|A!9BQj z8aUGYI_UZ^NYk81@gN|E$1QzTP^=O5_zA~}HIK#R?l4=)hZ7tc1bld^s zGU=dQmkag>Z0t+3QY4Q5HIl%)FGr>ey+y5PQFILY3ZG7Fn{eijGIgLYHt{cX^yE;!v$tviL68SL@`366s^e4AXY^H6XfUsp9PmD?&) zuQK)1q-Ksi-UO?;*Vx71!C?9>tRZpB#!Uo?^CP^ER< zYvuOODf(KrC)>4|J)iX?qY#s@#3rRe5~e+SB9fxdm49(!{oQ@?$F^x8fO#Upin-P@ zZUCu*tU*$WHvwSgLung)U(&2%l`jIGUO~EwsjZ;!>(pW*ogHaH~lw9|Nn?$irtS5c>pzqB~ zCi}E3KOm}R{usC$thsQVdp*uXIQ{P6&LFtx`N2gm1XMsTk^DjkHy7slANeEMdcw11 z9Yim;hbWzUaVAC&mKWdj>ep=2!ks}&TL8yJO*Sq-6T@Zf!bkEM?w95tF(Enx0VQ4S zz*W8pKItQl9o+fKE`2q|$)+# zv(A%0>-DrY6!uX4?#WYW)^mpVJ3W!eHxFd!u@$6~v5!J+7EqH*-Oi$T=j7mu)hhGi zrhovxEhk5fE+l=u?Mv$b)`BXRJ9N?!UITD!S~vrtu48iwX*o;@!BO=)0(ZrZx74T1 zT&~F+xfxwvZh3?2^V=CyNyyzuy^1l`|J19YCoum_Q@RRQLX82izn!NxAVzu*1Gz*y zl1vX`utKpva|wVHlm8PbmUDm<1F6J1^FbL3|yhc3=*f*q&NA7c|&ln7q6^vGsRk_-{y@fj1L3LYF zm-lJ_gw9^oQWqu;RGznPV))?e9MZI;i$LxB;QXbk!1>33^M5)4Hs-%f@>J=_p}LYx z0HFOFWPl8F)TcT+VI^1V@t}GrAtsrxKz;Kfx+cJna>ZfelxceZY279ALivFYL7qgy zK&p|Bs0Qjc)oU>5JZj$hA)l>Ic8s>c5oCB*{7s!9t&%ZT%CI#78ue@FlTWpoyJ^mA zjJLn9GK|Dl_-3CAf#D6^BcK8O9}D@yWKCTc^057_tZV89uO)KA&L3eGR(v_5$n2DK zQEU~+9*>gfz+LYF>gGIQIfkUTVamlJiS~>!$F4UL(4PpIlbdT_?$9J-?h3i7gDeS_z4eABS}nssAcqB_?H$b zbMC2kcK4H=(xyqA{i3~Wz9+$;)p_PO6+Py0NH&@T<@m%R9^HxNLMNwXZu?A-@|3>42`C*czVZv8U{}Baji7ePgnr$-j{C_BJtPE`*y+y%P-;Y2_j6yc5H}BF~a0YZu$)s{f%sD?Vs=a zM9pepqUKmN4ntEy4VZ*C_Lof`8HxN5{^KwYbB6r8HR?W%<2O7lv z*pCfIcg1)~Ci8B|pVj#rx^m8TcX@r>xMAf(j;p)g(n_|OL059ta5j?_L*-i{FQi__ z|IEF0(Veag=MW&WYn>Fc%M|%C6jVkS4VyA&h(=$pB^|1JkeS1?^xE=lIx4h)qT=ig z&b_-j1w*`+a6_u*YZX$`y+E@6lJ@&3!W%U9UBj3L^7N8^+LJ;kt@T?i-7*W%jjwo6#WNvTl9!J0e9Hu_saIG@rjh(<3gqc?X3;JKe z>XAB_%1ufU>M}6Xm;~ewTLUf93qMM{L$V^k4L~Rrxtb^YRZFteb~&+>l;(=mP9jVP z`<3^`cYcmkuH<)mq%JHeSn;ia0rkUS;Y{RxCokd}0UD_l9H{=RXstz-%ki6GZ<>Mc zYvZysCVw}Ge8Rof2#tvqC5lEY)`rTVBL#FEz9K1!c9^=Jb(?u);)70vKsdKK61i6sC|G~IVS?M-f=vrnG)aE!(4jlN|K)Hpm|jBZHhWDrmt37HRS znmyxG6XJP??uNoA<7c$9-v{|1 z4V%_%s~;sL=1;R%yT6)Jm1z+3O!oxsZgs(h@QKUD*0Lr(Sv_!3Af_=m9c%0xQ>(+- zFj~D~%H?z-{gvZXj|S{Xao0fqN8_jY&-o1`V|sfF*ij_uu0}92@F}Z{Pu6mYRJ3H7E- z$RE~2-8!Zov-K_lRhrlIMcA+D!GRNRyWAoQ)UjLM;8&O0!Ue~Ek;;#ajRcIY|7he= zP4=;XKS{2Bf2Hp17fuZ!`WO3G@WM1>-;N40zbc9fREDOC2smg^`p>5Gyd|P?n_Y4>4(-u-!aTLUq_g7zt# z(Tm^O>#(By56ki6k@LEGVnx?lk~myJy}!onj5#%p;^rCVFT!(6sYLH~N}Ii>!4CHm z-Qb&Cg&*iiSMwq!zQa0@JhUm~2c5>Upeq;g$pf>=i&JUStL;B_&9bfb_(D9rr_Q-R zKArsr;jFh{)?3JrhSwrDT#MG=eBm;lb=d(qxS5kBqxU;J?Af_R5hTMg>RkL0^yJ=Sm}KuFGLPZu!-m88lnyn7s|EJUtc{S?L@Fh_%W}3Ax$pq zyuppfDy1@XH?Cwd(0xT;fn%SEu@hn=!&!o2I=byeiBbN3^DM6tgvp-XH-cLFZs&|2 zx4Pc=B~76#&K0_baaD6h-Qx4@F;ug{ip2$98Vxm_UMuUECE}rQT1* z-`+NcUyFf&s4v!n!V1Q5N;>m*_dyvV_11V6g*zWBTC>Q{4`58ZW0o3K=eMV2d+GYD zt#i5H<6$}78j{FB_|)l{Ix-?;9Uw}Ov30*J_1I}aNL-JzEJ#}GfK`nQhVPv~ zk?u|;pqI;mW_O-B4TRAT5*UzLaQs3AFLKzS65QBY`IK}c%GtFp#E$hk8gDKQLeYsG zZ4f8^Z+;uV)z=!WmjX_1I8XIpOam(JB|)_z6UF&X@p`S7l|lCWJf>e}#Mp%CTdg0( zwg?5Jtp0B=$>#G-PciLp7H(V%Vuzc5- zzMJImoVGq60nhZ7Ga2{axHw}twEknvcT>cOvAM}xb)@nB)eDTzP%cbfztDe9L!A%3 zLCPQ^xG27$S`i&wU{XWtsd4{$LjQ|a#jk)p(eKB{ekGtoTIgs1IEB91tZGk@$9Jk{ zt>;LOE5b+ANBy;2+XzkpS`b;KYL$t`(c0L$Dp5(BzfizQC~n! zJd2L677+SQ;VRR6L<&L-j!n)#ObISCn}McozCY)t`3#M*B`6kjB=;*28y+DJ|B>W_4!UW&gS7xAPfpO~M?Hn$C&py;@ zo}+lUr#H#3$9}JZ7*9g`f>ofWHiR>tgE9DgImN|3tKF6AdnxbK7goG^uRCw&!bZZ2{AaI zza_-L(k%NTAjmdf_+kb{6$EPI7+N|j84xQ1RByUE$^19O1%TFe&iqH#yba(2@U4pn zeMuK5BhZH{s%V9gBw&;b^LQkYu^QN`bOaj1hg{4c_0w7#P_%RtM8rSMu|&WN|172M zZ4z@xy!cIGA_!-xxH9n&vD*KV^0Pr)J{bi;vIp0`|JeK`+i1 zB(W>gD^yHw~ zXWezkd~-qq-P*eS>ew`pQnU%g=Ke6Fjo$K%6bHczl|1%?Fb5}h_ zUcHoSO&Hi8Xesp#df6Hf-$GX5ln({a6=DL-1&#ZKtT{tvonr?XrzqHnOjTsZD!`li z?(;T*%Oyes^&2AZLBAjN68%((6o%L^12d*|QE)u7C1A)-hT-?o(>b(gUnV4$v+y^h zUkE}4t|u~sa+LIputxOOIZQGy%nfu*w5x0)%j5|NbZ~!q*H#iVl-hK+QTn}F2#p1e zcb_+E&7aEdZ_Veuk6PCbx)nHKcI#$LfbA={Nz?zq-kZQfx%d6!BP4|EB2AX4Y}thj zDUym7OT>f<32nA8m!<4WbWn*=6eW#a*0D<^`@W8C2$`{rVV3@1=iKM{-KTS&({n%1 zeLuhF|9j4Py}TS(b6s=IcdqYeeZSwI>Lsb;I~(U?$l4YYB~%>@=_T;M)-`A{BWM&Z zH3G8g_csN^x&dDm7?1BhGa#&5P|iVZIb*Fb>#Oe31WzDde`WRjJGl%H+aD5mTfG0FMuUK zj2KHpg6BT$ItJ^Er6W4vzq&FbxXI*+C+lSm7+@Iv=N zmKp)p%+X`dy$9<;%sVhs<_9&6W0eLb_sC5hG85|(hz`-+lV37{{*QfsE@UoT5>Q4U znHRZlp{nI}NfW3%%KFUZ&I1kMBR&V;&>d>upzPBOZW+nM-yY=(1iKkNU~#^%keoRl z*}B!q{=8$NHN{4a-)s+l{)xF{T+-|C9b0ZLbEbdz7d-KFv=e5U{riUtX+m!V^;fP~ zY@x}5G-#3sr>eGTD_?mIWVrQPlTC^$V`fp!b~o>&TA%V&+Fo%&WFHDBywt$pZLC$E$M*U)>s3kE)>AOynDoq9VgJuL*%uqG zI9%av)m|=Z;yVMM1=W@bzUJTOKn>=T7SFVZlvcOiNr+r2-TbhWs=f{-vZFc5g{ zO_GjSslofJEL*iSJ}jE~r`=zEIk4r8<3c&bjXXzcBzBj~w@{R=ZAQFLEG@X|7MUBP zcMPRpR%Q$qeR{>~byeGu^-fm?b3~6qpCO5xkU9Str}uX5xv1m#>4I%GhG)){X;}_# zzdrvFc2qa8k|1opQ^PbYjb&bfvWbdr$P>(S0xK$O04Pd+^>YgqJ+YlxcXl>qTQ zx+7!`ne9*a9Ywy6B?AmSHRE z4%?Bw{$p-d%@LoD=GhqUSNP1T_jEY|Mq>mh0ND3GZ*N-@p!?dVd|!OA6>ncqHO}F5 zxNWz$YrnG9XAAda)*GHig>j`fbR`*^-^Tn>>-__aJth?tW&RnI8kQ*qDt{!hw2*vm ztd*mlV%Jp^E-%SQu4-mH5c~4#=fgK=ZA(C4-$7`28%mL%ee%il^{H{WBlNF^jMLBo zhij!e0s4Uekc0uTZ&0qc=kyv5eoy-pn_J|G9xK-4U!H$u*@M1b&(amKVnUWOjGQk5 z^IKlSIge?$V76bgNbER@9%#NT;<_cy1)Y&hPWF2bl~oZCVe z(hHQ*Tt5OUzx{a#zsqgW8d;28z}}?jj0{k<$Ucr+pG}CoGoSWxOGq8~Fmm$VI{!}< ztx*A|`7S-U#q76`M)sn!B@+?G!S%}Cv2^Z_*4;dH{2wn^8s&vpDC)7sX&jbhs5HFG zT2e?+u%DWQv0WwTjgAElp!=ueY zLeTEp(bBcmJ&3o*8%rRU4CK(K#8=nmNFVzp{7&7J?34_4d|awCd-YO%LiO<1RFDP- zT_p!sqtpoaXsjbf;3QOlS(r+cJ8c&6WmrJh%JS`=qZTm7eP>Q9FRR@|0+sWvpXeVz z-h`^7V<;McPvq0#gC^6zY{Um<_#Fa~Byj2vjfsvlKWFrYH3#Y#s(CcOwH`4L0-v9w z*-soJ-^5W$C7-z207`+0qMW^?*yFf!zNy@v_(N=v4ZtcRmYh-GzA(|iff?Ijw81Tc z$ga8M3q(>J0gQgt9EL7XfN7!yY9A(u;J)~sAv?agCLEG%r+x5?+D7Ct$+$Q35;0Db zAsQjtwOQw19lD`cEsi~g7)u@HFItQkEy($aE?w$7Y*OddIkR-}!_-uY*zxJ`l;rr@ zLkUW^P7ghdcX>gpTfh&;T!IHGxvBW<}r`f)I%>3pGD+zi0}zcIE&nnJqYZ?7Z4Q3WpPg+@Y8Wx#=n8i^1peN-kUul1WfJ{psxsDRWzI{7#{{o&YYgG*p8K!xpT~ zGhFx$%}_=L`Q#A7{B%YLURIyCK*}@!YMWiUTlCN zxQuTtch4f*w|m6xl5ucl3g|z6RQgjme>m(CK6n;cYPRVOzMmBXM#cOgmhq2is~omXICVx9Q;+@J zG_EohI)@xYoJhz&hOij6QcgLYv0kA{uycy5Yo*}MR`c5o{)%d!zee}rroRi1_y)JAY zG`Qfvdhb*Gz7MyB243hh&<^(o(?6pNGV5adBXJESbx;WDbMc31-}2J;F4==GVJe+U z_nE_huArGIjDL}j=XYpkFlc?0Zg>i!*9syxRI@2vm@y1s94={s4MW;d2KcWzy6}GY zc?2jEg%Tp^4IjB^0y1Cjt~-!G$NHf@`4|u=n+l~z@~@;~(( z`H73l7fe5U2s9HYiJ-o9t#^g?`-5#PE3`ks$+HlXKw!2NEXn81Ox@0%mM)3{#JC!; zN|T=#u=854zQHu+2*z|afDM#RAtLFuqZ9*88h{A|fyHf38HD=0&e3nSv3xoFuk;)L zz_tHr;0}MVaQ|em{|~*h?bjamxeIUJCi$svYcHFO%8ufW3sT~`MmMEzoAZKl;S1e4 z-Xi0??5CKg(Z?!E#qQ~!uDFx>c~wu=GdfR;$LPGg^3&ecaO4)$06Z5^?gts}411nB zQz14XDre5rjVl(kcR6V6!&@`0Kg}$A(zWsg&)r~9X62*h(&d~+i$+1$jO<%`_eF&c zPv?`tqXk`8ALzN+FKwWRt z&QYxTi<^Y+pGo}bVF0XI5|Fh4J_mm<{+HE~+pf_7OiIaGrj` zpxU;k3m(Q7;L+TL`MmhsMoKz;m&Z6}L&g&TjUNK0yAxb-L*jd%3;)h3Tj0HZct0$E zUFYu_QNRDb{x_`;YNww~7ZwLRikm1SFtx!L?G~QtaZ#Sn)8e0BHg+}N^zabcG3i(3 z67-=i!1y{W7K8XcHp89(ZDsd9b{u;CUdg~h%rk`us>7=Wl24KEBl96c_Xmw4PK-uV zGCSDgzkbqP&sEe-*1-epnw+#raYtdF`AElS$q=F5uNgP9Hp%QTvk&{yHq)!0{l=!r z=Y@bD80VrqMy$Ai2>$y+rho1O{>=Tb@cRQnoSs+XmVS<}$kr#yiOl5&?bnnoYxa(- zSoWEF>hF;WEDV__@+|!GqyZj=AXvAq??rPU=9RsGzG-b0&`1pcUZ$mr7ZAgQ5j3Gy z0t;1OnQr7*XF}NyN^G+SF<8)~L98pg-ve%aYb;=H9)|ftzkc)|{kDaOj&vhr6n35+ z)az$e5@z`EpymGejU8S1?>jp`R4DtA5&viZ`#+x{#@|3*#jh5q)e_BQl<{7uAL+crlpSps;P&au!EwdR48{nQ8oRX`wXo)_%A5Q^@XP zkZY0E(&yGQg-_~CgS3)e?kRo*?M-z9p#vZKnYe3Ma$pcA!~!`%(Fj)7bR|*F8>`Cju_8iiTkZVguj=t5Z7o`2ns437mtYSgoj|{#oK(wZ3S z`#5vJpi_4&nt?PT!W9DsgU7_MBSWiXLgd^ov9_=7y71+|@QF94Z^~;uvO2cAX!ZWj z0WRu1RAWcwq)XG{4a1)y(Y3-0_m_*UX{V}v#Sb#DC|xx08SD&3C%gmk{@&hb98iCF z_QJ`omgHzQrE-PqK0NDvGRJDyffeSnukK~?J_bjGg zZOgvZ{?M&H|D8?#J=Z~_ix7%eFi@IWN@TST6Pm6WCVlHPXDved`Z zd+X(Q1;k4F3>nRBv&}G7@q`GLqdObqF@#avq`dY=Cxs%TUX=Cq`3Jfu7#|=!r55&-c&Q_6?;Ln(fJ}q2J%PpP3&h6%=wco`R35*GAtS z*LhQQJ@Jf+rz!u{3yT?N<;LxwzE+;Nz02;_-7z@6*A^r_u3S63Z!Jz7r61^hLsj&8 zCBjwI|A92%=XFK9#$DLpNET~n>?^q^3l@!-_t@!d(cfSQ+x=iC)FY^IhQB*~3x0+D7=%C)0->5SSo~ZAf~*k= zLNHLO_}$sM7wLgc82lW4hXVP4I+f;~37XV9K(2NS{=3uJ(QR`4R2ozZI;hMB-(YAA zA22@SaQgA|-=EHKdxQBo7LEDc3jM+9Y4VURXv~6E`}a!}`@7TsTHwEZKYuOoFVFv9 zPw+1W=1)w78AeA0sxeVzKQyoKf_Sv2SF!MH<3Pn;n_aF~B!KJGP+|T&N@s1~g#}HZ zv2=9=uc)xo0|OecIr0YCi=sv!MQ>Y`$#KMm z+F2qMl~|;7-JqYc2xRO6n>UJ)58jZ%YRj%8>J<~yAShI z-#}M@5wn73)FFiBa=d2mLOzduSri#|J~zU5 zaF{)9=VUMwi}{1+ujDiM#RB0SSN@;^0skcUw;Igi&m|Zq%q(dWtFoU3w2bRU*W>^; zsU6w50IEtL#$`6AWz*H^EoxYfHz@YGn;`ajT;#0a#_^(PD5_Yz%OFBNJ%QgOwIR%= ziIHhLes!SgGA#>ndGtP_2z~EOg(C5Q%p-Y~aLMG;U9uSf7-pn&|DoH|oy-2&l{=4X z=?U%G!9ZQ{HY9=2IVjUlxASzt*kMB?Q7ZMVgSw{sy*KV+oSLuVMGy5#bgFhkb--WL z*db=%z}->BKs{&uf{lAD5NJdebIrHB{RreDBO4x)V`Tht{iQrRRqcfE*Au6xrM<;n zaUYi{RX45CYygdqhU@Um!3n{rE}iYJp2h;)L&lS$5*O~RDOG>SeXl!Hby0FvPItE} zpz{HY4>Jhl)1XM;{z6Lti06tYIE-+1C(D?>Ai`_Qyu$Ca@-?RqUn^F$&1nt3R1z?# zJ(C|UC45V$(@yKD=E6>`5M?BQOm4xtK>$t=B0#d;O7s(WGDK!P-mT~W0dCV4L$Q9v z)g@cKMWoHoDH6*#*UxDM5~LLzHvg7ZnEWlRUFFwo#X^OOM#+!PV0gP=VJ6j|$5fB>^AL0cuC z%_}u4eWV9ZV&wS-JH+J35{PsKil>A#*r-vSDn?X`Ac$7^T;P3oshTPG%Z6yFBY5tU z^P+y`U|7%t&4u^#PK@X1tm_16aQ6k7^K*`b-tqHl1Q*jU7QM2?ovaf}_)F*Tup9!~=J%fp7?)zLb5^X|9y`FZ)__*9=^hlqXYXFq?PKl1u@(1maq1DI|Ecsc(V zj70e#)}fsf{0E>Uri>(R+C9Jh6e(|0m$I4A3*?7yY6Z`9op+1rQL7WtQmW?OvB%wS zYeL9rK5>Pa!yvo=K8@eZ>{H+{=nB+F!Qq=eE=EAxv|VX=6V|Z{EKh1wv$eX-${p~4 zMOa_ubpGmC;<`6e>KOd_3MgxTUqJu$Z`38I#ul1vDS}ftoS_X7i<ma&%qwBb>qp z#!?ISwxNH6R_tCLrShuNPu*&w*OrCSLkjtMS2zJkBklW6??;ph^+3ChDpj$!YUkX% zeM!%yzK5`#X{l;F`Uir$G9ogfz<*=C%Plw)UaCLX9FQNfyrfH|RrVs6)bzj&yh(SC z>D1*A)2*KPeRIoVdu@E~AUbU>c|WX86(q(y*-(!xj6CdCv+m;CC-E*U^>M#=KAwb&#sj_}j#WXphQ8;Rk?MaJ3+?ki00HP#SMg3JtapCU!fGl z;@}qa3*XeoFP!i0jk2&CpJ!R=>YraD0Z2uAE!h7Jc20qPvihqcDn z`l1`--W+pKE$rZE6Uos&aJYOTb6ZYuR0tq#=oEAz#(^)!d8xJ%jAUd36nT` zV-NJMkW$X3kF5sWi4tT(_s&yLT@{>Y`%ou%JX&ecqSGy;-_;FggnQtzp8h}yU2hRP zgUw%@U%(C9snX!Mk#Kj2mnFs|uW`gWCMu%(HFsYYlV_SNik&J!6pYb!pxh*DH6ppM zBTx5w&^$lWEv>eeu#47)hT5xZZ9nG88*jlBf7)a|c43epZAP9Ws7*K3?Wk-Gt2lI{ zEae{S+FLdm$y_ea=+r&M$=~c5MyRBH1>k%oL z3jSUDKkJTJ_~=^F{h-USgD>%<>uvIMUO$sI1rH8^#%X6c`sXc3%MS~b8R4D%d@pG& zK_5O$6+ER@wM-z$`{DJ-h*X6(KNgCH_qBIR7uerH||Edo{5 z_xq!IzaT&+<2V|G;87Lm(Y`qrEo0GzWcd1`NbknjcGyxDXCEV@`FWvO|6%Tc;=mLSVmY2FP!tYHm8<$8>P7uRR zQC-!CV~Y&l61eGPso1f@Tbk;6l#gHamX%8{j-J}H-_x=-s1J5|Ie_IFjLJ#N1T>># zv^a$GDtUBl(41b6?S`9ox)m2#v&yrpwnR6Y%2YPE{DtGU% zFvC@Po$ji?b|qv$#y87&Vw(FyXZ`WkBkzkgTwl*>nwMv`X3Tls_#&WgoNw(L)?=B0 zG&E$k^yRi?LCl9!lrGwe^y>)E1RXJ|wR&yXN>jcljf-G*m_`Nx4|$oqN(2}oYaAEz zsVmQ}aSvClt(^;b7wx(*`Pj07=ci-x5ur-zvIlPmKt|c|gropK+!Rskq;laF5Ot*1 z==MIVhU;$(J-J_h)xk@Z4g4Yb4+UD8AY7b$?#)iGu|RC-U|%TQ8wF@_0P4A9qXm#X z>6qysOYJf#w*l-r)2~+c-)Sl?k1nWFj*{I;wg5Wb;4a{tbhkK#dAKsY=E~GM!LHaSaQ^m5JEl7v~ z5sY-I^nGkWU$&ja^m<+cDzbM|1B_|(XC6c_ecFU3+)YFLQg{B7;w#JMS$_{H+4N-7 z#&kS__6*s%`tLfUUzh9u_HX{by33p2*k9e{KSDwM|GB&TUusPJEdBSrE%^Pxu0PiU z{IA@?{~5h>VRQ77`sV1RMbJCqqMsbZA!vc=0QM34H!4t~9{{f8w){#30tV@CR3Imi z+c~tc^sf9#RQ&e_=yxq6+raPpNH_fCz$O>N(73kH)=#8^38#);LAE;{l<7v|$ z)E)llx%|M?@XK(lb$$}sg zZf}ffhuzH3p*3NBO+SfA0w=@*8jKet#FBf5+k)&Oi?nF5Kc0mIryj_*JJ#{K4Q`8~s3 zH8+y4hM_HoPoWh2AdryAN%&{l(vmCA6)sH}^jb^wURTe1>z@GBuV;PRSO(xQpfU#% zYlTLrzKUwx1HDRPU3J`uOSyC2hG?UL;K(fjTEl+oB@{U*ul zJ4G!oE9Ji56QjpZRi<7fvk+s)3<_wWM3-8xkzFZ|-kderJ?Bi^yrE}5`KA|Z_4*K` zNwu_RSVx(#UR+Y7o7~C^*VY#{=pyI*%TuBKlOLYBEee1v|5^Z|_yF}BtrRhqD4f~2 z7*>K5(qX<<<`(^W?$~tC8&#vdApJ zTAop%wig83=Y8f~^Nb_TF~j{6JKx=NOxt2Nl#2n0Qcx(pw`v;!KOgSvLrAYDSxw>V z6J2>~@uy3Tx?Lwv^>$oU^8D%E`sEw%mtGnG?VyWRP|a>I`PZ6tdyghM)HOrG*=bel zo}@>kdEd0K^q zd2MwB`{lC3+FQFbHFJ^<(y~!46of9kz659hv5%wYyC<|1LgDZxvz&iB zRUV=7BL4V8pEzEZhaxwFJRjWUMz&RE&Odu@Nv~I;f97xy>e}08GkKMUrBJ{S zDIc`1aBVb6+_FWNf8C#AdB6_{_5{6Z8e8&f(HyE0r?aX5&blhi>y%|=;iZAo(!N&^chnTQVL_~nqS27<9gfXiUJLT= zE&r(>nPp{C2d%hD4Albh-a#^v_4PYeWwaI!$Qd{*Bziu)lf+wlM*Ecm8!T+sgP|ud zn)35!#YwFt5x3OV!*XIzMs@uVUi=&TZZUzN^2g$Go(y=)jITc3FItmmBd}Vn^~`BLjZq}4do1X<=cUn-ZlHFSsIZ6X zqtUX(aRixX(ZEcuqsUS1*p%4%;vBnCxsuU{-egAJcC+ zH!jMsx~{$NNlLcMje!?)eXI|L<=(`!XDo1fg*|SMyBPiAr+{1A(l{bErS$%w>$YZ+CG9k+% ztI=Eh(~pz2UfTE_={|Qr{`4G&qV-F^{_`zTTvx(mIOA_Wp?>N?IE|9`p%+H8hP-`o z^TJP@ZY#zZD{vki@JtqRA1kf$VhYlHvnFD4g^vM4xkGP4=EqihR#lOaM3ji@U7mXH z)AK1OCN=PY2z*S=o&LVe*0=DfZ{M)J$JF`2r#|VSLSmHW zj=1Aoj+nk!pznp4U!>^hi})!JY!X7!B7y6Wh@<=-yCWW zxabdwz?)LMCFrFSbYZCP8VDea?t}UhWeXTT;wKG)23b%C`&)njD#8uu^}Q%1a9AFE z4H+F4Aa(k!D%R-Y59!3Z@b=YBH;%drI65i|`sZjds~ReE&TcPk98TKjNgS^^8l`k3 z=w|#OOsD30Iw_x)o&(B!_s;oUsBA;l->C6*z1ovE#C{drt0W8OS-l!jMRFU^0y`}g2lch?I@rnK}s0O^m$F=HBXwDgne+ArD+ z4Tu;yT8EeyyO!UJ$l7*EkCHBWcy3nnAD(d{Lw-wM#Bm9E19(m+IeiQRo9z`Y#^!iW z)TBL>B9A zkNvBxri{*DTqBvkpdy;Zl0dIT zM0EQ=W1oZLRm&~nOdN{=9l6c5_m0t6{w9wm^D}$E{Tu9qp|&2F>>^E%zD7r96YEHz zR3CKL&xy=IvHXG-r6O~@QOmlO^xl5v=$4_R;DxVN+8M=hhSdl)NsR?y3S+;kaQ=qZ zvdl!||Wk04OdX->X|APIMXNL1~2 z1irK^blROW$C!T<&;3XL9e)3ZQ-9{xbHIIp{o8z$AV>>(3$6G=Cy=aIK!8^V1b8p{ zLqomoD?s8c@Za9_cXi`gnjfBWmdrTjSdY24wp)nyTs7ODrDoB~RW`{&GwSX|`*Ag? zy_BOb&hy;9->Sl}Dnq6d$ zzYJ))@u4}yxecGSCksFLgmR7!m!L>|gSDc=u>>37NhAUOe3gG)ACS%CXmQ_Q_}2PS zvV8C+Dj2|R-qYWIgDvC7-Km1W2sEoOY*ZdYfgM>l30;QPmx=0h_B2pueS*L(A@UK_ zb#gN3RJH+n4h)4kgZv3WScmt+>3-BxV4KYRtmxtwI}h51Iw&dNZ|=aU4|U<$T^3VZ zR2@T-A%LwQc%iW}F1it*{r`2Jf34^L!_Vg*6bffqzTXBw{c#|40v`xDBL=O(=H}B8 zl^bwc6BOppweGv894?19^Lt$o<;A2pF&Y7*mpT23xG5dPVq3*X=;-RWXfg@Mm?Kf2G7L{SP(Ek7BRFW9clk zCs@~59>3G%Q2zM=6?LoC^vG@T&cQ<`ahy%wjYCnpZ!hS=dtLuB`G2{lzg+$Q$dCl& zgg<6;pmsww#{sv7E9e`n2nwu#r}-&FnuRRtbvc&46OMkZ`|^-opBi7{$p?meCNMjv zu{ZuAg#34n46-A~*6HjbhD6`SjS9?~m-QrgmrEB`2FmYA@ZWuiQHCoE0}8bu3?>+2 z1F<`*Ks{d=n~WPNkh*9M>wR)Qx_9@vl4MR3@5z1X8ICVdRe(*>sL-_7XG}_9^!NGr(rTAAC2h%hkY(M8PV8>V&>zxrB`awtxI-ocwQ=&062;X`Js(4Ic$ zeSKQ1BGnE70p50S<%3`2iXeWsMTx?A#jEy4l9Tawkd)gHCcCH>) zbNcG~_R{TLxh=d~4GnC~5(4(bgkaw)kU_O)Auog84YCF(zrsk?9L-Ht=aJ98_D;`B zkgcRmdTrO(@pRh7P$9Nv^rRy9N>$d4y~GmBankj2BI91QDe7lqt=gw} zjmg$khFU2j+%$Z`0VrxX5)+0SK~z{lK4X-SrjG;W|ci498onM10U{ zD86p=u;w_q(eTo{i47^t^dOxTpFQN=Nm{HRq6)5G!gg8MbnRAq@03|OWOJOsXSaj= zHn<+L;}ml}{RYA*0U+)T>1;{PsBbX2iV;b|CklF+He9b}f9@WBYA-3uF|ZfAn1b4d z8j+yD>1^B?*NU+!IqIQq$59r3uEK$V!BKl!56!ip*4nlK%)%d{B7QB3^0#_tH&h7` zbV6^Ay9<3O9*}?NT+&ijo?v&MJ@*)MUtpp=Otcwhp}(nu@!Pq}x^Tl`>MgR?m9cx| zxZ4wQ`?VEiL-wt56&YNLH2cP+LSimRdz(I7zIY zH0CK4yO4h4E#CuOjRxp2BENTjmdHZFbjel7`DVAi^yaM@{8Ym1m7#EF+8|;?hd=YD zsDfP!rtFuQ^fzX~G^eBO{HR!6pB+`7fu&_=RZMg5?GRD{goPM!W80rO-A3;X4{ zMg}SW`xEYABU{L>FW=@(e1q+Q5h>d}9|TKk|Woq`-)gtsZApqGUW<-n_S}$im-lW&*+3HE$ zF+ISVQq^MTw2R*)0lEErU{Ol2vft=MzuO?mw5O_fR_*%oWUW?&c7kiiOi#zf7nPIE zH#uhRv#a=t^Tk!KTZ)#Bz2(#uMA+P2+J|XgBulY0VYtb?bsKB7TmmDfvRZIi-> zk3Yo}pFV2hW_-I?df?n!#)Pp05iQ^?o}$_ZM(@2@X$?fNS0&cYT_~Y=Vzgz?v?Lc8 z^5@=s&y!z%d@Em_Ui01&l)Kq3^bj*axvsgPY8hB@s zhw9u@B;)U37U;$t@`R{c9Ig~{G+{cz-1$8G5$tu&61Dm=HIH(h{?S&77F}@;w-|=t z^walJi}tu#+(?e&Ur3lY#(s=TXN#`XyL@Tu*v<<#l&UY%V`V<8Afn)GRO9jFMJ6)T zP8q~I znJO20IiMw@nRuq(((wuXCe<1qtBM#fZmQxT-{?$nXt1K^WQ66!^h-xfuddZwlg|`K z%=0DxlU#@DQ=tKKts}D4K1UPBU^H*c?XIH3 z>IS)JvmW{Jy!;}4$<9{s20o|C3&UY%jHS{bwjG1Wd%Z$eG_`J66(F!)1}4b z<(97LzEVypmz^BUWu!C)B(LHZufshn00`cQ7+F8ML+9Wu=uDQ}d+~T)#k0sb(z1EW zvRNe?aSI~KASO_`y`M53P`ZJJF9cE)jbBIa)RCM407$u9Mf4852AJSxm=Nnv#=J+9 z4bHGxKyZNj=L5syRGLqS#GJ?^^@cN;!&6^E7ir8*e7 z@4Xj)e1_%8S`n@*ur@m!07) zT}(9;jOEzeWX$a3g>kr$-;o}9**l`@*Vo*+IbG8p56<@p1lF8iHzPD07rD@1c2B&8(gocT9~xo-AC!7{#~&nE%0skC2#1Mv$%Q71joGstNrA}4B3+P z#s`~=c2=nFvbcMF`A}R>^ZW3HW;f4z$_bC^TNTV_bYt7GJbt!h>#;?i3*K4{DA`Vx z=-X$xlYM2fv&K`;C_kDw5T#f5?m%WK6WbKkhjN_CO1=WByS3^Q7!l`z)33KA7Wo)| z6tYWL7C6|b9b-Vfe7gfHgdKaik?=K|a@MJ=@v%ajkHfRs&bmDe#YX%sbrN=gEKH0+ z+b{;3NjB3^ca%iNj zI{n@|?z~PU)E>gZie}>DEQ12;p}}`RQ`}8oLSAockxL7mi>eunldeHK_e$P1ml#N2HG7MC8Hwf{THxWMAIy6cC)yaDWzcXpfb^A9In^hP3-k)dLN;n2 zRVg}go>wXky)9HTq$l`F%H-sBDHfK8y(Np?&=zyex369#xhgHW=u0sQU3OY?cK61!uL&P&VH792;gU-Pn3%ds1y?3BaEzV!7CyLe4AYpU5s$dE0T&)Tmn4sL^OV0C5>eR4szuVFMSV ze^MP(C=aL!tUVFQ=O|}C8M?kX5?qdHcJ~N?F;pZ;%P!?%M%)9jg-tbF1L_8v*jfa? z)tu6iz^~WC4~`B52T~{()ILO(PjMAC){92fJ)!T;b^uc%vADxmJL^#bU0(}bt!k(r zT|Vwl9T^wgbqM8<}_1MetOC1DaAc>GA=Kw z@RPb-)y1IADJbMko$N8L4S_qY)GeW)dj2Yh$~xw29QJiV`*5bdwydb6@o}^EpC*+a zlrrts<74RF96U+}^>JqKcJUUQa4L&m;Z(eTgHt(xmet6w^N3`Ao=24Z>pUV5%>JGH zvMwCi4z7p)j$Ho-s>xWW3ar@QjU&Rspv|$UWUW(;V^$3IZ?S?;r&EnAgYqe;0xk5~T zqRW0NHz_+@+=YGJLkQK@0_YW|WLwK666BgLF<{buqha?Cmjf|)Jc*x{i9+}p(n?n` zp*n|%RnOj#uZ>cCKjx-%)nczc2+Q3Pyu|smcZCDlvPk8n;2i50*+2=?V|0giX032z z^otK8;^)gNJHN^;^k>9HFi3{IHfDIB+^wj4!|8JyXUU+mB8jQNl9)TOD^E1(E?^`hV(E-FGsyO{(KI-+BdOLM z^x-)+`|v5C*xBBue|QrEqxb6@e=n=uy#XP`QG=reCEziy|mI;DLAs8v4P$i*zkRswPAN zj^^2sHK510HcIux8qTC~edZzqpv63B{vWx&Nat9@k{93v5Qg_9(eJa5Qbk4~;s6Z} zkU|;=9I$hF3N$uBELk0t;wK8Bx=f%)hTPy&fC%mYjMC(?f&!XMW1x3w0{9mz`)ok~ zF#@^(G3`J-2~7)!W=tzV>#-8tfheYZPY5xh8m{b<5>NfKLt^8O@L$?ID|iuyz3oHqVkE*O|-_b zwEI|6lm}%HZ5`b|@Az#>ExES>PIM%kwDloS zTH5Q!06vaB2oEm+P#Li)Y^@PTQB|EWL_%l<(W^&L}x%F=NBWuwL_f)0+M} zzjFbxzcc=**mhm`)8Dy(-x>d+4Fmu#-uRsh0ELF{jeqY9?XUa%&ZYjfp1%oB{?|O8 z6f~}+4$DrJBFM)Q@HM7~i$l}uNWRB2;$B=1Q@r$IRJO16tkJ6K(4Xtl{Trfj6QBDB zaNjhG@8up~f28+7kImk+AKxiF0QU6{L-p^|qkq=l|Bb?>44=F+Yjm~~TQUT8tG20t zmWj2s5&oFKGZ+4$n;#f#W(s7Ouk$Evy>Wc;f*PEyN^sVj{)vE!b&N?pjdtISSp129 zq+OS9L1h79Vv{J4XRVNNJPSc=J1_-s-Ah;sKOpN+`9kZh6o5L_3V;t;`JlaM=ROS< zFZX_IR363f(*!J*Ul5)P5`g|ywnb+KQe2P-g+Dtg|9-oA`-l*O+{c^ZJ2<^|95mUA zeLlb*P8~;$NK?QhZ0-zabF6H(dZ=qSO3*J_gfZ@6lt$~$xdzl)@0elJg``F-;XK9r zddbB)_i;35ZSy`!=G=(3i~DR*7n{UZ4xa{tCH-eMiN1^gAN%i*M<(BL53)v(aKDV|n$8?cnAUaBBH_>O^IsUz;KaGD%DhS))7H@8nr-IMn&FRuFIo>Zm4 zURs9qjWs@2_~+jFU_vJ82|4Ho5~jxL_K5*+DSQlwW-d!^G+ z&2@v7ly)3L;i5O7`5;F`xNvWbw(slN+k>{VO3v-pv&*NHm7BGd-ET`O!uXB_Mm;Pq zK1B8PAu_Cy=8X>cm{q2r*jEO^1^(ziNEM0~CJtx>uAxw$>|;l_{Bq95Z* z2M!IWD;>YPukWtg9+o_cATk9_I5%AYFW4uju`FVpx5JTs*Y&5TP_??nXCxCKg$Z+o zQn@m{-6y0&S49@AjRRd9=#RvHvXMN&zuOX@D4xGV=hn_Z#6d<4xH`QJ0oU1!8qv{i zql;4I36&q5t!`HoCmzsmk$k(QvSlb>{mgT&1S2g|MK9X{hG( znWFid6Ivc8zrpg~%azf1`DvWJW0_PwFikG9Y8Ta|nD(qnaPo%vo(xmQCYW_^#*(F zthoAsIbKMZuk-TL?Si~2(FY`Ud#<0fcxf_%`N$QEk|S3V_3IO{f^=EyQ|_Q%ebH^e zc12N-q~%#{Y}uE%L(Dy)OiRG^Q$zHY>vRAsah14*xU3+WjKtECUD0bC^;$q*c&K+m z@ISl@u+Pvt$(!4X&}u;tc+()Kt@qN{K_W0wPx}}nhI0Hg?yJoDSF#~dW;~S2yEfrj zkVfu2;+rhfVC{OO^R_{fC$aS4h}Cg}9tpYJxS*YJ#U%$g(6-GKc>sR78vQ(JoVq=R zEmyOfv`rhd40d;aLF9r;!Udus={7kIkSUtV-X7TGZ=0`kS1_J>MzQGEUmqd=UStCmhdr zW0zyz9{8hC!Z}l}g;UQN-)l+%gY?3JpT=ylya1}jsvx_`Yy&sPnhDLCH9&eQTQ+$C z_kToEy@B9&o5&?2C)j5L6$fVq?nMc{Vs)FG(7;D5W&=d1^Mbu7JNUGZvraAwQqPq= z-PfE_YaQtoqNMnKeb{HACjbGS&anqrW*uQaJ0i9oNxncA?DRWl2lS_PPHIM%4Q+e* z?ukhXe+}bQRM6cO%rKg4j|QmiaIO%DaDn9;j2D@JCg}U`wges|%63ou-X(mJeo@4! z%o|RLt5c?Zjf#?&1N@aV9oIetD^GKMf3_i25lEnXgB=a6Ly%=b zoMNRTNYx}8Qw+Rs*P*vp9CE-#%j0Vky>3}M6ZTe^+*n&{FT8IGvwY)}v9V7yd!rO1 zLgbpK5xg2k`QB71lM8A|PkJK@KMLKM(<*OR?itv&H}Q_*(aHLJmdCA+1R-4^23`0^ z_4#3t%nj4hu}-o1b5K7-2=PAcpF_)eBhfJ32t^DGhF^U@Wz7ygL~0Ng!h*Jt)Z+XE zMT?eXO$R)qd#w!*T-_I3?Rby}^r+@NLk*l<2l|{@s4J>#ASa>LfF~I-C6T%t%n<>b z(jbD4c3B6TFX#aTm^X*N!ETMxqxoq%ZQqAC|Kr8Erdg=`c+e3!15)}P&>wqNghKZ= zql+`=XLC?pXvViyyfea;RiR!t{hdAK-Ofx#$rT956dh`AX+3@^o{Jn~2K!xRt$V{e6?_YRZ2}CIcvrP}ITGbsWN_fzJMdJ_!X{ zL|(K&64!zP`7z`|%Z5n4`~bcMV@fO zkgrf=M#u%Y=}KQ*%T-ZplW4bjXA+ryV z&0Q;kd~C1MTW@72xF=0Fscy9XP~8Cj8zmU%AY%Ek0_7<{x8MH{j4t{OxA^Zy8vVB& zw;)(dKoq5Jc|v&f>`GLECv+%kIVtfR|Evwu)uv&pTKJsD1x$x6-&&k zxvAJbh<=I-CrFD-gm1`Nbglfi-SiI@ieYmQ?vL0KR8V6G3Wl2TeipzCmeAW{u)fe3 z@P4;;&kc1vGc8BB4k8|at6$H@4G_14F!Ad}Wz@5GE+ zO`9+)hmp}(g0=g-qy=9BWg_Jsmt0neJJ%N;Sh?MS1rNvgt&1b>?^vW6of_G9QK*Qb z?gXISNQYvLvNBS|OWH${6ulIiY1krHP9~&=lsRwKMYwFpP`d1v=dB+X`)fVPtsqX_ z)m7oKJ!sD4l!Hx=StKi>{rt(qXBYIv*lc%jF*?0J)Ci?TP_2M2k|5GCA&DbcH=95VO-=zfCdLZ3DJw`@e@(*gP)7USB0h7>!zSiRJVzKT*_E zn4MGQnlrgyo9Ftak7er2r+Z*A8hf}~+}d?_HQlS=2PF{LETT_`rCqj=L;aI8s@1_= zw8Fr!+EGRbmIvEnqRn+;cIJ7&A-E8kv3hhz;cADU7mmWU!%A$gJ+?RocP5tk;iq#& zm5m%eqJYy$QFdNKx|D#ybSeROo&m{b^qS-lHbiFT^*%-|qxO}Qt+jG8`uV}9^8@TP zjzg=(5M9vh&(h^1&=pW)a(}_-Axv zlAElggIAKdxG$>Y8)elvZ1=jpjBjx`kmcWTdGXN4%lJd8Ge4O!oL4<1W8AAU7cF63 z5NeuRcJ`t5kU@T(`0m#;W7%Bx2SA*@Oh;1=pjtL;Uzc^q*7gn}}x(DWJ z%nYKr6&7h^EY7439>#OQcmmBjYrDr^g}+#xWIFW7EcZm_qnx7CTXo0AZrafJU;)ew z=Pz*vl`8yjX+643Sw0totZ2R^bJ+jwDDG`o2jf#?r-xF|(8F7FA&gJQ9e-)P+{hDc z^J$%L(u88zoqB{G+&!>|jit>z8X$|ny!vAVOp|a+NmzyB7){!yTaMo$;tQ!!Bs`8g z@<3>u(ZCxcMh&KT&HsY8ijn)5oZ z^Z!4O|M5SL#MSeEr#wov2$;4Zq&%b?`3Nbl&u-a8?X;^+R-NXN=&SCDS`g}DXXCzo zZec0D{UgRdwFF-yrHXud1LoYrPdwtG5)bS)c?V*nt!i%z>bz$@eRxJDkm)ni3_}Mf zrQyTI*yB%A_(jko(qEcT&n14hW{{y-sKc@)cj=)^7 zq{HyXFnfa%%2ATZejO>5f` zn=sHT)BZLwwr|*c^VNFP?aoan`OuS;{JAR{N^uPKVz{$_v-3-BWndNm6n6~+d@Vgg zfStw*oWld7OGSjNH%BWR-kQBl&&^4C&UQ_EUs3yrMrS7H06$a#eZHEPfDKgRm}om$ zC%UUAwOi3zY)?sHVacf|O>TC^OE3ik(Cwjn&{uVPMAW`RCOat6I6?FPKiH^sJ4g|J zrMxGk2Nf8XXVz zG&6Iklc!)UFj-yO1T`r}HIn-jE(DhUHgy6KXu&%kRP^xmY!92>p|GBWsV7UJJc^U+SN%0Ktp~1WB?Z7&D`v-{q8CcCsekd#x`oK-SZ1d1tlV zqACOCjy0nEeudlon7#a|`Kk3c93}sf)6G$E?3lsE5jG=CPv`g*6X1OXzV57E`~ z!i2ya36d(s=}!mcTKyLS4$7xUFvO2ZvFJ`Pyj3xOGGq1la->bE7e3wmr$ zeTOJ%zzauC-A9g0MWUoW9LJDknrpoPOjM^%SZUxd9WPll9I}{!x}B832;oBCjt#%0 zUoQ`#Fn)sJ%j6jkzd(xE-AL!Qn)ihaPJ27tyer?iB%Jcx?aLO9!nZW}ap9sL0T>2a z*YEf;_hovqS^Uto>6FvW^kJffbkzNq;UA6rDSU40Z$8;OLAgic&H^{4P9ejvju)JWIQwdXijSc(uG2n9P4rR4_Q@!+C z{%PA@3%9W(DcRib5N_(Rv&0=Ets5M;hbFoxc*3Gd{v9tS*x@OdMHtcvJO%H)0!$Jd z#*PQ48XLkUcmrC_Nkly+vtN&V1xa}Zy2*Pvc(?ho4%;|xQ6wp0f%qKdN!Z+FX@h8` zy>WD&rFNxMtpM^ipv=cEZP^c1+Et^M0-HGDFYPiT3SUoLnBTY~-?Jl(mO7ui8yhIF{>udNDyLKEIWDejJzLy#^AW3b)YIT8;io7qGijP`_3#8#f)6UW#A>NkRN^7zJtY8|JhJD0nm~b> zwDG3*=l7r{W`dKEH%SsoSKavXR)#uQcLmy zj+-i@J_p^JRBguJF|7`3p;e0E1S_tfs1Fx22^Hs4WGbaC9ubumBJ>pRvfHTH5Z6}Q zglKG$FAP-RyO7ZG{J;W>d53$WT!gCU^lxFIWvjIi>X$T}z&!(!Om@xT4KQjtcsP&Q z)JfEP>9++vf-AE7n$cGu_D9-uq496W^RiyaRC*4*%}dYus9E(XUP@Y1GM=oG5Jkh= zbX+W?+w#;xUI!ZFhddW7C!eG3kj6dxLG6d8mki6gPknnRZ^K}A?$|0UKtln!pI8Da zFwFK5iZbX$S({KY%&4e*McCbVIm7wmNxwG8eXTzzWI>zYf5iF2*2<9%ouH(6HloRF z=tY6~jS?F3cL*r{dpZL|VANo<`4sf!>fb3p`>%$GkbY#;w(uR8nC}!ZTk=wPE|M6O z65SDuUKcC`sk99o4P1DZ?Z)|F3mlTPu=Btvg*gz=^$RB9)BtDLYCF{P1k* zu-IYoblEEeee_WpXt46Pr;S|QQ&~y47&8$}(Vs|i8ni1+Y`$DX=51YMSxPRM_*qmF$j$Yk zVHO)rLg)y1QRUqs*;sGBOpeEGA5G&ElPL<{H1B=req9&l_slG2Umaf7X^ZD)%A-F^ z(Ee`wy}u?=yjQ<{`&9SIX?`|(XlQV+u5FaR#I1+YADbbE=p|AhZ@DJ38xauX9)fl% zk$=(-%NLsJ@R~hFH_pyuLOgsAU$lKj zLTE;cGg3HIcw<&@#Km3ra%ExgPJsridO=8>Cu=X=61sSU!Yg-)m;|Tnrp8K?n#mm? z^L7N~);~yFJ*Om~I6eGjpPh5HC+7gQ6B4KqOy&TgK?-DMONC1yyc8WMhtK7PR=_y) z$$kFL=n-msHNyzfgn&x6eLX%r#}}8PMS4=1XZ@XcY!SzuwpB&=+j z_PhTkF7^*q-A%tREr4*!Fb23>NNRc!*ANqpPjVe}x&ReI+@vKnYTX($#BNE`(CC-- zu!qhyEjdzQ*e}^MmWe6!UMC;}lk21N={WVwRxNBoY#5>5n_s7N-eW>Agi=)swrZJ< ze+6QH<0tGW43C8^+YLoh0}63ejyE$n(m@0z@pB~gu{#%)!|AKRk0Z9-)S3h~9%VLQ z5Ul~cqG`bzyu}2Hk(d5}74Y|t6CDYJVMYL%L4*QK&hyCVE=~%bu19~+N+$Pkk z09&z2q!P9Bgr}51adBB;yjEw);*X0lSYM$9iJe6=Oz?@Ejo&WkrjpOv&SMj_BJpzWP zP-+W>{lY|Z18t^m7+NQGcIzJl_J3nO{a8u*hyP7urGR`^JPTC6uCrmct)k|BB+R_v zS}0kE0uHGr>1S{$GLOs!a=28A7+?k8t^W?u3fW5e<@fFAE7IfOI$!l^%^Vpjqai3O z8RMk%KF#pk@D)DgOBW#$>%6HyZl|8oQTma#;2yoA^+JUz^(7x`FfS5BBky;gc1GOH z(|oS6o8W4^0acR5I4X@}S<`ii?s^tyz)lJp%o5Zr+dWd1^iJ}#?SS6y z@QiCuxb6{}DZ5tS@$X}r(7cPzcOSL#VKOU8s1NyiYUWozhtLlTPy5US!tlYM3NHO= zC($6hQt!3y-pHet$CIDmRo+MaJ@9fK3KIdOYvyvptu`NC?BeL?y!x_czv3%8;8U28VV(jPUUFRu&Jt z|0Nu(n?!2pJrT@xl5v(^ft^O*fwq~g>SyG$`S!3zOcli(+seU1V+);pd14zAB1C^6 zs|>H}-_WzV*Mw|!SQ+nmxjOSnv4bQ7gqs~afS(BEXkRYs4p-Czga%L8yOAi{a!MuV z^OYK=InhY+0a|N08;c_LQOceG;~RCFX@iv^X3ol^RFQFi-&!JoDwHf&`V+~yN=6Cby4-h1^SCaPw+{y9xxDi?x^9U8J2s3|VKlv#N= z`PJS>mt$gll?5e_->7EK2OsbpRd4YqOF%=iA?!_s!tDJFDc@5Q8HJY&)Q@34Rp=QE zudFLoEh90Zdlfdx9-Gm+d?)UWei2)kZ(p#fK14siB{icbHv}kjhP`6PP7_1x4!s_B zl`XSnvxgA!&u2v3pMvJz+d?n$k~QX9d1lc(m2zGs9L2{ti){CKw2#ORUcBg4v(yOj zIlx<6Pc)baZ+OKZD%Uj~r_K7h{BfDxjXMy!WnFc~dvVobopRZhmNFMVP1?8jM%?sJ z7F32zRPC3$Dz0P~p|>F4D&gdE6P@G#{MN&UYRSo4TXizSC)%4S?qWSIoXm z)@ihsr+l1hiTHv_nE!lsV7|#UWt7A9elvv^x-Qx+(16=Dv}Jsab=bk;sqcveCjbhlhmeOWjD=As-Si4X;v8z?;^*a63^VENnsLIh7K9D!6Rr=YRQVRBm=#9x2z7`?HVBOP zBv_tkX(JZJH>9n%6R;MmaD{^O2tA(!QuF1`6a%&X@Od2J9M{0SsV=`kQUAR&)V&=> zvPPeqTh|kSzOu{dnclApfw zH{ReZ!I#)rsOz9=m2tAufSdPmoPkXB2K@~y72Pjt~+W3&YBFrhd}B|19>DKkEZNNuY>2&Ae6aP<#LSQ3KR|mVoGoQ4RM%l!quN@+W%YBwErtIyuar&R=z`i^usxYh5g`$ z4x!M2rDH!x`u?vSI%t+PU3m!71h7x}g^%9v>NISN0R5}}{y(I10Jcwmw7>Yz9H;-S z9-jUcMi6QC{bUO3L;H-Q?v>hl@RyrVeT535&~4}=Og`dqj0y2`&sC?yM~21m>6XHy zSJiC%FER&+44A|=F%bd~PbnSbJLp>7%H3ooEg!^V-Mt6Q?W!vdAl9PzYVs`J?$L5C z`OI@@;DvLrF2f5ZrtDc*Q3-|DX7XdnhAGsBkP_$9q!n}4#r=rMlv(pZ@D=vF-e{6_0f0hvav7vw`TK8@DFZMG3T&`NR z4>gP8N)Vq|s+&b>k+o;z60r~YzKtEeUeyzC>}#c>p(o^_t8iNQsFA;qJ2qA7H>`?$ zHE`5t6eofZD5|i@RO*nG&CDIDIilw#75(e=X&)ZDh~GYsY8MaMDar8e>I0{*tUu|B zX@a_;!}$8%KG^DUY|qwj3XuIC+4twiqzEV0#i-GsJMA&Fgs*^jY?8_Un_?upRdIRV zxZ-+uQ`6_c)XBrv@1|14p~YE-iEJYuP+QrWO5j3*HBE|`N*vK*A{BCzHUs3ULveGLzAKBFOXIw3+E!>;y zbN87v1*1-rhm)Y5yi{pZJdCnOipG+gi*|<7I&lS}7{!viALCWy1Mhu}Oo^G1@ywQn z%O7~x!5Uth|MlF-J$vnE8+l2MWL0#no_&`&l98-G)W-0MQO3xuq}WYC(cBzXcx}Y0 zR6H)2&Hkb2-M+2G39h{)oynb#nSv{K=UB`q8aK1J*&k^W#nT-*A5+v1Q+TeB!o1V} z+r58E@wvq|LSW3{xsLF)f@8vr^wGNN5Ue~YotQm=u6-axMogLpITzkCrZYoqzBb`z z*YdFQFwL%oO7*nat(}Pi77lSl%QFJst_vU9eb?tXT?{-KN-$U&;5kQ)y~j_?jtH;D z66MT;Bv!N;o;bwKb#AkMFSNfjHR|tcB6$&AAJKDA}i(lhpJ<+A0tUp4@UbU7apM9ZI2CqO#Jk$ zXzE6o!U8Ar^iAPQOuf*2#KM#lS&AYSIB`6#k|rL&Hk{O09z9}tVoR4H_0%#a^E{@m zIv)G6gCvoX{?ZYhH}*^;-A$(VO^&1EUQ-8wLs1v$Zd_C_;N7wZyY3?Nta5aJLQ6&p6M=&bUg17{%DGxp1xRW%Mn`#9l)%EHelLr+E96OBQ6NV zMQb4>_pgt~`%qJ_Yb+I?vw)7Cc9lrKvvxXygdLsx!$%!(aAp+X8!A8|n-$ot{eW!! z%L4X4EfM&2R~Pzz>@QuxU(%ChsDYqP*-6`t9w-CknY6|S25@Nqd3<&%W+^;?AlJFVoY#l?Ib+vH9Jrp$=HtnLFnj5iS9`R=zBCucPmWzpbFu1olS^h19^~B5lz6 z{`{zsGN%V{BN4Y))(vSt@2`7o=mmsLlfgEw7K*kRe);H*Cf5Aj?WIto*4+`9%HSp# zJGp<(8fci#`+zsPOih#s;yBqPD(i1OQdpvjd~M3)`^HAF-L)Ofby0Q+Fjfe8l%;nR zy-(LiD?Y5q$Ycy-Dk_9d*i|lsy6%tiMK`0jj;pQ2 z@kQuB7clei9+{sSs_SE6j)<+jNWt$AzoE$Q5UH17XRD7Ifv+lj9Ve+&fT?$%hPm~E z3*f{e$siVTbfYKXSzZA=&io@vwEc<7)d)a>WY&?G`7As$uR}SBrm=!2Qj>{h75yC& z4jL{m{lwa32Y4H!0jT<)E$Vl-VTX!&qhyDYO#KRlV)wgt?owVj{3`+-yl3$**T{{o zH3pc!N)F)lUxRJRMv?#-fh6_iF)cXVN#pP>_zdhOvGVf4soM7s&K#?NxJ4w)^lUu< zDu{r(X@?<6$5R<8_JF@I0;LqD(*(0=Yyk2;jRs$O!jlPD8i3UDFE9Q>s-ywwh5_ja zj>>c#Ly-oj9_U{s18rU4!PMlV?EZL~Fq zUWh{Cj*h&!Grhc1!>C=OhzbLgCeW?=!>BC4hPP)8~H03>yIULoh0P62X;Qh(nGz_UHdZz07?R^Il-nKzR$`r<@>X z#(i}l*wdICVU)L6z`TUeZs8_cfTi}=`-F}eFu~ui^#Fvgc9(`kfFa(Rz`bG1o6Yz0$S-U=t} zZ}NR|WQ=mHJnnjOn`ltftC_IFm3NgvmpBmL1=KuW*q+-kd=QuuQ-BFb-$CWb3fY8g7xXw@boYDafeVMidSV zU_zn}gnzZ+;6hyr2Anezpc%3B(rS710AR>sp-y>hX1Q9OpFo0Cd-rRv{?v zFgbBkp!syCsE%6yo3Ahcw|N=IQsnXaOgc9&y}rF#VR@JL^d+YFy#%^_TxYcI+wS?P z!1a@Ry$X03$~wT;H(jM}yP18_qs_pmsemK;lV|I{BbdL`k@}Me_$LwYpGHFbv2yh% z5%BLN0{)fR@jvC~cDVVKl>IdvJ_EWO)xm`xDFJ%dTIh2y?rKTkmAmn9sqz8wm{kdZ z_Ap!4uB=p2P@P-xQinu%n6&GglcK!3FF3n@m$HJb!3=xSSd-mz+T?1ngQCm!Whpie z-jh=I4w?&}+ohH1EK$5HKeO0MPfmRwUF}89L_5PG$4jy!4QgP6INe?u3iPE)gp9&~ zv}bvmEM3y2B#-yG{VIoUnq3j;F4&ccHo|RlO89+oc2!Cp&Cp%=BBD-1^RV1d$Yyb( zCHsW5=F5`=ltY4x1k!J(Cj=Qq-i@x6cFDqpPRT90+ON+irnD7c*8&E!elz)LduqVr)z_|-0gp~0K z4Zk-dPH}M?o+oa6`OK}tX7%nyg;5y0>tD|076Q^S7q_Kj04|oAi^BY2Bh~LND%8sEi%fBwtClBC>`HEwz-BKM zoHK7xkz8_%0o@8|f3*Kc?k9t5u2n-pP(8j6mh0!{G-n}*U5_+0#goqXD&E@B>FSo z=sM@CQ??#nL^ayZu5{{_NJl(FC@{4xU|z%`G@F|xU-c;=?bDnXHds9I(~aH9-t=62 zN{_EcTcN^`O4NL`Jy(dHumw}N^aA##Y+8>I;+;Y4zUybn#w`rH^OD5X`~0a`V*I)R zHJJwUE}$3_lrV4Rj)^t7@6mLA`8e_$SJ807%#o_6WmY$B7#?mq5shffvG_LaNo!4f ze}hofZu3w!5xYS|`ep+1l^e_BuFIY__wE)TUzC{a+l=faWk%fUzGbB=?5mLfJb>xR zX_3>}{rItL@@_C*=WupS90cMThn~K+ZO7v^?b2eKKt$pP%{^fn_WO#;=pyaKRgR*@ zze9>mYssX0*f3EMyePhCDrmLH?J)4#FhbLzfM!wZvxrx7-qS_zgmA;-p#i;GY0$~}h26b*0 zadwMCOE*%k?ZMOLxR*f=>Efjk;H%ygLgH$~5DamHZuLI>7kZ22xXU%_%H1Dgv(7mc zxMcV|Y*&6?3p?snPh%Z~#!HJrNcWK+F|t!eRd~+E7mrmWJN27XpA%ST7Dnxy_Kt;5 z%=7mhCzBs2e2~Mbe|*KRx8}KDX9n))is}RK9&AWdK{ZEk1RLnaM@uFQLgUuh$=ZA~2nHlT>>y^MLeIhBZObI}jCf&fYLl!{)RS=s4?Vl9%{Oy&Hk?-a zo8g4Zazk=cc5nFoarR4WZK+n)fyROQkBh=BT>UveYCaN}nGU_5qS-$Stp$d*rZtG2 zEr?eX)BfXJqF3vP@11*S2R<=5BHzynskLXeyUoZ!c4(}lFIMlR2$C)nJoWUU6X-{) zOB|LtdTc@`??ssJW;t>oN%Uq0_r;rbDiSjX{j6v{GvLH<4eI6WpwuEK( z9i;Eh{X3DX4obL;dkCVP8m)~V$3@!%_87y7l%eFub;A$R z#)jq3ELc~zJ`;=dz30wi?lxub5SsV_HsOf`w18AHK;1~kLBG;eEafl= zEu#pE>=MwQTZMU{fk{I#hH^KOWIYZFS6@+}pPt6y3?qF;vtGHxv>r~@0X{=IFKHdH zJ@=@v=Di|qeI1XBO4CAQ&i746>*PH%-feQmJmCpMI0oLrUki5IQzsTS-vYUGiUd)A z?6LSehsCZ(aq-HF>V-*ZF%fjO?`hmz$0q^2vKmf4fv&xYWSqq2lbqHgx?vOCy~B>^ z+Ya**Terh|6zW$xcA^JYOv{c)t(i8`WC$e@C?-;R1C70J@pzlqhGwq?TgBn%NfxF- z%(oW7Bd+5qC;JI4@N1}63xh?UyYBPjgH`Ug?%5WW?(OYLPE;x0eIl_$8?ow%kn-rO zji-5sbux{QZ(Btl`N=Ap%>n2+9>k^3(eC`pcnA3I{K)&Hn+AH6NVhi>d6M`P@D&<^ zMhLnauzgjH%XW-s?O-!-xV_vYVQhON>8T5ypLpp!d^->x$2nm`Fhzj@YOXIShJhC& zrt0J4TrHI_^uqyq3Db6*qPldor~T!7XVgslU_~30#P}vT@*ejNs#qKj6yoqnn8HAR zu<{IcOc^xb_)AmVaozZy+q`wHXe7Z(VB%t_FkckGW1l1geiv7t-0;Tw=N{P2NG1yh z4L&_<(@D9Yor@YDrrheCZ|4S9l%2J8e#R(n-CDKMcX_^y`l$LatrN2s#fELb4bfkC z*Zt+wTlR7V^V$Q2(X+$+;2SM#FhrI%c}XYS=Czds1txiFBGQw5pW@tybHoh69xOe& zUagRjVVLI_e|h)k!J2I4f?_)3&3(RN3^C$ztGVp*mJLrV)3(#v-KUI==^+qh2&ccV zL>dl?cH$x(;q9o$26IzbsYzrY50ooy#_wJ()9t8QF~U0K-^FsYjvbdF!rCKX}jb_&{Q z&AU8=>MGvmz}SYv*8{1T+!*UJebIGVUDKwy*{HdcpiFjz%}A-+1ksg z2_;2}npl+5>A82Z)lx>oSw>cz=VMo8{Na5m4*51lER5&Kln@A2D61mlR`t3WUxbrp?M=ycC9CI2hR(xLrgn|~`bD>5-+&SEnLzDc1 z!Xod#)~Db*84Z}cqQ{%A?(U?(9oU9z-b%gNc2R`Z@d1DjiJ8*n?rUo$_gU2lL zgPj+YnFc-hvV0~`=9^2l)9{r5miDD~8yVKJ_4F!^YhhD38u0$jOX*Zip2B-g=dfRT)^$ls#9{nWGhz?r z6iFSgM2rt9a@N=y+ql4YB0gY@WeMCIwImbnw=FhjTZK6W)S^y72N`*Z=mle9&lI*9 zzGI#yM%uvhk!#2_r($2dlwB%QK@P@B?l$ur=jOwx>1}so#;nN# zpJmyNOI8KNXYXWvUXiF3ze8vCL~0oY5?>JA7!MGu10@@9bV&ZfQi2zUv|(eKqv>T` zR(t7#{H%6t56h8aU0WNZ<5A7YzT-Xl|%&T8U3-OnzK?gIoHt zcD0VF=l&}`pgYhREdVbWr=%D(Mv^&nf%oA72-b&>sTyPS}s} zdBYNr5-RY`RIV{72raqU-yy>3G=9(8$Sr6k4D>Zx8qieWb+LQssGk6D_!~(clsGm5 zLDkaXS!9PZC7J|JqEbjj`Sm3DF0?=nD$weAU>Zc>PN z^oD(7Ck_q!rNW{+CDU{b2wD7TG4qod&HogOnV*vb{yTsF70GUu_19cHQ0Ihh_xHyd zfc}2Q?f(84c-I2|7_ju%k9yC;R=^y4sMaEU{j`v#@FLZObe&LA1K(N6n~hW_Ij7gB z-MW8+F+{*L?lz?R!_1dx)26FB2NIU`A>K?2jbw2Ol=_r%m>Nh}y0}v0)NuKg(}`2+ zYby~b`&r>r)b}=ZoEgXBJho83%~YiCpn*R4-!}$DeTQs!vPYXv$&q)=!M5}d{FW69 zgckX7TS{_U69fT7S%T1o(1pAP z2?mpL3skbQ1yY5So}KmV?dezJR*IK9ct0d$+upoJD3DVlJ`cM$3wP_U#}EwUBA~kf z@&X$yZ=RHFWE&{r@UT9=HTWfjd9g3uujWF==_|W96(Aibr*EqN(8KqW;LC5Qex>8N z7EHii)sZ$BBTsMAe;2f@OLK`e283&Tv()Rw{uq6 z=ggknt0nBmRvzB$3?EccVR9wc^f%MwK&B^$YTMgBTPZmS-A(eaDP$mPRgIR?FNs8- zIxsR;?ZT9a!gZa3ws(ZW@PT8B#dpVclCrbs4h`;i7g4no!<}F&7j*Hwjt+I4UWZJ3 zEr4u54*&#*nCd}GPaR#(kTo^PyA$-O%`Xpr(Vi10pDP@qWh_D$_tMf1z2N$M0}Vmf zTA+k9#dK|^go+BrW^Q&l<=j30B=hQQ#`@)oJbHoS+VG)l8tZ9%&pH<`31Px7(?iy) z>p)0CPq4-jBEn#SS?YJuZ!8XpbbJU1Cgc3g2i@U;CMV=Sd->DMF2aIr)8b)i~*n0yi2m6q%C63;a*x*(c z?c;g9S-|j$=z~U7(;PZ4e&N|07qV$BNXm4|*zFg#^ovTE)FKb?aCD3bug>^cJ?N+K z%&#j{`sHSlE@!;fP_S7^bK&&58D)lI8qTEqh`K%q(m2y<2C#yckl+nZri73$Z-4H1 z>uG)PB9A^+tDI`-EmBsU!p3@( z%{IA3jLySoB5q_BK({&ul&k zqw%yWc-l><$=9Bxg=!Mj>)sxnwYS?d?J@3^+k8>$K|Mf?7;>nAvg;2pHOI_vw_TE_T8#IsV9>beUh7DhG|%G9-cZ*n8yql z(1I~E6Zc4(e(rnpcB_uCyu@Kbac7R>9=&1%Fr-B#7ncN$EOQ3Qe=v45l*x99XBuzq6{YU62f=O^z ze%>|b!r@!z?1XeT#f774HJ+Qk`0aur(B?1-oC>7yay5^a6UGCP=bp5#e9I#g57gn> zG1WfWLr1e6#!lR8HD04)VE7y~&(O)xQZ*xIj+rGv`%l-pZx~DOygD*cTf#tJ&^n^B z|9=0PrWSZtWDgo2HC~K}#O_R;qWb9cOh0H}cD$dlQ6dxsU1>V(CxPd|k#t+8U}s47 zG-h|&TZ90`a&??BAd)t#oM?&J@TK;ru31oNX@1aHQ6_Nn)le-O0`5r-&DKPpl`=VO zV3Sg|wWi{7DcZ=+%Id{NPzi3(C~-y=tG2zyPh{acKT|7)ztq78u%t?y(aY$nx~fPe9%E? z=2fXkPSSm+illpmWkn^0PxR>N#Jx@O^+W|yI2Tgzrd?0zzIXlwYTs6+TpTBBQ0|hJ zdT%L_}u4iHW6PknMg2W;K7QO(7CL z`6$vAXeOW+aF53L%lfW`*IB_!m!)MYFiKx8Xfq2hFqA>v_0i52W1*Q|FPKgrm2=3!cf ztM6-d-xpk9CP`jyp(ThA!pdXMep z68XB9|8X^|SNdPyviVG}cu7&=<9+^n_`0tfD%Ti`)|kHO|0kOPGPPrVK);`W)tuHb zc7pVnAk~Bsq#Uu>YM!7jzIOCDmr-sprae9SmP=}P7|TwDJxk=*1GGvYL$Y~XE)2eY zXdbhrC4&Ox+*)qw#p6&2@@mf#UN6ZqsZHtdjabcQ2?S$Ze*Sa+92N#q<`-PAVfc8U zDZ>AdrbuD$LGd>_JEf7Z6Q7?enJV$kbX6Zm3d%truIa^>c89v$ z+HIMXdvN~w(<2u@J}bcYK~(AH(?F{?J5Ir#Ee)Jl&m~!tWhtkL1`SIgqC&k$<4F#~L77!@l3noF zc)O=hvqy68Qv7b;dP$*LWgAQx$hgM=FSG_438$yn5)zjt)=Li%cP1+;MUMy>?cx7$ zV(8+qtxQn%6*s@<^!=G$a0(B6O70=f79BCBMmkB>Y-9>d?-vzfalv*9pE%v_Zqo|Mf`NEJ|Z%D0mCgpaclM_oh$YSUM$ob{f{ikDU zUU0Q8k;Ig+uSSuQq?ifv(&=VedrK)wPfg_@IsjfOj#T zMsVV`xE`%p>*tt~ZqSI19yf+p6+q%xNIRZuAV6 ziKjL1i*=hn5(1e+;{h@NK)ZJu{9ZcvxFn5XEbmxxCi1m6nGfT~txQ{pDZReLf_<&A zW7fiRL7u#0mL^W*j5$8~<#9$b+O+5)o2$$sk-PG238X<}zw~`+?-^m*L5gfY1x!!I zUDOf{A>Ao2_D`xxn9Qt<2+!2Qu6;aLk8R0}4#v=zJf~B$!@UU&_3{M^NRWiQ&$pZ{mjB&Qf!#`3;D$)1BXtFa+Y2 zK~_;!Z?IInzjB6bP6Fe(r<|u$nC7p7LKZWVAMC4T51POUrHPesg+h0!$$*|(F`{I{ zo8-euI)th>W4iX&gAU$c&$IB$6XAXfdq%!t+X=vy1G6ys3kSqk8TL z+`f573&jl9l7k=x-$ETi56}Z3wx=;11@;HEiq%lB^oiyLcYLyVZ^>rC`7xD!t{BAR|7dr49>=2wLD`ZzS3+I3>uWY>ZIHFfJkFMD?ruiLr%0_@C&UvK7LMAm3* z#69~e^$js}Lym>k2V4w08sQM6P=DdLfFq03XNI1)Y2}yEcEqtpm#bnQB;58s7M#;| zZHFC3 zdheX-3j;g{C-`HU5GZ)5dTrE>NfbxiQ`H~s%ZYg1=IrCdriW1FBj-ej#)T@|M(um%nf2=B+L(b4&toi2M5)AtSf7Z<9U6Q4zze+7e4u`H!kt7Knc+-%3 zax7jIdPK=Eq+a%jL-~WaXP&(C4gIbOiIZ;hM}x(6WGmi})f{=)uHgaRT$&=sN zZ$b;Wzf=oF2w4pcIaCCQEUp$Djk`>%Z##T?Y4>87@3vqH<1T`2WCNOX*ffwP>gd+u zY(E)XDWBOPZMGsFvVIQlabeo!(sURxkru1QgU7NVRR}4ucm$(EmxHQk=nhK@%QHu# znlP_!+%=ETlb6<&NThk9Z{s%g;Ps$V;05;g=Tot>vc9l51nCHWgNURFwQ_MaQaH0I zdjihTm}Y4`P)9ajJYcr;Hd;9LvP53HsfUhoON{_It(C@ROyJTlczCggCOp6XsW>m^ zvKIe9wDQ#HqRUs#LyRF1CWvV(aO#rdBn9FF7mM)YLn?}mmEG=q-R{b4=+`MLn$gGO zhBPBsB9sgIkBrzea1Ema(1ivIxkRVB;uIM|WQdxY^S2w^EeJQ6+@jGunnbC;VZQbU zFBJjxE3x~YaLpj_xW;i#$|*3e7%%%tT%8Q}{F!bw(@Ev$X4T^=8smuRInGJ3)}jpW zOl4(O??$v)(TdJvpe+p_bh}E*wk5>WjLCHsAq2*oPu3$c<%XW-mJ1#*DJ#+T6TGZ@ zsCy@#n7DQIH|jE1<)m}gpd{(DaTbquj|{g>$zX+FiwJm&2wLuo%tc>X+wRP4zlr9D!j<1j2>-u(mFelMS=FoI&g(Lw?LtC#^t|tA=N&XKDmo>2JoVS~ z)4$?e|J6VLl;^$lyM)-^Y3cok&-;^={|g=HQ_zy$zf+#~zxpZpM?UYLOe{zJ;N)RC zOGR#w^=VxiLO}PBUyo?nm}onen|<07af@K?jY7@)pLxffe;5*hy59Wx6jTgdyB{~* zR=7mwaKvow?FXj=x4~mTH<6e662)ivlWw9c1>o@TfiWM~qcg-LIy#wd8XgY-m%kG*XjLtjl*JyN?h!?2#xD^~e5e7if z_uG2xWSxKIGx$cXYG61txM zghzK$`UB1stp;g}<2)BxZOb;#sR%abLXcon{G*-9+Di695g}=Uzc%vGI*pyQKer@E z?TG9*7qRjaPv8%wbJY5HEMKS6#b;l-%d)x!GMn#R{|0Gomt3%9C{k^^+9K58e$}&bEITE7dKLw$PlBM<&}Q1qDMAm zH1^;*gxD?XbUV{a6K1go%PvY1m^r!cL-z|9sy;b(38&~PxNW~?_?98_b=G!? z`k3fiqpUGhB`}pfNX;*^dv4L??H-QTvh>Z;hKmqL zhu|H^rCKj+B#_S&^Q;kipxNG6iI}u(Jvz9n>Gs6k>hS#~Nb-~PVP3L|7O3ZfB+?Jg zfwo7`)e~kWt|kt-L<)D@ozzY7wkv&l$oppW8C^Tp^e_ncBM%3yqqXOe`uGkGS`&Jg zQPUHdk^7aq2Fy-f{PaNE!8kYh!Nk3l_uulO;2!-AxZ;w9yW_?HcUu8IU>N_fhNQcDSv6?gbQ?7e#+l1(YMG<8$ zGD>#!lvI+WsU##M*)t^F_+2st)$?7 zMg#BDefLC=`I3+YasOcIr4NNU@`u{(Z`t#{7lj$+gyLV%^^Ou+=5)XwSLVq$AVaM^9l`hu)^)E#di~9vi@x1t7hLCVEmIc^h*_ zXl{>xSTvmA{xn!r=X9x=(E7NIM?dSX)7jkjG}p|`SCVyfe0Jj+1XI98f5+SYKrZ`# zvP|Yz@AW@fW}~J3kE?Oe0K#=_Syprmff8AxLC4V!ZzizM0*oTcj37_@TF_;~O#CXp z_b+7h`X3_*`PFAE|31P_9?ut+`n8FnoUot*xVHf4 z`V7xj*>VrgKBKvheAw>H3b+5jOplpduFJNCm$fz5e?LmRcO+eMoFmDGc@z){sRC+s zKpJu>=MUWHhpA)t`o4s^9vLklX39MSFv;6W*E)WPAo@`SU12cWSzlA+Mhdf6+NW#nll*RWa%qxz=66aipXCqSg>q_v z?mS?S%uTNl_!FW|8X<0>CY9TQf`D~}B-4?peyvr=8UGXIWwy88?s(Yuta@FtQrjVN zp!)K@iDNzyfCK8oUwYT7iNHnpd3xKBILT)xQKA%hjyqyr`!%hrdwtluzF2?U%sI`c z#gY1zFcnEPTs$=bJw8t4qoyHX-ul$iMRbssI-%l8Ht8Cc?A7!lvEvQq>ixSf*7?tI zJ?>iI)LxtG_rIAk=maW%;nRS(F9JEb`8atpury)-V4wMOuhIXC|DU-~n%eCiq}!py zX|S)b1NXW3(XO64_4}z8iE2`;MfBparFzT@eH?}ibsr$tAq%W%&(OA8WC^uCL){5v z?Vg)coVdHy2A|H~7CI(|f68}1^JAD4ZDSGHfwH}0iw@9VZW29y%RZ_;=gSsbx9)(r zD;%TBuVft5yx&yvRi1E0eVNQ(4CB@%a7MNlQMIrq3uU)eJM$hBw3={zDIt+p3qkOb zPSH4mo@uU=7okYU3%0RdZ72xtyhzGSZP2rG`um7L)q+U`T5#W-tSu z$5kA;Tmkb@u!Vh&8H-JS0*`g-`{F?08kwEb7i?(B2jEFDb~+???@{@X3Zc-4GPLOQ zS{nt{k$2U4`@(~sXmYuu?y4M|(s1=iGk9*|(e6rEz&22x3-QH!X$FT=U zUm~Oc&@&i>*qmt4swImxD{k|5ep-7|)jImlOMgE#=DcyKXAlI&77C>?lLB-mV7!{j z88W*r+82eW-!{=OT;8F{T%}cR`qb8QEF@QVeBxC=#S{LI_MLN2vx@3Ye`eoysdWiG zo^~3nF)`AZpi>IPq}?H<535`kr(9;w_F3bkZJ@!VVtoc0_VFuAJ3ziibO~tzawFhF zHC;qpA@l+6sQMot0_ugC!8G_~a5r-r_QJM%;`kZN<-Gj?GL~qgjhS_OeFF{ZAW}Ah z^#?RSt;;4u;0R6eC`!owv3%gXS0N@;)CdzBmOKEDbK}C6yIBj4=X%fsQuG)HZ7ULJ z3RYw0dH#?l%*f~HyACSwmR((lEfs1;1)xpwI;M4gL}d=OqeRU3n24FTZLRPJ{aMnZiA1tJ&Vutnau!|2R6cz1!ft zi+H$3Fl=ZPG!M6|fc}@c-dM_CR5hU_&otZ+h%;(1r|@Hs94?c`Ms!|20kY@sl9AeSxw zDqZ>QV}oyPrggq5p&VUk;UF10Sdjy<$2%lrKQD)Wv|D2Hb)HL7T>R@uOk;quC&xKe z8awD@UEUr|6AL0Er@&yd;BisQ&?&9@)a^xv0SPCDU=Ihx);{j-UB$&NFhSa9zt0pRx zXhhdU9Y79k0Gg;HTGqtNKod28Ul(`|er^i?ktHhkbcG@uw&X#p{o@nevDWa{k>RBP z)Pez#Om2z<9B(GigKTmXtR*5>kbX_T72&eIR=h@fHS-;Pq4=}dEj4p3>Fza$Prqp^ z3Blp`VgY5U~sWVhNdAq!G{w`A0O zt|v84vi>+#ZIN$uWfpV}5O=gC8Q{FFU9!s0^H@KFUrg*ZI1|6^!AS1G9{oc`d~<{&jSA0)ueybj9m`PAD}qC+`8AO4aiR^KJjwIENDFq%d^dy4FdkG)qEClITM z?XAl`GUzK_92&rOUvKHz9KJPeY} zW_jidwQ+KhRqOfW61mM@%59U~B=VCpjlG{hHX@B%k_~7bs?ldBg$~yd2YSYqT||%5 z7VoDtx$M_`V$Ae@Q=r~OK0jY)^2e{O!P+4r#_1*|klpFqAWW5zTYNjdA-wq&Oygg9 zEd!BcAd&W8% zw^loMxU%@yH+oPen1RgLx7mXGuuHSu*jjl0Qe^@%Kx=Qu^&)e!&!L&wmKO@!j-2PS zg1$ERw6}^a)CuzW~KEY-Sw&fmD89`vgak~1_NJQ}drm3EhNzZ^X3quj(eZf+~ zTkjfg9g%u$jeX`bOTVchj{6*bX*wwFOAig@^t?aJ@}wnav(cv8W?gJ{zBjx?z~DiU ztL=Z}4wheom6~pVEH$q=66^NLr}lN3Qq(?e$w)rKiwdkB zIQR!X#6`qA&|q{DO0t3VXq*7tK|>b6Kz;nlM`=6ZfWY(`mYP(H9KlXUS|Su_FK|mg zLmU71SmyseF6kfT!hZCA{_L#-0;e7rrGFtVoy^2>${I?nXL-D?^6`}1?T!@xBdxW>=m8sIqRm+XCz zP+u+IePO8<#jQMr<+9s5+2d{Jc1feRnOE=PROtqdvo4}C3tE`{u)&NA8whDN&~A_n zuTElWRp#cMY{|0yuqP?bZqMhc!>+P#q)o8bmeeX|wFO{c$#`c(k~O23}d6z|ac$Mqr96yoI9 z>m9V3tLWDY_9rE%Eh_(U{huo2K+w|hm!Rb~up&p$1FWxk5Gbcg(Za3)`3d?EQ#4*p zjeK5gsWh5iB&EO#C0}kv+SIEy*IZZ^D}0Gu^I3Z&GhB@-@I%*0r`!JQCBM1J(Z|llen(=?$A_c&GrYS?*Bh|SJYHBrXHKoDe)fnO4zrm;_oNOcrqw3& zuh&#pS}Jk1w-c=@g)dk`)NV#59sCkeSCK}S8viC2@@tC*BLlmNCP|4Ekzt5-&xrc& za%acfb8wZ65WW{)9(*pWvT4nV2JFB4>ZvcHKkld9LSBT0VTClU$li8ITm)nU*Um^+d92q@*A z{&z7vdkkG>2eOEwNYt-Fwu56H<9Qk5%diK=diw+8XZQz(|NYyV4~9K3?15nqjQH$( zHDL_@!0-?BqL3R|enW?Iu78WHE*x{!WXH`74I@fSWO>j^#?}BW8K=aR01XUgLqI^vTLA$nH-Kio zl_dctiAxDa)Sb#JJIGEpo?Z7~SJ1&i(4br7uv5lmZKG%Hb1r*~$M|RbCZTN>&ZNft z45^HTAjA<^Xn|ZzmJMoy=5NECa!uy(8X=Fy@XfL2&%%zdaw-?>T|T~nO$-3Ph&Fh$ zz&DAyweo%tq*0;i{Q5O>b+B&+`>$OF-=r2JNPh~~K+7Fjmo9ty_j$$CH8Pb#ZMagB1S1_jG8=3nJymM2vnfMD+)qOY zvH(<4zLkg4!r=9=fvXT6O?HCDun0{RUy(I5k-Ok!6;LEBh~qzVH1vgzg7jr3LFVUn zD;s~UBH|yF7ysZr?)$JGk<8dF93>97E~0r6Z{b^&U|e68gr29mr8d0iJy$c|BExp_ zke+XW$hsSBM`(5x5Q;Xn0(R9x=1iN`W_O($A&ZA4N^S!C-VMdcl^hj%ut~`7>ZQ5I zo4F38Gp|s;_6+|iI%a_5jN$`Eu>_+S<*yC!4FA9YI~ZUG1MK*J4D5(N$o@#((IYU2 zYzY#zCzaprUON2I`#^S!Rl3fOasZ0F*>386GDF{@>Do3cLqQgvPhG9ZpN~jwm{&H^ zOJL~~XV)y?M7LH@j0&f9?CX6yc**U0K1YIG!b{qRv&*ouB16gd&i*CLuYCm9C`Uz2 z7iH1juZ+L(0yUc5o*M3yO%5|>_SxGz+d_MExHXpC3`OGJE4U64(Wx?PmZbg?NgsMV zmbjUkgaAA<)bd4)FsZjPzAj|wOE3KFB1eKc@5j)&eIdLf4kAA7kHz|Rkvb;J>|`l0 zqKCDlygjKIE->GUUHE|r%V(c*v+`#e&hM$RKR|r4-SEo!y&|DL-05sn{agQ=yX%+R zp8wfBTYYZr# z!fR-f_@L<+pImpNGi7Zq+ls=JPmVQu>`60myI5(+a|RN=p=G1&~{K*n>xKzVv4dSOMNOJKIANDoKLIkbHTPGqtmA5{b}r`!!3O7EkuKM7t}U8 z)mJz`TUPuG+9N+B`6}UZgG%nQS%S^m%J-~x3kX2?-1(OdqzsV$cX!9u#kWwl%3Nq$ z3`3r%cws~`DDG`>k$n*smH1NcGDhA(4gKXDh8cNTzFUTPmt;(KvV(2W5_gvyF~HY0 z*=32&8`;ZVcd&>T+aA-)wV!w2a@TCk{~C^veJ*H*_|KmUzIkgLoU9I7 zii%)wIM8=qg_p_)rMxUFE)#IlF4>fGy|?&L_x;V4kxPO5$)+D%mLoy$fn(XLJirAx zll|PwBl=vsO8Ir;K#SW|BBB1lkQIO8G3sM`q!sec3Chm=E>z8pDlcdEl9T2kz`+`d zKadtXSS9LAGX1c)90y1$qB~qTjffiUQXLLfuVP%nj_Dl1Nv!AHRViy&XE==ced5

t(c;k6qEe=&wcnS&8qMZ*&Mh}D)^|=L2bDhyFFS%LY09BC?#03MxT4U1wZArs_Sx@5+ zl9i0_{l6d%W{fFgo&2p0RDjk;zc7j*DTXjkw0TWJ;#2i$eO+T6({lX1IDTSxu)t;9 z*!j0xy_wrxjV{66x@u7+E%hy;w0Ee2&n>mwK}T&?YAc>uUFRoCD6wh;T6t&Yi!6u- zijbYOw;}Pzjr?J}70ift{BUlX%miR3^Q7!gq*j~zqzsJn{61-1-ekKY;{JYzh8dZe z6;d^kowN!WiRdvWdghZ730yMLE^#g#y?XpSntLCV+}J$r1{fu)#>GAoeSRsL|H`3S zr3y$%C6dHfA*IoPQojgk^l^C^uR&wS;_C%FiM7x)P~^L#Sl$~5xPc8AU(&`vtI zZ8$9dDc*Te%omz`>)$Bo#J3E{8rB08D_W%kIj#H*?uX+UZRca&iLRcdEROp%05 zQKd`m;Br0}@!z0J?RY=m+ou4ecF-qoXfHsP<-cwGoEkUkwi^zr!h}NEaTG_q%wae; z>GIgj*l*WlFSR$k4Df5H+s6KT33RWNgx~@?1vHBOmjE>&ioh8IwVpFTtp`6p6y%4@ zEBCRdCeLBDLaeVfcTS%fr|#yr)H6vXfiQA`)it3jPYZP-~R?1 z_`b3O=_Byx0&lz`?ji-0WFmOnJ%!8c+zN|}O3LmxY48Ypn4Z)X+Zuj+I=r%#?JWv# z;0OfJG=KnF8ZEF02%xFcn&3he+bBLHBS!-FkXV4z5SIIxo3q2iSe0U86#K=Cl8(DO z4K1s;RP{&qZCp|KHG=_!eS-;rNIJ4*GqUe3sEIuX6v-&c)G*j(SdYYDJ?=>s1x^?U zaI_yXFYc^yGlrZ0UUNZ1NwOwl3{nlPj!e=X^M&0^|2I<8(X+Q3=jnt3>0ryZ10o8o zJs6{@aZHcnplLP8NecWb*tF|)D zNl|!(fi%i-P@&NY@`6ut5U8#eAVCDHghS|=PL4elLKLNrSr;EJjjWKDGxB4B!&g0e z$Axjj0UQ1H?kyF4i%yni{yQ-_Q}R9vlrARcHB=w&!f98#gzOQszq68ra^6VI3o*KT zu{iwFizJ0{(Rp~vS2@^k!-W5x(hL7;A}hxI|9vE4jIsaEAA8d=HVWT3@MoZ?=|v}K zZg>I+QM%rvisav{Li~~2w6^rQON`_eS3vQ#@A?a$PhfS@BAqz+!w$Fm#W^5Oa_Kou zX+w8_oYbHA+q}O3^}L-VUA*Pc0gS)a20Qy|5NtZa~MOJCkSisSf7(z{S)Dog7 z2M+jLvp-Vz`OMfQ7R32YM=7H`kHywQ+_7tHNi`_EQOj*D7&Q@QZ4L{>@IEQ=KNobe zqSUJRkr6ajV;xEjV$B>0w{K}i%9Z}~m zFL$TxR1Mr0>M|~IV1E}*6tS6B14Lb{aZ&_l;>S)cUT+;j5V7PA4p~eolbd87Tqw_WR5~3I1_1zS#iW;Dv{r0E@s_VY$1pd2Dyd)jkMpZ(> zuxX`ZcgoT*+nX>;GB-Tv)Qb{xR6%dw@UBFEbgZ*T-W;1nsc4uF79I!K?%!#g=vakV zzCO;mzsEeo^lbaKY=aMbjU_hSPkwwQ+=jUgHuxANSh2&~gBmYeA?|J32`f=vF47>o zmXZ&=F1{Slv3=RKtriMd?2Yd0>R+T|m@M}T=@@4A9>+^`46|cNG8kZ(1`V2_{_#}R zWbVK6-2aTOpy|R#z@T4%qHJ~6r6wb7gn?Gn36ju=PFbFXj_Nx^EKh;$Y*Em|-$H#! zk|KmJ@MpJvN%Vjb4*%ze!;Cp%%+dd}4VYG<@FycepqptNh-SB>{UmtK(PX<9*NQhk z>o_X>BK)eAi_Gp<_x014M7ppp%OIz;{t{?r*-UHj3`|V6T@s4v?o{gB+y_A+ftnJ3<7wLD|poTte6fg z(r?O=1a8WWXz)ZNqCta<{ijaloqPLg@C|&4H(Xz`vE|hwVp^GgB?f|{P|Ot@XjKC| zq*<>KBR{d<+Gc1HN4mzmj>_}oKDog%eXLa_a9Hm|w#V|#m!si!3tX5-P|{AI`SsMf z2D+qxoahA9mH$?~=n5NZHpXTO-VGnxisg5_B|s3_d!g5?DBEpaH|B#OQ}u)UH}ro; zC@;K5Ttrq~K#smEf|5CHkt@O{*gm52gK)xWG_dV$*n`+(+appT7CbyN*4lG)f1g58 zfqkE(iMGxyAK3tW zFQN-o7(`J>t13fc*+YhXN&0Mh+{RBT?$k7rJt{b$ zH6uKRi>HNT%~=wG0wc&JwvQ~~4rX{WcWVBY_T`*U1o3%qgH+6z(1KH z?1g`4G2omvE`z+^4qhZxIfcHn!isi$(%%O^XAzR{z(a!LdTlW7_)X_pA8Wce*J-pc z>8m&wau^1gTKOu@RXEW(|NAStI9C~~v8*f*=L#KMO6I;o4PJ#BwNZU(s@^t)SR5hOU89x;Be0ex>#Rc9S>Np^ zo0OFnz;_;!sNrpF3-g~DLR|QSo_K{xKz*7TDhaP(qV@NnB50~M^X^-!&qb30H7tzI z^;%kM7<|63uT*PXvTj3_dcz_g8B~pMa^N{=!SFyN-q~suQg6fDHIWNHPQdsrMsOV) z5tV|s&)(v=_0Z5|n_SAncf63_v)i+_y{TgwIC|-ZbMP>1=m~-e*>)7)LgTuQA>kmN zRK9AG`LJWPqe%ZT%Rv;st(JIciIKwDLNeK_5R+3*hH8Nm11)2-gf?_9+b`=?)Eo4pTs`FQO< z$g=rbXVg*I{Jp0S>)tO=+{bi!ouL|E=`iXZhO?a0g)<_o-0Ud{F*c;xkkTQK!-=-5 z^<^n$$lTY{7fj|F4xZyC*}#3_3nwXx0A@~*r)gNcZV96ZSPeOic)dd!`B#TVHYlsh zrI>GzdSUSCJUh9(qmp)ayEowV-9+~+?F;yOL(iXMi8@)+ElDHjZs$ew{ocfa2O=5~d z_BK`y`fVy@5lbmj;toY_C^yMEnCuR(7vj&$=y#a2zrLtpesA>Cv)(IC4b8nKM@6_G zl6Yz657KB2fSYpR+^ zA$2qkJR*j*Y`7n3)TY#%c6Rtx#jD^RcF~SknT9gE3ly^ULPmWJ22FXsV=TJBSSW^1 zT85EaQ!9v!RCA9cdt6S(sb2lhoF5$Gwxvvq6dN%|pV$9p#FDw;DKB9hlv5n=bs*L-l`}qtEZqnBP0K_7BfmjiB+&-`*kcW2?Euz zs;Y8%wOc)3CzOYeZ_G?Nd6f4CWHf$K)0Dtbsl`l|r$}k-XxG#rf?iB(THEWu_V;te zvvZVji(WUmX=_2jNonR6Ddqk&A%Q^w>Q>AqplLD&giyn~9Km|T0D*=f0nqn}B_oZ% zA7|@0Y8o-UqPD>G9oDA*Gp>Jb+*Psu9Ar`&4Qi?b14!#vA-eL_Fp_#cDC#ciBApr* z@b&f}qDcqKk3M$~nby#1qb7QBVxz}qy2m`^dfn{pOY3epwYIWd()AF*Kxxdqjd&x1 zM78l-VLmJS%gqWe)WrSe4zi`^a0-kVRkiS5d$txoV4HbU~SSV1=x=EFtL`HEa+Zmuz})w7TeNqEJ{-N3Vcz z!jQ*lq+*EpYZ^P(j zeOBKC+k@xSDqs}0moei0!}@_wqx3drzN(G8die%q-T5e^gBC%-vSDqVHDjo zcTbICI%UyId;T-p94v|of#N*(_h0qh-zd}Y=SAo0w@4TMdD}cZ}uG=jM!>>;kSovCeJPQuW%IKJAeOK zja=BIL6_Dw^+df{qP8r`ciTj*Gh zI0CN@7)*^6q>DJnUi!D*tL|=|JTl4?S!eg?TN5v2np!ZG$^<{W2O5;)ZIMF?Io`e? z^YLEJB1g!}%Y|{)7KCl%t~@VXmLK@Jap~zgq1LjO&y+nQ$n~VLPP zhZb6sGKKnet_6phnJ@bLzn8iDo$pU1xeYJ9K?+PpP9U}ra_{9_Q^-;RXv6 z9+=)oRv)6QTZN>=4xuO_-AF>uG{s~D%k0i8<3Vu*23_6OUm&|Dy}#NfKOb`Tx3~xU zAfdP0m6$q>FC6^Fm)e!Impcx=wychJuQ%*kAos89-w|b@$C3W5;IQu&8Kud(y?MTF zQJ>C>0OJV$*065T_(=))-g?zjXGv1E@kV#$?Ax0k?$M$R;Mx?!nqQP{E)~B*h=rp=^Fbl`#+hq2}hA^(WXR2T!nN8lQ&+BX&|$@X8%3S}KIA(L>~f zJM-H^PGCw3-F_drnf?B0?mT+YTB>-9*ns-5S!JpgH^gNekzmNek#DEU zmA0L}WkR1F%q5k;+ZT?heCbrnMdzjz`{}#LWkwxo%x=zXhNz44s%>TJSld|Iyd4KI zyf)4xRP`$wD$}NOz&gRp&SfjHZCU45!gD?8GPXU@mER3jB0z23R@yca_C&NH9v-sb zgIJLcon=L(oA)2dli3}+YfHiUO^qBpq4za{A+tYeGA1~!1KymNSAli)!}39L4h|NU z!P1Lfsve^6`AiN!eJDA|^j3*!YO5)97<2am6CT=)98xF9#SLLg9LX?Z*tANLTTBTu zc6|R#Z>gK5T0@KR`JlCT9wg})1?;&SFcg5@^sXQ&5(%0uI9QP`aZf_A16glY-eERY zBQI%x*LR{8*c;U5a^{hiBIFy+VGG3?_@!=Q#PrlB2MS8nwMFr*Yj2sI{d-0K@W4`a z1lS5nVg^xQD@ct}+1=Pv`^f6zjf1S`{kV;_#Y%NOl0pb+gX8y#}W|EY&{q3q6%OICup{6mxXeh!@dBXxq$Q`5KD*x>6@%z_=0$yk5Y*CcH z5%ntXWMk#tZ4DB6$y{@5MW_`;bPwpL<6lGD-nBrL*DIZ)Wc6mHCl^P*6He__e4#g> zeqKy}`uHcSNgj zW95w#iG6$PFUWlFa)BZEJt!B&ZBLEZ27+G9u%JEPIjeSba+yuc+l}WBO`qoFVxI~g zUxbFESPH`qjT?Tf5GQ#yt~8qZxl4V_k$AYv@^Wr?n7EbIMV(y`zr8lhAHkCPPyxJf zmy3+OunzFTi^Al3o+?Mj%${RPx&3@RlCsTS5JhvbU39Nr*>&O*sTX+lpc#iP&44wW zHqh3u7}i*@KdO}FiqluGzE*76J(+X>uM#vg1sAYGY;P~5Zd|-<=W}RyW?Rlqvo3Ev z_5M58MG;owiPe#jYtMKYlEWQag-|bwCs*yKp@uo^+~z*V8odg}@emTTTMHnc#0Tet zzZct<3}HjC>=byW{u7c>-K(8N>5bWM?X5D*&%bZsEL3Dv% zaJ$~baZU9`k8HLt|Awnpo3tgwZ@v3YT(K}KOp-r~A062D!G)@|q+McoCDzz)a@X}X zahAq+`}0C&FN%&93$Ur>N_=w!Xk|mUoreWffX$EauA@_n9c+HX90|APcZ3*WO3H5i zKH@AZS0iU2sChMS5grDR^@Tl0#`U^1eTJy}Y44hyd;6{x-+#9SZF0+lS%zycw3RL0 z_=0Jg_bt+ntV#Z!aLQm{HL1_vgX)I<~K>KKYZAXLs4-vpSEQ5ZS$mGGLYUr z61K|qf`FK6dDj9-Q|+yY|EZ|S9r?RtSi%CTE{Szr-I(UeF3xof;<9zkfJcTN@XW~E zqws)$r)FP|LiW2fi)zr3XCv@<7zlmpE6xg<^p3xT#lbW`4F{ zQ#RKI@uvF+CuL4))5G)HbA>`|0Tibvc0!4rvpYM1F8a%)W5E{1`XZt4icaTM4z4p= zM-R{EF+AQENJc}PCy#O0CMmZmS?KLJ<7L->q^HNlRPdzbL*0uTMn!JQY*~B0Fn2+y zg<<=0!meTuaHCs}y>bbc$(l;fNsjD4n)x&;KTC0U0fftdC4BwbN0-kFa9}bu@gg)S z{0wMAidQa)8yo29*F8{(IMKiJOJuNIRnrx`x^m4oF(PDfMaCbGsf7h=R zLg5v`5*k%$ll)63vyU`BYdkz%rK8|iV=2isod5GiV`NL18nmZ&sHnhryw#nZZ%&CA z>t}VBHO$<|%-kWFCk5d$%uTIDf_-lg29BO@oWaIdjd8#dkvQQ`_L{RY4yzFP_b&Y< zH)racB9l^lH8!phyZ}#~K$55NfGrqEkJ6c4^;JAjJ;-Yz)8?wz`-x!#7y7YsZkC!Z z$y_OGZDPm(t>4_)>QMlbxg%Wj%KC0abpa0A-Swh>S<3;JWF3VoaqaAVY|SRh331(O z8rehu=j{XKXqHt7ucop{WZs!GowC(3%zhgbMmKu~x3(daC?bTN8IDg#ZuOpA8R`9b zKmx(#sM&aoujHLDN!fbHW2V**4yv7L0w;hebPj;c9FLp_`5(t^i$xwvfQ|KO2&%j>G^ zhTeJ^|Ax`;B}jTUM^|fYZ!hxMUYqD3FFP~zvN#~+pv1i$VR2hC_g-ZQ z-Nk~p``&hhBH0bsziAPignt^~COizw#ZKF5wOmT?wK$gNR9bv^x>rx{_+CYm#Z?GB zktCC4lpG98Hh z-n<`_ITEGLko`)CDGWdREG+odex=(nF}F881qsj$B{ba~FZLhl)oY~y zcdSPv)Z8%doh*(%CR3t&*pNd}ta20G9skRWb&b5tmBQ=a zEu7t4Z_`+ylIX_SQ}dqv#2Ej@r@lA1&znO3;pQvN+$D`xZPnUG*-ki?yf>?2C8kxJBjP1r@pe%%o~F0j+Uj-W1aIXfrYEbt5)%Vw(v@x`r7Qx8nz zCNM$>F8AouC9M>Vk=JaeI~+b`t8~74mObBC$$LmIir>=rIAlUbrxr?2I7pm`FeIdb zgu}X(vv3ze2x6DX;*-s0W(uX##e;;yTBn)bsSCgj1}>;+8V&D38hu!> zL&-P3e)TGt!&bBH)qw&H7WS~XIg46AI$ihaMcmKo&Y$qolu?k_D*R*7ykmWiMAs=CHbviGSAem$eCm6XMo z08DTs!3Q0!9d z&V^<+KkPggQ`9`2u27@DHMEB5ts2OlrWjvv8k^uMBhZ zjwr z8Bi|+>ixTNKnB#yfO;8F?{|u=7*H<*>SaK^45*g@^)jH|Z=0eRQ14GP7_E)gGN4`t z)Vp?Q!GL-hP%i`OWk9_QsFwltGN4`t)XRW+8Bi|+>Sc8GGP-*Ir@DG&#<+XtiGUR* zFpH$>jE|=A%9C_yZC-XBL~OoK3H0jaUQP7=O*1VP6i=arpQ?byr&0JEhYwe5q{ij$ z(cCxuu+{tY%0rYUPd8TBLc7-^G|C`EIQzMlYXNs`UNUzY>TI+xn`D>q zPg56Yiw3OKyza+t!u&LI!PoxH&tF^X2Tc8tptU~S;8T{29si}6+f^)rH#Z#6 zwXylB?hR??cIjO1pYPxp|1<3koffR=C)+o^OW`8*PxNmn*R5GA-|}+=2AM0XkUH3% z3w$5bS~ZPF!Z`FhaIVXpCGm+l<%HQ2Qx)oxXDk}40`F}^(R1Qdvx_dv+(qi8wsN6V>@9s zoS!eMNgtotW{Lh@2SXMfT@w+6(0Dk^&ElSVgs`bNI7i`2SF1(gkO{M15YxsTnpcW*o* zb<^R!JsQuQO3zrIG!+jGS3F<8c31y_Q{bn!QV!S+;~ZFpuTlVa2V{1XgZ2)z^Z8$vj4e{M6pP~dFAB4W zh&g?SoUDAd8bhaf!JLlJ!J`1pi=AP$)7}m3Qha(^Ps!NG%{I!la>tQq? z!m5lkZy7=I*baz=&*!3#@t_5wW;~ZU6w0camifyYc*3>B!d3IDKU|Ry8WOv zbYO!2i)_^7UJKnl@sf}@v$Qk&0z#R~AHEqJ{~nV}8os0k8#ti;D!#r-OZJ2JL1%}g zD|xm{#3hG&j~%Q6J_WpGmIABV2aCR zZiKkir9Phch~xAwAk&ULg^5z23kdCqwlWa+<2?7l(CVSy#=Cd(`LA9T-|cHC1seHJ z1`knA5mbjtZs$J9Bb0eA!W@2Y4&*lT9iQga7KvYzrSL7u6c`tw&vXk(0x*i) zwJfwUOE_K@h`VgflJ3m0_oTKOTdqvRk2~NJCZOABJnmU%{I&6zaHTz#r(X4s_H<9W zmTY;OceJ=+Br>iz^P!pu#BlI*(od0GeC-KZs~d?R0J@O|sk=GdY60s{ifzg9yjm9@ z7rsJXjSQF5QD;o9TAjJb^cE6Xxpq&IMMVgmKqguTRYMa6%m=6RisvWZ3OsdcsqL_}yAt)#z@*7Nv(u#piI1eO zo;VwN<#yi*HjkgQ7xYiC79?|@O?kNr;iDYFbNkW&^B#N4y<06@)G}l@pv!i=R5)Lr zQ%1q4ce(aAl?m4Ac?OTJxdG8TCI>MjT|n5UO2f8;KO)^@NLy;AeBSl`0=FJj4=*?G z(*96F=bFvb@FW!eR7=nW4N@w3_oNs<$>oU!Ddg<_+{C?w`FG!`hpTfDHbf9>Yo5NIK{-;&`WLDX`ppSVLyY&aM62ipImaXP2Y)h4)z9->UX@ zVn5G22q5aLlBYr4M}}gj(N;sSyPll0LCTL@><{S`Vs7Kc*d=!9^$dz;?-z?%fx@>V zYG8#%_h8qczCAXL@ZHdA^PcyWxHX?88Uxj?xiymk7g*}_?)}fsMY=&?$ zzqivWB>95rybk=s()5#6h%}ObriN�kK&n$s<3#12Q2KL^K94KHNjj!M$BpA(!8-LO%3*l9-8|tB~lIq}QP1QU!FqruO)% zuR^*tXvY%VErt{VC)+Ng3DVi=k1i45~1wW)L6GCAk zSX08zwNMd#$cZ|13HV{L&^aF@;ELK~pqUMzCV2flhaH&ESSP-tBN4{O8 z^OC;oqQ%YHFt@@6(Xcs>{jt>Bg%~RT$K~5_FaUT{)UsycQCejWoECD8QVE1hyl5M# zKwcP*U!v97ORYkVsV-7lyB5HZtw(?%kSBT32QM0YZ^8=A1`x<=fiLqwHY(6$Rv~>K z!Rs%lmaEVcWE%TtFmp{L^eV*Jlomhob(Z1a;e%0gw+42B5mF$pLXPiu@kWxth`$}B zoCPD9xD`ko$AS0KME6&~s1>V_(K&=H^3ydVDw8yG1I!w;ERW%Y zOYTN50##3MmDT=TtlDW)wLr znY&Vm7J(UOM=k+A&A!h_HDxLk`JNYyD+jnqBZPLtZi!6WV7UrWE}(tJP~Vyka)LoX zR&7F#6(jq=OqA{hvrfueh3qh@gA)ZikPFX`tRQLZV7R1;@Fg@1-Rc8}D=z-{Grn5M z;0J&KeB17BPOE8xQq?Avmzm&T{P5>6U^F<8pW?uyq>1Wb#HIfjcc3-^27W8DFAI!& zsmveaz7c4EfpIrph1^8b;zz%Zd+ID{ha=^Yd@Y8&X=)X+VwDeG#;p0OFU%B7QW$2= zO#nrdorJAu^^_xN8zaCk>LMu81~Dtv`RI$_QB}aB_FX`JDga8<6Xa_IuqJoIC@6SM z3mNd4eo1Qd)e^^}faR)E5xw9ewP@@tNlu;;jA4gOErH2yb1fn<55bm&3LRIdU@+T4 z;213aJbalC!Hevg0i&9NvteEy0>(6yMtLGP5A1Z7hLX$r`aA#brF3TSTK_ERw=PN; zoPQNE)ua#Z7i&V&ftO+NI_PDgiao0k;IP2@kV|Kt3nGHMgY^s?gC*)Na$)4aEohz* zC9(bulq^`-LKQ1&p>@LMZeays)6N8L+GFcneBb))WWs)9DLLAB;l}@~sbE zo*E*t3OVW^L-4~`gYhdy0oMbp=yC(y^#oHcCUZ}aiQqpE47_E5ovZp`76Nwz16$Fe z=EFYoLOEd5oP%iUZ^4+yLJ(RkhVHzy%YpN<+z0L;<4LozTe$$fZ3h_jb%ZWzX#Fb0 zAZLp14coxTO9K6&q^>)#sU|Rc9|Xaa_`N{B#kpf><%JQ{-vT3k_z7Q_n#|Y5Jb-*F zOJ6gW5-`-Ae3Zei1+WCaIz%(z5UW*@E5A$Ahx>~c5%}u|Fj5zAY$C`Bftat3f-H?} zgi`FlPj3l_EpUKUWMfA}(l*3`Me={zJJ+Bnt~3mHIdLta+)2g43+_6lu7V=LNHE4# zdQ)sAModUH8Y&WPukkW*2}vwXB%FyCynvagQP%4a$qFvUj3Q z?|TW0-1@4YX*q!Ce6B53wzEdkbUB2Pur%v1kXdvf^HT(AY?x;~SL|p@DeE3*12aNb zg9jsphXJs8o_ZwAtL3ESl{TBa|0pU5O5wC0e`Vg-p^Fj-oWJ^DH?6$Da0@@7=#-Np8thUg-tL=6&aQW3xps(%n9C~xq{M|h7ff*cE>tkU87;vL z3J{##hX|glb$^n;1nSLxB*71ZQCD{LMCas06uKDsE(n-w1Z758eK*U*-5BI(ZZkWG zy?w-URlYdIDt}Z;gCN1w{i&@BOLTvR@GvS$=}YZ7Qsyd?GVqYEO!v;h+){(zq8$34W34cGFp0x^RhA-oGlG z@mXjs)H0dEH!{_i7gPU?bVcx*cUai{v>Mgz5?u3Xgtf$i{7t*BfCs%Drl(6B?sZZy zhJCp_YKUi4s_($1yHXa$Y(1H4G~vWNoY?7`sm#=#mcj;Oe?Rk8MINba73AzWqK5IC zr$+dhQ7>=9@J^y+Jhse$Vi=m`XGZcDlDyJQwqx$6WNMsbBqeF&_G}NLk2`Ibvx@k; znHDzBnQ3!vI8!DX%z`)lbfDa*4_VmNiVBKL$0YL6xXY*KCf_l(E~4q&z7y&EFy`R$ zyV5+||L@6H7450u8Igvr;yz-FtccCaN#gH(38;!g8x6VARss!(ORBCNCXJnn22_C< z8gQ%%;|OiMdJx4U4ZwJwvu2e1;)u2r@)xxsfa;Jy-O)gt%7ot042&q;`$H(W7i*G6 z7UvqF2SJWHC=}P)dALDOui`2R%K|WhVq!D{9Mo&~fa2=E0e04g# z2lIn<%JoN0xIBhlJ_?sFz~u~$l46@HF+QABz2@{Y&u$C|8Y)wFD>C((L*5x8KqKRd$+gMPfH+krZds*1E@5vt(+j7Y)tDCjj2iG3c4cb! zAFWvqckBw{omOD5NjrtxP3H&T#qZWTrO72+z6Qq=h?@3IO0zH{AI(Ca9}x@FbB$ky z%Q@#KlPiNCcc4{3jOIs$DAiHLaTKYK`9`b&is(rGlHiq4C_Cm>^u+9M1;lJIjagRC zIt?vo8iW8l3U0UBa<8bNuG1uh4urLDSvjeXOt*X*Svp!E=!n zw~?lUZp$=-9P$9y>^wrMh5<=8R&sv}^Ac3gcMx->-w7=P{th8@lK&d3g%Zy@Tuc0n zk+i(_E$JvH+_eED(b1Od?4IanHVN+7<*awPfDVFYlP4kvuxJpZYn<2oK@SwvEMI2m zq$_cRrcZ-S92kR6B|q7YAK7#G4klf)iUW|Oxuf5EUxV<%TWW$^rfL!XJp|lp-whUz zo9`ONWwR$7=)p2=?$~7xZ%SB387X;7_~#bDxfGPCM(0A=_Xf(<8=Q2#r}wLmfrb?K z=NchEiBOlxK1gBjBmIBamBCax(f-(F^HS+55qDszO<5+_EFUIm!2x8=jKzFDa%Z2z zwU!Gd&29ww^uOW*lV3k6(s0k{iRJ)|jY7-JFZ$5GS5PL`|XZ)8LTaS%z{k@t%c^BxV!DTec12Z4=67Y~@oTdOE1L5mc_A z6y0csdeh*(kzHs5J2tRL0h@Q)G6NRc6P=G(!XMa;4X$s`Mv12by*2EPNOPXW?tov9(q~pq9*5 zHJCAG6}+{l7yxf%a`au_jxuD;&@&+V0`B?%@{B^l(NvhmuJ$vZz#%}@&k>dwyA%A( zHYn3}yZrt;0YH2XhXJ20V9*{U?VrvnHDxIRNM(2U=JVfE3CI1+C$Yq(SDsh&KILl~6oIQF^VAApb zpTJyT(2)6@2F&w7<%b~IcQ(@pN~G~pY;9$(@+?%>suf(&LUw#g+1fiSkSVWh}coN#2_Zy^K=Ro8>{*JBZv$3gOMyjU#CE^U>&|7B*G% zH2P}W!xGQ_ifRV%e@%JAFAL(fB~D?$bw9H=0wOv>zGZI1LghKTFK|)`>3T8{k`Fcr zw2{C6CP=_2WEtXTuDuS^BNf)dY|r}kMe+<=w$Kt=0AS4AMc4nK$MB+vx;Gv2?ZWnc z4R+ryb||S@`XXXvPo$v<@i?Xng64w5E77nr=>@VgTzY3v>M~!4O)R^30V?=3J>GGt&*$PE; zT?DG+J+B7@o#t=>?S3k7t3e`~zY@}$-e%fyVzBRVUS)d}YvJx$e}TB-mN_~_%w^sn z3yVs4N`M=PKA!(^5!}G)%77aipY!W6h|LQ-K zF0PlOkn92dGb$PS$8`hxXGulI3o2`auh1GN-3cP*xwJfeF-TqZB?{v~%M@c(KAQ7- zBcgt(Y@H)w5>*7Y1e4pW1-FGA0-HhWd1%3{Cb4gAE#L z%n#kcMwQeZY~8`8A3^F4cCS`-k9ndtwsi+vcd%p7?&uD-?qL7t(@5RH{so&`cd&H_ zTlHb2?qDB5Q|8tkY~8`u9qb@uTX(Q?X6p{N?qKVCuvR2<&+w+wNeP?ao|G~~$`la=C38Zx2a)&#- Sg6)Xo%e}rYw8t3#_x}RxVfIe| diff --git a/src/wiredancer/c/wd_f1.c b/src/wiredancer/c/wd_f1.c deleted file mode 100644 index 5e116a01d07..00000000000 --- a/src/wiredancer/c/wd_f1.c +++ /dev/null @@ -1,430 +0,0 @@ -#ifndef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE -#endif - -#include "wd_f1.h" - -// private functions -uint32_t _wd_read_32 (wd_pci_t* pci, uint32_t addr); -void _wd_write_32 (wd_pci_t* pci, uint32_t addr, uint32_t v); -void _wd_write_256 (wd_pci_t* pci, uint64_t off, void const* buf); -inline void _wd_stream_256 (wd_wksp_t* wd, uint32_t slot, void const* buf); -void _wd_stream_flush (wd_wksp_t* wd, uint32_t slot); -uint32_t _wd_next_slot (wd_wksp_t* wd, uint32_t slot); - -// PPPPPPPPPPPPPPPPP CCCCCCCCCCCCCIIIIIIIIII -// P::::::::::::::::P CCC::::::::::::CI::::::::I -// P::::::PPPPPP:::::P CC:::::::::::::::CI::::::::I -// PP:::::P P:::::P C:::::CCCCCCCC::::CII::::::II -// P::::P P:::::P C:::::C CCCCCC I::::I -// P::::P P:::::PC:::::C I::::I -// P::::PPPPPP:::::P C:::::C I::::I -// P:::::::::::::PP C:::::C I::::I -// P::::PPPPPPPPP C:::::C I::::I -// P::::P C:::::C I::::I -// P::::P C:::::C I::::I -// P::::P C:::::C CCCCCC I::::I -// PP::::::PP C:::::CCCCCCCC::::CII::::::II -// P::::::::P CC:::::::::::::::CI::::::::I -// P::::::::P CCC::::::::::::CI::::::::I -// PPPPPPPPPP CCCCCCCCCCCCCIIIIIIIIII - -int wd_init_pci(wd_wksp_t* wd, uint64_t slots) -{ - wd->pci_slots = slots; - wd->stream_buf = mmap(0, 32, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_LOCKED, -1, 0); - - fpga_mgmt_state.initialized = true; - - for (uint32_t slot = 0; slot < WD_N_PCI_SLOTS; slot ++) - { - wd_pci_t* pci = &wd->pci[slot]; - - pci->bar0 = PCI_BAR_HANDLE_INIT; - pci->bar4 = PCI_BAR_HANDLE_INIT; - pci->bar4_addr = 0; - - fpga_mgmt_state.slots[slot].handle = PCI_BAR_HANDLE_INIT; - - if ((wd->pci_slots & (1UL<bar0); - if (rc) - { - FD_LOG_ERR(( "Unable to attach to the AFI on slot id %d", slot )); - return -1; - } - rc = fpga_pci_attach((int)slot, FPGA_APP_PF, APP_PF_BAR4, BURST_CAPABLE, &pci->bar4); - if (rc) - { - FD_LOG_ERR (( "Unable to attach to the AFI on slot id %d", slot )); - return -1; - } - - fpga_pci_get_address(pci->bar4, 0, 1024*1024, (void**)&pci->bar4_addr); - - for (uint32_t si = 0; si < WD_N_PCI_STREAMS; si ++) - { - pci->stream[si].a = 0x0; - pci->stream[si].b = 1+si; - pci->stream[si].b <<= 32; - pci->stream[si].m = (1L << 20); - } - } - - return 0; -} - -int wd_free_pci (wd_wksp_t* wd) -{ - (void)wd; - return 0; -} - -uint32_t _wd_read_32(wd_pci_t* pci, uint32_t addr) -{ - int rc; - uint32_t value; - rc = fpga_pci_peek(pci->bar0, addr, &value); - if (rc) - FD_LOG_ERR (("Unable to read from the fpga !" )); - return value; -} - -void _wd_write_32(wd_pci_t* pci, uint32_t addr, uint32_t v) -{ - fpga_pci_poke(pci->bar0, addr, v); -} - -void _wd_write_256(wd_pci_t* pci, uint64_t off, void const* buf) -{ - uint32_t* data = (uint32_t*)buf; - volatile uint32_t* addr = (volatile uint32_t*)pci->bar4_addr; - addr += (off >> 2); - if (0) - { - addr[0] = data[0]; - addr[1] = data[1]; - addr[2] = data[2]; - addr[3] = data[3]; - addr[4] = data[4]; - addr[5] = data[5]; - addr[6] = data[6]; - addr[7] = data[7]; - } else - { - __m256i v; - v = _mm256_load_si256((__m256i*)data); - _mm256_stream_si256((__m256i*)(addr), v); - } -} - -inline void _wd_stream_256(wd_wksp_t* wd, uint32_t slot, void const* buf) -{ - wd_pci_st_t* pci_st = &wd->pci[slot].stream[0]; - _wd_write_256(&wd->pci[slot], pci_st->a | pci_st->b, buf); - pci_st->a += 32; - if (pci_st->a == pci_st->m) - { - _wd_stream_flush(wd, slot); - pci_st->a = 0; - } - else if ((pci_st->a & 0xFC0) == 0xFC0) - { - _wd_stream_flush(wd, slot); - } -} - -void _wd_stream_flush(wd_wksp_t* wd, uint32_t slot) -{ - (void)wd; - (void)slot; - _mm_sfence(); -} - -// MMMMMMMM MMMMMMMMIIIIIIIIII SSSSSSSSSSSSSSS CCCCCCCCCCCCC -// M:::::::M M:::::::MI::::::::I SS:::::::::::::::S CCC::::::::::::C -// M::::::::M M::::::::MI::::::::IS:::::SSSSSS::::::S CC:::::::::::::::C -// M:::::::::M M:::::::::MII::::::IIS:::::S SSSSSSS C:::::CCCCCCCC::::C -// M::::::::::M M::::::::::M I::::I S:::::S C:::::C CCCCCC -// M:::::::::::M M:::::::::::M I::::I S:::::S C:::::C -// M:::::::M::::M M::::M:::::::M I::::I S::::SSSS C:::::C -// M::::::M M::::M M::::M M::::::M I::::I SS::::::SSSSS C:::::C -// M::::::M M::::M::::M M::::::M I::::I SSS::::::::SS C:::::C -// M::::::M M:::::::M M::::::M I::::I SSSSSS::::S C:::::C -// M::::::M M:::::M M::::::M I::::I S:::::SC:::::C -// M::::::M MMMMM M::::::M I::::I S:::::S C:::::C CCCCCC -// M::::::M M::::::MII::::::IISSSSSSS S:::::S C:::::CCCCCCCC::::C -// M::::::M M::::::MI::::::::IS::::::SSSSSS:::::S CC:::::::::::::::C -// M::::::M M::::::MI::::::::IS:::::::::::::::SS CCC::::::::::::C -// MMMMMMMM MMMMMMMMIIIIIIIIII SSSSSSSSSSSSSSS CCCCCCCCCCCCC - -void wd_rst_cntrs(wd_wksp_t* wd, uint32_t slot) -{ - if (!(wd->pci_slots & (1UL << slot))) - return; - _wd_write_32(&wd->pci[slot], 0x20<<2, 1); -} -void wd_snp_cntrs(wd_wksp_t* wd, uint32_t slot) -{ - if (!(wd->pci_slots & (1UL << slot))) - return; - _wd_write_32(&wd->pci[slot], 0x20<<2, 2); -} -uint32_t wd_rd_cntr(wd_wksp_t* wd, uint32_t slot, uint32_t ci) -{ - if (!(wd->pci_slots & (1UL << slot))) - return 0; - _wd_write_32(&wd->pci[slot], 0x10<<2, ci); - return _wd_read_32(&wd->pci[slot], 0x20<<2); -} - -uint64_t wd_rd_ts(wd_wksp_t* wd, uint32_t slot) -{ - if (!(wd->pci_slots & (1UL << slot))) - return 0; - uint64_t ts = _wd_read_32(&wd->pci[slot], (0x12+0)<<2); - ts <<= 32; - ts |= _wd_read_32(&wd->pci[slot], (0x11+0)<<2); - return ts; -} - -void wd_zprintf(const char* format, ...) -{ - char s[512]; - va_list argptr; - va_start(argptr, format); - vsnprintf (s, 512, format, argptr); - for (int i = 0; s[i]; i ++) if (s[i] == '0') s[i] = '_'; - printf ("%s", s); - va_end(argptr); -} - -uint32_t _wd_next_slot(wd_wksp_t* wd, uint32_t slot) -{ - for (int i = 0; i < WD_N_PCI_SLOTS; i ++) - { - slot ++; - if (slot >= WD_N_PCI_SLOTS) - slot = 0; - if (wd->pci_slots & (1UL << slot)) - break; - } - return slot; -} - -int _wd_set_vdip_64(wd_wksp_t* wd, uint32_t slot, uint32_t vi, uint64_t v) -{ - (void)wd; - for (uint32_t i = 0; i < 8; i ++) - { - uint32_t vdip = 0xf; - vdip |= ((vi * 8) + i) << 4; - vdip |= (uint32_t)((v & 0xff) << 8); - v >>= 8; - if (fpga_mgmt_set_vDIP((int)slot, (uint16_t)vdip)) - { - FD_LOG_ERR (( "Unable to set privileged bytes for slot id %d", slot )); - return -1; - } - } - return 0; -} - - -// DDDDDDDDDDDDD MMMMMMMM MMMMMMMM AAA -// D::::::::::::DDD M:::::::M M:::::::M A:::A -// D:::::::::::::::DD M::::::::M M::::::::M A:::::A -// DDD:::::DDDDD:::::D M:::::::::M M:::::::::M A:::::::A -// D:::::D D:::::D M::::::::::M M::::::::::M A:::::::::A -// D:::::D D:::::DM:::::::::::M M:::::::::::M A:::::A:::::A -// D:::::D D:::::DM:::::::M::::M M::::M:::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M::::M M::::M M::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M::::M::::M M::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M:::::::M M::::::M A:::::AAAAAAAAA:::::A -// D:::::D D:::::DM::::::M M:::::M M::::::M A:::::::::::::::::::::A -// D:::::D D:::::D M::::::M MMMMM M::::::M A:::::AAAAAAAAAAAAA:::::A -// DDD:::::DDDDD:::::D M::::::M M::::::M A:::::A A:::::A -// D:::::::::::::::DD M::::::M M::::::M A:::::A A:::::A -// D::::::::::::DDD M::::::M M::::::M A:::::A A:::::A -// DDDDDDDDDDDDD MMMMMMMM MMMMMMMMAAAAAAA AAAAAAA - -uint64_t _wd_get_phys(void* p) -{ - uint64_t PAGE_SIZE = (uint64_t)sysconf(_SC_PAGESIZE); - int pagemap_fd; - uint64_t vaddr; - uintptr_t vpn; - uint64_t pfn; - - pagemap_fd = open("/proc/self/pagemap", O_RDONLY); - if (pagemap_fd < 0) - { - FD_LOG_ERR (( "cannot open pagemap file: %d ", pagemap_fd )); - return 0; - } - - vaddr = (uint64_t)p; - vpn = vaddr / PAGE_SIZE; - for (size_t nread = 0; nread < sizeof(pfn); ) - { - ssize_t ret = pread(pagemap_fd, &pfn, sizeof(pfn) - nread, (off_t)((vpn * sizeof(pfn)) + nread)); - if (ret <= 0) - { - FD_LOG_ERR (( "pread error: %lu ", ret )); - close(pagemap_fd); - return 0; - } - nread += (size_t)ret; - } - pfn &= (1UL << 55) - 1; - pfn = (pfn * (long unsigned int)PAGE_SIZE) + (vaddr % (long unsigned int)PAGE_SIZE); - - close(pagemap_fd); - - return pfn; -} - -// SSSSSSSSSSSSSSS VVVVVVVV VVVVVVVV -// SS:::::::::::::::SV::::::V V::::::V -// S:::::SSSSSS::::::SV::::::V V::::::V -// S:::::S SSSSSSSV::::::V V::::::V -// S:::::S V:::::V V:::::V -// S:::::S V:::::V V:::::V -// S::::SSSS V:::::V V:::::V -// SS::::::SSSSS V:::::V V:::::V -// SSS::::::::SS V:::::V V:::::V -// SSSSSS::::S V:::::V V:::::V -// S:::::S V:::::V:::::V -// S:::::S V:::::::::V -// SSSSSSS S:::::S V:::::::V -// S::::::SSSSSS:::::S V:::::V -// S:::::::::::::::SS V:::V -// SSSSSSSSSSSSSSS VVV - -void -wd_ed25519_verify_init_req( wd_wksp_t * wd, - uint8_t send_fails, - uint64_t mcache_depth, - void* mcache_addr) -{ - wd->sv.req_slot = _wd_next_slot(wd, 0); - wd->sv.req_depth = mcache_depth; - uint64_t dma_phys = _wd_get_phys(mcache_addr); - - for (uint32_t slot = 0; slot < WD_N_PCI_SLOTS; slot ++) - { - if (!(wd->pci_slots & (1UL << slot))) - continue; - - // setup threshold levels for pipe-chain - for (uint32_t i = 0; i < 5; i ++) - { - _wd_write_32(&wd->pci[slot], 0x10<<2, i); - _wd_write_32(&wd->pci[slot], 0x13<<2, 0); - _wd_write_32(&wd->pci[slot], 0x14<<2, (200 << 0) | (200 << 12)); - } - // sha_pad thresholds - _wd_write_32(&wd->pci[slot], 0x10<<2, 0); - _wd_write_32(&wd->pci[slot], 0x13<<2, 0); - _wd_write_32(&wd->pci[slot], 0x14<<2, 10 | (10 << 12)); - // send fails back - _wd_write_32(&wd->pci[slot], 0x11<<2, send_fails); - - _wd_set_vdip_64(wd, slot, 0, dma_phys); - _wd_set_vdip_64(wd, slot, 1, ((wd->sv.req_depth-1) << 5) | 0x1f); - } -} - -void -wd_ed25519_verify_init_resp( wd_wksp_t * wd) -{ - (void)wd; -} - -int -wd_ed25519_verify_req( wd_wksp_t * wd, - void const * msg, - ulong sz, - void const * sig, - void const * public_key, - uint64_t m_seq, - uint32_t m_chunk, - uint16_t m_ctrl, - uint16_t m_sz) -{ - uint32_t slot = wd->sv.req_slot; - uint32_t src = 0; - - // Every sixteen requests we check for backpressure - // this check is one PCIe RTT (~1us), we try to avoid - // it as much as possible - if (!(m_seq & 0xf)) - { - int i; - // we cycle through all PCIe slots available to us - // whichever slot is not backpressured we use that next - for (i = 0; i < WD_TRY_LIMIT; i ++, slot = _wd_next_slot(wd, slot)) - { - uint32_t fill = _wd_read_32(&wd->pci[slot], (0x21+src)<<2); - // PCIe buffer level - if ((fill & 0xfff) > 0) - continue; - // number of pending transactions in pipe-chain - if (((fill >> 12) & 0x3ff) > 256) - continue; - // DMA buffer level - if (((fill >> 22) & 0x3ff) > 256) - continue; - break; - } - // timeout - if (i == WD_TRY_LIMIT) - return -1; - } - - uint64_t dma_addr = fd_mcache_line_idx(m_seq, wd->sv.req_depth) << 5; - - wd->stream_buf[0] = WD_PCI_MAGIC; - wd->stream_buf[1] = src | (((uint32_t)sz + 32 + 32) << 16); - wd->stream_buf[2] = (uint32_t)((uint32_t)(m_sz) | (((uint32_t)m_ctrl) << 16)); - wd->stream_buf[3] = (uint32_t)((dma_addr >> 0) & 0xFFFFFFFF); - wd->stream_buf[4] = (uint32_t)((dma_addr >> 32) & 0xFFFFFFFF); - wd->stream_buf[5] = (uint32_t)((m_seq >> 0) & 0xFFFFFFFF); - wd->stream_buf[6] = (uint32_t)((m_seq >> 32) & 0xFFFFFFFF); - wd->stream_buf[7] = m_chunk; - - _wd_stream_256(wd, slot, wd->stream_buf); - - // unfortunately we cannot avoid this copy as we don't - // make assumptions about the incoming buffer's - // alignment, which is required for avx streaming - memcpy(wd->stream_buf, ((uint8_t*)sig)+0, 32); - _wd_stream_256(wd, slot, wd->stream_buf); - - memcpy(wd->stream_buf, ((uint8_t*)sig)+32, 32); - _wd_stream_256(wd, slot, wd->stream_buf); - - memcpy(wd->stream_buf, public_key, 32); - _wd_stream_256(wd, slot, wd->stream_buf); - uint32_t i; - for (i = 0; i < sz; i += 32) - { - memcpy(wd->stream_buf, ((uint8_t*)msg) + i, 32); - _wd_stream_256(wd, slot, wd->stream_buf); - } - - // pad the stream for the sake of 512-bit wide PCIe endpoint in AWS-F1 - if ((i / 32) & 1) - _wd_stream_256(wd, slot, wd->stream_buf); - - // flush write-combining buffers - _wd_stream_flush(wd, slot); - - wd->sv.req_slot = slot; - - return 0; -} diff --git a/src/wiredancer/c/wd_f1.h b/src/wiredancer/c/wd_f1.h deleted file mode 100644 index 9970376be7c..00000000000 --- a/src/wiredancer/c/wd_f1.h +++ /dev/null @@ -1,114 +0,0 @@ -#ifndef HEADER_fd_src_wiredancer_c_wd_f1_h -#define HEADER_fd_src_wiredancer_c_wd_f1_h - -#include -#include -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#include -#pragma GCC diagnostic pop -#include -#include -#include - -#include "../../tango/mcache/fd_mcache.h" - -#define WD_PCI_MAGIC 0xACE0FBAC -#define WD_N_PCI_SLOTS 8 -#define WD_N_PCI_STREAMS 32 - -#define WD_TRY_LIMIT 1000000 - -typedef struct { - - uint64_t a; // address - uint64_t b; // base - uint64_t m; // mask - -} wd_pci_st_t; - -typedef struct { - - pci_bar_handle_t bar0; - pci_bar_handle_t bar4; - void* bar4_addr; - wd_pci_st_t stream[WD_N_PCI_STREAMS]; - -} wd_pci_t; - -typedef struct { - - uint32_t req_slot; - uint64_t req_depth; - -} wd_ed25519_verify_t; - -typedef struct { - - int initialized; - uint64_t pci_slots; - uint32_t *stream_buf; - wd_pci_t pci[32]; - wd_ed25519_verify_t sv; -} wd_wksp_t; - -int wd_init_pci (wd_wksp_t* wd, uint64_t slots); -int wd_free_pci (wd_wksp_t* wd); - -void wd_rst_cntrs (wd_wksp_t* wd, uint32_t slot); -void wd_snp_cntrs (wd_wksp_t* wd, uint32_t slot); -uint32_t wd_rd_cntr (wd_wksp_t* wd, uint32_t slot, uint32_t ci); -uint64_t wd_rd_ts (wd_wksp_t* wd, uint32_t slot); - -uint64_t wd_get_phys (void* p); -void wd_zprintf (const char* format, ...); - -/* wd_ed25519_verify_init_req initializes the internal state - of the request path. */ -void -wd_ed25519_verify_init_req( wd_wksp_t * wd, - uint8_t send_fails, - uint64_t mcache_depth, - void* mcache_addr); - -/* wd_ed25519_verify_init_resp initializes the internal state - of the response path. */ -void -wd_ed25519_verify_init_resp( wd_wksp_t * wd); - - -/* wd_ed25519_verify_req sends a verification request to the underlying - hardware to verify the message according to the ED25519 standard. - The function blocks until the request can be sent to the hardware. - msg is assumed to point to the first byte of a sz byte memory region - which holds the message to verify (sz==0 fine, msg==NULL fine if - sz==0). - sig is assumed to point to the first byte of a 64 byte memory region - which holds the signature of the message. - public_key is assumed to point to first byte of a 32-byte memory - region that holds the public key to use to verify this message. - Does no input argument checking. This function takes a read - interest in msg, sig, public_key and private_key for the duration - the call. - ctrl shows start_of_packet and end_of_packet boundaries. - ctrl[0] == sop - ctrl[1] == eop - Returns zero on success. */ - -int -wd_ed25519_verify_req( wd_wksp_t * wd, - void const * msg, - ulong sz, - void const * sig, - void const * public_key, - uint64_t m_seq, - uint32_t m_chunk, - uint16_t m_ctrl, - uint16_t m_sz); - -#endif /* HEADER_fd_src_wiredancer_c_wd_f1_h */ diff --git a/src/wiredancer/platform/f1/constraints/cl_pnr_user.xdc b/src/wiredancer/platform/f1/constraints/cl_pnr_user.xdc deleted file mode 100644 index 901edff5246..00000000000 --- a/src/wiredancer/platform/f1/constraints/cl_pnr_user.xdc +++ /dev/null @@ -1,101 +0,0 @@ -# This contains the CL specific constraints for Top level PNR - -create_pblock pblock_CL_top -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/CL_TST_DDR_A*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_A_TST_AXI4_REG_SLC_2*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/gen_ddr_tst[0].*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_cores.DDR4_0*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_inst[0].*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_stat[0].*}] -resize_pblock [get_pblocks pblock_CL_top] -add {CLOCKREGION_X0Y10:CLOCKREGION_X5Y14} -set_property PARENT pblock_CL [get_pblocks pblock_CL_top] - -create_pblock pblock_CL_mid -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/CL_TST_DDR_B*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells [list WRAPPER_INST/CL/CL_DMA_PCIS_SLV/CL_TST_DDR_B/CL_TST/sync_rst_n_reg]] -clear_locs -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_A_TST_AXI4_REG_SLC_1*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_B_TST_AXI4_REG_SLC_1*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_D_TST_AXI4_REG_SLC_1*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_B_TST_AXI4_REG_SLC_2*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/gen_ddr_tst[1].*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_cores.DDR4_1*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_inst[1].*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_stat[1].*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/CL_TST_DDR_C}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_C_TST_AXI4_REG_SLC}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_C_TST_AXI4_REG_SLC_1}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/AXI_CROSSBAR}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_PCIM_MSTR/CL_TST_PCI}] -clear_locs -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells [list WRAPPER_INST/CL/CL_PCIM_MSTR/PCI_AXI4_REG_SLC]] -#resize_pblock [get_pblocks pblock_CL_mid] -add {CLOCKREGION_X0Y5:CLOCKREGION_X3Y9} -resize_pblock [get_pblocks pblock_CL_mid] -add {SLICE_X88Y300:SLICE_X107Y599} -resize_pblock [get_pblocks pblock_CL_mid] -add {DSP48E2_X11Y120:DSP48E2_X13Y239} -resize_pblock [get_pblocks pblock_CL_mid] -add {LAGUNA_X12Y240:LAGUNA_X15Y479} -resize_pblock [get_pblocks pblock_CL_mid] -add {RAMB18_X7Y120:RAMB18_X7Y239} -resize_pblock [get_pblocks pblock_CL_mid] -add {RAMB36_X7Y60:RAMB36_X7Y119} -resize_pblock [get_pblocks pblock_CL_mid] -add {URAM288_X2Y80:URAM288_X2Y159} -resize_pblock [get_pblocks pblock_CL_mid] -add {CLOCKREGION_X0Y5:CLOCKREGION_X2Y9} -set_property SNAPPING_MODE ON [get_pblocks pblock_CL_mid] -set_property PARENT pblock_CL [get_pblocks pblock_CL_mid] - -create_pblock pblock_CL_bot -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/CL_TST_DDR_D*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/CL_DMA_PCIS_SLV/DDR_D_TST_AXI4_REG_SLC_2*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/gen_ddr_tst[2].*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_cores.DDR4_2*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_inst[2].*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -quiet -hierarchical -filter {NAME =~ WRAPPER_INST/CL/SH_DDR/ddr_stat[2].*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells [list WRAPPER_INST/CL/CL_DMA_PCIS_SLV/PCI_AXL_REG_SLC WRAPPER_INST/CL/CL_OCL_SLV/AXIL_OCL_REG_SLC WRAPPER_INST/CL/CL_SDA_SLV/AXIL_SDA_REG_SLC]] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -hierarchical -filter { NAME =~ "*CL/CL_OCL_SLV/slv_tst_wdata_reg[*][*]*" && PRIMITIVE_TYPE =~ REGISTER.*.* }] -#Reassign select cells to parent Pblock for better QoR -set pblock_cells [get_cells {WRAPPER_INST/CL/CL_DMA_PCIS_SLV/CL_TST_DDR_D/CL_TST/sync_rst_n_reg WRAPPER_INST/CL/CL_DMA_PCIS_SLV/CL_TST_DDR_D/CL_TST/pre_sync_rst_n_reg}] -add_cells_to_pblock [get_pblocks pblock_CL] $pblock_cells -#resize_pblock [get_pblocks pblock_CL_bot] -add {CLOCKREGION_X0Y0:CLOCKREGION_X3Y4} -resize_pblock [get_pblocks pblock_CL_bot] -add {SLICE_X88Y0:SLICE_X107Y299} -resize_pblock [get_pblocks pblock_CL_bot] -add {DSP48E2_X11Y0:DSP48E2_X13Y119} -resize_pblock [get_pblocks pblock_CL_bot] -add {LAGUNA_X12Y0:LAGUNA_X15Y239} -resize_pblock [get_pblocks pblock_CL_bot] -add {RAMB18_X7Y0:RAMB18_X7Y119} -resize_pblock [get_pblocks pblock_CL_bot] -add {RAMB36_X7Y0:RAMB36_X7Y59} -resize_pblock [get_pblocks pblock_CL_bot] -add {URAM288_X2Y0:URAM288_X2Y79} -resize_pblock [get_pblocks pblock_CL_bot] -add {CLOCKREGION_X0Y0:CLOCKREGION_X2Y4} -set_property SNAPPING_MODE ON [get_pblocks pblock_CL_bot] -set_property PARENT pblock_CL [get_pblocks pblock_CL_bot] - -#set_clock_groups -name TIG_SRAI_1 -asynchronous -group [get_clocks -of_objects [get_pins static_sh/SH_DEBUG_BRIDGE/inst/bsip/inst/USE_SOFTBSCAN.U_TAP_TCKBUFG/O]] -group [get_clocks -of_objects [get_pins WRAPPER_INST/SH/kernel_clks_i/clkwiz_sys_clk/inst/CLK_CORE_DRP_I/clk_inst/mmcme3_adv_inst/CLKOUT0]] -#set_clock_groups -name TIG_SRAI_2 -asynchronous -group [get_clocks -of_objects [get_pins static_sh/SH_DEBUG_BRIDGE/inst/bsip/inst/USE_SOFTBSCAN.U_TAP_TCKBUFG/O]] -group [get_clocks drck] -#set_clock_groups -name TIG_SRAI_3 -asynchronous -group [get_clocks -of_objects [get_pins static_sh/SH_DEBUG_BRIDGE/inst/bsip/inst/USE_SOFTBSCAN.U_TAP_TCKBUFG/O]] -group [get_clocks -of_objects [get_pins static_sh/pcie_inst/inst/gt_top_i/diablo_gt.diablo_gt_phy_wrapper/phy_clk_i/bufg_gt_userclk/O]] - - - - - - - - - -#create_pblock pblock_CL_mid_left -#resize_pblock [get_pblocks pblock_CL_mid_left] -add {CLOCKREGION_X0Y5:CLOCKREGION_X1Y9} -#set_property PARENT pblock_CL_mid [get_pblocks pblock_CL_mid_left] - - -#set_property EXCLUDE_PLACEMENT 1 [get_pblocks {pblock_CL_top}] -#add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/ed25519_sigverify_1_inst/SLR*}] - -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sha512_pre_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/pad_th_inst*}] - -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sha_f_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sha512_modq_meta_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_bot] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sha_th_inst*}] - -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sv0_f_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/ed25519_sigverify_0_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_mid] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sv0_th_inst*}] - -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sv1_f_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/ed25519_sigverify_1_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sv1_th_inst*}] - -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sv2_i_pipe_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/sv2_f_inst*}] -add_cells_to_pblock [get_pblocks pblock_CL_top] [get_cells -hierarchical -filter {NAME =~ WRAPPER_INST/CL/top_inst/ed25519_sigverify_2_inst*}] diff --git a/src/wiredancer/platform/f1/design/cl_dram_dma.sv b/src/wiredancer/platform/f1/design/cl_dram_dma.sv deleted file mode 100644 index ddc1e5610c0..00000000000 --- a/src/wiredancer/platform/f1/design/cl_dram_dma.sv +++ /dev/null @@ -1,545 +0,0 @@ -// Amazon FPGA Hardware Development Kit -// -// Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Amazon Software License (the "License"). You may not use -// this file except in compliance with the License. A copy of the License is -// located at -// -// http://aws.amazon.com/asl/ -// -// or in the "license" file accompanying this file. This file is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -// implied. See the License for the specific language governing permissions and -// limitations under the License. - -// DDR address space -// The addressing uses ROW/COLUMN/BANK mapping of AXI address to DRAM Row/Col/BankGroup - -module cl_dram_dma #(parameter NUM_DDR=4) - -( - `include "cl_ports.vh" - -); - -`include "cl_common_defines.vh" // CL Defines for all examples -`include "cl_id_defines.vh" // Defines for ID0 and ID1 (PCI ID's) -`include "cl_dram_dma_defines.vh" - -// TIE OFF ALL UNUSED INTERFACES -// Including all the unused interface to tie off -// This list is put in the top of the fie to remind -// developers to remove the specific interfaces -// that the CL will use - -`include "unused_flr_template.inc" -`include "unused_ddr_a_b_d_template.inc" -`include "unused_ddr_c_template.inc" -// `include "unused_pcim_template.inc" -// `include "unused_dma_pcis_template.inc" -`include "unused_cl_sda_template.inc" -`include "unused_sh_bar1_template.inc" -`include "unused_apppf_irq_template.inc" - -// Define the addition pipeline stag -// needed to close timing for the various -// place where ATG (Automatic Test Generator) -// is defined - - localparam NUM_CFG_STGS_CL_DDR_ATG = 8; - localparam NUM_CFG_STGS_SH_DDR_ATG = 4; - localparam NUM_CFG_STGS_PCIE_ATG = 4; - -// To reduce RTL simulation time, only 8KiB of -// each external DRAM is scrubbed in simulations - -`ifdef SIM - localparam DDR_SCRB_MAX_ADDR = 64'h1FFF; -`else - localparam DDR_SCRB_MAX_ADDR = 64'h3FFFFFFFF; //16GB -`endif - localparam DDR_SCRB_BURST_LEN_MINUS1 = 15; - -`ifdef NO_CL_TST_SCRUBBER - localparam NO_SCRB_INST = 1; -`else - localparam NO_SCRB_INST = 0; -`endif - -logic clk; -(* dont_touch = "true" *) logic pipe_rst_n; -logic pre_sync_rst_n; -(* dont_touch = "true" *) logic sync_rst_n; -logic sh_cl_flr_assert_q; - -logic [3:0] all_ddr_scrb_done; -logic [3:0] all_ddr_is_ready; -logic [2:0] lcl_sh_cl_ddr_is_ready; - -logic dbg_scrb_en; -logic [2:0] dbg_scrb_mem_sel; - -//---------------------------- -// End Internal signals -//---------------------------- - -assign cl_sh_status0 = 32'h0; -assign cl_sh_status1 = 32'h0; - -assign cl_sh_id0[31:0] = `CL_SH_ID0; -assign cl_sh_id1[31:0] = `CL_SH_ID1; - -// Unused 'full' signals -assign cl_sh_dma_rd_full = 1'b0; -assign cl_sh_dma_wr_full = 1'b0; - -assign clk = clk_main_a0; - -//reset synchronizer -lib_pipe #(.WIDTH(1), .STAGES(4)) PIPE_RST_N (.clk(clk), .rst_n(1'b1), .in_bus(rst_main_n), .out_bus(pipe_rst_n)); - -always_ff @(negedge pipe_rst_n or posedge clk) - if (!pipe_rst_n) - begin - pre_sync_rst_n <= 0; - sync_rst_n <= 0; - end - else - begin - pre_sync_rst_n <= 1; - sync_rst_n <= pre_sync_rst_n; - end - - - - - - - - -logic [1-1:0] rst; - -always_ff@(posedge clk_main_a0) - rst <= ~sync_rst_n; - -logic [1-1:0] clk_f; -logic [1-1:0] rst_f; - -assign clk_f = clk_extra_c1; - -areset_sync rst_f_areset_sync_inst ( - .areset (rst), - .dclk (clk_f), - .dreset (rst_f) -); - - - - - - - - - - -// AAA VVVVVVVV VVVVVVVVMMMMMMMM MMMMMMMMMMMMMMMM MMMMMMMM -// A:::A V::::::V V::::::VM:::::::M M:::::::MM:::::::M M:::::::M -// A:::::A V::::::V V::::::VM::::::::M M::::::::MM::::::::M M::::::::M -// A:::::::A V::::::V V::::::VM:::::::::M M:::::::::MM:::::::::M M:::::::::M -// A:::::::::A V:::::V V:::::V M::::::::::M M::::::::::MM::::::::::M M::::::::::M -// A:::::A:::::A V:::::V V:::::V M:::::::::::M M:::::::::::MM:::::::::::M M:::::::::::M -// A:::::A A:::::A V:::::V V:::::V M:::::::M::::M M::::M:::::::MM:::::::M::::M M::::M:::::::M -// A:::::A A:::::A V:::::V V:::::V M::::::M M::::M M::::M M::::::MM::::::M M::::M M::::M M::::::M -// A:::::A A:::::A V:::::V V:::::V M::::::M M::::M::::M M::::::MM::::::M M::::M::::M M::::::M -// A:::::AAAAAAAAA:::::A V:::::V V:::::V M::::::M M:::::::M M::::::MM::::::M M:::::::M M::::::M -// A:::::::::::::::::::::A V:::::V:::::V M::::::M M:::::M M::::::MM::::::M M:::::M M::::::M -// A:::::AAAAAAAAAAAAA:::::A V:::::::::V M::::::M MMMMM M::::::MM::::::M MMMMM M::::::M -// A:::::A A:::::A V:::::::V M::::::M M::::::MM::::::M M::::::M -// A:::::A A:::::A V:::::V M::::::M M::::::MM::::::M M::::::M -// A:::::A A:::::A V:::V M::::::M M::::::MM::::::M M::::::M -// AAAAAAA AAAAAAA VVV MMMMMMMM MMMMMMMMMMMMMMMM MMMMMMMM - -localparam DMA_N = 1; -localparam NO_AVMM_MASTERS = 1; -localparam NO_BASE_ENGINES = 1; -localparam NO_DBG_TAPS = 1; -localparam DBG_WIDTH = 1024*2; -localparam DDR_SIM = 0; - -logic [NO_DBG_TAPS-1:0][DBG_WIDTH-1:0] dbg_wires; - -logic [NO_AVMM_MASTERS-1:0][1-1:0] avmm_fh_read = 0; -logic [NO_AVMM_MASTERS-1:0][1-1:0] avmm_fh_write = 0; -logic [NO_AVMM_MASTERS-1:0][32-1:0] avmm_fh_address; -logic [NO_AVMM_MASTERS-1:0][32-1:0] avmm_fh_writedata; -logic [NO_AVMM_MASTERS-1:0][32-1:0] avmm_fh_readdata; -logic [NO_AVMM_MASTERS-1:0][1-1:0] avmm_fh_readdatavalid; -logic [NO_AVMM_MASTERS-1:0][1-1:0] avmm_fh_waitrequest; - -logic [3-1:0] st_ocl; -logic [32-1:0] ocl_addr; -logic [3-1:0] ocl_mi; - -assign ocl_sh_arready = (st_ocl == 0); -assign ocl_sh_awready = (st_ocl == 0) & ~sh_ocl_arvalid; -assign ocl_sh_wready = (st_ocl == 3); -assign ocl_sh_rresp = '0; -assign ocl_sh_bresp = '0; - -always_ff@(posedge clk_main_a0) begin - integer i; - - case (st_ocl) - 0: begin - if (sh_ocl_arvalid) begin - ocl_mi <= sh_ocl_araddr[10+:3]; - for (i = 0; i < NO_AVMM_MASTERS; i ++) begin - avmm_fh_address [i] <= sh_ocl_araddr[0+:10]; - if (sh_ocl_araddr[10+:3] == i) begin - avmm_fh_read [i] <= sh_ocl_arvalid; - end - end - st_ocl <= 1; - end else - if (sh_ocl_awvalid) begin - ocl_mi <= sh_ocl_awaddr[10+:3]; - for (i = 0; i < NO_AVMM_MASTERS; i ++) begin - avmm_fh_address [i] <= sh_ocl_awaddr[0+:10]; - end - st_ocl <= 3; - end - end - 1: begin - if (~avmm_fh_waitrequest[ocl_mi]) - avmm_fh_read [ocl_mi] <= 0; - - if (avmm_fh_readdatavalid[ocl_mi]) begin - ocl_sh_rvalid <= 1; - ocl_sh_rdata <= avmm_fh_readdata[ocl_mi]; - - st_ocl <= 2; - end - end - 2: begin - if (sh_ocl_rready) begin - ocl_sh_rvalid <= 0; - st_ocl <= 0; - end - end - - 3: begin - avmm_fh_write [ocl_mi] <= sh_ocl_wvalid; - avmm_fh_writedata [ocl_mi] <= sh_ocl_wdata; - - if (sh_ocl_wvalid) begin - st_ocl <= 4; - end - end - - 4: begin - if (~avmm_fh_waitrequest[ocl_mi]) begin - avmm_fh_write [ocl_mi] <= 0; - ocl_sh_bvalid <= 1; - st_ocl <= 5; - end - end - - 5: begin - if (sh_ocl_bready) begin - ocl_sh_bvalid <= 0; - - st_ocl <= 0; - end - end - - endcase - - if (rst) begin - st_ocl <= 0; - ocl_sh_rvalid <= '0; - ocl_sh_bvalid <= '0; - avmm_fh_read <= '0; - avmm_fh_write <= '0; - end -end - -logic [4-1:0] vdip_func; -logic [4-1:0] vdip_sel; -logic [8-1:0] vdip_byte; -logic [16-1:0][8-1:0] vdip_bytes; - -assign {vdip_byte, vdip_sel, vdip_func} = sh_cl_status_vdip; - -always_ff@(posedge clk_main_a0) begin - cl_sh_status_vled <= {vdip_bytes[vdip_sel], vdip_sel, vdip_func}; - if (vdip_func == 4'hf) begin - vdip_bytes[vdip_sel] <= vdip_byte; - end -end - - -// DDDDDDDDDDDDD DDDDDDDDDDDDD RRRRRRRRRRRRRRRRR -// D::::::::::::DDD D::::::::::::DDD R::::::::::::::::R -// D:::::::::::::::DD D:::::::::::::::DD R::::::RRRRRR:::::R -// DDD:::::DDDDD:::::D DDD:::::DDDDD:::::D RR:::::R R:::::R -// D:::::D D:::::D D:::::D D:::::D R::::R R:::::R -// D:::::D D:::::D D:::::D D:::::D R::::R R:::::R -// D:::::D D:::::D D:::::D D:::::D R::::RRRRRR:::::R -// D:::::D D:::::D D:::::D D:::::D R:::::::::::::RR -// D:::::D D:::::D D:::::D D:::::D R::::RRRRRR:::::R -// D:::::D D:::::D D:::::D D:::::D R::::R R:::::R -// D:::::D D:::::D D:::::D D:::::D R::::R R:::::R -// D:::::D D:::::D D:::::D D:::::D R::::R R:::::R -// DDD:::::DDDDD:::::D DDD:::::DDDDD:::::D RR:::::R R:::::R -// D:::::::::::::::DD D:::::::::::::::DD R::::::R R:::::R -// D::::::::::::DDD D::::::::::::DDD R::::::R R:::::R -// DDDDDDDDDDDDD DDDDDDDDDDDDD RRRRRRRR RRRRRRR - - - - - - - - - -// PPPPPPPPPPPPPPPPP CCCCCCCCCCCCCIIIIIIIIIIEEEEEEEEEEEEEEEEEEEEEE -// P::::::::::::::::P CCC::::::::::::CI::::::::IE::::::::::::::::::::E -// P::::::PPPPPP:::::P CC:::::::::::::::CI::::::::IE::::::::::::::::::::E -// PP:::::P P:::::P C:::::CCCCCCCC::::CII::::::IIEE::::::EEEEEEEEE::::E -// P::::P P:::::P C:::::C CCCCCC I::::I E:::::E EEEEEE -// P::::P P:::::PC:::::C I::::I E:::::E -// P::::PPPPPP:::::P C:::::C I::::I E::::::EEEEEEEEEE -// P:::::::::::::PP C:::::C I::::I E:::::::::::::::E -// P::::PPPPPPPPP C:::::C I::::I E:::::::::::::::E -// P::::P C:::::C I::::I E::::::EEEEEEEEEE -// P::::P C:::::C I::::I E:::::E -// P::::P C:::::C CCCCCC I::::I E:::::E EEEEEE -// PP::::::PP C:::::CCCCCCCC::::CII::::::IIEE::::::EEEEEEEE:::::E -// P::::::::P CC:::::::::::::::CI::::::::IE::::::::::::::::::::E -// P::::::::P CCC::::::::::::CI::::::::IE::::::::::::::::::::E -// PPPPPPPPPP CCCCCCCCCCCCCIIIIIIIIIIEEEEEEEEEEEEEEEEEEEEEE - -logic [1-1:0] st_addr_v; -logic [3-1:0] st_data_v; -logic [2-1:0] st_v; -logic [1-1:0] st_p; -logic [64-1:0] st_addr; -logic [2-1:0][256-1:0] st_data; - -assign cl_sh_dma_pcis_awready = 1'b1; -assign cl_sh_dma_pcis_wready = 1'b1; -assign cl_sh_dma_pcis_bresp = '0; - -always_ff@(posedge clk) cl_sh_dma_pcis_bvalid <= sh_cl_dma_pcis_wvalid & sh_cl_dma_pcis_wlast; -always_ff@(posedge clk) cl_sh_dma_pcis_bid <= sh_cl_dma_pcis_awid; - -showahead_fifo #( - .WIDTH ($bits(sh_cl_dma_pcis_awaddr)), - .DEPTH (32) -) st_in_addr_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (sh_cl_dma_pcis_awvalid & cl_sh_dma_pcis_awready), - .wr_full (), - .wr_data ({sh_cl_dma_pcis_awaddr[64-1:6], 6'h0}), - - .rd_clk (clk), - .rd_req (st_p), - .rd_empty (), - .rd_not_empty (st_addr_v), - .rd_count (), - .rd_data ({st_addr}) -); - -showahead_fifo #( - .WIDTH ($bits({sh_cl_dma_pcis_wdata, sh_cl_dma_pcis_wstrb[32], sh_cl_dma_pcis_wstrb[0]})), - .DEPTH (32) -) st_in_data_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (sh_cl_dma_pcis_wvalid & cl_sh_dma_pcis_wready), - .wr_full (), - .wr_data ({sh_cl_dma_pcis_wdata, sh_cl_dma_pcis_wstrb[32], sh_cl_dma_pcis_wstrb[0]}), - - .rd_clk (clk), - .rd_req (st_p), - .rd_empty (), - .rd_not_empty (st_data_v[2]), - .rd_count (), - .rd_data ({st_data, st_data_v[0+:2]}) -); - -assign st_p = st_addr_v & st_data_v[2]; -assign st_v[0] = st_addr_v & st_data_v[2] & st_data_v[0]; -assign st_v[1] = st_addr_v & st_data_v[2] & st_data_v[1]; - - - - - - -// DDDDDDDDDDDDD MMMMMMMM MMMMMMMM AAA -// D::::::::::::DDD M:::::::M M:::::::M A:::A -// D:::::::::::::::DD M::::::::M M::::::::M A:::::A -// DDD:::::DDDDD:::::D M:::::::::M M:::::::::M A:::::::A -// D:::::D D:::::D M::::::::::M M::::::::::M A:::::::::A -// D:::::D D:::::DM:::::::::::M M:::::::::::M A:::::A:::::A -// D:::::D D:::::DM:::::::M::::M M::::M:::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M::::M M::::M M::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M::::M::::M M::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M:::::::M M::::::M A:::::AAAAAAAAA:::::A -// D:::::D D:::::DM::::::M M:::::M M::::::M A:::::::::::::::::::::A -// D:::::D D:::::D M::::::M MMMMM M::::::M A:::::AAAAAAAAAAAAA:::::A -// DDD:::::DDDDD:::::D M::::::M M::::::M A:::::A A:::::A -// D:::::::::::::::DD M::::::M M::::::M A:::::A A:::::A -// D::::::::::::DDD M::::::M M::::::M A:::::A A:::::A -// DDDDDDDDDDDDD MMMMMMMM MMMMMMMMAAAAAAA AAAAAAA - -logic [1-1:0] dma_r; -logic [1-1:0] dma_push; -logic [64-1:0] dma_push_a; -logic [64-1:0] dma_push_b; -logic [1-1:0] dma_full_a; -logic [1-1:0] dma_full_d; -logic [256-1:0] dma_push_d; - -logic [256-1:0] cl_sh_pcim_wdata_h; - -assign dma_r = (~dma_full_a) & (~dma_full_d); - -assign cl_sh_pcim_awid = '0; -assign cl_sh_pcim_awlen = '0; -assign cl_sh_pcim_awsize = 'b110; -assign cl_sh_pcim_wlast = '1; -assign cl_sh_pcim_wdata = {2{cl_sh_pcim_wdata_h}}; -assign cl_sh_pcim_bready = '1; - -showahead_fifo #( - .WIDTH ($bits({dma_push_a})), - .FULL_THRESH (512-64), - .DEPTH (512) -) dma_addr_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (dma_push & dma_r), - .wr_full (dma_full_a), - .wr_data ({dma_push_a}), - - .rd_clk (clk), - .rd_req (cl_sh_pcim_awvalid & sh_cl_pcim_awready), - .rd_empty (), - .rd_not_empty (cl_sh_pcim_awvalid), - .rd_count (), - .rd_data ({cl_sh_pcim_awaddr}) -); - -showahead_fifo #( - .WIDTH ($bits({dma_push_b, dma_push_d})), - .FULL_THRESH (512-64), - .DEPTH (512) -) dma_data_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (dma_push & dma_r), - .wr_full (dma_full_d), - .wr_data ({dma_push_b, dma_push_d}), - - .rd_clk (clk), - .rd_req (cl_sh_pcim_wvalid & sh_cl_pcim_wready), - .rd_empty (), - .rd_not_empty (cl_sh_pcim_wvalid), - .rd_count (), - .rd_data ({cl_sh_pcim_wstrb, cl_sh_pcim_wdata_h}) -); - - - - - - - - - - - - - - - - -// AAA PPPPPPPPPPPPPPPPP PPPPPPPPPPPPPPPPP -// A:::A P::::::::::::::::P P::::::::::::::::P -// A:::::A P::::::PPPPPP:::::P P::::::PPPPPP:::::P -// A:::::::A PP:::::P P:::::PPP:::::P P:::::P -// A:::::::::A P::::P P:::::P P::::P P:::::P -// A:::::A:::::A P::::P P:::::P P::::P P:::::P -// A:::::A A:::::A P::::PPPPPP:::::P P::::PPPPPP:::::P -// A:::::A A:::::A P:::::::::::::PP P:::::::::::::PP -// A:::::A A:::::A P::::PPPPPPPPP P::::PPPPPPPPP -// A:::::AAAAAAAAA:::::A P::::P P::::P -// A:::::::::::::::::::::A P::::P P::::P -// A:::::AAAAAAAAAAAAA:::::A P::::P P::::P -// A:::::A A:::::A PP::::::PP PP::::::PP -// A:::::A A:::::A P::::::::P P::::::::P -// A:::::A A:::::A P::::::::P P::::::::P -// AAAAAAA AAAAAAAPPPPPPPPPP PPPPPPPPPP - -`ifndef TOP_NAME -`define TOP_NAME top_f1 -`endif - -`TOP_NAME #( - .DBG_WIDTH(DBG_WIDTH), - .DMA_N (DMA_N) -) top_inst ( - - .avmm_read (avmm_fh_read[0]), - .avmm_write (avmm_fh_write[0]), - .avmm_address (avmm_fh_address[0]), - .avmm_writedata (avmm_fh_writedata[0]), - .avmm_readdata (avmm_fh_readdata[0]), - .avmm_readdatavalid (avmm_fh_readdatavalid[0]), - .avmm_waitrequest (avmm_fh_waitrequest[0]), - - .priv_bytes (vdip_bytes), - - .pcie_v (st_v), - .pcie_a (st_addr), - .pcie_d (st_data), - - .dma_r (dma_r), - .dma_v (dma_push), - .dma_a (dma_push_a), - .dma_b (dma_push_b), - .dma_f (dma_full_a | dma_full_d), - .dma_d (dma_push_d), - - // .ddr_rd_en (ddr_rd_en), - // .ddr_rd_pop (ddr_rd_pop), - // .ddr_rd_addr (ddr_rd_addr), - // .ddr_rd_sz (ddr_rd_sz), - // .ddr_rd_v (ddr_rd_v), - // .ddr_rd_data (ddr_rd_data), - - // .ddr_wr_en (ddr_wr_en), - // .ddr_wr_pop (ddr_wr_pop), - // .ddr_wr_res (ddr_wr_res), - // .ddr_wr_addr (ddr_wr_addr), - // .ddr_wr_data (ddr_wr_data), - - .dbg_wire (dbg_wires[0]), - - .clk_f(clk_f), - .rst_f(rst_f), - - .clk(clk), - .rst(rst) -); - -endmodule diff --git a/src/wiredancer/platform/f1/scripts/synth_cl_dram_dma.tcl b/src/wiredancer/platform/f1/scripts/synth_cl_dram_dma.tcl deleted file mode 100644 index 85f71101c4b..00000000000 --- a/src/wiredancer/platform/f1/scripts/synth_cl_dram_dma.tcl +++ /dev/null @@ -1,179 +0,0 @@ -# Amazon FPGA Hardware Development Kit -# -# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Amazon Software License (the "License"). You may not use -# this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/asl/ -# -# or in the "license" file accompanying this file. This file is distributed on -# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or -# implied. See the License for the specific language governing permissions and -# limitations under the License. - -#Param needed to avoid clock name collisions -set_param sta.enableAutoGenClkNamePersistence 0 -set CL_MODULE $CL_MODULE -set VDEFINES $VDEFINES - -create_project -in_memory -part [DEVICE_TYPE] -force - -######################################## -## Generate clocks based on Recipe -######################################## - -puts "AWS FPGA: ([clock format [clock seconds] -format %T]) Calling aws_gen_clk_constraints.tcl to generate clock constraints from developer's specified recipe."; - -source $HDK_SHELL_DIR/build/scripts/aws_gen_clk_constraints.tcl - -############################# -## Read design files -############################# - -#Convenience to set the root of the RTL directory -set ENC_SRC_DIR $CL_DIR/build/src_post_encryption -set TARGET_DIR $CL_DIR/build/src_post_encryption -set UNUSED_TEMPLATES_DIR $HDK_SHELL_DESIGN_DIR/interfaces - -puts "AWS FPGA: ([clock format [clock seconds] -format %T]) Reading developer's Custom Logic files post encryption."; - -#---- User would replace this section ----- - -# Reading the .sv and .v files, as proper designs would not require -# reading .v, .vh, nor .inc files - -# read_verilog -sv [glob $ENC_SRC_DIR/*.?v] - -#---- End of section replaced by User ---- - -puts "AWS FPGA: Reading AWS Shell design"; - -#Read AWS Design files -read_verilog -sv [ list \ - $HDK_SHELL_DESIGN_DIR/lib/lib_pipe.sv \ - $HDK_SHELL_DESIGN_DIR/lib/bram_2rw.sv \ - $HDK_SHELL_DESIGN_DIR/lib/flop_fifo.sv \ - $HDK_SHELL_DESIGN_DIR/sh_ddr/synth/sync.v \ - $HDK_SHELL_DESIGN_DIR/sh_ddr/synth/flop_ccf.sv \ - $HDK_SHELL_DESIGN_DIR/sh_ddr/synth/ccf_ctl.v \ - $HDK_SHELL_DESIGN_DIR/sh_ddr/synth/mgt_acc_axl.sv \ - $HDK_SHELL_DESIGN_DIR/sh_ddr/synth/mgt_gen_axl.sv \ - $HDK_SHELL_DESIGN_DIR/sh_ddr/synth/sh_ddr.sv \ - $HDK_SHELL_DESIGN_DIR/interfaces/cl_ports.vh -] - -file copy -force $UNUSED_TEMPLATES_DIR/unused_sh_bar1_template.inc $CL_DIR/design -file copy -force $UNUSED_TEMPLATES_DIR/unused_flr_template.inc $CL_DIR/design -file copy -force $UNUSED_TEMPLATES_DIR/unused_cl_sda_template.inc $CL_DIR/design -file copy -force $UNUSED_TEMPLATES_DIR/unused_apppf_irq_template.inc $CL_DIR/design - -read_verilog -sv $CL_DIR/design/cl_dram_dma.sv -# read_verilog -sv $CL_DIR/design/cl_hello_world_defines.vh -# read_verilog -sv $CL_DIR/design/cl_id_defines.vh - -read_verilog -sv $CL_DIR/design/areset_sync.sv -read_verilog -sv $CL_DIR/design/simple_dual_port_ram.sv -read_verilog -sv $CL_DIR/design/showahead_fifo.sv -read_verilog -sv $CL_DIR/design/dual_clock_showahead_fifo.sv -read_verilog -sv $CL_DIR/design/wd_pkg.sv -read_verilog -sv $CL_DIR/design/pcie_inorder.sv -read_verilog -sv $CL_DIR/design/pcie_tr_ext.sv -read_verilog -sv $CL_DIR/design/tid_inorder.sv -read_verilog -sv $CL_DIR/design/key_store.sv -read_verilog -sv $CL_DIR/design/dma_result.sv -read_verilog -sv $CL_DIR/design/sha512_pre.sv -read_verilog -sv $CL_DIR/design/sha512_sch.sv -read_verilog -sv $CL_DIR/design/sha512_msgseq.sv -read_verilog -sv $CL_DIR/design/sha512_round.sv -read_verilog -sv $CL_DIR/design/sha512_block.sv -read_verilog -sv $CL_DIR/design/sha512_modq.sv -read_verilog -sv $CL_DIR/design/sha512_modq_meta.sv -read_verilog -sv $CL_DIR/design/mul_wide.sv -read_verilog -sv $CL_DIR/design/schl_cpu_instr_rom.sv -read_verilog -sv $CL_DIR/design/schl_cpu.sv -read_verilog -sv $CL_DIR/design/ed25519_add_modp.sv -read_verilog -sv $CL_DIR/design/ed25519_sub_modp.sv -read_verilog -sv $CL_DIR/design/ed25519_mul_modp.sv -read_verilog -sv $CL_DIR/design/ed25519_point_add.sv -read_verilog -sv $CL_DIR/design/ed25519_point_dbl.sv -read_verilog -sv $CL_DIR/design/ed25519_sigverify_dsdp_mul.sv -read_verilog -sv $CL_DIR/design/ed25519_sigverify_ecc.sv -read_verilog -sv $CL_DIR/design/ed25519_sigverify_0.sv -read_verilog -sv $CL_DIR/design/ed25519_sigverify_1.sv -read_verilog -sv $CL_DIR/design/ed25519_sigverify_2.sv -read_verilog -sv $CL_DIR/design/top_f1.sv - - -puts "AWS FPGA: Reading IP blocks"; - -#Read DDR IP -read_ip [ list \ - $HDK_SHELL_DESIGN_DIR/ip/ddr4_core/ddr4_core.xci -] - -#Read IP for axi register slices -read_ip [ list \ - $HDK_SHELL_DESIGN_DIR/ip/src_register_slice/src_register_slice.xci \ - $HDK_SHELL_DESIGN_DIR/ip/dest_register_slice/dest_register_slice.xci \ - $HDK_SHELL_DESIGN_DIR/ip/axi_clock_converter_0/axi_clock_converter_0.xci \ - $HDK_SHELL_DESIGN_DIR/ip/axi_register_slice/axi_register_slice.xci \ - $HDK_SHELL_DESIGN_DIR/ip/axi_register_slice_light/axi_register_slice_light.xci -] - -#Read IP for virtual jtag / ILA/VIO -read_ip [ list \ - $HDK_SHELL_DESIGN_DIR/ip/cl_debug_bridge/cl_debug_bridge.xci \ - $HDK_SHELL_DESIGN_DIR/ip/ila_1/ila_1.xci \ - $HDK_SHELL_DESIGN_DIR/ip/ila_vio_counter/ila_vio_counter.xci \ - $HDK_SHELL_DESIGN_DIR/ip/vio_0/vio_0.xci -] - -# Additional IP's that might be needed if using the DDR -read_bd [ list \ - $HDK_SHELL_DESIGN_DIR/ip/cl_axi_interconnect/cl_axi_interconnect.bd -] - -puts "AWS FPGA: Reading AWS constraints"; - -#Read all the constraints -# -# cl_clocks_aws.xdc - AWS auto-generated clock constraint. ***DO NOT MODIFY*** -# cl_ddr.xdc - AWS provided DDR pin constraints. ***DO NOT MODIFY*** -# cl_synth_user.xdc - Developer synthesis constraints. -read_xdc [ list \ - $CL_DIR/build/constraints/cl_clocks_aws.xdc \ - $HDK_SHELL_DIR/build/constraints/cl_ddr.xdc \ - $HDK_SHELL_DIR/build/constraints/cl_synth_aws.xdc \ - $CL_DIR/build/constraints/cl_synth_user.xdc -] - -#Do not propagate local clock constraints for clocks generated in the SH -set_property USED_IN {synthesis implementation OUT_OF_CONTEXT} [get_files cl_clocks_aws.xdc] -set_property PROCESSING_ORDER EARLY [get_files cl_clocks_aws.xdc] - -######################## -# CL Synthesis -######################## -puts "AWS FPGA: ([clock format [clock seconds] -format %T]) Start design synthesis."; - -update_compile_order -fileset sources_1 -puts "\nRunning synth_design for $CL_MODULE $CL_DIR/build/scripts \[[clock format [clock seconds] -format {%a %b %d %H:%M:%S %Y}]\]" -# eval [concat synth_design -include_dirs "{$CL_DIR/design}" -top $CL_MODULE -verilog_define XSDB_SLV_DIS $VDEFINES -part [DEVICE_TYPE] -mode out_of_context $synth_options -directive $synth_directive -strategy strategy] -eval [concat synth_design -verilog_define PATH_TO_INSTR_ROM_MIF=\"$CL_DIR/design/schl_cpu_instr_rom.mif\" -include_dirs "{$CL_DIR/design}" -top $CL_MODULE -verilog_define XSDB_SLV_DIS $VDEFINES -part [DEVICE_TYPE] -mode out_of_context $synth_options -directive $synth_directive] - -set failval [catch {exec grep "FAIL" failfast.csv}] -if { $failval==0 } { - puts "AWS FPGA: FATAL ERROR--Resource utilization error; check failfast.csv for details" - exit 1 -} - -#read_xdc $CL_DIR/build/constraints/cl_pnr_user.xdc - -puts "AWS FPGA: ([clock format [clock seconds] -format %T]) writing post synth checkpoint."; -write_checkpoint -force $CL_DIR/build/checkpoints/${timestamp}.CL.post_synth.dcp - -close_project -#Set param back to default value -set_param sta.enableAutoGenClkNamePersistence 1 diff --git a/src/wiredancer/py/ed25519_lib.py b/src/wiredancer/py/ed25519_lib.py deleted file mode 100755 index 43bda794f8e..00000000000 --- a/src/wiredancer/py/ed25519_lib.py +++ /dev/null @@ -1,1001 +0,0 @@ - -import random -import numbers - -import ref_ed25519 - -class Expr: - ''' - Expr class is used to generate the sequence of machine instructions - needed to compute the ed25519 verify procedure using the processor - implemented inside hw-sigverify. - - Expr class tracks the sequence of python expressions executed as an - arbitrary piece of python code is evaluated. Optimizations include - constant propagation, deadcode elimination, and minimum register - allocation. - - If statements are not supported as the entire datapath of the code - is meant to be executed regardless of the input data. Instead, - ternary operator is introduced which allows for Expr class to track - both branches of the datapath. In hardware terminology, all paths are - executed and the correct data is multiplexed using ternary operators. - - To start an expression tracker: - - Expr.reset() - - Simply invoke the target python code with all input variables - initialized as Expr(var=True): - - ab = add_a_and_b(Expr(var=True), Expr(var=true)) - - Provide all the outputs of the executed code to Expr.outputs(). - This step is necessary as Expr has no other means of tracking what - constitutes the intended output list. - - Expr.outputs(ab) - - ''' - trace_q = None - op_hist = None - op_args = None - func_const_prop = True - def reset(): - Expr.func_const_prop = False - Expr.maddr = 0 - Expr.caddr = 0 - Expr.trace_q = list() - Expr.op_hist = dict() - Expr.op_args = dict() - Expr.outs = list() - - def outputs(outs): - if isinstance(outs, Expr): - outs = [outs] - Expr.outputs = outs - for t in Expr.outputs: - t.out = True - Expr.outs.append(t) - Expr.opt_trace() - - def __init__( - self, - left = None, - op = None, - right = None, - cond = None, - func = None, - args = None, - var = False, - out = False, - ): - self.v = None - self.t = None - self.c = False - self.func = None - self.op = None - self.args = None - self.addr = None - self.out = False - - if func != None: - self.func = func - self.args = args - self.c = Expr.func_const_prop - e_args = list() - for a in args: - e_a = a if isinstance(a, Expr) else Expr(a) - e_args.append(e_a) - if not e_a.c: - self.c = False - self.args = e_args - if self.c: - self.c = False - left = self.eval() - self.c = True - self.func = None - self.args = [left] - elif op == None: - if var == True: - e_left = left if isinstance(left, Expr) else Expr(left) - self.func = 'input' - self.args = [] - else: - if isinstance(left, numbers.Integral): - # if type(left) in [int, long]: - self.c = True - self.args = [left] - else: - print (type(left)) - WTF - elif op == 'if': - e_left = left if isinstance(left, Expr) else Expr(left) - e_right = right if isinstance(right, Expr) else Expr(right) - e_cond = cond if isinstance(cond, Expr) else Expr(cond) - if e_cond.c: - if e_cond.eval() != 0: - self.copy(e_left) - else: - self.copy(e_right) - else: - self.func = 'ternary' - self.args = [e_left, e_right, e_cond] - else: - e_left = left if isinstance(left, Expr) else Expr(left) - e_right = right if isinstance(right, Expr) else Expr(right) - self.func = 'op' - self.op = op - self.args = (e_left, e_right) - # constant propagation - if e_left.c and e_right.c: - left = self.eval() - self.func = None - self.op = None - self.c = True - self.args = [left] - - if not self.c: - Expr.trace_q.append(self) - self.eval() - - def copy(self, r): - self.v = r.v - self.t = r.t - self.c = r.c - self.op = r.op - self.func = r.func - self.args = r.args - self.addr = r.addr - - def eval(self): - if self.v == None: - if False: - pass - elif self.c == True: - self.v = self.args[0] - elif self.func == 'input': - self.v = 0 - elif self.func == 'if': - left = self.args[0].eval() - right = self.args[1].eval() - cond = self.args[2].eval() - self.v = eval('{} if {} else {}'.format(left, cond, right)) - elif self.func == 'op': - left = self.args[0].eval() - right = self.args[1].eval() - self.v = eval('{} {} {}'.format(left, self.op, right)) - elif self.func != None: - self.v = eval('{}({})'.format(self.func, ','.join([str(e.eval()) for e in self.args]))) - else: - WTF - - self.v = int(self.v) - return self.v - - def __add__(self, v): - return Expr(self, '+', v) - - def __sub__(self, v): - return Expr(self, '-', v) - - def __rshift__(self, v): - return Expr(self, '>>', v) - - def __lshift__(self, v): - return Expr(self, '<<', v) - - def __and__(self, v): - return Expr(self, '&', v) - - def __eq__(self, v): - return Expr(self, '==', v) - - def __ne__(self, v): - return Expr(self, '!=', v) - - def __ge__(self, v): - return Expr(self, '>=', v) - - def __int__(self): - return self.eval() - - def __repr__(self): - return '[{} {} {} {} {}]'.format( - self.func, - self.op, - [a.addr for a in self.args], - self.c, - self.addr, - ) - - def __hash__(self): - return id(self) - - def mem(self): - if self.addr == None: - WTF - return '0x{:x}'.format(self.expr[0]) - elif self.c: - return 'c{}'.format(self.caddr) - else: - return 'm{}'.format(self.addr) - - # def eval_trace(inputs): - # inputs = inputs[::] - # mem = dict() - # con = dict() - - # # store inputs in the first spots - # for i in range(len(inputs)): - # mem[str(i)] = inputs[i] - - # def get_v(t): - # if t.addr[0] == 'c': - # return t.v#con[t.addr[1:]] - # return mem[Expr.mmap[t.addr[1:]]] - # def put_v(t, r): - # mem[Expr.mmap[t.addr[1:]]] = r - - # for t in Expr.trace_q: - # if t.c: - # r = None - # elif t.func == 'input': - # r = None - # elif t.func == 'op' and t.op == '+': - # r = get_v(t.args[0]) + get_v(t.args[1]) - # elif t.func == 'op' and t.op == '-': - # r = get_v(t.args[0]) - get_v(t.args[1]) - # elif t.func == 'op' and t.op == '&': - # r = get_v(t.args[0]) & get_v(t.args[1]) - # elif t.func == 'op' and t.op == '<<': - # r = get_v(t.args[0]) << get_v(t.args[1]) - # elif t.func == 'op' and t.op == '>>': - # r = get_v(t.args[0]) >> get_v(t.args[1]) - # elif t.func == 'op' and t.op == '==': - # r = int(get_v(t.args[0]) == get_v(t.args[1])) - # elif t.func == 'op' and t.op == '!=': - # r = int(get_v(t.args[0]) != get_v(t.args[1])) - # elif t.func == 'op' and t.op == '>=': - # r = int(get_v(t.args[0]) >= get_v(t.args[1])) - # elif t.func == 'mul_modp': - # r = (get_v(t.args[0]) * get_v(t.args[1])) % ref_ed25519.p - # elif t.func == 'add_modp': - # r = add_modp(get_v(t.args[0]), get_v(t.args[1]), ref_ed25519.p) - # elif t.func == 'sub_modp': - # r = sub_modp(get_v(t.args[0]), get_v(t.args[1]), ref_ed25519.p) - # elif t.func == 'ternary': - # # we only check the first bit - # r = get_v(t.args[0]) if (get_v(t.args[2]) & 1) else get_v(t.args[1]) - # elif t.func == 'dsdp_sel': - # # we only check the first bit - # r = dsdp_sel(get_v(t.args[0]), get_v(t.args[1])) - # elif t.func == 'ternary_dsdp_x': - # r = ternary_dsdp_x(get_v(t.args[0]), get_v(t.args[1]), get_v(t.args[2])) - # elif t.func == 'ternary_dsdp_y': - # r = ternary_dsdp_y(get_v(t.args[0]), get_v(t.args[1]), get_v(t.args[2])) - # elif t.func == 'ternary_dsdp_z': - # r = ternary_dsdp_z(get_v(t.args[0]), get_v(t.args[1]), get_v(t.args[2])) - # elif t.func == 'ternary_dsdp_t': - # r = ternary_dsdp_t(get_v(t.args[0]), get_v(t.args[1]), get_v(t.args[2])) - # else: - # WTF - - # # dead code - # if t.addr == None: - # r = None - # if r != None: - # # print ('mem[{}] = {:x}'.format(t.addr, r)) - # put_v(t, r) - # t.r = r - # rs = list() - # for o in Expr.outs: - # rs.append(o.r) - # return tuple(rs) - - def opt_trace(): - mfree = list() - mmap = dict() - cmap = dict() - Expr.consts = list() - Expr.mmap = dict() - Expr.max_i = 0 - Expr.max_c = 0 - Expr.max_m = 0 - Expr.n_dead = 0 - - def tmap(t): - if t.c: - if t.args[0] not in cmap: - cmap[t.args[0]] = Expr.max_c - Expr.consts.append(t.args[0]) - Expr.max_c += 1 - t.addr = 'c{}'.format(cmap[t.args[0]]) - else: - if t not in mmap: - if len(mfree) == 0: - mfree.append(Expr.max_m) - Expr.max_m += 1 - mmap[t] = mfree.pop(0) - t.addr = 'm{}'.format(mmap[t]) - def tfree(t): - if t.c: - WTF - else: - if t in mmap: - mfree.append(mmap[t]) - del mmap[t] - else: - t.dead = True - Expr.n_dead += 1 - - # allocate inputs, outputs, and consts - for t in Expr.trace_q: - if t.func == 'input': - # t.addr_i = 'i{}'.format(Expr.max_i) - tmap(t) - Expr.max_i += 1 - elif t.c: - tmap(t) - for t in Expr.outputs: - tmap(t) - - for ti in range(len(Expr.trace_q)-1, -1, -1): - t = Expr.trace_q[ti] - op = t.func - - targs = [] - if t.c: - WTF - elif t.func == 'input': - pass - elif t.func.startswith('ternary'): - targs = t.args - elif t.func == 'op': - op = t.op - targs = t.args - elif t.func != None: - targs = t.args - else: - WTF - - # free dest - if t.func != 'input': - tfree(t) - - # allocate args - for targ in targs: - tmap(targ) - - # opcode and arg histogram - Expr.op_hist.setdefault(op, 0) - Expr.op_hist[op] += 1 - Expr.op_args.setdefault(op, dict()) - args = [_.v if _.c else 'var' for _ in targs] - args = tuple(args) - Expr.op_args[op].setdefault(args, 0) - Expr.op_args[op][args] += 1 - - Expr.trace_q[ti] = t - - for op in Expr.op_hist: - print ('{}: {}'.format(op, Expr.op_hist[op])) - if op in Expr.op_args: - for a in Expr.op_args[op]: - print (' {}: {}'.format(Expr.op_args[op][a], a)) - - Expr.max_mem = Expr.max_i + Expr.max_m + Expr.max_c - print ('{} = {} + {} + {}'.format( - Expr.max_mem, - Expr.max_i, - Expr.max_m, - Expr.max_c, - )) - print ('n_dead: {}'.format(Expr.n_dead)) - - # Expr.mmap = dict() - # for i in range(Expr.max_mem): - # Expr.mmap[str(i)] = None - # for t in Expr.trace_q: - # if t.func == 'input': - # Expr.mmap[t.addr[1:]] = t.addr_i[1:] - # _i = Expr.max_i - # for i in range(Expr.max_mem): - # if Expr.mmap[str(i)] == None: - # Expr.mmap[str(i)] = str(_i) - # _i += 1 - - # n = number of copies - # r = number of rows per copy - def dump_const_hex(n, r, format='hex'): - hex_st = '' - h_i = 0 - for i in range(n): - if format == 'hex': - # hex_st = '@{:08x}\n'.format(Expr.max_m) - hex_st = '@{:08x}\n'.format(4) - if format == 'coe': - hex_st = 'memory_initialization_radix=16;\n' - hex_st += 'memory_initialization_vector=\n' - # hex_st += '@{:08x}\n'.format(i*r) - for ci in range(len(Expr.consts)): - if format == 'hex': - hex_st += '{:064x}\n'.format(Expr.consts[ci]) - if format == 'coe': - if h_i > 0: - hex_st += ',\n' - h_i += 1 - hex_st += '{:064x}'.format(Expr.consts[ci]) - if format == 'coe': - hex_st += ';' - return hex_st - - # // Instruction ROM - # // { 5 [op], 6 [INA], 6[INB], 6[INC], 6[OUT] } - def dump_instr_hex(format='hex'): - if format == 'hex': - hex_st = '@{:08x}\n'.format(0) - if format == 'coe': - hex_st = 'memory_initialization_radix=16;\n' - hex_st += 'memory_initialization_vector=\n' - if format == 'mif': - hex_st = '' - - mmap = dict() - for i in range(4): - mmap['m{}'.format(i)] = i - for i in range(Expr.max_c): - mmap['c{}'.format(i)] = 4+i - for i in range(Expr.max_m): - mmap['m{}'.format(4+i)] = 0x24+i - - h_i = 0 - for t in Expr.trace_q: - - a = 'm0' - b = 'm0' - c = 'm0' - if t.c: - WTF - elif t.func == 'input': - continue - elif t.func.startswith('ternary'): - a = t.args[0].addr - b = t.args[1].addr - c = t.args[2].addr - elif t.func == 'op': - a = t.args[0].addr - b = t.args[1].addr - elif t.func != None: - a = t.args[0].addr - b = t.args[1].addr - else: - WTF - a = mmap[a] - b = mmap[b] - c = mmap[c] - d = mmap[t.addr] - - if t.func == 'op' and t.op == '&': - op = 0 - elif t.func == 'op' and t.op == '==': - op = 1 - elif t.func == 'op' and t.op == '!=': - op = 2 - elif t.func == 'op' and t.op == '>=': - op = 3 - elif t.func == 'op' and t.op == '<<': - op = 4 - elif t.func == 'op' and t.op == '>>': - op = 5 - elif t.func == 'op' and t.op == '+': - op = 6 - elif t.func == 'op' and t.op == '-': - op = 7 - elif t.func == 'add_modp': - op = 8 - elif t.func == 'sub_modp': - op = 9 - elif t.func == 'mul_modp': - op = 10 - elif t.func == 'ternary': - op = 0x1B - else: - WTF - - h = (op << (4*6)) | (a << (3*6)) | (b << (2*6)) | (c << (1*6)) | (d << (0*6)) - if format == 'mif': - hex_st += '{:029b}\n'.format(h) - if format == 'hex': - hex_st += '{:08x}\n'.format(h) - if format == 'coe': - if h_i > 0: - hex_st += ',\n' - h_i += 1 - hex_st += '{:08x}'.format(h) - - for ti in range(len(Expr.outputs)): - t = Expr.outputs[ti] - op = 0xC - a = mmap[t.addr] - b = mmap[t.addr] - c = mmap[t.addr] - d = ti - - h = (op << (4*6)) | (a << (3*6)) | (b << (2*6)) | (c << (1*6)) | (d << (0*6)) - if format == 'mif': - hex_st += '{:029b}\n'.format(h) - if format == 'hex': - hex_st += '{:08x}\n'.format(h) - if format == 'coe': - if h_i > 0: - hex_st += ',\n' - h_i += 1 - hex_st += '{:08x}'.format(h) - - if format == 'coe': - hex_st += ';' - return hex_st - - def eval_hex(const_hex, instr_hex, inputs): - mem = dict() - rs = list() - - for i in range(len(inputs)): - mem[i] = inputs[i] - - off_c = None - for l in const_hex.split("\n"): - if l == '': - continue - if off_c == None: - if l == 'memory_initialization_radix=16;': - continue - elif l == 'memory_initialization_vector=': - off_c = 4 - else: - off_c = int(l[1:], 16) - else: - l = l.rstrip(',;') - mem[off_c] = int(l, 16) - off_c += 1 - - for l in instr_hex.split("\n")[1:]: - if l == '': - continue - if l == 'memory_initialization_radix=16;': - continue - if l == 'memory_initialization_vector=': - continue - l = l.rstrip(',;') - instr = int(l, 16) - - d = bits(instr, 6, 0*6) - c = bits(instr, 6, 1*6) - b = bits(instr, 6, 2*6) - a = bits(instr, 6, 3*6) - o = bits(instr, 5, 4*6) - - c = mem[c] - b = mem[b] - a = mem[a] - - if o == 0: - r = a & b - elif o == 1: - r = int(a == b) - elif o == 2: - r = int(a != b) - elif o == 3: - r = int(a >= b) - elif o == 4: - r = bits(a << b, 256, 0) - elif o == 5: - r = bits(a >> b, 256, 0) - elif o == 6: - r = a + b - elif o == 7: - r = a - b - elif o == 8: - r = add_modp(a, b, ref_ed25519.p) - elif o == 9: - r = sub_modp(a, b, ref_ed25519.p) - elif o == 10: - r = mul_modp3(a, b, ref_ed25519.p) - elif o == 0x1B: - r = a if c!=0 else b - elif o == 0xC: - rs.append(a) - else: - WTF - - mem[d] = r - - return tuple(rs) - - - - - -def bits(n, b, s): - return (n >> s) & ((1<= (p2<= (p2<= (p<= p: - Cp -= p - - return Cp - -def mul_modp3(x, y, p): - - x0 = bits(x, 128, 0) - x1 = bits(x, 127, 128) - - y0 = bits(y, 128, 0) - y1 = bits(y, 127, 128) - - A = bits(x0 * y0, 256, 0) - B = bits(x0 * y1, 255, 0) - C = bits(x1 * y0, 255, 0) - D = bits(x1 * y1, 254, 0) - - E = A + (B << 128) + (C << 128) + (D << 1) + (D << 2) + (D << 5) - - El = bits(E, 255, 0) - Eh = bits(E, 132, 255) - - F = El + (Eh << 0) + (Eh << 1) + (Eh << 4) - - G = F - (p if F >= p else 0) - - # print ('A = {:x}'.format(A)) - # print ('B = {:x}'.format(B)) - # print ('C = {:x}'.format(C)) - # print ('D = {:x}'.format(D)) - # print ('E = {:x}'.format(E)) - # print ('F = {:x}'.format(F)) - # print ('G = {:x}'.format(G)) - - return G - -def flip(n, b): - nn = 0 - for i in range(b): - _ = 0 if (n & 1) else 1 - nn |= _ << i - n >>= 1 - return nn - -def mul_modp4(x, y, p): - - x0 = bits(x, 128, 0) - x1 = bits(x, 127, 128) - - y0 = bits(y, 128, 0) - y1 = bits(y, 127, 128) - - A = bits(x0 + x1, 129, 0) - B = bits(y0 + y1, 129, 0) - - C = bits(x0 * y0, 256, 0) - D = bits(x1 * y1, 254, 0) - - M = bits(A * B, 258, 0) - N = bits((D << 1) + (D << 2) + (D << 5), 260, 0) - - print ('A: {:x}'.format(A)) - print ('B: {:x}'.format(B)) - - print ('C: {:x}'.format(C)) - print ('D: {:x}'.format(D)) - - print ('M: {:x}'.format(M)) - print ('N: {:x}'.format(N)) - - E = C + N + (M << 128) - (C << 128) - (D << 128) - - print ('E: {:x}'.format(E)) - - El = bits(E, 255, 0) - Eh = bits(E, 132, 255) - - F = El + (Eh << 0) + (Eh << 1) + (Eh << 4) - - G = F - (p if F >= p else 0) - - # print ('A = {:x}'.format(A)) - # print ('B = {:x}'.format(B)) - # print ('C = {:x}'.format(C)) - # print ('D = {:x}'.format(D)) - # print ('E = {:x}'.format(E)) - # print ('F = {:x}'.format(F)) - # print ('G = {:x}'.format(G)) - - return G - -def kpow_2k(x, y, p): - for i in range(y): - x = Expr(func='mul_modp', args=(x, x, p)) - return x - -def kpow_ed250(x, p): - t0 = Expr(func='mul_modp', args=(x, x, p)) # 1 - t00 = Expr(func='mul_modp', args=(t0, t0, p)) # 2 - t1 = Expr(func='mul_modp', args=(t00, t00, p)) # 3 - t2 = Expr(func='mul_modp', args=(x, t1, p)) # x^(2^(1001)) - t3 = Expr(func='mul_modp', args=(t0, t2, p)) # 3,1,0 - t4 = Expr(func='mul_modp', args=(t3, t3, p)) # 4,2,1 - t5 = Expr(func='mul_modp', args=(t2, t4, p)) # 4,3,2,1,0 - t6 = kpow_2k(t5, 5, p) # 9,8,7,6,5 - t7 = Expr(func='mul_modp', args=(t6, t5, p)) # 9,8,7,6,5,4,3,2,1,0 - t8 = kpow_2k(t7, 10, p) # 19..10 - t9 = Expr(func='mul_modp', args=(t8, t7, p)) # 19..0 - t10 = kpow_2k(t9, 20, p) # 39..20 - t11 = Expr(func='mul_modp', args=(t10, t9, p)) # 39..0 - t12 = kpow_2k(t11, 10, p) # 49..10 - t13 = Expr(func='mul_modp', args=(t12, t7, p)) # 49..0 - t14 = kpow_2k(t13, 50, p) # 99..50 - t15 = Expr(func='mul_modp', args=(t14, t13, p)) # 99..0 - t16 = kpow_2k(t15, 100, p) # 199..100 - t17 = Expr(func='mul_modp', args=(t16, t15, p)) # 199..0 - t18 = kpow_2k(t17, 50, p) # 249..50 - t19 = Expr(func='mul_modp', args=(t18, t13, p)) # 249..0 - - return t0, t3, t19 - -def kpow_ed255192(x, p): - t0, t3, t19 = kpow_ed250(x, p) - t20 = kpow_2k(t19, 5, p) # 254..5 - t21 = Expr(func='mul_modp', args=(t20, t3, p)) # 254..5,3,1,0 - return t21 - -def kpow_ed2551938(x, p): - t0, t3, t19 = kpow_ed250(x, p) - t20 = kpow_2k(t19, 2, p) # 251..2 - t21 = Expr(func='mul_modp', args=(t20, t0, p)) # 251..2,1 - - return t21 - -def kpow(x, y, p): - a = Expr(1) - y = Expr(y) - wy = y.eval().bit_length() - for i in range(wy): - xx = ternary(y & 1, x, 1) - a = Expr(func='mul_modp', args=(a, xx, p)) - x = Expr(func='mul_modp', args=(x, x, p)) - y = y >> 1 - return a - -def add_modp(a, b, p): - return (a + b) % p - - -def sub_modp(a, b, p): - if a >= b: - return a - b - else: - return a - b + p - -def dsdp_sel(sh2, h): - b0 = bits(sh2, 1, 255) - b1 = bits(h, 1, 255) - b0b1 = b0 & b1 - if b0b1: - return 3 - if b1: - return 2 - if b0: - return 1 - return 0 - -def ternary_dsdp_x(sel, A, T): - if sel == 0: - return 0 - if sel == 1: - return ref_ed25519.G[0] - if sel == 2: - return A - return T - -def ternary_dsdp_y(sel, A, T): - if sel == 0: - return 1 - if sel == 1: - return ref_ed25519.G[1] - if sel == 2: - return A - return T - -def ternary_dsdp_z(sel, A, T): - if sel == 0: - return 1 - if sel == 1: - return ref_ed25519.G[2] - if sel == 2: - return A - return T - -def ternary_dsdp_t(sel, A, T): - if sel == 0: - return 0 - if sel == 1: - return ref_ed25519.G[3] - if sel == 2: - return A - return T - - - - - - -def ed25519_dsdp_mul(A, As, Gs, W_S=256): - - Z = (0, 1, 1, 0) - Z = ref_ed25519.point_add(Z, Z) - G = ref_ed25519.G - T = ref_ed25519.point_add(A, G) - - for i in range(W_S): - - # taking advantage of the fact - # ternary only checks LSB - b0 = (Gs >> 255) & 0x1 - b1 = (As >> 255) & 0x1 - b0b1 = b0 & b1 - - As = As << 1 - Gs = Gs << 1 - - qx = G[0] if (b0 == 1) else 0 - qx = A[0] if (b1 == 1) else qx - qx = T[0] if (b0b1 == 1) else qx - - qy = G[1] if (b0 == 1) else 1 - qy = A[1] if (b1 == 1) else qy - qy = T[1] if (b0b1 == 1) else qy - - qz = G[2] if (b0 == 1) else 1 - qz = A[2] if (b1 == 1) else qz - qz = T[2] if (b0b1 == 1) else qz - - qt = G[3] if (b0 == 1) else 0 - qt = A[3] if (b1 == 1) else qt - qt = T[3] if (b0b1 == 1) else qt - - Q = (qx, qy, qz, qt) - - # if i > 0: - # print ('{:b} {:b}'.format(b1, b0)) - # print ("{}_0: {:x} {:x}".format(i, Z[0]&0xFFFFFFFF, Z[0]&0xFFFFFFFF)) - # if i > 0: - Z = ref_ed25519.point_add(Z, Z) - # print ("{}_1: {:x} {:x}".format(i, Z[0]&0xFFFFFFFF, Q[0]&0xFFFFFFFF)) - Z = ref_ed25519.point_add(Z, Q) - - return Z - - - - - - - - - - - - - - - - - - -if __name__ == '__main__': - - from ref_ed25519 import p - - while True: - x = random.choice([ - 0, - 1, - p-random.randint(1, 5), - rand_int(256) % p, - (rand_int(255-128) << 128) | ((1<<128)-1) - ]) - y = random.choice([ - 0, - 1, - p-random.randint(1, 5), - rand_int(256) % p, - (rand_int(255-128) << 128) | ((1<<128)-1) - ]) - xy = x * y - - m_0 = xy % p - m_1 = mul_modp4(x, y, p) - - print ('{:x} =?= \n{:x}'.format(m_0, m_1)) - if m_0 != m_1: - WTF diff --git a/src/wiredancer/py/point_decomp.py b/src/wiredancer/py/point_decomp.py deleted file mode 100755 index 738055f70f7..00000000000 --- a/src/wiredancer/py/point_decomp.py +++ /dev/null @@ -1,94 +0,0 @@ -import random - -from ref_ed25519 import modp_sqrt_m1 -from ref_ed25519 import d -from ref_ed25519 import p -from ref_ed25519 import q -from ref_ed25519 import point_decompress - -from ed25519_lib import mul_modp -from ed25519_lib import kpow_ed255192 -from ed25519_lib import kpow_ed2551938 -from ed25519_lib import Expr -from ed25519_lib import ternary - -PM1 = p-1 - -def kpoint_decomp(y, d, p, ERR): - - sign = y >> 255 - y = y & ((1 << 255) - 1) - - yy = Expr(func='mul_modp', args=(d, y, p)) - yy = Expr(func='mul_modp', args=(yy, y, p)) - yy = ternary(yy == PM1, 0, yy+1) - yy = kpow_ed255192(yy, p) - - x2 = Expr(func='mul_modp', args=(y, y, p)) - x2 = ternary(x2 == 0, PM1, x2-1) - x2 = Expr(func='mul_modp', args=(x2, yy, p)) - x = kpow_ed2551938(x2, p) - xx = Expr(func='mul_modp', args=(x, x, p)) - xp = Expr(func='mul_modp', args=(x, modp_sqrt_m1, p)) - x = ternary(xx != x2, xp, x) - x = ternary((x & 1) != sign, p - x, x) - xx = Expr(func='mul_modp', args=(x, x, p)) - - x2z = x2 == 0 - sz = sign == 0 - snz = sign != 0 - x2zsz = x2z & sz - x2zsnz = x2z & snz - - r = ternary(xx != x2, ERR, x) - r = ternary(x2zsz, ERR, r) - r = ternary(x2zsnz, 0, r) - r = ternary(y >= p, ERR, r) - - return r - - - - - -if __name__ == '__main__': - - - if False: - - Expr.reset() - os = kpoint_decomp(Expr(0, var=True), Expr(d), Expr(p), Expr(p)) - Expr.outputs(os) - - # print (Expr.get_io_addrs()) - hex_const = Expr.dump_const_hex(1, 16, format='coe') - hex_instr = Expr.dump_instr_hex(format='coe') - - with open('hex_const.coe', 'w') as f: - f.write(hex_const) - with open('hex_instr.coe', 'w') as f: - f.write(hex_instr) - - while True: - - rstr = "" - for _ in range(64): - rstr += random.choice(['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']) - keyP = bytes.fromhex(rstr) - inputs = [int.from_bytes(keyP, 'little')] - - a = point_decompress(keyP) - if a == None: - a = p - else: - a = a[0] - - # b = Expr.eval_trace(inputs)[-1] - # b = Expr.eval_hex(hex_const, hex_instr, inputs)[-1] - # print ('\nf({:x}): \n{:x} =?=\n{:x}, {} {}'.format(inputs[0], a, b, len(Expr.trace_q), Expr.max_mem)) - - b = kpoint_decomp(Expr(inputs[0]), Expr(d), Expr(p), Expr(p)).eval() - print ('\nf({:x}): \n{:x} =?=\n{:x}'.format(inputs[0], a, b)) - - if a != b: - WTF diff --git a/src/wiredancer/py/point_mul.py b/src/wiredancer/py/point_mul.py deleted file mode 100755 index 33bab647d0d..00000000000 --- a/src/wiredancer/py/point_mul.py +++ /dev/null @@ -1,123 +0,0 @@ -import random - -import ref_ed25519 -from ref_ed25519 import modp_sqrt_m1 -from ref_ed25519 import d -from ref_ed25519 import p -from ref_ed25519 import q -from ref_ed25519 import point_decompress -from ref_ed25519 import point_add -from ref_ed25519 import point_mul - -from ed25519_lib import mul_modp -from ed25519_lib import kpow -from ed25519_lib import Expr -from ed25519_lib import ternary -from ed25519_lib import rand_int - -def kpoint_add(P, Q, d, p): - A0 = Expr(func='sub_modp', args=(P[1], P[0], p)) - B0 = Expr(func='add_modp', args=(P[1], P[0], p)) - - A1 = Expr(func='sub_modp', args=(Q[1], Q[0], p)) - B1 = Expr(func='add_modp', args=(Q[1], Q[0], p)) - - A = Expr(func='mul_modp', args=(A0, A1, p)) - B = Expr(func='mul_modp', args=(B0, B1, p)) - - # A, B = (P[1]-P[0]) * (Q[1]-Q[0]) % p, (P[1]+P[0]) * (Q[1]+Q[0]) % p; - - C = Expr(func='mul_modp', args=(P[3], Q[3], p)) - C = Expr(func='mul_modp', args=(C, d, p)) - C = Expr(func='add_modp', args=(C, C, p)) - - D = Expr(func='mul_modp', args=(P[2], Q[2], p)) - D = Expr(func='add_modp', args=(D, D, p)) - - # C, D = 2 * P[3] * Q[3] * d % p, 2 * P[2] * Q[2] % p; - - F = Expr(func='sub_modp', args=(D, C, p)) - G = Expr(func='add_modp', args=(D, C, p)) - E = Expr(func='sub_modp', args=(B, A, p)) - H = Expr(func='add_modp', args=(B, A, p)) - # E, F, G, H = B-A, D-C, D+C, B+A; - - x = Expr(func='mul_modp', args=(E, F, p), out=True) - y = Expr(func='mul_modp', args=(G, H, p), out=True) - z = Expr(func='mul_modp', args=(F, G, p), out=True) - t = Expr(func='mul_modp', args=(E, H, p), out=True) - # return (E*F, G*H, F*G, E*H); - - return (x, y, z, t) - - -def kpoint_mul(P, s, d, p): - - Q = (Expr(0), Expr(1), Expr(1), Expr(0)) # Neutral element - for i in range(256): - a = s & 1 - s >>= 1 - Q2 = kpoint_add(Q, P, d, p) - P = kpoint_add(P, P, d, p) - x = ternary(a, Q2[0], Q[0]) - y = ternary(a, Q2[1], Q[1]) - z = ternary(a, Q2[2], Q[2]) - t = ternary(a, Q2[3], Q[3]) - Q = (x, y, z, t) - return Q - - - - -if __name__ == '__main__': - - Expr.reset() - kpoint_mul( - ( - Expr(0, var=True), - Expr(0, var=True), - Expr(0, var=True), - Expr(0, var=True), - ), - Expr(0, var=True), - Expr(d), Expr(p) - ) - # Expr.dump_instrs() - Expr.shrink_trace() - # Expr.dump_instrs() - - # print (Expr.get_io_addrs()) - # print (Expr.dump_const_hex(32, 16)) - # print (Expr.dump_instr_hex()) - - while True: - - rstr = "" - for _ in range(64): - rstr += random.choice(['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']) - keyP = bytes.fromhex(rstr) - keyP = ref_ed25519.secret_to_public(keyP) - - s = rand_int(256) - a = point_decompress(keyP) - if a == None: - continue - - aa = point_mul(s, a) - - b = Expr.eval_trace([ - a[0], - a[1], - a[2], - a[3], - s, - ])[-4:] - - bb = tuple(b) - for _, __ in zip(aa, bb): - print ('{:x} =?=\n{:x}'.format(_, __)) - print ('{} {}'.format(len(Expr.trace_q), Expr.max_mem)) - - if aa != bb: - WTF - diff --git a/src/wiredancer/py/ref_ed25519.py b/src/wiredancer/py/ref_ed25519.py deleted file mode 100755 index 7d0d8a68c77..00000000000 --- a/src/wiredancer/py/ref_ed25519.py +++ /dev/null @@ -1,390 +0,0 @@ -# RFC8032 reference implementation pages 20-23 -# + test vectors pages 24-27 - -# NOTE: it needs python3 (3.6) to run - -import hashlib - -def sha512(s): - return hashlib.sha512(s).digest() - -# Base field Z_p -p = 2**255 - 19 - -def modp_inv(x): - return pow(x, p-2, p) - -# Curve constant -d = -121665 * modp_inv(121666) % p - -# Group order -q = 2**252 + 27742317777372353535851937790883648493 - -def sha512_modq(s): - return int.from_bytes(sha512(s), "little") % q - -## Then follows functions to perform point operations. -# Points are represented as tuples (X, Y, Z, T) of extended -# coordinates, with x = X/Z, y = Y/Z, x*y = T/Z -def point_add(P, Q): - A, B = ((P[1]-P[0]) * (Q[1]-Q[0])) % p, ((P[1]+P[0]) * (Q[1]+Q[0])) % p; - C, D = (2 * P[3] * Q[3] * d) % p, (2 * P[2] * Q[2]) % p; - E, F, G, H = (B-A)%p, (D-C)%p, (D+C)%p, (B+A)%p; - return ((E*F) % p, (G*H) % p, (F*G) % p, (E*H) % p); - - - -# Computes Q = s * Q -def point_mul(s, P): - Q = (0, 1, 1, 0) # Neutral element - while s > 0: - if s & 1: - Q = point_add(Q, P) - P = point_add(P, P) - s >>= 1 - return (Q[0]%p, Q[1]%p, Q[2]%p, Q[3]%p) - -def point_equal(P, Q): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - if (P[0] * Q[2] - Q[0] * P[2]) % p != 0: - return False - if (P[1] * Q[2] - Q[1] * P[2]) % p != 0: - return False - return True - -## Now follows functions for point compression. -# Square root of -1 -modp_sqrt_m1 = pow(2, (p-1) // 4, p) -# Compute corresponding x-coordinate, with low bit corresponding to -# sign, or return None on failure -def recover_x(y, sign): - if y >= p: - return None - x2 = (y*y-1) * modp_inv(d*y*y+1) - if x2 == 0: - if sign: - return None - else: - return 0 - - # Compute square root of x2 - x = pow(x2, (p+3) // 8, p) - if (x*x - x2) % p != 0: - x = x * modp_sqrt_m1 % p - if (x*x - x2) % p != 0: - return None - if (x & 1) != sign: - x = p - x - return x - -# Base point -g_y = 4 * modp_inv(5) % p -g_x = recover_x(g_y, 0) -G = (g_x, g_y, 1, g_x * g_y % p) - -def point_compress(P): - zinv = modp_inv(P[2]) - x = P[0] * zinv % p - y = P[1] * zinv % p - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - -def point_decompress(s): - if len(s) != 32: - raise Exception("Invalid input length for decompression") - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - - x = recover_x(y, sign) - if x is None: - return None - else: - return (x, y, 1, x*y % p) - -## These are functions for manipulating the private key. -def secret_expand(secret): - if len(secret) != 32: - raise Exception("Bad size of private key") - h = sha512(secret) - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= (1 << 254) - return (a, h[32:]) - -def secret_to_public(secret): - (a, dummy) = secret_expand(secret) - return point_compress(point_mul(a, G)) - -## The signature function works as below. -def sign(secret, msg): - a, prefix = secret_expand(secret) - A = point_compress(point_mul(a, G)) - r = sha512_modq(prefix + msg) - R = point_mul(r, G) - Rs = point_compress(R) - h = sha512_modq(Rs + A + msg) - s = (r + h * a) % q - return Rs + int.to_bytes(s, 32, "little") - -## And finally the verification function. -def verify2(public, msg, signature, h=None): - if len(public) != 32: - raise Exception("Bad public key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - if s >= q: return False - if h == None: - h = sha512_modq(Rs + public + msg) - sB = point_mul(s, G) - hA = point_mul(h, A) - return point_equal(sB, point_add(R, hA)) - -## And finally the verification function. -def verify(public, msg, signature, h=None): - if len(public) != 32: - raise Exception("Bad public key length") - if len(signature) != 64: - Exception("Bad signature length") - A = point_decompress(public) - if not A: - return False - Rs = signature[:32] - R = point_decompress(Rs) - if not R: - return False - s = int.from_bytes(signature[32:], "little") - if s >= q: return False - if h == None: - h = sha512_modq(Rs + public + msg) - # sB = point_mul(s, G) - # hA = point_mul(h, A) - # return point_equal(sB, point_add(R, hA)) - - P0 = point_mul(s, G) - P1 = point_mul(h, A) - P0P1 = point_add(P0, P1) - - An = ((p-A[0]), A[1], A[2], (p-A[3])) - T = point_add(G, An) - Z = (0, 1, 1, 0) - for i in range(255, -1, -1): - Z = point_add(Z, Z) - b0 = (s >> i) & 1 - b1 = (h >> i) & 1 - - Q = (0, 1, 1, 0) - if b0 == 1 and b1 == 1: - Q = T - elif b0 == 1: - Q = G - elif b1 == 1: - Q = An - - Z = point_add(Z, Q) - - # print (point_equal(P0P1, Z)) - - return point_equal(Z, R) - - -# ############################################################### -# Test Vectors - -def TEST_VERIF( n, keyP, msg, sigt ): - print( " " ) - print( "TEST {}".format(n) ) - print( "Public Key : 0x{}".format(keyP[ :16].hex())) - print( " {}".format(keyP[16: ].hex())) - print( "Message : 0x{}".format(msg [ :16].hex())) - for i in range(16,len(msg),16): - print( " {}".format(msg[i:i+16].hex())) - print( "Signature : 0x{}".format(sigt[ :16].hex())) - print( " {}".format(sigt[16:32].hex())) - print( " {}".format(sigt[32:48].hex())) - print( " {}".format(sigt[48:64].hex())) - v = verify(keyP, msg, sigt) - print( "Verif pass? : {}".format(v) ) - assert v, "!" - - -# https://stackoverflow.com/questions/5649407/hexadecimal-string-to-byte-array-in-python -if __name__ == '__main__': - - print ('{:x} {}'.format(G[0], G[0].bit_length())) - print ('{:x} {}'.format(G[1], G[1].bit_length())) - print ('{:x} {}'.format(G[2], G[2].bit_length())) - print ('{:x} {}'.format(G[3], G[3].bit_length())) - print ('{:x} {}'.format(d, d.bit_length())) - print ('{:x} {}'.format((d+d)%p, d.bit_length())) - - # -----TEST 1 - # ALGORITHM: - # Ed25519 - # SECRET KEY: - keyS = bytes.fromhex("9d61b19deffd5a60ba844af492ec2cc4") - keyS+= bytes.fromhex("4449c5697b326919703bac031cae7f60") - # PUBLIC KEY: - keyP = bytes.fromhex("d75a980182b10ab7d54bfed3c964073a") - keyP+= bytes.fromhex("0ee172f3daa62325af021a68f707511a") - # MESSAGE (length 0 bytes): - msg = b'' - # SIGNATURE: - sigt = bytes.fromhex("e5564300c360ac729086e2cc806e828a") - sigt+= bytes.fromhex("84877f1eb8e5d974d873e06522490155") - sigt+= bytes.fromhex("5fb8821590a33bacc61e39701cf9b46b") - sigt+= bytes.fromhex("d25bf5f0595bbe24655141438e7a100b") - TEST_VERIF( 1, keyP, msg, sigt ) - - - # -----TEST 2 - # ALGORITHM: - # Ed25519 - # SECRET KEY: - keyS = bytes.fromhex("4ccd089b28ff96da9db6c346ec114e0f") - keyS+= bytes.fromhex("5b8a319f35aba624da8cf6ed4fb8a6fb") - # PUBLIC KEY: - keyP = bytes.fromhex("3d4017c3e843895a92b70aa74d1b7ebc") - keyP+= bytes.fromhex("9c982ccf2ec4968cc0cd55f12af4660c") - # MESSAGE (length 1 byte): - msg = bytes.fromhex("72") - # SIGNATURE: - sigt = bytes.fromhex("92a009a9f0d4cab8720e820b5f642540") - sigt+= bytes.fromhex("a2b27b5416503f8fb3762223ebdb69da") - sigt+= bytes.fromhex("085ac1e43e15996e458f3613d0f11d8c") - sigt+= bytes.fromhex("387b2eaeb4302aeeb00d291612bb0c00") - TEST_VERIF( 2, keyP, msg, sigt ) - - - # -----TEST 3 - # ALGORITHM: - # Ed25519 - # SECRET KEY: - keyS = bytes.fromhex("c5aa8df43f9f837bedb7442f31dcb7b1") - keyS+= bytes.fromhex("66d38535076f094b85ce3a2e0b4458f7") - # PUBLIC KEY: - keyP = bytes.fromhex("fc51cd8e6218a1a38da47ed00230f058") - keyP+= bytes.fromhex("0816ed13ba3303ac5deb911548908025") - # MESSAGE (length 2 bytes): - msg = bytes.fromhex("af82") - # SIGNATURE: - sigt = bytes.fromhex("6291d657deec24024827e69c3abe01a3") - sigt+= bytes.fromhex("0ce548a284743a445e3680d7db5ac3ac") - sigt+= bytes.fromhex("18ff9b538d16f290ae67f760984dc659") - sigt+= bytes.fromhex("4a7c15e9716ed28dc027beceea1ec40a") - TEST_VERIF( 3, keyP, msg, sigt ) - - - # -----TEST 1024 - # ALGORITHM: - # Ed25519 - # SECRET KEY: - keyS = bytes.fromhex("f5e5767cf153319517630f226876b86c") - keyS+= bytes.fromhex("8160cc583bc013744c6bf255f5cc0ee5") - # PUBLIC KEY: - keyP = bytes.fromhex("278117fc144c72340f67d0f2316e8386") - keyP+= bytes.fromhex("ceffbf2b2428c9c51fef7c597f1d426e") - # MESSAGE (length 1023 bytes): - msg = bytes.fromhex("08b8b2b733424243760fe426a4b54908") - msg += bytes.fromhex("632110a66c2f6591eabd3345e3e4eb98") - msg += bytes.fromhex("fa6e264bf09efe12ee50f8f54e9f77b1") - msg += bytes.fromhex("e355f6c50544e23fb1433ddf73be84d8") - msg += bytes.fromhex("79de7c0046dc4996d9e773f4bc9efe57") - msg += bytes.fromhex("38829adb26c81b37c93a1b270b20329d") - msg += bytes.fromhex("658675fc6ea534e0810a4432826bf58c") - msg += bytes.fromhex("941efb65d57a338bbd2e26640f89ffbc") - msg += bytes.fromhex("1a858efcb8550ee3a5e1998bd177e93a") - msg += bytes.fromhex("7363c344fe6b199ee5d02e82d522c4fe") - msg += bytes.fromhex("ba15452f80288a821a579116ec6dad2b") - msg += bytes.fromhex("3b310da903401aa62100ab5d1a36553e") - msg += bytes.fromhex("06203b33890cc9b832f79ef80560ccb9") - msg += bytes.fromhex("a39ce767967ed628c6ad573cb116dbef") - msg += bytes.fromhex("efd75499da96bd68a8a97b928a8bbc10") - msg += bytes.fromhex("3b6621fcde2beca1231d206be6cd9ec7") - msg += bytes.fromhex("aff6f6c94fcd7204ed3455c68c83f4a4") - msg += bytes.fromhex("1da4af2b74ef5c53f1d8ac70bdcb7ed1") - msg += bytes.fromhex("85ce81bd84359d44254d95629e9855a9") - msg += bytes.fromhex("4a7c1958d1f8ada5d0532ed8a5aa3fb2") - msg += bytes.fromhex("d17ba70eb6248e594e1a2297acbbb39d") - msg += bytes.fromhex("502f1a8c6eb6f1ce22b3de1a1f40cc24") - msg += bytes.fromhex("554119a831a9aad6079cad88425de6bd") - msg += bytes.fromhex("e1a9187ebb6092cf67bf2b13fd65f270") - msg += bytes.fromhex("88d78b7e883c8759d2c4f5c65adb7553") - msg += bytes.fromhex("878ad575f9fad878e80a0c9ba63bcbcc") - msg += bytes.fromhex("2732e69485bbc9c90bfbd62481d9089b") - msg += bytes.fromhex("eccf80cfe2df16a2cf65bd92dd597b07") - msg += bytes.fromhex("07e0917af48bbb75fed413d238f5555a") - msg += bytes.fromhex("7a569d80c3414a8d0859dc65a46128ba") - msg += bytes.fromhex("b27af87a71314f318c782b23ebfe808b") - msg += bytes.fromhex("82b0ce26401d2e22f04d83d1255dc51a") - msg += bytes.fromhex("ddd3b75a2b1ae0784504df543af8969b") - msg += bytes.fromhex("e3ea7082ff7fc9888c144da2af58429e") - msg += bytes.fromhex("c96031dbcad3dad9af0dcbaaaf268cb8") - msg += bytes.fromhex("fcffead94f3c7ca495e056a9b47acdb7") - msg += bytes.fromhex("51fb73e666c6c655ade8297297d07ad1") - msg += bytes.fromhex("ba5e43f1bca32301651339e22904cc8c") - msg += bytes.fromhex("42f58c30c04aafdb038dda0847dd988d") - msg += bytes.fromhex("cda6f3bfd15c4b4c4525004aa06eeff8") - msg += bytes.fromhex("ca61783aacec57fb3d1f92b0fe2fd1a8") - msg += bytes.fromhex("5f6724517b65e614ad6808d6f6ee34df") - msg += bytes.fromhex("f7310fdc82aebfd904b01e1dc54b2927") - msg += bytes.fromhex("094b2db68d6f903b68401adebf5a7e08") - msg += bytes.fromhex("d78ff4ef5d63653a65040cf9bfd4aca7") - msg += bytes.fromhex("984a74d37145986780fc0b16ac451649") - msg += bytes.fromhex("de6188a7dbdf191f64b5fc5e2ab47b57") - msg += bytes.fromhex("f7f7276cd419c17a3ca8e1b939ae49e4") - msg += bytes.fromhex("88acba6b965610b5480109c8b17b80e1") - msg += bytes.fromhex("b7b750dfc7598d5d5011fd2dcc5600a3") - msg += bytes.fromhex("2ef5b52a1ecc820e308aa342721aac09") - msg += bytes.fromhex("43bf6686b64b2579376504ccc493d97e") - msg += bytes.fromhex("6aed3fb0f9cd71a43dd497f01f17c0e2") - msg += bytes.fromhex("cb3797aa2a2f256656168e6c496afc5f") - msg += bytes.fromhex("b93246f6b1116398a346f1a641f3b041") - msg += bytes.fromhex("e989f7914f90cc2c7fff357876e506b5") - msg += bytes.fromhex("0d334ba77c225bc307ba537152f3f161") - msg += bytes.fromhex("0e4eafe595f6d9d90d11faa933a15ef1") - msg += bytes.fromhex("369546868a7f3a45a96768d40fd9d034") - msg += bytes.fromhex("12c091c6315cf4fde7cb68606937380d") - msg += bytes.fromhex("b2eaaa707b4c4185c32eddcdd306705e") - msg += bytes.fromhex("4dc1ffc872eeee475a64dfac86aba41c") - msg += bytes.fromhex("0618983f8741c5ef68d3a101e8a3b8ca") - msg += bytes.fromhex("c60c905c15fc910840b94c00a0b9d0") - # SIGNATURE: - sigt = bytes.fromhex("0aab4c900501b3e24d7cdf4663326a3a") - sigt+= bytes.fromhex("87df5e4843b2cbdb67cbf6e460fec350") - sigt+= bytes.fromhex("aa5371b1508f9f4528ecea23c436d94b") - sigt+= bytes.fromhex("5e8fcd4f681e30a6ac00a9704a188a03") - TEST_VERIF( 4, keyP, msg, sigt ) - - - # -----TEST SHA(abc) - # ALGORITHM: - # Ed25519 - # SECRET KEY: - keyS = bytes.fromhex("833fe62409237b9d62ec77587520911e") - keyS+= bytes.fromhex("9a759cec1d19755b7da901b96dca3d42") - # PUBLIC KEY: - keyP = bytes.fromhex("ec172b93ad5e563bf4932c70e1245034") - keyP+= bytes.fromhex("c35467ef2efd4d64ebf819683467e2bf") - # MESSAGE (length 64 bytes): - msg = bytes.fromhex("ddaf35a193617abacc417349ae204131") - msg += bytes.fromhex("12e6fa4e89a97ea20a9eeee64b55d39a") - msg += bytes.fromhex("2192992a274fc1a836ba3c23a3feebbd") - msg += bytes.fromhex("454d4423643ce80e2a9ac94fa54ca49f") - # SIGNATURE: - sigt = bytes.fromhex("dc2a4459e7369633a52b1bf277839a00") - sigt+= bytes.fromhex("201009a3efbf3ecb69bea2186c26b589") - sigt+= bytes.fromhex("09351fc9ac90b3ecfdfbc7c66431e030") - sigt+= bytes.fromhex("3dca179c138ac17ad9bef1177331a704") - TEST_VERIF( 4, keyP, msg, sigt ) - - - print("\nTEST(s): pass") - diff --git a/src/wiredancer/py/sigverify.py b/src/wiredancer/py/sigverify.py deleted file mode 100755 index a69e492c87b..00000000000 --- a/src/wiredancer/py/sigverify.py +++ /dev/null @@ -1,271 +0,0 @@ -import random - -import ref_ed25519 -from ref_ed25519 import modp_sqrt_m1 -from ref_ed25519 import d -from ref_ed25519 import p -from ref_ed25519 import q -from ref_ed25519 import G -from ref_ed25519 import point_decompress - -from ed25519_lib import mul_modp -from ed25519_lib import kpow -from ed25519_lib import Expr -from ed25519_lib import ternary -from ed25519_lib import ed25519_dsdp_mul - -from point_decomp import kpoint_decomp -from point_mul import kpoint_add -from point_mul import kpoint_mul - -def kpoint_equal(P, Q, p): - # x1 / z1 == x2 / z2 <==> x1 * z2 == x2 * z1 - x1z2 = Expr(func='mul_modp', args=(P[0], Q[2], p)) - x2z1 = Expr(func='mul_modp', args=(P[2], Q[0], p)) - y1z2 = Expr(func='mul_modp', args=(P[1], Q[2], p)) - y2z1 = Expr(func='mul_modp', args=(P[2], Q[1], p)) - r = 1 - r = ternary(x1z2 != x2z1, 0, r) - r = ternary(y1z2 != y2z1, 0, r) - return r - -def ksigverify(public, sl, sh, h, d, p, q): - Ax = kpoint_decomp(public, d, p, q) - Ay = public & ((1 << 255) - 1) - At = Expr(func='mul_modp', args=(Ax, Ay, p)) - A = (Ax, Ay, 1, At) - - Rx = kpoint_decomp(sl, d, p, q) - Ry = sl & ((1 << 255) - 1) - Rt = Expr(func='mul_modp', args=(Rx, Ry, p)) - R = (Rx, Ry, 1, Rt) - - shG = kpoint_mul(G, sh, d, p) - hA = kpoint_mul(A, h, d, p) - - RhA = kpoint_add(R, hA, d, p) - - r = kpoint_equal(shG, RhA, p) - r = ternary(sh >= q, 0, r) - r = ternary(Ax == p, 0, r) - r = ternary(Rx == p, 0, r) - - return r - -def ksigverify2(public, sl, sh, h, d, p, q): - Ax = kpoint_decomp(public, d, p, q) - Axn = Expr(func='sub_modp', args=(p, Ax, p)) - Ay = public & ((1 << 255) - 1) - At = Expr(func='mul_modp', args=(Axn, Ay, p)) - A = (Axn, Ay, 1, At) - - # double scalar, double point - T = kpoint_add(G, A, d, p) - Z = (0, 1, 1, 0) - sh2 = sh - for i in range(256): - - # taking advantage of the fact - # ternary only checks LSB - sel = Expr(func='dsdp_sel', args=(sh2, h)) - # b0 = sh2 >> 255 - # b1 = h >> 255 - # b0b1 = b0 & b1 - - sh2 = sh2 << 1 - h = h << 1 - - qx = Expr(func='ternary_dsdp_x', args=(sel, A[0], T[0])) - qy = Expr(func='ternary_dsdp_y', args=(sel, A[1], T[1])) - qz = Expr(func='ternary_dsdp_z', args=(sel, A[2], T[2])) - qt = Expr(func='ternary_dsdp_t', args=(sel, A[3], T[3])) - - # qx = ternary(b0, G[0], 0) - # qx = ternary(b1, A[0], qx) - # qx = ternary(b0b1, T[0], qx) - - # qy = ternary(b0, G[1], 1) - # qy = ternary(b1, A[1], qy) - # qy = ternary(b0b1, T[1], qy) - - # qz = ternary(b0, G[2], 1) - # qz = ternary(b1, A[2], qz) - # qz = ternary(b0b1, T[2], qz) - - # qt = ternary(b0, G[3], 0) - # qt = ternary(b1, A[3], qt) - # qt = ternary(b0b1, T[3], qt) - - Q = (qx, qy, qz, qt) - - if i > 0: - Z = kpoint_add(Z, Z, d, p) - Z = kpoint_add(Z, Q, d, p) - - Rx = kpoint_decomp(sl, d, p, q) - Ry = sl & ((1 << 255) - 1) - R = (Rx, Ry, 1, 0) - - # checks - r = kpoint_equal(Z, R, p) - r = ternary(sh >= q, 0, r) - r = ternary(Ax == p, 0, r) - r = ternary(Rx == p, 0, r) - - return r - - -def ksigverify_split0(public, sl, sh, d, p, q): - Ax = kpoint_decomp(public, d, p, q) - Axn = Expr(func='sub_modp', args=(p, Ax, p)) - Ay = public & ((1 << 255) - 1) - At = Expr(func='mul_modp', args=(Axn, Ay, p)) - A = (Axn, Ay, 1, At) - - Rx = kpoint_decomp(sl, d, p, q) - - T = kpoint_add(A, G, d, p) - - # checks - r = 1 - r = ternary(sh >= q, 0, r) - r = ternary(Ax == p, 0, r) - r = ternary(Rx == p, 0, r) - - return r, Axn, At, Rx, T[0], T[1], T[2], T[3] - -def ksigverify_split1(r, Ax, At, Rx, Tx, Ty, Tz, Tt, public, sl, sh, h): - if r == 0: - return 0 - - Ay = public & ((1 << 255) - 1) - A = (Ax, Ay, 1, At) - - Ry = sl & ((1 << 255) - 1) - R = (Rx, Ry, 1, None) - - Z = ed25519_dsdp_mul(A, h, sh) - - RxZz = mul_modp(R[0], Z[2], p) - RzZx = Z[0] # Rz == 1 - RyZz = mul_modp(R[1], Z[2], p) - RzZy = Z[1] # Rz == 1 - - if RxZz != RzZx: - return 0 - if RyZz != RzZy: - return 0 - return 1 - - - -if __name__ == '__main__': - - if False: - Expr.reset() - outs = ksigverify_split0( - Expr(0, var=True), - Expr(0, var=True), - Expr(0, var=True), - Expr(d), Expr(p), Expr(q), - ) - Expr.outputs(outs) - - hex_const = Expr.dump_const_hex(1, 16, format='mif') - hex_instr = Expr.dump_instr_hex(format='mif') - - with open('sig_hex_const.mif', 'w') as f: - f.write(hex_const) - with open('sig_hex_instr.mif', 'w') as f: - f.write(hex_instr) - - END - - if True: - sec = ''.join([random.choice(['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']) for i in range(32)]).encode('utf-8') - pub = ref_ed25519.secret_to_public(sec) - ml = 0#random.randint(0, 1024) - msg = ''.join([chr(random.randint(0, 255)) for i in range(ml)]).encode('utf-8') - sig = ref_ed25519.sign(sec, msg) - - public = int.from_bytes(pub, 'little') - sl = int.from_bytes(sig[:32], 'little') - sh = int.from_bytes(sig[32:], 'little') - - h = ref_ed25519.sha512_modq(sig[:32] + pub + msg) - outs = ksigverify_split0( - Expr(public), - Expr(sl), - Expr(sh), - Expr(d), Expr(p), Expr(q), - ) - outs = [_.eval() for _ in outs] - - s = list() - for i in range(64): - s.append('0x{:02x}'.format(sig[i])) - for i in range(32): - s.append('0x{:02x}'.format(pub[i])) - print (','.join(s)) - - print ('{:x}'.format(h)) - for o in outs: - print ('{:x}'.format(o)) - - - Axn = outs[1] - At = outs[2] - Rx = outs[3] - Ay = public & ((1 << 255) - 1) - A = (Axn, Ay, 1, At) - - Ry = sl & ((1 << 255) - 1) - R = (Rx, Ry, 1, None) - - Z = ed25519_dsdp_mul(A, h, sh) - for o in Z: - print ('{:x}'.format(o)) - - END - - while True: - - sec = ''.join([random.choice(['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']) for i in range(32)]).encode('utf-8') - pub = ref_ed25519.secret_to_public(sec) - ml = random.randint(0, 1024) - msg = ''.join([chr(random.randint(0, 255)) for i in range(ml)]).encode('utf-8') - sig = ref_ed25519.sign(sec, msg) - - err = random.randint(0, 100) - if err < 10: - msg = msg + b' ' - elif err < 20: - sig = sig[::-1] - elif err < 30: - pub = pub[::-1] - - h = ref_ed25519.sha512_modq(sig[:32] + pub + msg) - v0 = ref_ed25519.verify(pub, msg, sig, h) - v0 = int(v0) - - public = int.from_bytes(pub, 'little') - sl = int.from_bytes(sig[:32], 'little') - sh = int.from_bytes(sig[32:], 'little') - - inputs = [public, sl, sh, d, p, q] - ret = Expr.eval_hex(hex_const, hex_instr, inputs) - r, Ax, At, Rx, Tx, Ty, Tz, Tt = ret - - print ('inputs:') - for i in range(3): - print ('{:x}'.format(inputs[i])) - print ('outputs:') - for i in range(8): - print ('{:x}'.format(ret[i])) - - v1 = ksigverify_split1(r, Ax, At, Rx, Tx, Ty, Tz, Tt, public, sl, sh, h) - - print ('{} =?= {:x}, {} {}'.format(v0, v1, len(Expr.trace_q), Expr.max_mem)) - - if v0 != v1: - WTF diff --git a/src/wiredancer/py/wd_cocotil.py b/src/wiredancer/py/wd_cocotil.py deleted file mode 100755 index fb46e01a46b..00000000000 --- a/src/wiredancer/py/wd_cocotil.py +++ /dev/null @@ -1,1678 +0,0 @@ - -import random -import hashlib - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue -from cocotb.utils import get_sim_time - -# import sha512_lib -import ed25519_lib -import ref_ed25519 -import sigverify - -PCIE_MAGIC = 0xACE0FBAC -PCIE_ADDR_W = 20 - -meta0 = [16, 64] -meta1 = meta0 + [256, 256, 256] -meta2 = meta1 + [16, 6, 1, 512] -meta3 = meta1 + [1, 4, 1024] -meta4 = meta1 + [256] -meta5 = meta4 + ([256]*8) -meta6 = meta0 + ([256]*5) + [1] -meta7 = meta0 + [1] - -def get_cycle(): - return get_sim_time('ns') - -async def f1_write_32x16(dut, clk, addr, data): - - split = 0#random.randint(0, 1) - b = BinaryValue(bits=512, bigEndian=False) - b[511:0] = data - - if split == 0: - dut.pcie_v = 0x3 - dut.pcie_a = addr - dut.pcie_d[0] = b[256*1-1:256*0] - dut.pcie_d[1] = b[256*2-1:256*1] - await RisingEdge(clk) - dut.pcie_v = 0x0 - else: - dut.pcie_v = 0x1 - dut.pcie_a = addr - dut.pcie_d[0] = b[256*1-1:256*0] - await RisingEdge(clk) - dut.pcie_v = 0x2 - dut.pcie_a = addr + 32 - dut.pcie_d[1] = b[256*2-1:256*1] - await RisingEdge(clk) - dut.pcie_v = 0x0 - -def sha512_modq_from_bytes(s): - h = hashlib.sha512(s).digest() - q = 2**252 + 27742317777372353535851937790883648493 - return bytes_to_little(h) % ref_ed25519.q - -def sha512_modq_from_ints(s): - return sha512_modq_from_bytes(bytearray(s)) - -def bytes_to_little(s): - n = 0 - for i in range(len(s)): - n += s[i] << (i*8) - return n - -def str_to_little(s): - n = 0 - for i in range(len(s)): - n += ord(s[i]) << (i*8) - return n - -def little_to_str(n, bs): - s = '' - for i in range(bs): - s += chr(bits(n, 8, i*8)) - return s - -def little_to_ints(n, bs): - return [bits(n, 8, i*8) for i in range(bs)] - -def lfsr_32(lfsr): - fb = bits(lfsr, 1, 0) ^ bits(lfsr, 1, 1) ^ bits(lfsr, 1, 21) ^ bits(lfsr, 1, 31) - return (bits(lfsr, 31, 0) << 1) | fb - -def log2(n): - return int(math.log(n) / math.log(2)) - -def bits(n, b, s): - return (n >> s) & ((1< 0: - c = (s[0]) - s = s[1:] - blk |= c << (bi*8) - bi += 1 - if bi == (512//8): - blks.append(blk) - blk = 0 - bi = 0 - if bi != 0: - blks.append(blk) - - return blks - -def build_pcie_tr_o(tr): - blks = list() - - blk = 0 - blk |= tr['sig_l'] << 0 - blk |= tr['pub'] << 256 - blks.append(blk) - - blk = 0 - bi = 0 - s = tr['msg_s'] - while len(s) > 0: - c = (s[0]) - s = s[1:] - blk |= c << (bi*8) - bi += 1 - if bi == (512//8): - blks.append(blk) - blk = 0 - bi = 0 - if bi != 0: - blks.append(blk) - - return blks - -async def mon_pcie_tr_ext(ddut, dut, clk, q_i=None, q_o=None, do_log=False): - - refs_i = list() - refs_o = list() - - while True: - - await RisingEdge(clk) - - if q_i != None and str(dut.pcie_v) == '1': - - if len(refs_i) == 0: - tr = q_i.pop(0) - refs_i.extend(build_pcie_tr_i(tr)) - - i_d = int(dut.pcie_d) - e_d = refs_i.pop(0) - - if do_log: - ddut._log.info ('mon_pcie_tr_ext_i: \n{:x} =?= \n{:x}'.format(e_d, i_d)) - - assert str(dut.pcie_f) != '1' - assert e_d == i_d - - if str(dut.o_v) == '1' and str(dut.o_r) == '1' and q_o != None: - - o_m0 = int(dut.o_m0) - o_m1 = int(dut.o_m1) - o_e = int(dut.o_e) - - ( - o_src, - o_tid, - o_sig_l, - o_sig_h, - o_pub, - o_size, - o_emp, - o_sop, - o_data, - ) = extr_meta(meta2, o_m0) - - if len(refs_o) == 0: - tr = q_o.pop(0) - blks = build_pcie_tr_o(tr) - - # first blk is pcie header - for i in range(len(blks)): - - e_src = tr['src'] - e_tid = tr['tid'] - e_sig_l = tr['sig_l'] - e_sig_h = tr['sig_h'] - e_pub = tr['pub'] - e_size = len(tr['sha_msg']) - e_emp = bits((512//8) - bits(len(tr['sha_msg']), 6, 0), 6, 0) - e_sop = int(i == 0) - e_e = int(i == len(blks)-1) - e_data = blks[i] - - refs_o.append(( - e_e, - e_src, - e_tid, - e_sig_l, - e_sig_h, - e_pub, - e_size, - e_emp, - e_sop, - e_data, - )) - - ( - e_e, - e_src, - e_tid, - e_sig_l, - e_sig_h, - e_pub, - e_size, - e_emp, - e_sop, - e_data, - ) = refs_o.pop(0) - - e_m1 = build_pcie_tr_i(tr)[0] - - if do_log: - ddut._log.info ('mon_pcie_tr_ext_o: e : \n{:x} =?=\n{:x}'.format(e_e , o_e)) - ddut._log.info ('mon_pcie_tr_ext_o: src : \n{:x} =?=\n{:x}'.format(e_src , o_src)) - ddut._log.info ('mon_pcie_tr_ext_o: tid : \n{:x} =?=\n{:x}'.format(e_tid , o_tid)) - ddut._log.info ('mon_pcie_tr_ext_o: sig_l: \n{:x} =?=\n{:x}'.format(e_sig_l , o_sig_l)) - ddut._log.info ('mon_pcie_tr_ext_o: sig_h: \n{:x} =?=\n{:x}'.format(e_sig_h , o_sig_h)) - ddut._log.info ('mon_pcie_tr_ext_o: pub : \n{:x} =?=\n{:x}'.format(e_pub , o_pub)) - ddut._log.info ('mon_pcie_tr_ext_o: size : \n{:x} =?=\n{:x}'.format(e_size , o_size)) - ddut._log.info ('mon_pcie_tr_ext_o: emp : \n{:x} =?=\n{:x}'.format(e_emp , o_emp)) - ddut._log.info ('mon_pcie_tr_ext_o: sop : \n{:x} =?=\n{:x}'.format(e_sop , o_sop)) - ddut._log.info ('mon_pcie_tr_ext_o: data : \n{:x} =?=\n{:x}'.format(e_data , o_data)) - ddut._log.info ('mon_pcie_tr_ext_o: m1 : \n{:x} =?=\n{:x}'.format(e_m1 , o_m1)) - - assert e_e == o_e - assert e_src == o_src - assert e_tid == o_tid - assert e_sig_l == o_sig_l - assert e_sig_h == o_sig_h - assert e_pub == o_pub - assert e_size == o_size - assert e_emp == o_emp - assert e_sop == o_sop - assert e_data == o_data - assert e_m1 == o_m1 - - - - - - - - - - - - - - - - - - -def build_sha_pre_meta(tr): - m = 0 - m |= tr['src'] << (0) - m |= tr['tid'] << (0+32) - m |= tr['sig_h'] << (0+32+64) - m |= tr['sig_l'] << (0+32+64+256) - m |= tr['pub'] << (0+32+64+256+256) - return m - -def build_sha_pre_o(tr): - sha_pre_o = gen_blocks_from_msg_str(tr['sha_msg']) - return [( - tr['src'], - tr['tid'], - tr['sig_l'], - tr['sig_h'], - tr['pub'], - int(i == 0), - int(i == len(sha_pre_o)-1), - len(sha_pre_o), - sha_pre_o[i], - ) for i in range(len(sha_pre_o))] - -async def mon_sha_pre(ddut, dut, clk, q_i=None, q_o=None, do_log=False): - - q_bo = list() - - while True: - - await RisingEdge(clk) - - if str(dut.o_v) == '1': - - if q_o != None: - - if len(q_bo) == 0: - tr = q_o.pop(0) - tr['time_1'] = get_cycle() - q_bo.extend(build_sha_pre_o(tr)) - ddut._log.info ('mon_sha_pre_o: tid.{:x} sz.{} err.{} {} {}'.format( - tr['tid'], - len(tr['msg_s']), - tr['err'], - tr['time_0'], - tr['time_1'], - )) - - o_m = int(dut.o_m) - o_e = int(dut.o_e) - - ( - o_src, - o_tid, - o_sig_l, - o_sig_h, - o_pub, - o_f, - o_c, - o_d, - ) = extr_meta(meta3, o_m) - - ( - e_src, - e_tid, - e_sig_l, - e_sig_h, - e_pub, - e_f, - e_e, - e_c, - e_d, - ) = q_bo.pop(0) - - if do_log: - ddut._log.info ('mon_sha_pre_o: src: \n{:x} =?=\n{:x}'.format(e_src, o_src)) - ddut._log.info ('mon_sha_pre_o: tid: \n{:x} =?=\n{:x}'.format(e_tid, o_tid)) - ddut._log.info ('mon_sha_pre_o: sig_l: \n{:x} =?=\n{:x}'.format(e_sig_l, o_sig_l)) - ddut._log.info ('mon_sha_pre_o: sig_h: \n{:x} =?=\n{:x}'.format(e_sig_h, o_sig_h)) - ddut._log.info ('mon_sha_pre_o: pub: \n{:x} =?=\n{:x}'.format(e_pub, o_pub)) - ddut._log.info ('mon_sha_pre_o: f: \n{:x} =?=\n{:x}'.format(e_f, o_f)) - ddut._log.info ('mon_sha_pre_o: e: \n{:x} =?=\n{:x}'.format(e_e, o_e)) - ddut._log.info ('mon_sha_pre_o: c: \n{:x} =?=\n{:x}'.format(e_c, o_c)) - ddut._log.info ('mon_sha_pre_o: d: \n{:x} =?=\n{:x}'.format(e_d, o_d)) - - assert e_src == o_src - assert e_tid == o_tid - assert e_sig_l == o_sig_l - assert e_sig_h == o_sig_h - assert e_pub == o_pub - assert e_f == o_f - assert e_e == o_e - assert e_c == o_c - assert e_d == o_d - - - - - - - - - - - - - - - - - - - - - - - - - -def build_sha_modq_o(tr): - return ( - tr['src'], - tr['tid'], - tr['sig_l'], - tr['sig_h'], - tr['pub'], - tr['sha_modq'], - ) - -def build_sha_modq_meta(tr): - m = 0 - m |= tr['src'] << (0) - m |= tr['tid'] << (0+32) - m |= tr['sig_h'] << (0+32+64) - m |= tr['sig_l'] << (0+32+64+256) - m |= tr['pub'] << (0+32+64+256+256) - return m - -def build_sha_modq_meta_i(tr): - blks = gen_blocks_from_msg_str(tr['sha_msg']) - - return [( - int(i==0), - int(i==len(blks)-1), - len(blks), - tr['tid'], - tr['sha_modq_meta'], - blks[i], - ) for i in range(len(blks))] - -async def mon_sha_modq_meta(ddut, dut, clk, q_i=None, q_o=None, do_log=False): - - i_cnt = 0 - q_bi = list() - - while True: - - await RisingEdge(clk) - - # no gaps allowed - assert i_cnt == 0 or str(dut.i_v) == '1', 'i_cnt: {}'.format(i_cnt) - - if str(dut.i_v) == '1' and str(dut.i_r) == '1': - - i_e = int(dut.i_e) - i_m = int(dut.i_m) - - ( - i_src, - i_tid, - i_sig_l, - i_sig_h, - i_pub, - i_f, - i_c, - i_d, - ) = extr_meta(meta3, i_m) - - if i_f == 1: - i_cnt = i_c - i_cnt -= 1 - - if q_i != None: - - # if first, q_bi should have drained by now - assert (len(q_bi) == 0 and i_f == 1) or (i_f == 0) - - if i_f: - tr = q_i.pop(0) - q_bi = build_sha_modq_meta_i(tr) - - (e_f, - e_l, - e_c, - e_t, - e_m, - e_d) = q_bi.pop(0) - - if do_log: - ddut._log.info ('mon_sha_modq_meta: i_f: \n{:x} =?= \n{:x}'.format(e_f, i_f)) - ddut._log.info ('mon_sha_modq_meta: i_l: \n{:x} =?= \n{:x}'.format(e_l, i_l)) - ddut._log.info ('mon_sha_modq_meta: i_c: \n{:x} =?= \n{:x}'.format(e_c, i_c)) - ddut._log.info ('mon_sha_modq_meta: i_t: \n{:x} =?= \n{:x}'.format(e_t, i_t)) - ddut._log.info ('mon_sha_modq_meta: i_m: \n{:x} =?= \n{:x}'.format(e_m, i_m)) - ddut._log.info ('mon_sha_modq_meta: i_d: \n{:x} =?= \n{:x}'.format(e_d, i_d)) - - assert e_f == i_f - assert e_c == i_c - assert e_t == i_t - assert e_m == i_m - assert e_d == i_d - - if str(dut.o_v) == '1': - - if q_o != None: - - o_m = int(dut.o_m) - - ( - o_src, - o_tid, - o_sig_l, - o_sig_h, - o_pub, - o_h, - ) = extr_meta(meta4, o_m) - - tr = q_o[o_tid] - del q_o[o_tid] - - tr['time_2'] = get_cycle() - ddut._log.info ('mon_sha_modq_meta_o: tid.{:x} sz.{} err.{} {} {} {}'.format( - tr['tid'], - len(tr['msg_s']), - tr['err'], - tr['time_0'], - tr['time_1'], - tr['time_2'], - )) - - ( - e_src, - e_tid, - e_sig_l, - e_sig_h, - e_pub, - e_h, - ) = build_sha_modq_o(tr) - - if do_log: - ddut._log.info ('mon_sha_modq_o: src: \n{:x} =?=\n{:x}'.format(e_src, o_src)) - ddut._log.info ('mon_sha_modq_o: tid: \n{:x} =?=\n{:x}'.format(e_tid, o_tid)) - ddut._log.info ('mon_sha_modq_o: sig_l: \n{:x} =?=\n{:x}'.format(e_sig_l, o_sig_l)) - ddut._log.info ('mon_sha_modq_o: sig_h: \n{:x} =?=\n{:x}'.format(e_sig_h, o_sig_h)) - ddut._log.info ('mon_sha_modq_o: pub: \n{:x} =?=\n{:x}'.format(e_pub, o_pub)) - ddut._log.info ('mon_sha_modq_o: h: \n{:x} =?=\n{:x}'.format(e_h, o_h)) - - assert e_src == o_src - assert e_tid == o_tid - assert e_sig_l == o_sig_l - assert e_sig_h == o_sig_h - assert e_pub == o_pub - assert e_h == o_h - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -@cocotb.coroutine -def mon_ed25519_sigverify_dsdp_mul(dut, clk, q_i=None, q_o=None, do_print=False, self_test=False): - - while True: - - yield RisingEdge(clk) - - if str(dut.i_v) == '1' and str(dut.i_r) == '1': - - if q_i != None: - - i_m = int(dut.i_m) - - i_Ax = int(dut.i_Ax) - i_Ay = int(dut.i_Ay) - i_Az = int(dut.i_Az) - i_At = int(dut.i_At) - - i_ApGx = int(dut.i_ApGx) - i_ApGy = int(dut.i_ApGy) - i_ApGz = int(dut.i_ApGz) - i_ApGt = int(dut.i_ApGt) - - i_As = int(dut.i_As) - i_Gs = int(dut.i_Gs) - - tr = q_i[i_m] - del q_i[i_m] - - ( - e_pub, - e_sig_l, - e_sig_h, - e_h, - e_m, - ( - e_res, - e_Ax, - e_At, - e_Rx, - e_ApGx, - e_ApGy, - e_ApGz, - e_ApGt, - ) - ) = build_ed25519_sigverify_0_o(tr) - - e_Ay = bits(e_pub, 255, 0) - e_Az = 1 - e_As = e_h - e_Gs = e_sig_h - - if do_print: - print ('mon_ed25519_sigverify_dsdp_mul_i: i_Ax:\n{:x} =?=\n{:x}'.format(e_Ax, i_Ax)) - print ('mon_ed25519_sigverify_dsdp_mul_i: i_Ay:\n{:x} =?=\n{:x}'.format(e_Ay, i_Ay)) - print ('mon_ed25519_sigverify_dsdp_mul_i: i_Az:\n{:x} =?=\n{:x}'.format(e_Az, i_Az)) - print ('mon_ed25519_sigverify_dsdp_mul_i: i_At:\n{:x} =?=\n{:x}'.format(e_At, i_At)) - - print ('mon_ed25519_sigverify_dsdp_mul_i: i_ApGx:\n{:x} =?=\n{:x}'.format(e_ApGx, i_ApGx)) - print ('mon_ed25519_sigverify_dsdp_mul_i: i_ApGy:\n{:x} =?=\n{:x}'.format(e_ApGy, i_ApGy)) - print ('mon_ed25519_sigverify_dsdp_mul_i: i_ApGz:\n{:x} =?=\n{:x}'.format(e_ApGz, i_ApGz)) - print ('mon_ed25519_sigverify_dsdp_mul_i: i_ApGt:\n{:x} =?=\n{:x}'.format(e_ApGt, i_ApGt)) - - print ('mon_ed25519_sigverify_dsdp_mul_i: i_As:\n{:x} =?=\n{:x}'.format(e_As, i_As)) - print ('mon_ed25519_sigverify_dsdp_mul_i: i_Gs:\n{:x} =?=\n{:x}'.format(e_Gs, i_Gs)) - - assert e_Ax == i_Ax - assert e_Ay == i_Ay - assert e_Az == i_Az - assert e_At == i_At - - assert e_ApGx == i_ApGx - assert e_ApGy == i_ApGy - assert e_ApGz == i_ApGz - assert e_ApGt == i_ApGt - - assert e_As == i_As - assert e_Gs == i_Gs - - - - if str(dut.o_v) == '1': - - if q_o != None: - - o_m = int(dut.o_m) - o_d = [0]*4 - o_d[0] = int(dut.o_Cx) - o_d[1] = int(dut.o_Cy) - o_d[2] = int(dut.o_Cz) - o_d[3] = int(dut.o_Ct) - - e_d = q_o[o_m] - del q_o[o_m] - - if do_print: - print ('ed25519_sigverify_dsdp_mul: ox: \n{:x} =?= \n{:x}'.format(e_d[0], o_d[0])) - print ('ed25519_sigverify_dsdp_mul: oy: \n{:x} =?= \n{:x}'.format(e_d[1], o_d[1])) - print ('ed25519_sigverify_dsdp_mul: oz: \n{:x} =?= \n{:x}'.format(e_d[2], o_d[2])) - print ('ed25519_sigverify_dsdp_mul: ot: \n{:x} =?= \n{:x}'.format(e_d[3], o_d[3])) - - assert ref_ed25519.point_equal(e_d, o_d) - - # t =?= (x*y)/z - xy = ed25519_lib.mul_modp(o_d[0], o_d[1], ref_ed25519.p) - zi = ref_ed25519.modp_inv(o_d[2]) - xyzi = ed25519_lib.mul_modp(xy, zi, ref_ed25519.p) - assert xyzi == o_d[3] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -def build_ed25519_sigverify_0_o(tr): - os = sigverify.ksigverify_split0( - ed25519_lib.Expr(tr['pub']), - ed25519_lib.Expr(tr['sig_l']), - ed25519_lib.Expr(tr['sig_h']), - ed25519_lib.Expr(ref_ed25519.d), - ed25519_lib.Expr(ref_ed25519.p), - ed25519_lib.Expr(ref_ed25519.q), - ) - - os = tuple(_.eval() for _ in os) - - return ( - tr['src'], - tr['tid'], - tr['sig_l'], - tr['sig_h'], - tr['pub'], - tr['sha_modq'], - os, - ) - -async def mon_ed25519_sigverify_0(ddut, dut, clk, q_i=None, q_o=None, do_log=False): - - N_SCH_O = 8 - - while True: - - await RisingEdge(clk) - - if str(dut.i_v) == '1' and str(dut.i_r) == '1': - ddut._log.info ('mon_ed25519_sigverify_0_i') - - if str(dut.o_v) == '1': - - o_m = int(dut.o_m) - - o_os = [None]*8 - - ( - o_src, - o_tid, - o_sig_l, - o_sig_h, - o_pub, - o_h, - o_os[0], - o_os[1], - o_os[2], - o_os[3], - o_os[4], - o_os[5], - o_os[6], - o_os[7], - ) = extr_meta(meta5, o_m) - - if q_o != None: - tr = q_o[o_tid] - del q_o[o_tid] - - tr['time_3'] = get_cycle() - ddut._log.info ('mon_ed25519_sigverify_0_o: tid.{:x} sz.{} err.{} {} {} {} {}'.format( - tr['tid'], - len(tr['msg_s']), - tr['err'], - tr['time_0'], - tr['time_1'], - tr['time_2'], - tr['time_3'], - )) - - ( - e_src, - e_tid, - e_sig_l, - e_sig_h, - e_pub, - e_h, - e_os, - ) = build_ed25519_sigverify_0_o(tr) - - if do_log: - ddut._log.info ('mon_ed25519_sigverify_0_o: e_src : \n{:x} =?= \n{:x}'.format(e_src, o_src)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_tid : \n{:x} =?= \n{:x}'.format(e_tid, o_tid)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_sig_l : \n{:x} =?= \n{:x}'.format(e_sig_l, o_sig_l)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_sig_h : \n{:x} =?= \n{:x}'.format(e_sig_h, o_sig_h)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_pub : \n{:x} =?= \n{:x}'.format(e_pub, o_pub)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_h : \n{:x} =?= \n{:x}'.format(e_h, o_h)) - for i in range(N_SCH_O): - ddut._log.info ('mon_ed25519_sigverify_0_o: e_os[{}]: \n{:x} =?= \n{:x}'.format(i, e_os[i], o_os[i])) - - assert e_src == o_src - assert e_tid == o_tid - assert e_sig_l == o_sig_l - assert e_sig_h == o_sig_h - assert e_pub == o_pub - assert e_h == o_h - for i in range(N_SCH_O): - assert e_os[i] == o_os[i] - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -def build_ed25519_sigverify_1_o(tr, DSDP_WS=256): - - ( - i_src, - i_tid, - i_sig_l, - i_sig_h, - i_pub, - i_h, - ( - i_res, - i_Ax, - i_At, - i_Rx, - i_Tx, - i_Ty, - i_Tz, - i_Tt, - ) - ) = build_ed25519_sigverify_0_o(tr) - - A = (i_Ax, bits(i_pub, 255, 0), 1, i_At) - - As = i_h - Gs = i_sig_h - - Z = ed25519_lib.ed25519_dsdp_mul(A, As, Gs, W_S=DSDP_WS) - - return ( - tr['src'], - tr['tid'], - tr['sig_l'], - Z[2], - Z[1], - Z[0], - i_Rx, - i_res, - ) - -async def mon_ed25519_sigverify_1(ddut, dut, clk, q_i=None, q_o=None, do_log=False, self_test=False): - - q_self = list() - DSDP_WS = int(dut.DSDP_WS) - - while True: - - await RisingEdge(clk) - - # if str(dut.i_r) == '1': - # print ('mon_ed25519_sigverify_1_i_r') - - if str(dut.i_v) == '1' and str(dut.i_r) == '1': - - ddut._log.info ('mon_ed25519_sigverify_1_i') - - if self_test: - i_m = int(dut.i_m) - i_sig_l = int(dut.i_sig_l) - i_sig_h = int(dut.i_sig_h) - i_pub = int(dut.i_pub) - i_h = int(dut.i_h) - i_res = int(dut.i_res) - i_Rx = int(dut.i_Rx) - i_Ax = int(dut.i_Ax) - i_At = int(dut.i_At) - i_Tx = int(dut.i_Tx) - i_Ty = int(dut.i_Ty) - i_Tz = int(dut.i_Tz) - i_Tt = int(dut.i_Tt) - - A = (i_Ax, bits(int(dut.i_pub), 255, 0), 1, i_At) - - As = i_h - Gs = i_sig_h - - Z = ed25519_lib.ed25519_dsdp_mul(A, As, Gs, W_S=DSDP_WS) - - q_self.append(( - i_m, - i_sig_l, - i_res, - i_Rx, - Z[0], - Z[1], - Z[2], - )) - - if str(dut.o_v) == '1': - - o_m = int(dut.o_m) - - ( - o_src, - o_tid, - o_sig_l, - o_Zz, - o_Zy, - o_Zx, - o_Rx, - o_res, - ) = extr_meta(meta6, o_m) - - if q_o != None: - tr = q_o[o_tid] - del q_o[o_tid] - - tr['time_4'] = get_cycle() - ddut._log.info ('mon_ed25519_sigverify_1_o: tid.{:x} sz.{} err.{} {} {} {} {} {}'.format( - tr['tid'], - len(tr['msg_s']), - tr['err'], - tr['time_0'], - tr['time_1'], - tr['time_2'], - tr['time_3'], - tr['time_4'], - )) - - ( - e_src, - e_tid, - e_sig_l, - e_Zz, - e_Zy, - e_Zx, - e_Rx, - e_res, - ) = build_ed25519_sigverify_1_o(tr) - - if do_log: - ddut._log.info ('mon_ed25519_sigverify_0_o: e_src : \n{:x} =?= \n{:x}'.format(e_src, o_src)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_tid : \n{:x} =?= \n{:x}'.format(e_tid, o_tid)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_sig_l : \n{:x} =?= \n{:x}'.format(e_sig_l, o_sig_l)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_Zx : \n{:x} =?= \n{:x}'.format(e_Zx, o_Zx)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_Zy : \n{:x} =?= \n{:x}'.format(e_Zy, o_Zy)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_Zz : \n{:x} =?= \n{:x}'.format(e_Zz, o_Zz)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_Rx : \n{:x} =?= \n{:x}'.format(e_Rx, o_Rx)) - ddut._log.info ('mon_ed25519_sigverify_0_o: e_res : \n{:x} =?= \n{:x}'.format(e_res, o_res)) - - assert e_src == o_src - assert e_tid == o_tid - assert e_sig_l == o_sig_l - assert e_Zz == o_Zz - assert e_Zy == o_Zy - assert e_Zx == o_Zx - assert e_Rx == o_Rx - assert e_res == o_res - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -async def mon_ed25519_sigverify_2(ddut, dut, clk, q_i=None, q_o=None, do_log=False, self_test=False): - - while True: - - await RisingEdge(clk) - - if str(dut.o_v) == '1': - - o_m = int(dut.o_m) - - ( - o_src, - o_tid, - o_res, - ) = extr_meta(meta7, o_m) - - if q_o != None: - tr = q_o[o_tid] - del q_o[o_tid] - - tr['time_5'] = get_cycle() - print ('mon_ed25519_sigverify_2_o: tid.{:x} sz.{} err.{} {} {} {} {} {} {}'.format( - tr['tid'], - len(tr['msg_s']), - tr['err'], - tr['time_0'], - tr['time_1'], - tr['time_2'], - tr['time_3'], - tr['time_4'], - tr['time_5'], - )) - - e_src = tr['src'] - e_tid = tr['tid'] - e_res = tr['sigverify'] - - if do_log: - ddut._log.info ('mon_ed25519_sigverify_2_o: src:\n{:x} =?=\n{:x}'.format(e_src, o_src)) - ddut._log.info ('mon_ed25519_sigverify_2_o: tid:\n{:x} =?=\n{:x}'.format(e_tid, o_tid)) - ddut._log.info ('mon_ed25519_sigverify_2_o: res:\n{:x} =?=\n{:x}'.format(e_res, o_res)) - - assert e_src == o_src - assert e_tid == o_tid - assert e_res == o_res - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -async def mon_dma(ddut, dut, clk, q_o=None, do_log=False): - - while True: - - await RisingEdge(clk) - - if str(dut.dma_v) == '1' and str(dut.dma_r) == '1': - - ddut._log.info ('mon_dma_o') - - if q_o != None: - - tr = q_o.pop(0) - - o_a = int(dut.dma_a) - o_b = int(dut.dma_b) - o_d = int(dut.dma_d) - - e_a = bits(tr['dma_addr'], 64-6, 6) << 6 - e_b = 0xFFFFFFFF00000000 if bits(tr['dma_addr'], 1, 5) else 0x00000000FFFFFFFF - e_ctrl = tr['dma_ctrl'] - e_ctrl |= (0 if tr['sigverify'] else 1) << 2 - - e_d = 0 - e_d |= tr['dma_seq'] << (0) - e_d |= bits(tr['sig_l'], 64, 0) << (0+64) - e_d |= tr['dma_chunk'] << (0+64+64) - e_d |= tr['dma_size'] << (0+64+64+32) - e_d |= e_ctrl << (0+64+64+32+16) - - if do_log: - ddut._log.info ('mon_dma: o_a: \n{:x} =?= \n{:x}'.format(e_a, o_a)) - ddut._log.info ('mon_dma: o_b: \n{:x} =?= \n{:x}'.format(e_b, o_b)) - ddut._log.info ('mon_dma: o_d: \n{:x} =?= \n{:x}'.format(e_d, o_d)) - - assert e_a == o_a - assert e_b == o_b - assert e_d == o_d - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -async def model_schl_cpu(ddut, dut, clk, do_log=False): - - D = 1024 - W_HASH = int(dut.W_HASH) - W_T = int(dut.W_T) - W_IN_MEM = int(dut.W_IN_MEM) - MAX_INFLIGHT = int(dut.MAX_INFLIGHT) - - await cocotb.start(random_toggle(clk, dut.in_hash_ready, 50)) - - ins = list() - outs = list() - - dut.out_hash_valid.value = 0 - - while True: - await RisingEdge(clk) - - if str(dut.in_hash_valid) == '1' and str(dut.in_hash_ready) == '1': - - i_r = int(dut.in_hash_ref) - i_d = int(dut.in_hash_data) - - ddut._log.info ('model_schl_cpu_i: {:x} {:x}'.format(i_r, i_d)) - - ins.append(i_d) - - if len(ins) == 3: - os = sigverify.ksigverify_split0( - ed25519_lib.Expr(ins.pop(0)), - ed25519_lib.Expr(ins.pop(0)), - ed25519_lib.Expr(ins.pop(0)), - ed25519_lib.Expr(ref_ed25519.d), - ed25519_lib.Expr(ref_ed25519.p), - ed25519_lib.Expr(ref_ed25519.q), - ) - os = tuple(_.eval() for _ in os) - - outs.append(( - get_cycle() + D, - i_r, - os, - 0 - )) - - dut.out_hash_valid.value = 0 - - for out in outs: - if out[0] < get_cycle(): - - ( - _, - ref, - os, - oa, - ) = out - - outs.remove(out) - - ddut._log.info ('model_schl_cpu_o: {:x} {:x} {:x}'.format(ref, oa, os[oa])) - - b_d = BinaryValue(bits=W_HASH, bigEndian=False) - b_r = BinaryValue(bits=W_T, bigEndian=False) - b_a = BinaryValue(bits=W_IN_MEM, bigEndian=False) - - b_d[W_HASH-1:0] = os[oa] - b_r[W_T-1:0] = ref - b_a[W_IN_MEM-1:0] = oa - - dut.out_hash_valid.value = 1 - dut.out_hash_data.value = b_d - dut.out_ref.value = b_r - dut.out_d_addr.value = b_a - - oa += 1 - - if oa < len(os): - outs.append(( - _ + MAX_INFLIGHT, - ref, - os, - oa - )) - - break - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -async def model_dsdp(ddut, dut, clk, do_log=False): - N_TH = int(dut.N_TH) - W_M = int(dut.W_M) - W_S = int(dut.W_S) - - await cocotb.start(random_toggle(clk, dut.i_r, 50)) - - outs = list() - - while True: - await RisingEdge(clk) - - if str(dut.i_v) == '1' and str(dut.i_r) == '1': - - i_m = int(dut.i_m) - - A = ( - int(dut.i_Ax), - int(dut.i_Ay), - int(dut.i_Az), - int(dut.i_At), - ) - - As = int(dut.i_As) - Gs = int(dut.i_Gs) - - Z = ed25519_lib.ed25519_dsdp_mul(A, As, Gs, W_S=W_S) - - outs.append(( - get_cycle() + (W_S * N_TH // 10), - i_m, - Z[0], - Z[1], - Z[2], - Z[3], - )) - - - dut.o_v.value = 0 - - for out in outs: - if out[0] < get_cycle(): - - ( - _, - i_m, - Zx, - Zy, - Zz, - Zt, - ) = out - - outs.remove(out) - - b_m = BinaryValue(bits=W_M, bigEndian=False) - b_zx = BinaryValue(bits=255, bigEndian=False) - b_zy = BinaryValue(bits=255, bigEndian=False) - b_zz = BinaryValue(bits=255, bigEndian=False) - b_zt = BinaryValue(bits=255, bigEndian=False) - - b_m[W_M-1:0] = i_m - b_zx[255-1:0] = Zx - b_zy[255-1:0] = Zy - b_zz[255-1:0] = Zz - b_zt[255-1:0] = Zt - - dut.o_v.value = 1 - dut.o_m.value = b_m - dut.o_Cx.value = b_zx - dut.o_Cy.value = b_zy - dut.o_Cz.value = b_zz - dut.o_Ct.value = b_zt - - break diff --git a/src/wiredancer/rtl/areset_sync.sv b/src/wiredancer/rtl/areset_sync.sv deleted file mode 100644 index 1a4195c738d..00000000000 --- a/src/wiredancer/rtl/areset_sync.sv +++ /dev/null @@ -1,98 +0,0 @@ -`default_nettype none - -module areset_sync( - input wire areset, - input wire dclk, - output wire dreset -); - - - -// XPM_CDC instantiation template for Asynchronous Reset Synchronizer configurations -// Refer to the targeted device family architecture libraries guide for XPM_CDC documentation -// ======================================================================================================================= - -// Parameter usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Parameter name | Data type | Restrictions, if applicable | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | DEST_SYNC_FF | Integer | Range: 2 - 10. Default value = 4. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Number of register stages used to synchronize signal in the destination clock domain. | -// | This parameter also determines the minimum width of the asserted reset signal. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | INIT_SYNC_FF | Integer | Allowed values: 0, 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | 0- Disable behavioral simulation initialization value(s) on synchronization registers. | -// | 1- Enable behavioral simulation initialization value(s) on synchronization registers. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | RST_ACTIVE_HIGH | Integer | Allowed values: 0, 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Defines the polarity of the asynchronous reset signal. | -// | | -// | 0- Active low asynchronous reset signal | -// | 1- Active high asynchronous reset signal | -// +---------------------------------------------------------------------------------------------------------------------+ - -// Port usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Port name | Direction | Size, in bits | Domain | Sense | Handling if unused | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | dest_arst | Output | 1 | dest_clk| NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | src_arst asynchronous reset signal synchronized to destination clock domain. This output is registered. | -// | NOTE: Signal asserts asynchronously but deasserts synchronously to dest_clk. Width of the reset signal is at least | -// | (DEST_SYNC_FF*dest_clk) period. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | dest_clk | Input | 1 | NA | Rising edge | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Destination clock. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | src_arst | Input | 1 | NA | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Source asynchronous reset signal. | -// +---------------------------------------------------------------------------------------------------------------------+ - - -// xpm_cdc_async_rst : In order to incorporate this function into the design, -// Verilog : the following instance declaration needs to be placed -// instance : in the body of the design code. The instance name -// declaration : (xpm_cdc_async_rst_inst) and/or the port declarations within the -// code : parenthesis may be changed to properly reference and -// : connect this function to the design. All inputs -// : and outputs must be connected. - -// Please reference the appropriate libraries guide for additional information on the XPM modules. - -// <-----Cut code below this line----> - - // xpm_cdc_async_rst: Asynchronous Reset Synchronizer - // Xilinx Parameterized Macro, version 2019.1 - - xpm_cdc_async_rst #( - .DEST_SYNC_FF(2), // DECIMAL; range: 2-10 - .INIT_SYNC_FF(0), // DECIMAL; 0=disable simulation init values, 1=enable simulation init values - .RST_ACTIVE_HIGH(1) // DECIMAL; 0=active low reset, 1=active high reset - ) - xpm_cdc_async_rst_inst ( - .dest_arst(dreset), // 1-bit output: src_arst asynchronous reset signal synchronized to destination - // clock domain. This output is registered. NOTE: Signal asserts asynchronously - // but deasserts synchronously to dest_clk. Width of the reset signal is at least - // (DEST_SYNC_FF*dest_clk) period. - - .dest_clk(dclk), // 1-bit input: Destination clock. - .src_arst(areset) // 1-bit input: Source asynchronous reset signal. - ); - - // End of xpm_cdc_async_rst_inst instantiation - - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/dma_result.sv b/src/wiredancer/rtl/dma_result.sv deleted file mode 100644 index 9b80ba3992d..00000000000 --- a/src/wiredancer/rtl/dma_result.sv +++ /dev/null @@ -1,154 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module dma_result #( - N_PCIE = 2 -) ( - input wire [1-1:0] dma_r, - output logic [1-1:0] dma_v, - output logic [64-1:0] dma_a, - output logic [64-1:0] dma_b, - input wire [1-1:0] dma_f, - output logic [256-1:0] dma_d, - - input wire [N_PCIE-1:0][1-1:0] ext_v, - input wire [N_PCIE-1:0][1-1:0] ext_r, - input wire [N_PCIE-1:0][1-1:0] ext_e, - input wire [N_PCIE-1:0][$bits(pcie_meta_t)-1:0] ext_m, - - input wire [N_PCIE-1:0][1-1:0] res_v, - input wire [N_PCIE-1:0][64-1:0] res_t, - input wire [N_PCIE-1:0][1-1:0] res_d, - output logic [N_PCIE-1:0][16-1:0] res_c, - output logic [N_PCIE-1:0][1-1:0] res_f, - output logic [N_PCIE-1:0][1-1:0] res_p, - - input wire [64-1:0] priv_base, - input wire [64-1:0] priv_mask, - - input wire [1-1:0] send_fails, - - input wire clk, - input wire rst -); - -logic [N_PCIE-1:0][1-1:0] dma_p_r; -logic [N_PCIE-1:0][1-1:0] dma_p_v; -mcache_pcim_t [N_PCIE-1:0] dma_p_dab; - -logic [64-1:0] dma_aa; - -assign dma_a = {dma_aa[64-1:6], 6'h0}; - -generate - - for (genvar g_i = 0; g_i < N_PCIE; g_i ++) begin: P_IN - - logic [1-1:0] ext_p_v, ext_pp_v; - pcie_meta_t ext_p_m, ext_pp_m; - - logic [1-1:0] dma_m_v; - logic [16-1:0] dma_m_ctrl; - logic [1-1:0] res_o_v; - logic [1-1:0] res_o_d; - - assign dma_p_v[g_i] = res_o_v & dma_m_v & (|dma_p_dab[g_i].pcim_addr) & (send_fails | res_o_d); - assign dma_p_dab[g_i].pcim_strb = dma_p_dab[g_i].pcim_addr[5] ? 64'hFFFF_FFFF_0000_0000 : 64'h0000_0000_FFFF_FFFF; - assign dma_p_dab[g_i].ctrl[1:0] = dma_m_ctrl[1:0]; - assign dma_p_dab[g_i].ctrl[2] = ~res_o_d; - assign dma_p_dab[g_i].ctrl[15:3] = dma_m_ctrl[15:3]; - assign dma_p_dab[g_i].tsorig = '0; - assign dma_p_dab[g_i].tspub = '0; - - always_ff@(posedge clk) res_p [g_i] <= dma_p_r[g_i] & dma_p_v[g_i]; - - (* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({ext_m[g_i], ext_v[g_i] & ext_r[g_i] & ext_e[g_i]})), - .DEPTH (2) - ) ext_pipe_inst ( - .in ({ext_m[g_i], ext_v[g_i] & ext_r[g_i] & ext_e[g_i]}), - .out ({ext_p_m, ext_p_v}), - - .clk (clk), - .reset (rst) - ); - - always_ff@(posedge clk) begin - ext_pp_v <= ext_p_v; - ext_pp_m <= ext_p_m; - ext_pp_m.dma_addr <= (ext_p_m.dma_addr & priv_mask) + priv_base; - end - - showahead_fifo #( - .WIDTH ($bits({ext_pp_m.sig_l[0+:64], ext_pp_m.dma_chunk, ext_pp_m.dma_seq, ext_pp_m.dma_addr, ext_pp_m.dma_ctrl, ext_pp_m.dma_size})), - .DEPTH (512) - ) dma_m_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (ext_pp_v), - .wr_full (), - .wr_full_b (), - .wr_count (), - .wr_data ({ext_pp_m.sig_l[0+:64], ext_pp_m.dma_chunk, ext_pp_m.dma_seq, ext_pp_m.dma_addr, ext_pp_m.dma_ctrl, ext_pp_m.dma_size}), - - .rd_clk (clk), - .rd_req (dma_m_v & res_o_v & dma_p_r[g_i]), - .rd_empty (), - .rd_not_empty (dma_m_v), - .rd_count (), - .rd_data ({dma_p_dab[g_i].sig, dma_p_dab[g_i].chunk, dma_p_dab[g_i].seq, dma_p_dab[g_i].pcim_addr, dma_m_ctrl, dma_p_dab[g_i].sz}) - ); - - tid_inorder #( - .W ($bits({res_d[g_i]})), - .D (2048) - ) tid_inorder_inst ( - .i_v (res_v [g_i]), - .i_a (res_t [g_i][0+:11]), - .i_f (res_f [g_i]), - .i_c (res_c [g_i]), - .i_d (res_d [g_i]), - - .o_r (dma_m_v & res_o_v & dma_p_r[g_i]), - .o_v (res_o_v), - .o_d (res_o_d), - - .clk (clk), - .rst (rst) - ); - - end -endgenerate - -rrb_merge #( - .W ($bits({dma_p_dab[0]})), - .N (N_PCIE) -) dma_merge_inst ( - .i_r (dma_p_r), - .i_v (dma_p_v), - .i_e ({N_PCIE{1'b1}}), - .i_m (dma_p_dab), - - .o_r (dma_r), - .o_v (dma_v), - .o_e (), - .o_m ({dma_d, dma_b, dma_aa}), - - .clk (clk), - .rst (rst) -); - -always_ff@(posedge clk) -if (|dma_p_v) -$display("%t: %m: %b %b - %b %b", $time -, dma_p_r -, dma_p_v -, dma_r -, dma_v -); -endmodule - - -`default_nettype wire diff --git a/src/wiredancer/rtl/dual_clock_showahead_fifo.sv b/src/wiredancer/rtl/dual_clock_showahead_fifo.sv deleted file mode 100644 index 9c7f25ba736..00000000000 --- a/src/wiredancer/rtl/dual_clock_showahead_fifo.sv +++ /dev/null @@ -1,469 +0,0 @@ - -`default_nettype none - -module dual_clock_showahead_fifo #(parameter LIB="xpm", WIDTH=32, DEPTH=32, D_L=$clog2(DEPTH))( - input wire wr_clk, - input wire wr_req, - input wire [WIDTH-1:0] wr_data, - output wire wr_full, - output wire wr_full_b, - output wire [D_L+1-1:0] wr_count, - - input wire rd_clk, - input wire rd_req, - output wire [WIDTH-1:0] rd_data, - output wire rd_empty, - output wire rd_not_empty, - output wire [D_L+1-1:0] rd_count, - - input wire aclr -); - -generate - if (LIB == "xpm") begin - -assign wr_full_b = ~wr_full; -assign rd_not_empty = ~rd_empty; - -// XPM_FIFO instantiation template for Asynchronous FIFO configurations -// Refer to the targeted device family architecture libraries guide for XPM_FIFO documentation -// ======================================================================================================================= - -// Parameter usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Parameter name | Data type | Restrictions, if applicable | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | CDC_SYNC_STAGES | Integer | Range: 2 - 8. Default value = 2. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the number of synchronization stages on the CDC path | -// | | -// | Must be < 5 if FIFO_WRITE_DEPTH = 16 | -// +---------------------------------------------------------------------------------------------------------------------+ -// | DOUT_RESET_VALUE | String | Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Reset value of read data path. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | ECC_MODE | String | Allowed values: no_ecc, en_ecc. Default value = no_ecc. | -// |---------------------------------------------------------------------------------------------------------------------| -// | | -// | "no_ecc" - Disables ECC | -// | "en_ecc" - Enables both ECC Encoder and Decoder | -// | | -// | NOTE: ECC_MODE should be "no_ecc" if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | FIFO_MEMORY_TYPE | String | Allowed values: auto, block, distributed. Default value = auto. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Designate the fifo memory primitive (resource type) to use. | -// | | -// | "auto"- Allow Vivado Synthesis to choose | -// | "block"- Block RAM FIFO | -// | "distributed"- Distributed RAM FIFO | -// | | -// | NOTE: There may be a behavior mismatch if Block RAM or Ultra RAM specific features, like ECC or Asymmetry, are selected with FIFO_MEMORY_TYPE set to "auto".| -// +---------------------------------------------------------------------------------------------------------------------+ -// | FIFO_READ_LATENCY | Integer | Range: 0 - 10. Default value = 1. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Number of output register stages in the read data path. | -// | | -// | If READ_MODE = "fwft", then the only applicable value is 0. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | FIFO_WRITE_DEPTH | Integer | Range: 16 - 4194304. Default value = 2048. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Defines the FIFO Write Depth, must be power of two. | -// | | -// | In standard READ_MODE, the effective depth = FIFO_WRITE_DEPTH-1 | -// | In First-Word-Fall-Through READ_MODE, the effective depth = FIFO_WRITE_DEPTH+1 | -// | | -// | NOTE: The maximum FIFO size (width x depth) is limited to 150-Megabits. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | FULL_RESET_VALUE | Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Sets full, almost_full and prog_full to FULL_RESET_VALUE during reset | -// +---------------------------------------------------------------------------------------------------------------------+ -// | PROG_EMPTY_THRESH | Integer | Range: 3 - 4194301. Default value = 10. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the minimum number of read words in the FIFO at or below which prog_empty is asserted. | -// | | -// | Min_Value = 3 + (READ_MODE_VAL*2) | -// | Max_Value = (FIFO_WRITE_DEPTH-3) - (READ_MODE_VAL*2) | -// | | -// | If READ_MODE = "std", then READ_MODE_VAL = 0; Otherwise READ_MODE_VAL = 1. | -// | NOTE: The default threshold value is dependent on default FIFO_WRITE_DEPTH value. If FIFO_WRITE_DEPTH value is | -// | changed, ensure the threshold value is within the valid range though the programmable flags are not used. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | PROG_FULL_THRESH | Integer | Range: 5 - 4194301. Default value = 10. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the maximum number of write words in the FIFO at or above which prog_full is asserted. | -// | | -// | Min_Value = 3 + (READ_MODE_VAL*2*(FIFO_WRITE_DEPTH/FIFO_READ_DEPTH))+CDC_SYNC_STAGES | -// | Max_Value = (FIFO_WRITE_DEPTH-3) - (READ_MODE_VAL*2*(FIFO_WRITE_DEPTH/FIFO_READ_DEPTH)) | -// | | -// | If READ_MODE = "std", then READ_MODE_VAL = 0; Otherwise READ_MODE_VAL = 1. | -// | NOTE: The default threshold value is dependent on default FIFO_WRITE_DEPTH value. If FIFO_WRITE_DEPTH value is | -// | changed, ensure the threshold value is within the valid range though the programmable flags are not used. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | RD_DATA_COUNT_WIDTH | Integer | Range: 1 - 23. Default value = 1. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the width of rd_data_count. To reflect the correct value, the width should be log2(FIFO_READ_DEPTH)+1. | -// | | -// | FIFO_READ_DEPTH = FIFO_WRITE_DEPTH*WRITE_DATA_WIDTH/READ_DATA_WIDTH | -// +---------------------------------------------------------------------------------------------------------------------+ -// | READ_DATA_WIDTH | Integer | Range: 1 - 4096. Default value = 32. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Defines the width of the read data port, dout | -// | | -// | Write and read width aspect ratio must be 1:1, 1:2, 1:4, 1:8, 8:1, 4:1 and 2:1 | -// | For example, if WRITE_DATA_WIDTH is 32, then the READ_DATA_WIDTH must be 32, 64,128, 256, 16, 8, 4. | -// | | -// | NOTE: | -// | | -// | READ_DATA_WIDTH should be equal to WRITE_DATA_WIDTH if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior. | -// | The maximum FIFO size (width x depth) is limited to 150-Megabits. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | READ_MODE | String | Allowed values: std, fwft. Default value = std. | -// |---------------------------------------------------------------------------------------------------------------------| -// | | -// | "std"- standard read mode | -// | "fwft"- First-Word-Fall-Through read mode | -// +---------------------------------------------------------------------------------------------------------------------+ -// | RELATED_CLOCKS | Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies if the wr_clk and rd_clk are related having the same source but different clock ratios | -// +---------------------------------------------------------------------------------------------------------------------+ -// | SIM_ASSERT_CHK | Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | 0- Disable simulation message reporting. Messages related to potential misuse will not be reported. | -// | 1- Enable simulation message reporting. Messages related to potential misuse will be reported. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | USE_ADV_FEATURES | String | Default value = 0707. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Enables data_valid, almost_empty, rd_data_count, prog_empty, underflow, wr_ack, almost_full, wr_data_count, | -// | prog_full, overflow features. | -// | | -// | Setting USE_ADV_FEATURES[0] to 1 enables overflow flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[1] to 1 enables prog_full flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[2] to 1 enables wr_data_count; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[3] to 1 enables almost_full flag; Default value of this bit is 0 | -// | Setting USE_ADV_FEATURES[4] to 1 enables wr_ack flag; Default value of this bit is 0 | -// | Setting USE_ADV_FEATURES[8] to 1 enables underflow flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[9] to 1 enables prog_empty flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[10] to 1 enables rd_data_count; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[11] to 1 enables almost_empty flag; Default value of this bit is 0 | -// | Setting USE_ADV_FEATURES[12] to 1 enables data_valid flag; Default value of this bit is 0 | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WAKEUP_TIME | Integer | Range: 0 - 2. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | | -// | 0 - Disable sleep | -// | 2 - Use Sleep Pin | -// | | -// | NOTE: WAKEUP_TIME should be 0 if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WRITE_DATA_WIDTH | Integer | Range: 1 - 4096. Default value = 32. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Defines the width of the write data port, din | -// | | -// | Write and read width aspect ratio must be 1:1, 1:2, 1:4, 1:8, 8:1, 4:1 and 2:1 | -// | For example, if WRITE_DATA_WIDTH is 32, then the READ_DATA_WIDTH must be 32, 64,128, 256, 16, 8, 4. | -// | | -// | NOTE: | -// | | -// | WRITE_DATA_WIDTH should be equal to READ_DATA_WIDTH if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior. | -// | The maximum FIFO size (width x depth) is limited to 150-Megabits. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WR_DATA_COUNT_WIDTH | Integer | Range: 1 - 23. Default value = 1. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the width of wr_data_count. To reflect the correct value, the width should be log2(FIFO_WRITE_DEPTH)+1. | -// +---------------------------------------------------------------------------------------------------------------------+ - -// Port usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Port name | Direction | Size, in bits | Domain | Sense | Handling if unused | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | almost_empty | Output | 1 | rd_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Almost Empty : When asserted, this signal indicates that only one more read can be performed before the FIFO goes to| -// | empty. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | almost_full | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Almost Full: When asserted, this signal indicates that only one more write can be performed before the FIFO is full.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | data_valid | Output | 1 | rd_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Data Valid: When asserted, this signal indicates that valid data is available on the output bus (dout). | -// +---------------------------------------------------------------------------------------------------------------------+ -// | dbiterr | Output | 1 | rd_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Double Bit Error: Indicates that the ECC decoder detected a double-bit error and data in the FIFO core is corrupted.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | din | Input | WRITE_DATA_WIDTH | wr_clk | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Data: The input data bus used when writing the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | dout | Output | READ_DATA_WIDTH | rd_clk | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Data: The output data bus is driven when reading the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | empty | Output | 1 | rd_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Empty Flag: When asserted, this signal indicates that the FIFO is empty. | -// | Read requests are ignored when the FIFO is empty, initiating a read while empty is not destructive to the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | full | Output | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Full Flag: When asserted, this signal indicates that the FIFO is full. | -// | Write requests are ignored when the FIFO is full, initiating a write when the FIFO is full is not destructive | -// | to the contents of the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | injectdbiterr | Input | 1 | wr_clk | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Double Bit Error Injection: Injects a double bit error if the ECC feature is used on block RAMs or | -// | UltraRAM macros. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | injectsbiterr | Input | 1 | wr_clk | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Single Bit Error Injection: Injects a single bit error if the ECC feature is used on block RAMs or | -// | UltraRAM macros. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | overflow | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Overflow: This signal indicates that a write request (wren) during the prior clock cycle was rejected, | -// | because the FIFO is full. Overflowing the FIFO is not destructive to the contents of the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | prog_empty | Output | 1 | rd_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Programmable Empty: This signal is asserted when the number of words in the FIFO is less than or equal | -// | to the programmable empty threshold value. | -// | It is de-asserted when the number of words in the FIFO exceeds the programmable empty threshold value. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | prog_full | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Programmable Full: This signal is asserted when the number of words in the FIFO is greater than or equal | -// | to the programmable full threshold value. | -// | It is de-asserted when the number of words in the FIFO is less than the programmable full threshold value. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rd_clk | Input | 1 | NA | Rising edge | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read clock: Used for read operation. rd_clk must be a free running clock. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rd_data_count | Output | RD_DATA_COUNT_WIDTH | rd_clk | NA | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Data Count: This bus indicates the number of words read from the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rd_en | Input | 1 | rd_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Enable: If the FIFO is not empty, asserting this signal causes data (on dout) to be read from the FIFO. | -// | | -// | Must be held active-low when rd_rst_busy is active high. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rd_rst_busy | Output | 1 | rd_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Reset Busy: Active-High indicator that the FIFO read domain is currently in a reset state. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rst | Input | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Reset: Must be synchronous to wr_clk. The clock(s) can be unstable at the time of applying reset, but reset must be released only after the clock(s) is/are stable.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | sbiterr | Output | 1 | rd_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Single Bit Error: Indicates that the ECC decoder detected and fixed a single-bit error. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | sleep | Input | 1 | NA | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Dynamic power saving: If sleep is High, the memory/fifo block is in power saving mode. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | underflow | Output | 1 | rd_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Underflow: Indicates that the read request (rd_en) during the previous clock cycle was rejected | -// | because the FIFO is empty. Under flowing the FIFO is not destructive to the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_ack | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Acknowledge: This signal indicates that a write request (wr_en) during the prior clock cycle is succeeded. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_clk | Input | 1 | NA | Rising edge | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write clock: Used for write operation. wr_clk must be a free running clock. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_data_count | Output | WR_DATA_COUNT_WIDTH | wr_clk | NA | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Data Count: This bus indicates the number of words written into the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_en | Input | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Enable: If the FIFO is not full, asserting this signal causes data (on din) to be written to the FIFO. | -// | | -// | Must be held active-low when rst or wr_rst_busy is active high. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_rst_busy | Output | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Reset Busy: Active-High indicator that the FIFO write domain is currently in a reset state. | -// +---------------------------------------------------------------------------------------------------------------------+ - - -// xpm_fifo_async : In order to incorporate this function into the design, -// Verilog : the following instance declaration needs to be placed -// instance : in the body of the design code. The instance name -// declaration : (xpm_fifo_async_inst) and/or the port declarations within the -// code : parenthesis may be changed to properly reference and -// : connect this function to the design. All inputs -// : and outputs must be connected. - -// Please reference the appropriate libraries guide for additional information on the XPM modules. - -// <-----Cut code below this line----> - - // xpm_fifo_async: Asynchronous FIFO - // Xilinx Parameterized Macro, version 2019.1 - - xpm_fifo_async #( - .CDC_SYNC_STAGES(2), // DECIMAL - .DOUT_RESET_VALUE("0"), // String - .ECC_MODE("no_ecc"), // String - .FIFO_MEMORY_TYPE("auto"), // String - .FIFO_READ_LATENCY(0), // DECIMAL - .FIFO_WRITE_DEPTH(DEPTH), // DECIMAL - .FULL_RESET_VALUE(0), // DECIMAL - .PROG_EMPTY_THRESH(10), // DECIMAL - .PROG_FULL_THRESH(10), // DECIMAL - .RD_DATA_COUNT_WIDTH(1+D_L), // DECIMAL - .READ_DATA_WIDTH(WIDTH), // DECIMAL - .READ_MODE("fwft"), // String - .RELATED_CLOCKS(0), // DECIMAL - .SIM_ASSERT_CHK(0), // DECIMAL; 0=disable simulation messages, 1=enable simulation messages - .USE_ADV_FEATURES("0707"), // String - .WAKEUP_TIME(0), // DECIMAL - .WRITE_DATA_WIDTH(WIDTH), // DECIMAL - .WR_DATA_COUNT_WIDTH(1+D_L) // DECIMAL - ) - xpm_fifo_async_inst ( - .almost_empty(), // 1-bit output: Almost Empty : When asserted, this signal indicates that - // only one more read can be performed before the FIFO goes to empty. - - .almost_full(), // 1-bit output: Almost Full: When asserted, this signal indicates that - // only one more write can be performed before the FIFO is full. - - .data_valid(), // 1-bit output: Read Data Valid: When asserted, this signal indicates - // that valid data is available on the output bus (dout). - - .dbiterr(), // 1-bit output: Double Bit Error: Indicates that the ECC decoder detected - // a double-bit error and data in the FIFO core is corrupted. - - .dout(rd_data), // READ_DATA_WIDTH-bit output: Read Data: The output data bus is driven - // when reading the FIFO. - - .empty(rd_empty), // 1-bit output: Empty Flag: When asserted, this signal indicates that the - // FIFO is empty. Read requests are ignored when the FIFO is empty, - // initiating a read while empty is not destructive to the FIFO. - - .full(wr_full), // 1-bit output: Full Flag: When asserted, this signal indicates that the - // FIFO is full. Write requests are ignored when the FIFO is full, - // initiating a write when the FIFO is full is not destructive to the - // contents of the FIFO. - - .overflow(), // 1-bit output: Overflow: This signal indicates that a write request - // (wren) during the prior clock cycle was rejected, because the FIFO is - // full. Overflowing the FIFO is not destructive to the contents of the - // FIFO. - - .prog_empty(), // 1-bit output: Programmable Empty: This signal is asserted when the - // number of words in the FIFO is less than or equal to the programmable - // empty threshold value. It is de-asserted when the number of words in - // the FIFO exceeds the programmable empty threshold value. - - .prog_full(), // 1-bit output: Programmable Full: This signal is asserted when the - // number of words in the FIFO is greater than or equal to the - // programmable full threshold value. It is de-asserted when the number of - // words in the FIFO is less than the programmable full threshold value. - - .rd_data_count(rd_count), // RD_DATA_COUNT_WIDTH-bit output: Read Data Count: This bus indicates the - // number of words read from the FIFO. - - .rd_rst_busy(), // 1-bit output: Read Reset Busy: Active-High indicator that the FIFO read - // domain is currently in a reset state. - - .sbiterr(), // 1-bit output: Single Bit Error: Indicates that the ECC decoder detected - // and fixed a single-bit error. - - .underflow(), // 1-bit output: Underflow: Indicates that the read request (rd_en) during - // the previous clock cycle was rejected because the FIFO is empty. Under - // flowing the FIFO is not destructive to the FIFO. - - .wr_ack(), // 1-bit output: Write Acknowledge: This signal indicates that a write - // request (wr_en) during the prior clock cycle is succeeded. - - .wr_data_count(wr_count), // WR_DATA_COUNT_WIDTH-bit output: Write Data Count: This bus indicates - // the number of words written into the FIFO. - - .wr_rst_busy(), // 1-bit output: Write Reset Busy: Active-High indicator that the FIFO - // write domain is currently in a reset state. - - .din(wr_data), // WRITE_DATA_WIDTH-bit input: Write Data: The input data bus used when - // writing the FIFO. - - .injectdbiterr(1'b0), // 1-bit input: Double Bit Error Injection: Injects a double bit error if - // the ECC feature is used on block RAMs or UltraRAM macros. - - .injectsbiterr(1'b0), // 1-bit input: Single Bit Error Injection: Injects a single bit error if - // the ECC feature is used on block RAMs or UltraRAM macros. - - .rd_clk(rd_clk), // 1-bit input: Read clock: Used for read operation. rd_clk must be a free - // running clock. - - .rd_en(rd_req), // 1-bit input: Read Enable: If the FIFO is not empty, asserting this - // signal causes data (on dout) to be read from the FIFO. Must be held - // active-low when rd_rst_busy is active high. - - .rst(aclr), // 1-bit input: Reset: Must be synchronous to wr_clk. The clock(s) can be - // unstable at the time of applying reset, but reset must be released only - // after the clock(s) is/are stable. - - .sleep(1'b0), // 1-bit input: Dynamic power saving: If sleep is High, the memory/fifo - // block is in power saving mode. - - .wr_clk(wr_clk), // 1-bit input: Write clock: Used for write operation. wr_clk must be a - // free running clock. - - .wr_en(wr_req) // 1-bit input: Write Enable: If the FIFO is not full, asserting this - // signal causes data (on din) to be written to the FIFO. Must be held - // active-low when rst or wr_rst_busy is active high. - - ); - - // End of xpm_fifo_async_inst instantiation - - -end else if (LIB == "exablaze") begin -assign wr_full_b = ~wr_full; -assign rd_empty = ~rd_not_empty; -assign rd_count = '0; - -async_fifo #( - .WIDTH(WIDTH) -) async_fifo_inst ( - .clk_write (wr_clk), - .data_in (wr_data), - .wren (wr_req), - .full (wr_full), - - .clk_read (rd_clk), - .data_out (rd_data), - .vld (rd_not_empty), - .rden (rd_req), - .almost_empty () -); -end -endgenerate - - - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_add_modp.sv b/src/wiredancer/rtl/ed25519_add_modp.sv deleted file mode 100644 index 5beaed1633c..00000000000 --- a/src/wiredancer/rtl/ed25519_add_modp.sv +++ /dev/null @@ -1,61 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module ed25519_add_modp #( - W = 255, - M=128 -)( - input wire clk, - input wire rst, - - input wire [W-1:0] in0, - input wire [W-1:0] in1, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - output logic [W-1:0] out0 -); - -logic [M-1:0] m_o_p; -logic [W+1-1:0] c_2_AB; -logic [1-1:0] c_2_AB_ge_p; -logic [1-1:0] out0_; - -assign c_2_AB_ge_p = c_2_AB >= ED25519_P; - -piped_adder #( - .W(W+1), - .R(0), - .C(1), - .M(M) -) c0_addmodp_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0({1'b0, in0}), - .in1({1'b0, in1}), - .out0(c_2_AB), - .cout0(), - .m_i(m_i), - .m_o(m_o_p) -); -piped_adder #( - .W(W+1), - .R(0), - .C(1), - .M(M) -) c2_addmodp_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_2_AB), - .in1(c_2_AB_ge_p ? ED25519_P_N : {(W+1){1'b0}}), - .out0({out0_, out0}), - .cout0(), - .m_i(m_o_p), - .m_o(m_o) -); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_mul_modp.sv b/src/wiredancer/rtl/ed25519_mul_modp.sv deleted file mode 100644 index 1bcca2bad29..00000000000 --- a/src/wiredancer/rtl/ed25519_mul_modp.sv +++ /dev/null @@ -1,255 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module ed25519_mul_modp #( - T=32'h007F_CCC2, - CT = T[0 +: 4], - ST = T >> 4, - R_I=0, - M=128 -)( - input wire clk, - input wire rst, - - input wire [255-1:0] in0, - input wire [255-1:0] in1, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - output logic [255-1:0] out0 -); - -logic [255-1:0] in0_r; -logic [255-1:0] in1_r; -logic [M-1:0] m_i_r; - -generate; - if (R_I) begin - always_ff@(posedge clk) in0_r <= in0; - always_ff@(posedge clk) in1_r <= in1; - always_ff@(posedge clk) m_i_r <= m_i; - end else begin - assign in0_r = in0; - assign in1_r = in1; - assign m_i_r = m_i; - end - - if (CT == 2) begin - -localparam A0_R0 = 0; -localparam A0_R1 = 0; -localparam A0_R = 1; -localparam A0_C = 0; - -localparam A1_R0 = 0; -localparam A1_R1 = 0; -localparam A1_R = 1; -localparam A1_C = 0; - -localparam A2_R0 = 0; -localparam A2_R1 = 0; -localparam A2_R = 1; -localparam A2_C = 1; - -localparam A3_R0 = 0; -localparam A3_R1 = 0; -localparam A3_R = 1; -localparam A3_C = 1; - -localparam A4_R0 = 0; -localparam A4_R1 = 0; -localparam A4_R = 1; -localparam A4_C = 1; - -logic [128-1:0] c_0_x0, c_0_y0; -logic [127-1:0] c_0_x1, c_0_y1; - -logic [129-1:0] c_a_A; -logic [129-1:0] c_a_B; - -logic [256-1:0] c_m_C; -logic [254-1:0] c_m_D; - -logic [256-1:0] c_ma_C; -logic [254-1:0] c_ma_D; -logic [258-1:0] c_am_M; -logic [260-1:0] c_ma_N; - -logic [255+132-1:0] c_maa_E; -logic [255-1:0] c_maa_El; -logic [132-1:0] c_maa_Eh; -logic [256-1:0] c_maab_F; -logic [256-1:0] c_maabc_G; - -logic [5-1:0][M-1:0] m_o_p; - -piped_adder #( - .W(128), - .R(A0_R), - .C(A0_C), - .M(1) -) aK_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_0_x0), - .in1({1'b0, c_0_x1}), - .out0(c_a_A[0+:128]), - .cout0(c_a_A[128]), - .m_i(), - .m_o() -); -piped_adder #( - .W(128), - .R(A0_R), - .C(A0_C), - .M(1) -) aL_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_0_y0), - .in1({1'b0, c_0_y1}), - .out0(c_a_B[0+:128]), - .cout0(c_a_B[128]), - .m_i(), - .m_o() -); - -mul_wide #(.W(128), .T(ST), .M(M)) m0_inst (.clk(clk), .rst(rst), .in0(c_0_x0), .in1(c_0_y0), .out0(c_m_C ), .m_i(m_i), .m_o(m_o_p[0])); -mul_wide #(.W(127), .T(ST)) m1_inst (.clk(clk), .rst(rst), .in0(c_0_x1), .in1(c_0_y1), .out0(c_m_D ), .m_i( '0), .m_o() ); -mul_wide #(.W(129), .T(ST)) m2_inst (.clk(clk), .rst(rst), .in0( c_a_A), .in1( c_a_B), .out0(c_am_M), .m_i( '0), .m_o() ); - -shift_adder_3 #( - .W (260), - .S0 (1), - .S1 (2), - .S2 (5), - .M ($bits({c_m_D, c_m_C, m_o_p[0]})), - .R0 (A1_R0), - .R (A1_R), - .C (A1_C) -) a_N_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0({6'h0, c_m_D}), - .in1({6'h0, c_m_D}), - .in2({6'h0, c_m_D}), - .out0(c_ma_N), - .cout0(), - .m_i({c_m_D, c_m_C, m_o_p[0]}), - .m_o({c_ma_D, c_ma_C, m_o_p[1]}) -); - -shift_adder_6 #( - .W (255+132), - .S0 (0), - .S1 (0), - .S2 (128), - .S3 (128), - .S4 (128), - .S5 (128), - .M (M), - .R0 (A2_R0), - .R1 (A2_R1), - .R (A2_R), - .C (A2_C) -) a0_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0({140'h0, c_ma_C}), - .in1({140'h0, c_ma_N}), - .in2({140'h0, c_am_M}), - .in3({{400{1'b1}}, ~c_ma_C}), - .in4({{400{1'b1}}, ~c_ma_D}), - .in5({400'h2}), - .out0(c_maa_E), - .cout0(), - .m_i(m_o_p[1]), - .m_o(m_o_p[2]) -); - -shift_adder_6 #( - .W (256), - .S0 (0), - .S1 (0), - .S2 (1), - .S3 (4), - .S4 (0), - .S5 (0), - .M (M), - .R0 (A3_R0), - .R1 (A3_R1), - .R (A3_R), - .C (A3_C) -) a1_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0({ 1'h0, c_maa_El}), - .in1({124'h0, c_maa_Eh}), - .in2({124'h0, c_maa_Eh}), - .in3({124'h0, c_maa_Eh}), - .in4({256'h0}), - .in5({256'h0}), - .out0(c_maab_F), - .cout0(), - .m_i(m_o_p[2]), - .m_o(m_o_p[3]) -); - -piped_adder #( - .W(256), - .M(M), - .R(A4_R), - .C(A4_C) -) a3_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_maab_F), - .in1(c_maab_F >= ED25519_P ? ED25519_P_N : 256'h0), - .out0(c_maabc_G), - .cout0(), - .m_i(m_o_p[3]), - .m_o(m_o_p[4]) -); - -always_comb begin - c_0_x0 = in0_r[ 0+:128]; - c_0_x1 = in0_r[128+:127]; - c_0_y0 = in1_r[ 0+:128]; - c_0_y1 = in1_r[128+:127]; - - c_maa_El = c_maa_E[ 0+:255]; - c_maa_Eh = c_maa_E[255+:132]; -end - -assign out0 = c_maabc_G; -assign m_o = m_o_p[4]; - -// always_ff@(posedge clk) $display("%t: c_a_A: %x", $time, c_a_A); -// always_ff@(posedge clk) $display("%t: c_a_B: %x", $time, c_a_B); - -// always_ff@(posedge clk) $display("%t: c_m_C: %x", $time, c_m_C); -// always_ff@(posedge clk) $display("%t: c_m_D: %x", $time, c_m_D); - -// always_ff@(posedge clk) $display("%t: c_ma_C: %x", $time, c_ma_C); -// always_ff@(posedge clk) $display("%t: c_ma_D: %x", $time, c_ma_D); -// always_ff@(posedge clk) $display("%t: c_am_M: %x", $time, c_am_M); -// always_ff@(posedge clk) $display("%t: c_ma_N: %x", $time, c_ma_N); -// always_ff@(posedge clk) $display("%t: c_maa_E: %x", $time, c_maa_E); - - end // T2 - -endgenerate - -// always_ff@(posedge clk) $display("%t: %m: %x x %x = %x", $time, in0, in1, out0); - - -endmodule - - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_point_add.sv b/src/wiredancer/rtl/ed25519_point_add.sv deleted file mode 100644 index a5e0baeb9fe..00000000000 --- a/src/wiredancer/rtl/ed25519_point_add.sv +++ /dev/null @@ -1,97 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -`define ADD(___, _, __) ed25519_add_modp #(.W(255)) add_modp_``___`` (.clk(clk), .rst(rst),.in0(_),.in1(__),.out0(___),.m_i('0),.m_o()); -`define SUB(___, _, __) ed25519_sub_modp #(.W(255)) sub_modp_``___`` (.clk(clk), .rst(rst),.in0(_),.in1(__),.out0(___),.m_i('0),.m_o()); -`define MUL(___, _, __) ed25519_mul_modp #(.T(T)) mul_modp_``___`` (.clk(clk), .rst(rst),.in0(_),.in1(__),.out0(___),.m_i('0),.m_o()); -`define PIP(___, _, __, _W) piped_wire #(.WIDTH(_W),.DEPTH(__)) wide_pip_``___`` (.clk(clk),.reset(rst), .in(_), .out(___) ); - -(* keep_hierarchy = "yes" *) module ed25519_point_add #( - T=32'h007F_CCC2, - D_M = 15, - D_A = 4, - D_S = 2, - CT = T[0 +: 4], - ST = T >> 4, - R_I=0, - M=128 -)( - input wire clk, - input wire rst, - - input wire [255-1:0] in0_x, - input wire [255-1:0] in0_y, - input wire [255-1:0] in0_z, - input wire [255-1:0] in0_t, - input wire [255-1:0] in1_x, - input wire [255-1:0] in1_y, - input wire [255-1:0] in1_z, - input wire [255-1:0] in1_t, - output logic [255-1:0] out0_x, - output logic [255-1:0] out0_y, - output logic [255-1:0] out0_z, - output logic [255-1:0] out0_t, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o -); - -logic [255-1:0] R1_s; -logic [255-1:0] R2_s; -logic [255-1:0] R3_a; -logic [255-1:0] R4_a; -logic [255-1:0] R5_sm; -logic [255-1:0] R6_am; -logic [255-1:0] R7_m; -logic [255-1:0] R8_m; -logic [255-1:0] R9_mm; -logic [255-1:0] R5_am; -logic [255-1:0] R10_ma; -logic [255-1:0] R11_ams; -logic [255-1:0] R12_mms; -logic [255-1:0] R13_mma; -logic [255-1:0] R14_ama; -logic [255-1:0] R10_mm; -logic [255-1:0] R11_mma; -logic [255-1:0] R12_mma; -logic [255-1:0] R14_mma; - -`SUB(R1_s, in0_y, in0_x) -`SUB(R2_s, in1_y, in1_x) -`ADD(R3_a, in0_y, in0_x) -`ADD(R4_a, in1_y, in1_x) - -`MUL(R5_sm, R1_s, R2_s) -`MUL(R6_am, R3_a, R4_a) -`MUL(R7_m, in0_t, in1_t) -`MUL(R8_m, in0_z, in1_z) -`MUL(R9_mm, ED25519_2D, R7_m) - -`PIP(R5_am, R5_sm, (D_A+D_M)-(D_S+D_M), 255) - -`ADD(R10_ma, R8_m, R8_m) -`SUB(R11_ams, R6_am, R5_am) -`SUB(R12_mms, R10_mm, R9_mm) -`ADD(R13_mma, R10_mm, R9_mm) -`ADD(R14_ama, R6_am, R5_am) - -`PIP(R10_mm, R10_ma, D_M-D_A, 255) -`PIP(R11_mma, R11_ams, (D_M+D_M+D_A) - (D_A+D_M+D_S), 255) -`PIP(R12_mma, R12_mms, (D_M+D_M+D_A) - (D_M+D_M+D_S), 255) -`PIP(R14_mma, R14_ama, D_M-D_A, 255) - -`MUL(out0_x, R11_mma, R12_mma) -`MUL(out0_y, R13_mma, R14_mma) -`MUL(out0_t, R11_mma, R14_mma) -`MUL(out0_z, R12_mma, R13_mma) - -`PIP(m_o, m_i, D_M+D_M+D_A+D_M, M); - -endmodule - -`undef ADD -`undef SUB -`undef MUL -`undef PIP - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_point_dbl.sv b/src/wiredancer/rtl/ed25519_point_dbl.sv deleted file mode 100644 index b3c472b7fc9..00000000000 --- a/src/wiredancer/rtl/ed25519_point_dbl.sv +++ /dev/null @@ -1,91 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -`define ADD(___, _, __) ed25519_add_modp #(.W(255)) add_modp_``___`` (.clk(clk), .rst(rst),.in0(_),.in1(__),.out0(___),.m_i('0),.m_o()); -`define SUB(___, _, __) ed25519_sub_modp #(.W(255)) sub_modp_``___`` (.clk(clk), .rst(rst),.in0(_),.in1(__),.out0(___),.m_i('0),.m_o()); -`define MUL(___, _, __) ed25519_mul_modp #(.T(T)) mul_modp_``___`` (.clk(clk), .rst(rst),.in0(_),.in1(__),.out0(___),.m_i('0),.m_o()); -`define PIP(___, _, __, _W) piped_wire #(.WIDTH(_W),.DEPTH(__)) wide_pip_``___`` (.clk(clk),.reset(rst), .in(_), .out(___) ); - -(* keep_hierarchy = "yes" *) module ed25519_point_dbl #( - T=32'h007F_CCC2, - D_M = 15, - D_A = 4, - D_S = 2, - CT = T[0 +: 4], - ST = T >> 4, - R_I=0, - M=128 -)( - input wire clk, - input wire rst, - - input wire [255-1:0] in0_x, - input wire [255-1:0] in0_y, - input wire [255-1:0] in0_z, - input wire [255-1:0] in0_t, - output logic [255-1:0] out0_x, - output logic [255-1:0] out0_y, - output logic [255-1:0] out0_z, - output logic [255-1:0] out0_t, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o -); - -logic [255-1:0] R1_s; -logic [255-1:0] R2_s; -logic [255-1:0] R3_a; -logic [255-1:0] R4_a; -logic [255-1:0] R5_sm; -logic [255-1:0] R6_am; -logic [255-1:0] R7_m; -logic [255-1:0] R8_m; -logic [255-1:0] R9_mm; -logic [255-1:0] R5_am; -logic [255-1:0] R10_ma; -logic [255-1:0] R11_ams; -logic [255-1:0] R12_mms; -logic [255-1:0] R13_mma; -logic [255-1:0] R14_ama; -logic [255-1:0] R10_mm; -logic [255-1:0] R11_mma; -logic [255-1:0] R12_mma; -logic [255-1:0] R14_mma; - -`SUB(R1_s, in0_y, in0_x) -`ADD(R3_a, in0_y, in0_x) - -`MUL(R5_sm, R1_s, R1_s) -`MUL(R6_am, R3_a, R3_a) -`MUL(R7_m, in0_t, in0_t) -`MUL(R8_m, in0_z, in0_z) -`MUL(R9_mm, ED25519_2D, R7_m) - -`PIP(R5_am, R5_sm, (D_A+D_M)-(D_S+D_M), 255) - -`ADD(R10_ma, R8_m, R8_m) -`SUB(R11_ams, R6_am, R5_am) -`SUB(R12_mms, R10_mm, R9_mm) -`ADD(R13_mma, R10_mm, R9_mm) -`ADD(R14_ama, R6_am, R5_am) - -`PIP(R10_mm, R10_ma, D_M-D_A, 255) -`PIP(R11_mma, R11_ams, (D_M+D_M+D_A) - (D_A+D_M+D_S), 255) -`PIP(R12_mma, R12_mms, (D_M+D_M+D_A) - (D_M+D_M+D_S), 255) -`PIP(R14_mma, R14_ama, D_M-D_A, 255) - -`MUL(out0_x, R11_mma, R12_mma) -`MUL(out0_y, R13_mma, R14_mma) -`MUL(out0_t, R11_mma, R14_mma) -`MUL(out0_z, R12_mma, R13_mma) - -`PIP(m_o, m_i, D_M+D_M+D_A+D_M, M); - -endmodule - -`undef ADD -`undef SUB -`undef MUL -`undef PIP - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_sigverify_0.sv b/src/wiredancer/rtl/ed25519_sigverify_0.sv deleted file mode 100644 index 21c5d830a57..00000000000 --- a/src/wiredancer/rtl/ed25519_sigverify_0.sv +++ /dev/null @@ -1,309 +0,0 @@ - -/* - - --+ +-----------+ +------+ +-- - / +---> Processor +---+ FIFO +---> \ - | | +-----------+ +------+ | | - | | | | - +-------+ | | +-----------+ +------+ | | +--------+ ------> FIFO +---+---> +---> Processor +---+ FIFO +---> +---+-----> FIFO +-------+---> - +-------+ | | | +-----------+ +------+ | | | +--------+ | - | | | | | | | - | | | +-----------+ +------+ | | | +--------------+ | - | \ +---> Processor +---+ FIFO +---> / +---> Reassemble +---+ - | --+ +-----------+ +------+ +-- +--------------+ | - | | - | | - | +------+ | - +-----------------------------> Meta +--------------------------------+ - +------+ - - Dispatch and merge of sigverify-split-0 jobs to processors. - - Dispatch serializes the three inputs of each invocation - into the same processor. Job distribution logic is a - simple round-robin policy. Pointer moves when a job - is completely taken by a processor. - - Merge is a simple multiplexer with a round robin pocliy for - selection. As multipler processors can produce results at - the same time, each requires a separate fifo to accept - backrepssure from the multiplexer. - - Each transaction has multiple (eight) results. As the - results of a transaction will be produced in the span tens of - cycles, reassembly is required to provide all results of a single - transaction to the output in a single cycle. The final result - is stored in a fifo to support backpressure. - - Metadata is stored in a key-storage to avoid sending - wide metadata into the processors. - -*/ - -`default_nettype none - -import wd_sigverify::*; - -module ed25519_sigverify_0 #( - MUL_T = 32'h007F_CCC2, - MUL_D = 15, - N_SCH = 2, - KEY_D = 512, - KEY_D_L = $clog2(KEY_D) -) ( - output logic [1-1:0] i_r, // backpressure - input wire [1-1:0] i_w, // wait - input wire [1-1:0] i_v, - input wire [$bits(sv_meta4_t)-1:0] i_m, - - output logic [1-1:0] o_v, - output logic [$bits(sv_meta5_t)-1:0] o_m, - - input wire clk, - input wire rst -); - -logic [KEY_D_L-1:0] i_k; -sv_meta4_t i_mm; -sv_meta5_t o_mm; - -logic [1-1:0] key_i_r; -logic [1-1:0] key_i_v; - -logic [$clog2(N_SCH):0] sch_i_rrb; // round robin index -logic [N_SCH-1:0] sch_i_r; // individual ready signals from schedulers - -logic [1-1:0] sch_o_r [N_SCH-1:0]; -logic [1-1:0] sch_o_v [N_SCH-1:0]; -logic [6-1:0] sch_o_a [N_SCH-1:0]; -logic [KEY_D_L-1:0] sch_o_k [N_SCH-1:0]; -logic [256-1:0] sch_o_d [N_SCH-1:0]; -logic [$clog2(N_SCH):0] sch_o_rrb; - -logic [1-1:0] sch_m_v; -logic [6-1:0] sch_m_a; -logic [KEY_D_L-1:0] sch_m_k; -logic [256-1:0] sch_m_d; - -logic [1-1:0] sch_a_v; - -assign i_r = |sch_i_r; -assign i_mm = i_m; -assign o_m = o_mm; - -assign key_i_v = |sch_i_r; - -key_store #( - .D (KEY_D), - .W ($bits({i_mm})) -) keystore_inst ( - .i_r (key_i_r), - .i_v (key_i_v), - .i_k (i_k), - .i_d ({i_mm}), - - .o_r (sch_m_v & (sch_m_a == (N_SCH_O-1))), - .o_k (sch_m_k), - .o_d ({o_mm.m}), - - .clk (clk), - .rst (rst) -); - -always_ff@(posedge clk) begin - if (i_r) - sch_i_rrb <= (sch_i_rrb == N_SCH-1) ? 0 : sch_i_rrb + 1; - if (rst) - sch_i_rrb <= 0; -end - -generate; - for (genvar g_i = 0; g_i < N_SCH; g_i ++) begin: G_SCH - - logic [1-1:0] rst_r; - - logic [3-1:0] sch_ii_st; - logic [1-1:0] sch_ii_r; - logic [1-1:0] sch_ii_v; - logic [KEY_D_L-1:0] sch_ii_k; - logic [256-1:0] sch_ii_d; - - logic [1-1:0] sch_oo_v; - logic [KEY_D_L-1:0] sch_oo_k; - logic [6-1:0] sch_oo_a; - logic [256-1:0] sch_oo_d; - - // serialize scheduler inputs - always_ff@(posedge clk) begin - case (sch_ii_st) - 0: begin - if (i_v & (~i_w) & key_i_r & (sch_i_rrb == g_i)) begin - sch_ii_v <= 1; - sch_ii_k <= i_k; - sch_ii_d <= i_mm.m.pub; - sch_ii_st <= 1; - end - end - 1: begin - if (sch_ii_r) begin - sch_ii_d <= i_mm.m.sig_l; - sch_ii_st <= 2; - end - end - 2: begin - if (sch_ii_r) begin - sch_i_r[g_i] <= 1; - sch_ii_d <= i_mm.m.sig_h; - sch_ii_st <= 3; - end - end - 3: begin - sch_i_r[g_i] <= 0; - if (sch_ii_r) begin - sch_ii_v <= 0; - sch_ii_st <= 0; - end - end - endcase - - if (rst) begin - sch_ii_st <= 0; - sch_ii_v <= 0; - sch_i_r[g_i] <= 0; - end - end - - (* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({rst})), - .DEPTH (2) - ) rst_pipe_inst ( - .in ({rst}), - .out ({rst_r}), - - .clk (clk), - .reset (rst) - ); - - shcl_cpu #( - .MUL_T (MUL_T), - .MUL_D (MUL_D), - .W_HASH (256), - .W_IN_MEM (6), - .W_T ($bits(sch_ii_k)) - ) schl_cpu_inst ( - .in_hash_ready (sch_ii_r), - .in_hash_valid (sch_ii_v), - .in_hash_ref (sch_ii_k), - .in_hash_data (sch_ii_d), - - .out_hash_valid (sch_oo_v), - .out_ref (sch_oo_k), - .out_d_addr (sch_oo_a), - .out_hash_data (sch_oo_d), - - .clk (clk), - .rst (rst_r) - ); - - showahead_fifo #( - .WIDTH ($bits({sch_oo_d, sch_oo_a, sch_oo_k})), - .DEPTH (512) - ) sch_o_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (sch_oo_v), - .wr_full (), - .wr_full_b (), - .wr_count (), - .wr_data ({sch_oo_d, sch_oo_a, sch_oo_k}), - - .rd_clk (clk), - .rd_req (sch_o_v[g_i] & sch_o_r[g_i]), - .rd_empty (), - .rd_not_empty (sch_o_v[g_i]), - .rd_count (), - .rd_data ({sch_o_d[g_i], sch_o_a[g_i], sch_o_k[g_i]}) - ); - - assign sch_o_r[g_i] = (sch_o_rrb == g_i); - - always_ff@(posedge clk) if (sch_oo_v | sch_ii_v) $display("%t: %m: %x %x %x %x - %x %x %x %x - %x %x", $time - , sch_ii_r - , sch_ii_v - , sch_ii_k - , sch_ii_d[0+:64] - - , sch_oo_v - , sch_oo_a - , sch_oo_k - , sch_oo_d[0+:64] - - , sch_o_v [g_i] - , sch_o_k [g_i] - ); - end -endgenerate - -// merge scheduler outputs -always_ff@(posedge clk) begin - sch_o_rrb <= (sch_o_rrb == N_SCH-1) ? 0 : sch_o_rrb + 1; - if (rst) - sch_o_rrb <= 0; - - sch_m_v <= sch_o_v[sch_o_rrb]; - sch_m_a <= sch_o_a[sch_o_rrb]; - sch_m_k <= sch_o_k[sch_o_rrb]; - sch_m_d <= sch_o_d[sch_o_rrb]; -end - -// reassemble sch outputs, no need to store the last output -generate - for (genvar g_i = 0; g_i < N_SCH_O-1; g_i ++) begin: G_O - simple_dual_port_ram #( - .WRITE_MODE ("read_first"), - .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH (KEY_D_L), - .DATA_WIDTH (256) - ) reassemble_ram_inst ( - .wr_clock (clk), - .wr_en (sch_m_v & (sch_m_a == g_i)), - .wr_byteenable ('1), - .wr_address (sch_m_k), - .data (sch_m_d), - - .rd_clock (clk), - .rd_address (sch_m_k), - .q (o_mm.os[g_i]), - .rd_en (1'b1) - ); - end -endgenerate - -always_ff@(posedge clk) begin - o_v <= sch_m_v & (sch_m_a == (N_SCH_O-1)); - o_mm.os[N_SCH_O-1] <= sch_m_d; - if (rst) - o_v <= 0; -end - -always_ff@(posedge clk) if ((i_v & i_r) | sch_m_v | o_v) -$display("%t: %m: %x %x %x %x - %x %x %x %x - %x %x", $time - , i_v - , i_w - , sch_i_rrb - , sch_i_r - - , sch_m_v - , sch_m_a - , sch_m_k - , sch_m_d - - , o_v - , o_m -); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_sigverify_1.sv b/src/wiredancer/rtl/ed25519_sigverify_1.sv deleted file mode 100644 index 9892b523974..00000000000 --- a/src/wiredancer/rtl/ed25519_sigverify_1.sv +++ /dev/null @@ -1,139 +0,0 @@ -/* - - +-----------------------------------------------------------+ - | SLR | - | | - | +----------+ +-------------------------+ | +----------+ - ------|R|-------|R|---> CDC-FIFO |--> DSDP |---|R|-------|R|---> CDC-FIFO |-------> - | | +----------+ +-------------------------+ | +----------+ ^ - | | | | - | +-----------------------------------------------------------+ | - | | - | | - | +------+ | - +------------------------------------------> Meta +--------------------------------------------+ - +------+ - - DSDP runs at a marginally higher clock rate to keep up with the previous - stage throughput. - - DSDP is placed in a separate SLR to assist with placement and routing. - To further assist with timing closure, two sets of registers are used - before and after SLR corssings. - - Metadata is stored in a key-storage to avoid sending wide metadata - into DSDP pipeline. - -*/ - - -`default_nettype none - -import wd_sigverify::*; - -module ed25519_sigverify_1 #( - // parameter logic [32-1:0] MUL_T = 32'h0000_00802, // 8-cycle mock mul_wide - parameter logic [32-1:0] MUL_T = 32'h007F_CCC2, - parameter integer MUL_D = 15, - parameter integer DSDP_WS = 2, - parameter integer KEY_D = 512, - parameter integer KEY_D_L = $clog2(KEY_D) -) ( - output logic [1-1:0] i_r, - input wire [1-1:0] i_w, - input wire [1-1:0] i_v, - input wire [$bits(sv_meta5_t)-1:0] i_m, - - output logic [1-1:0] o_v, - output logic [$bits(sv_meta6_t)-1:0] o_m, - - input wire clk, - input wire rst -); - -logic [KEY_D_L-1:0] i_k; -logic [2-1:0] i_rr; -sv_meta5_t i_mm; -sv_meta6_t o_mm; - -logic [1-1:0] dsdp_o_v; -logic [KEY_D_L-1:0] dsdp_o_k; -logic [255-1:0] dsdp_o_Zx; -logic [255-1:0] dsdp_o_Zy; -logic [255-1:0] dsdp_o_Zz; - -assign i_r = i_rr[0] & i_rr[1] & (~i_w); -assign i_mm = i_m; -assign o_m = o_mm; - -always_ff@(posedge clk) begin - o_v <= dsdp_o_v; - o_mm.Zx <= dsdp_o_Zx; - o_mm.Zy <= dsdp_o_Zy; - o_mm.Zz <= dsdp_o_Zz; - if (rst) - o_v <= 0; -end - -key_store #( - .D (KEY_D), - .W ($bits({i_mm.os[SCH_O_RES][0], i_mm.os[SCH_O_RX], i_mm.m.m.sig_l, i_mm.m.m.m})) -) keystore_inst ( - .i_r (i_rr[0]), - .i_v (i_v & i_r), - .i_k (i_k), - .i_d ({i_mm.os[SCH_O_RES][0], i_mm.os[SCH_O_RX], i_mm.m.m.sig_l, i_mm.m.m.m}), - - .o_r (dsdp_o_v), - .o_k (dsdp_o_k), - .o_d ({o_mm.res, o_mm.Rx, o_mm.sig_l, o_mm.m}), - - .clk (clk), - .rst (rst) -); - -ed25519_sigverify_dsdp_mul #( - .MUL_T (MUL_T), - .MUL_D (MUL_D), - .W_S (DSDP_WS), - .W_M (KEY_D_L) -) ed25519_sigverify_dsdp_mul_inst ( - .i_r (i_rr[1]), - .i_v (i_v & i_rr[0] & (~i_w)), - .i_m (i_k), - - .i_Ax (i_mm.os[SCH_O_AX]), - .i_Ay (i_mm.m.m.pub[0+:255]), - .i_Az (255'h1), - .i_At (i_mm.os[SCH_O_AT]), - - .i_ApGx (i_mm.os[SCH_O_TX]), // A+G - .i_ApGy (i_mm.os[SCH_O_TY]), - .i_ApGz (i_mm.os[SCH_O_TZ]), - .i_ApGt (i_mm.os[SCH_O_TT]), - - .i_As (i_mm.m.h), - .i_Gs (i_mm.m.m.sig_h), - - .o_v (dsdp_o_v), - .o_m (dsdp_o_k), - - .o_Cx (dsdp_o_Zx), - .o_Cy (dsdp_o_Zy), - .o_Cz (dsdp_o_Zz), - .o_Ct (), - - .clk (clk), - .rst (rst) -); - -// always_ff@(posedge clk) if (i_v | oo_v[3]) -// $display("%t: %m: %x - %x", $time -// , i_t - -// , o_t -// ); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_sigverify_2.sv b/src/wiredancer/rtl/ed25519_sigverify_2.sv deleted file mode 100644 index 8a6795bbca9..00000000000 --- a/src/wiredancer/rtl/ed25519_sigverify_2.sv +++ /dev/null @@ -1,95 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module ed25519_sigverify_2 #( - MUL_T = 32'h007F_CCC2 -) ( - output logic [1-1:0] i_r, - input wire [1-1:0] i_w, - input wire [1-1:0] i_v, - input wire [$bits(sv_meta6_t)-1:0] i_m, - - output logic [1-1:0] o_v, - output logic [$bits(sv_meta7_t)-1:0] o_m, - - input wire clk, - input wire rst -); - -logic [1-1:0] st_i, st_o; - -sv_meta6_t i_mm; -sv_meta7_t o_mm; - -sv_meta6_t mul_i_m; -logic [255-1:0] mul_i_A; -logic [255-1:0] mul_i_B; - -logic [1-1:0] mul_o_v; -sv_meta6_t mul_o_m; -logic [255-1:0] mul_o_C; - -assign i_r = st_i == 1; -assign i_mm = i_m; -assign o_m = o_mm; - -always_ff@(posedge clk) begin - case (st_i) - 0: begin - mul_i_m <= i_mm; - mul_i_A <= i_mm.Zz; - mul_i_B <= i_mm.Rx; - if (i_v & ~i_w) begin - st_i <= 1; - end - end - 1: begin - mul_i_B <= i_mm.sig_l[0+:255]; - st_i <= 0; - end - endcase - if (rst) begin - st_i <= 0; - end -end - -ed25519_mul_modp #( - .T (MUL_T), - .M ($bits({mul_i_m, st_i})) -) mul_modp_inst ( - .in0 (mul_i_A), - .in1 (mul_i_B), - .m_i ({mul_i_m, st_i}), - - .out0 (mul_o_C), - .m_o ({mul_o_m, mul_o_v}), - - .clk (clk), - .rst (rst) -); - -always_ff@(posedge clk) begin - case (st_o) - 0: begin - o_v <= 0; - o_mm.m <= mul_o_m.m; - o_mm.res <= mul_o_m.res & (mul_o_m.Zx == mul_o_C); - if (mul_o_v) - st_o <= 1; - end - 1: begin - o_v <= 1; - o_mm.res <= o_mm.res & (mul_o_m.Zy == mul_o_C); - st_o <= 0; - end - endcase - if (rst) begin - o_v <= 0; - st_o <= 0; - end -end - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_sigverify_dsdp_mul.sv b/src/wiredancer/rtl/ed25519_sigverify_dsdp_mul.sv deleted file mode 100644 index fc4dd7ee712..00000000000 --- a/src/wiredancer/rtl/ed25519_sigverify_dsdp_mul.sv +++ /dev/null @@ -1,317 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module ed25519_sigverify_dsdp_mul #( - MUL_T = 32'h007F_CCC2, - MUL_D = 15, - ADD_D = 4, - PIPE_D = (MUL_D*3) + ADD_D, - N_TH = 2+PIPE_D+PIPE_D, - W_M = 64, - W_S = 2 -) ( - output logic [1-1:0] i_r, - input wire [1-1:0] i_v, - input wire [W_M-1:0] i_m, - - input wire [255-1:0] i_Ax, - input wire [255-1:0] i_Ay, - input wire [255-1:0] i_Az, - input wire [255-1:0] i_At, - - input wire [255-1:0] i_ApGx, // A+G - input wire [255-1:0] i_ApGy, - input wire [255-1:0] i_ApGz, - input wire [255-1:0] i_ApGt, - - input wire [256-1:0] i_As, - input wire [256-1:0] i_Gs, - - output logic [1-1:0] o_v, - output logic [W_M-1:0] o_m, - - output logic [255-1:0] o_Cx, - output logic [255-1:0] o_Cy, - output logic [255-1:0] o_Cz, - output logic [255-1:0] o_Ct, - - input wire clk, - input wire rst -); - -localparam N_TH_L = $clog2(N_TH*2); - -typedef struct packed { - - logic [N_TH_L-1:0] t_i; - logic [1-1:0] v; - logic [10-1:0] PC; - logic [W_M-1:0] m; - logic [256-1:0] As; - logic [256-1:0] Gs; - -} meta_t; - -logic [1-1:0] c_0_we; -meta_t c_0_m; - -meta_t c_1_m; -logic [2-1:0] c_1_sel; -logic [255-1:0] c_1_Ax; -logic [255-1:0] c_1_Ay; -logic [255-1:0] c_1_Az; -logic [255-1:0] c_1_At; -logic [255-1:0] c_1_ApGx; -logic [255-1:0] c_1_ApGy; -logic [255-1:0] c_1_ApGz; -logic [255-1:0] c_1_ApGt; -logic [255-1:0] c_1_Cx; -logic [255-1:0] c_1_Cy; -logic [255-1:0] c_1_Cz; -logic [255-1:0] c_1_Ct; -logic [4-1:0][255-1:0] c_1_Px; -logic [4-1:0][255-1:0] c_1_Py; -logic [4-1:0][255-1:0] c_1_Pz; -logic [4-1:0][255-1:0] c_1_Pt; - -meta_t c_2_m; -logic [255-1:0] c_2_Ax; -logic [255-1:0] c_2_Ay; -logic [255-1:0] c_2_Az; -logic [255-1:0] c_2_At; -logic [255-1:0] c_2_Bx; -logic [255-1:0] c_2_By; -logic [255-1:0] c_2_Bz; -logic [255-1:0] c_2_Bt; - -meta_t c_a_m; -meta_t c_a_m_; -logic [255-1:0] c_a_Cx; -logic [255-1:0] c_a_Cy; -logic [255-1:0] c_a_Cz; -logic [255-1:0] c_a_Ct; - -meta_t po_m; -logic [255-1:0] po_Cx; -logic [255-1:0] po_Cy; -logic [255-1:0] po_Cz; -logic [255-1:0] po_Ct; - -always_comb begin - - case (po_m.v) - 0: begin - i_r = 1; - - c_0_we = i_v; - - c_0_m.v = i_v; - c_0_m.PC = 0; - c_0_m.m = i_m; - - c_0_m.As = i_As; - c_0_m.Gs = i_Gs; - end - - // 1: begin - default: begin - i_r = 0; - - c_0_we = 0; - - c_0_m.v = po_m.v; - c_0_m.PC = po_m.PC + 1; - c_0_m.m = po_m.m; - c_0_m.As = po_m.As; - c_0_m.Gs = po_m.Gs; - end - endcase - - c_1_Px[0] = ED25519_Ix; - c_1_Px[1] = ED25519_Gx; - c_1_Px[2] = c_1_Ax; - c_1_Px[3] = c_1_ApGx; - - c_1_Py[0] = ED25519_Iy; - c_1_Py[1] = ED25519_Gy; - c_1_Py[2] = c_1_Ay; - c_1_Py[3] = c_1_ApGy; - - c_1_Pz[0] = ED25519_Iz; - c_1_Pz[1] = ED25519_Gz; - c_1_Pz[2] = c_1_Az; - c_1_Pz[3] = c_1_ApGz; - - c_1_Pt[0] = ED25519_It; - c_1_Pt[1] = ED25519_Gt; - c_1_Pt[2] = c_1_At; - c_1_Pt[3] = c_1_ApGt; - - if (rst) - c_0_m.v = 0; -end - -always_comb begin - c_a_m_ = c_a_m; - if (c_a_m.PC == (W_S)) - c_a_m_.v = 0; -end - -always_ff@(posedge clk) begin - - o_v <= c_a_m.v & ~c_a_m_.v; - o_m <= c_a_m.m; - o_Cx <= c_a_Cx; - o_Cy <= c_a_Cy; - o_Cz <= c_a_Cz; - o_Ct <= c_a_Ct; - - c_0_m.t_i <= (c_0_m.t_i == N_TH-1) ? 0 : c_0_m.t_i + 1; - - c_1_m <= c_0_m; - - c_1_Cx <= po_Cx; - c_1_Cy <= po_Cy; - c_1_Cz <= po_Cz; - c_1_Ct <= po_Ct; - - casez({ - c_0_m.PC == 0, - po_m.As[255], - po_m.Gs[255] - }) - 3'b1_zz: c_1_sel <= 0; // I+I - 3'b0_00: c_1_sel <= 0; // Z+I - 3'b0_01: c_1_sel <= 1; // Z+G - 3'b0_10: c_1_sel <= 2; // Z+A - 3'b0_11: c_1_sel <= 3; // Z+ApG - endcase - - c_2_m <= c_1_m; - c_2_m.As <= (c_1_m.PC == 0) ? c_1_m.As : c_1_m.As << 1; - c_2_m.Gs <= (c_1_m.PC == 0) ? c_1_m.Gs : c_1_m.Gs << 1; - - c_2_Ax <= (c_1_m.PC == 0) ? ED25519_Ix : c_1_Cx; - c_2_Ay <= (c_1_m.PC == 0) ? ED25519_Iy : c_1_Cy; - c_2_Az <= (c_1_m.PC == 0) ? ED25519_Iz : c_1_Cz; - c_2_At <= (c_1_m.PC == 0) ? ED25519_It : c_1_Ct; - - c_2_Bx <= c_1_Px[c_1_sel]; - c_2_By <= c_1_Py[c_1_sel]; - c_2_Bz <= c_1_Pz[c_1_sel]; - c_2_Bt <= c_1_Pt[c_1_sel]; - - if (rst) begin - c_0_m.t_i <= 0; - c_1_m <= '0; - end -end - -simple_dual_port_ram #( - .WRITE_MODE ("read_first"), - .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH ($bits({c_0_m.t_i})), - .DATA_WIDTH ($bits({i_Ax, i_Ay, i_Az, i_At, i_ApGx, i_ApGy, i_ApGz, i_ApGt})) -) mem_ram_inst ( - .wr_clock (clk), - .wr_en (c_0_we), - .wr_byteenable ('1), - .wr_address ({c_0_m.t_i}), - .data ({i_Ax, i_Ay, i_Az, i_At, i_ApGx, i_ApGy, i_ApGz, i_ApGt}), - - .rd_clock (clk), - .rd_address ({c_0_m.t_i}), - .q ({c_1_Ax, c_1_Ay, c_1_Az, c_1_At, c_1_ApGx, c_1_ApGy, c_1_ApGz, c_1_ApGt}), - .rd_en (1'b1) -); - -ed25519_point_add #( - .T (MUL_T), - .D_M (MUL_D), - .M ($bits(c_2_m)) -) point_add_inst ( - - .in0_x (c_2_Ax), - .in0_y (c_2_Ay), - .in0_z (c_2_Az), - .in0_t (c_2_At), - .in1_x (c_2_Bx), - .in1_y (c_2_By), - .in1_z (c_2_Bz), - .in1_t (c_2_Bt), - .out0_x (c_a_Cx), - .out0_y (c_a_Cy), - .out0_z (c_a_Cz), - .out0_t (c_a_Ct), - .m_i (c_2_m), - .m_o (c_a_m), - .clk (clk), - .rst (rst) -); - -ed25519_point_add #( - .T (MUL_T), - .D_M (MUL_D), - .M ($bits(c_2_m)) -) point_dbl_inst ( - - .in0_x (c_a_Cx), - .in0_y (c_a_Cy), - .in0_z (c_a_Cz), - .in0_t (c_a_Ct), - .in1_x (c_a_Cx), - .in1_y (c_a_Cy), - .in1_z (c_a_Cz), - .in1_t (c_a_Ct), - .out0_x (po_Cx), - .out0_y (po_Cy), - .out0_z (po_Cz), - .out0_t (po_Ct), - .m_i (c_a_m_), - .m_o (po_m), - .clk (clk), - .rst (rst) -); - -always_ff@(posedge clk) -if (i_v | po_m.v | c_0_m.v | c_1_m.v | c_2_m.v | c_a_m.v) -$display("%t: i %x %x %x - po %x %x %x - c0 %x %x %x %x %x %x - c1 %x %x %x %x - c2 %x %x %x %x %x - ca %x %x %x %x %x %x %x", $time - , i_r - , i_v - , i_m - - , po_m.t_i - , po_m.v - , po_m.PC - - , c_0_m.t_i - , c_0_m.v - , c_0_we - , c_0_m.PC - , po_m.As[255] - , po_m.Gs[255] - - , c_1_m.t_i - , c_1_m.v - , c_1_m.PC - , c_1_sel - - , c_2_m.t_i - , c_2_m.v - , c_2_m.PC - , c_2_Ay - , c_2_By - - , c_a_m.t_i - , c_a_m.v - , c_a_m.PC - , c_a_Cx - , c_a_Cy - , c_a_Cz - , c_a_Ct -); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_sigverify_ecc.sv b/src/wiredancer/rtl/ed25519_sigverify_ecc.sv deleted file mode 100644 index 4e0011b7677..00000000000 --- a/src/wiredancer/rtl/ed25519_sigverify_ecc.sv +++ /dev/null @@ -1,235 +0,0 @@ -`default_nettype none - -/* - - Multiple sub-pipelines combined into a single-input single-output - pipeline. Different operations have different depths. Sub-results - are combined into a single output at various stages (1, 3, 5, M). - - - - - -*/ - -module ed25519_sigverify_ecc #( - // MUL_T = 32'h07F2_BBB0, - MUL_T = 32'h007F_CCC2, - MUL_D = 15, - W_M = 128, - W_D = 256 -)( - input wire [5-1:0] i_o, - input wire [W_D-1:0] i_a, - input wire [W_D-1:0] i_b, - input wire [1-1:0] i_c, - input wire [W_M-1:0] i_m, - - output logic [W_D-1:0] o_d, - output logic [W_M-1:0] o_m, - - input wire clk, - input wire rst -); - -localparam OP_AND = 0; -localparam OP_EQ = 1; -localparam OP_NE = 2; -localparam OP_GE = 3; -localparam OP_SHL = 4; -localparam OP_SHR = 5; -localparam OP_ADD = 6; -localparam OP_SUB = 7; -localparam OP_ADD_MODP = 8; -localparam OP_SUB_MODP = 9; -localparam OP_MUL_MODP = 10; -localparam OP_TERNARY = 5'h1B; - -logic [255-1:0] ED25519_P = 255'h7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed; -logic [256-1:0] ED25519_P_N = 256'h1 + ~256'h7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed; - -logic [5-1:0] c_1_o; -logic [5-1:0] c_2_o; -logic [5-1:0] c_3_o; -logic [5-1:0] c_4_o; - -logic [256-1:0] c_1_d; -logic [4-1:0][256-1:0] c_2_d; -logic [256-1:0] c_3_d; -logic [2-1:0][256-1:0] c_4_d; -logic [256-1:0] c_5_d; - -logic [5-1:0] c_m_o; -logic [W_M-1:0] c_m_m; - -logic [MUL_D-1:0][256-1:0] c_x_d; -logic [2-1:0][256-1:0] c_m_d; - -always_ff@(posedge clk) begin - integer i; - - c_1_o <= i_o; - c_2_o <= c_1_o; - c_3_o <= c_2_o; - c_4_o <= c_3_o; - - c_2_d[0] <= c_1_d; - c_4_d[0] <= c_3_d; - - c_x_d[6] <= c_5_d; - for (i = 7; i < MUL_D; i ++) - c_x_d[i] <= c_x_d[i-1]; - c_m_d[0] <= c_x_d[MUL_D-1]; - - case(i_o) - OP_AND: c_1_d <= i_a & i_b; - OP_EQ: c_1_d <= i_a == i_b; - OP_NE: c_1_d <= i_a != i_b; - OP_GE: c_1_d <= i_a >= i_b; - OP_SHL: c_1_d <= {i_a[0+:255], 1'b0}; - OP_SHR: c_1_d <= {255'h0, i_a[255]}; - OP_TERNARY: c_1_d <= i_c ? i_a : i_b; - endcase - - case (c_2_o) - OP_ADD: c_3_d <= c_2_d[1]; - OP_SUB: c_3_d <= c_2_d[2]; - OP_SUB_MODP:c_3_d <= c_2_d[3]; - default: c_3_d <= c_2_d[0]; - endcase - - case (c_4_o) - OP_ADD_MODP:c_5_d <= c_4_d[1]; - default: c_5_d <= c_4_d[0]; - endcase - - case (c_m_o) - OP_MUL_MODP:o_d <= c_m_d[1]; - default: o_d <= c_m_d[0]; - endcase - - o_m <= c_m_m; - -end - -// add -piped_adder #( - .W(W_D), - .R(0), - .C(1), - .M(1) -) c0_add_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(i_a), - .in1(i_b), - .out0(c_2_d[1]), - .cout0(), - .m_i('0), - .m_o() -); - -// sub -piped_adder #( - .W(W_D), - .R(0), - .C(1), - .M(1) -) c0_sub_inst ( - .clk(clk), - .rst(rst), - .cin0(1'b1), - .in0(i_a), - .in1(~i_b), - .out0(c_2_d[2]), - .cout0(), - .m_i('0), - .m_o() -); - -// sub_modp -logic [1-1:0] c_0_a_lt_b; - -assign c_0_a_lt_b = i_a < i_b; - -shift_adder_3 #( - .W (W_D), - .S0 (0), - .S1 (0), - .S2 (0), - .C (1), - .M (1), - .R (0), - .R0 (0) -) sub_modp_inst ( - .clk(clk), - .rst(rst), - .cin0(1'b1), - .in0(i_a), - .in1(~i_b), - .in2(c_0_a_lt_b ? ED25519_P : {(W_D){1'b0}}), - .out0(c_2_d[3]), - .m_i('0), - .m_o() -); - - -// add_modp -logic [W_D+1-1:0] c_2_AB; -logic [1-1:0] c_2_AB_ge_p; - -assign c_2_AB_ge_p = c_2_AB >= ED25519_P; - -piped_adder #( - .W(W_D+1), - .R(0), - .C(1), - .M(1) -) c0_addmodp_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0({1'b0, i_a}), - .in1({1'b0, i_b}), - .out0(c_2_AB), - .cout0(), - .m_i('0), - .m_o() -); -piped_adder #( - .W(W_D+1), - .R(0), - .C(1), - .M(1) -) c2_addmodp_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_2_AB), - .in1(c_2_AB_ge_p ? ED25519_P_N : {(W_D+1){1'b0}}), - .out0(c_4_d[1]), - .cout0(), - .m_i('0), - .m_o() -); - -// mul_modp -ed25519_mul_modp #( - .T(MUL_T), - .R_I(0), - .M($bits({i_o, i_m})) -) c0_mul_inst ( - .clk(clk), - .rst(rst), - .in0(i_a), - .in1(i_b), - .out0(c_m_d[1]), - .m_i({i_o, i_m}), - .m_o({c_m_o, c_m_m}) -); - -endmodule - - -`default_nettype wire diff --git a/src/wiredancer/rtl/ed25519_sub_modp.sv b/src/wiredancer/rtl/ed25519_sub_modp.sv deleted file mode 100644 index 30926bf6954..00000000000 --- a/src/wiredancer/rtl/ed25519_sub_modp.sv +++ /dev/null @@ -1,55 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module ed25519_sub_modp #( - W = 255, - M=128 -)( - input wire clk, - input wire rst, - - input wire [W-1:0] in0, - input wire [W-1:0] in1, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - output logic [W-1:0] out0 -); - -logic [M-1:0] m_o_p; -logic [W+1-1:0] c_2_AB; -logic [1-1:0] c_2_AB_ge_p; - -assign c_2_AB_ge_p = c_2_AB >= ED25519_P; - -logic [1-1:0] c_0_a_lt_b; - -assign c_0_a_lt_b = in0 < in1; - -shift_adder_3 #( - .W (W), - .S0 (0), - .S1 (0), - .S2 (0), - .C (1), - .M (M), - .R (0), - .R0 (0) -) a3_inst ( - .clk(clk), - .rst(rst), - .cin0(1'b1), - .in0(in0), - .in1(~in1), - .in2(c_0_a_lt_b ? ED25519_P : {(W){1'b0}}), - .out0(out0), - .cout0(), - .m_i(m_i), - .m_o(m_o) -); - -// always_ff@(posedge clk) $display("%t: %m: %x - %x = %x", $time, in0, in1, out0); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/key_store.sv b/src/wiredancer/rtl/key_store.sv deleted file mode 100644 index 517e12617ed..00000000000 --- a/src/wiredancer/rtl/key_store.sv +++ /dev/null @@ -1,107 +0,0 @@ -`default_nettype none - -module key_store #( - D = 512, - D_L = $clog2(D), - W = 1 -) ( - output logic [1-1:0] i_r, // has empty space - input wire [1-1:0] i_v, // push - output logic [D_L-1:0] i_k, // key provided to pusher - input wire [W-1:0] i_d, // data to be stored - - input wire [1-1:0] o_r, // pop - input wire [D_L-1:0] o_k, // key to pop - output logic [W-1:0] o_d, // popped data, next cycle - - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -logic [2-1:0] idx_st; - -logic [2-1:0] idx_we; -logic [16-1:0] idx_wd [2-1:0]; - -logic [1-1:0] idx_rr; -logic [1-1:0] idx_rv; -logic [D_L-1:0] idx_rd; - -assign i_r = idx_rv; -assign i_k = idx_rd; - -assign idx_we[1] = o_r; -assign idx_wd[1] = o_k; -assign idx_rr = i_v & idx_rv; - -always_ff@(posedge clk) begin - case (idx_st) - // XPM fifo reset wait - 0: begin - idx_wd[0] <= idx_wd[0] + 1; - if (idx_wd[0] == 1024) begin - idx_we[0] <= 1'b1; - idx_wd[0] <= '0; - idx_st <= 1; - end - end - // fill idx fifo - 1: begin - idx_wd[0] <= idx_wd[0] + 1; - if (idx_wd[0] == D-1) begin - idx_we[0] <= 1'b0; - idx_wd[0] <= '0; - idx_st <= 2; - end - end - 2: begin - end - endcase - if (rst) begin - idx_st <= 0; - idx_we[0] <= 1'b0; - idx_wd[0] <= 0; - end -end - -showahead_fifo #( - .WIDTH (D_L), - .DEPTH (D) -) idx_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (|idx_we), - .wr_full (), - .wr_data (idx_we[0] ? idx_wd[0] : idx_wd[1]), - - .rd_clk (clk), - .rd_req (idx_rr), - .rd_empty (), - .rd_not_empty (idx_rv), - .rd_count (), - .rd_data (idx_rd) -); - -simple_dual_port_ram #( - .ADDRESS_WIDTH (D_L), - .DATA_WIDTH (W), - .REGISTER_OUTPUT (0), - .CLOCKING_MODE ("common_clock") -) meta_ram_inst( - - .wr_clock (clk), - .wr_address (idx_rd), - .data (i_d), - .wr_en (idx_rr), - - .rd_clock (clk), - .rd_address (o_k), - .q (o_d), - .rd_en (1'b1) -); - -endmodule - -`default_nettype wire - diff --git a/src/wiredancer/rtl/mul_const_ED25519_L0_133.svh b/src/wiredancer/rtl/mul_const_ED25519_L0_133.svh deleted file mode 100644 index 06623065515..00000000000 --- a/src/wiredancer/rtl/mul_const_ED25519_L0_133.svh +++ /dev/null @@ -1,36 +0,0 @@ -logic [258-1:0] out_p; -logic [258-1:0] out_n; -logic [M-1:0] p_m_o; -logic [M-1:0] n_m_o; -logic [2-1:0][M-1:0] p_m_o_p; -logic [288-1:0] p_s_1_00; -logic [167-1:0] p_s_0_00; -logic [160-1:0] p_s_0_06; -logic [138-1:0] p_s_0_18; -logic [167-1:0] p_s_0_12; -`SHADD_6_1C(167, p_s_0_00, 0, 10, 12, 24, 31, 33, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}); -`SHADD_6_1C(160, p_s_0_06, 0, 5, 10, 12, 15, 26, {160'b0, in0[0+:133]}, {160'b0, in0[0+:133]}, {160'b0, in0[0+:133]}, {160'b0, in0[0+:133]}, {160'b0, in0[0+:133]}, {160'b0, in0[0+:133]}); -`SHADD_6_1C(167, p_s_0_12, 0, 5, 18, 21, 23, 33, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}, {167'b0, in0[0+:133]}); -`SHADD_6_1C(138, p_s_0_18, 0, 2, 4, 0, 0, 0, {138'b0, in0[0+:133]}, {138'b0, in0[0+:133]}, {138'b0, in0[0+:133]}, {138'b0, '0}, {138'b0, '0}, {138'b0, '0}); -`SHADD_6_1C(288, p_s_1_00, 0, 37, 72, 120, 0, 0, {288'b0, p_s_0_00}, {288'b0, p_s_0_06}, {288'b0, p_s_0_12}, {288'b0, p_s_0_18}, {288'b0, '0}, {288'b0, '0}); -assign out_p = p_s_1_00 << 0; -assign p_m_o = p_m_o_p[2-1]; -always_ff@(posedge clk) p_m_o_p[0] <= m_i; -always_ff@(posedge clk) p_m_o_p[1] <= p_m_o_p[1-1]; -logic [2-1:0][M-1:0] n_m_o_p; -logic [166-1:0] n_s_0_06; -logic [253-1:0] n_s_1_00; -logic [158-1:0] n_s_0_00; -logic [152-1:0] n_s_0_12; -logic [163-1:0] n_s_0_18; -`SHADD_6_1C(158, n_s_0_00, 0, 2, 12, 15, 17, 24, {158'b0, in0[0+:133]}, {158'b0, in0[0+:133]}, {158'b0, in0[0+:133]}, {158'b0, in0[0+:133]}, {158'b0, in0[0+:133]}, {158'b0, in0[0+:133]}); -`SHADD_6_1C(166, n_s_0_06, 0, 6, 11, 16, 30, 32, {166'b0, in0[0+:133]}, {166'b0, in0[0+:133]}, {166'b0, in0[0+:133]}, {166'b0, in0[0+:133]}, {166'b0, in0[0+:133]}, {166'b0, in0[0+:133]}); -`SHADD_6_1C(152, n_s_0_12, 0, 2, 4, 9, 14, 18, {152'b0, in0[0+:133]}, {152'b0, in0[0+:133]}, {152'b0, in0[0+:133]}, {152'b0, in0[0+:133]}, {152'b0, in0[0+:133]}, {152'b0, in0[0+:133]}); -`SHADD_6_1C(163, n_s_0_18, 0, 9, 13, 19, 24, 29, {163'b0, in0[0+:133]}, {163'b0, in0[0+:133]}, {163'b0, in0[0+:133]}, {163'b0, in0[0+:133]}, {163'b0, in0[0+:133]}, {163'b0, in0[0+:133]}); -`SHADD_6_1C(253, n_s_1_00, 0, 27, 63, 86, 0, 0, {253'b0, n_s_0_00}, {253'b0, n_s_0_06}, {253'b0, n_s_0_12}, {253'b0, n_s_0_18}, {253'b0, '0}, {253'b0, '0}); -assign out_n = n_s_1_00 << 2; -assign n_m_o = n_m_o_p[2-1]; -always_ff@(posedge clk) n_m_o_p[0] <= m_i; -always_ff@(posedge clk) n_m_o_p[1] <= n_m_o_p[1-1]; -always_ff@(posedge clk) out0 <= out_p - out_n; -always_ff@(posedge clk) m_o <= p_m_o; diff --git a/src/wiredancer/rtl/mul_const_ED25519_L0_260.svh b/src/wiredancer/rtl/mul_const_ED25519_L0_260.svh deleted file mode 100644 index 7cfecc47d03..00000000000 --- a/src/wiredancer/rtl/mul_const_ED25519_L0_260.svh +++ /dev/null @@ -1,40 +0,0 @@ -logic [385-1:0] out_p; -logic [385-1:0] out_n; -logic [M-1:0] p_m_o; -logic [M-1:0] n_m_o; -logic [4-1:0][M-1:0] p_m_o_p; -logic [415-1:0] p_s_1_00; -logic [294-1:0] p_s_0_00; -logic [287-1:0] p_s_0_06; -logic [265-1:0] p_s_0_18; -logic [294-1:0] p_s_0_12; -`SHADD_6_2C(294, p_s_0_00, 0, 10, 12, 24, 31, 33, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}); -`SHADD_6_2C(287, p_s_0_06, 0, 5, 10, 12, 15, 26, {287'b0, in0[0+:260]}, {287'b0, in0[0+:260]}, {287'b0, in0[0+:260]}, {287'b0, in0[0+:260]}, {287'b0, in0[0+:260]}, {287'b0, in0[0+:260]}); -`SHADD_6_2C(294, p_s_0_12, 0, 5, 18, 21, 23, 33, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}, {294'b0, in0[0+:260]}); -`SHADD_6_2C(265, p_s_0_18, 0, 2, 4, 0, 0, 0, {265'b0, in0[0+:260]}, {265'b0, in0[0+:260]}, {265'b0, in0[0+:260]}, {265'b0, '0}, {265'b0, '0}, {265'b0, '0}); -`SHADD_6_2C(415, p_s_1_00, 0, 37, 72, 120, 0, 0, {415'b0, p_s_0_00}, {415'b0, p_s_0_06}, {415'b0, p_s_0_12}, {415'b0, p_s_0_18}, {415'b0, '0}, {415'b0, '0}); -assign out_p = p_s_1_00 << 0; -assign p_m_o = p_m_o_p[4-1]; -always_ff@(posedge clk) p_m_o_p[0] <= m_i; -always_ff@(posedge clk) p_m_o_p[1] <= p_m_o_p[1-1]; -always_ff@(posedge clk) p_m_o_p[2] <= p_m_o_p[2-1]; -always_ff@(posedge clk) p_m_o_p[3] <= p_m_o_p[3-1]; -logic [4-1:0][M-1:0] n_m_o_p; -logic [293-1:0] n_s_0_06; -logic [380-1:0] n_s_1_00; -logic [285-1:0] n_s_0_00; -logic [279-1:0] n_s_0_12; -logic [290-1:0] n_s_0_18; -`SHADD_6_2C(285, n_s_0_00, 0, 2, 12, 15, 17, 24, {285'b0, in0[0+:260]}, {285'b0, in0[0+:260]}, {285'b0, in0[0+:260]}, {285'b0, in0[0+:260]}, {285'b0, in0[0+:260]}, {285'b0, in0[0+:260]}); -`SHADD_6_2C(293, n_s_0_06, 0, 6, 11, 16, 30, 32, {293'b0, in0[0+:260]}, {293'b0, in0[0+:260]}, {293'b0, in0[0+:260]}, {293'b0, in0[0+:260]}, {293'b0, in0[0+:260]}, {293'b0, in0[0+:260]}); -`SHADD_6_2C(279, n_s_0_12, 0, 2, 4, 9, 14, 18, {279'b0, in0[0+:260]}, {279'b0, in0[0+:260]}, {279'b0, in0[0+:260]}, {279'b0, in0[0+:260]}, {279'b0, in0[0+:260]}, {279'b0, in0[0+:260]}); -`SHADD_6_2C(290, n_s_0_18, 0, 9, 13, 19, 24, 29, {290'b0, in0[0+:260]}, {290'b0, in0[0+:260]}, {290'b0, in0[0+:260]}, {290'b0, in0[0+:260]}, {290'b0, in0[0+:260]}, {290'b0, in0[0+:260]}); -`SHADD_6_2C(380, n_s_1_00, 0, 27, 63, 86, 0, 0, {380'b0, n_s_0_00}, {380'b0, n_s_0_06}, {380'b0, n_s_0_12}, {380'b0, n_s_0_18}, {380'b0, '0}, {380'b0, '0}); -assign out_n = n_s_1_00 << 2; -assign n_m_o = n_m_o_p[4-1]; -always_ff@(posedge clk) n_m_o_p[0] <= m_i; -always_ff@(posedge clk) n_m_o_p[1] <= n_m_o_p[1-1]; -always_ff@(posedge clk) n_m_o_p[2] <= n_m_o_p[2-1]; -always_ff@(posedge clk) n_m_o_p[3] <= n_m_o_p[3-1]; -always_ff@(posedge clk) out0 <= out_p - out_n; -always_ff@(posedge clk) m_o <= p_m_o; diff --git a/src/wiredancer/rtl/mul_const_ED25519_L0_6.svh b/src/wiredancer/rtl/mul_const_ED25519_L0_6.svh deleted file mode 100644 index 3b2d2e076fe..00000000000 --- a/src/wiredancer/rtl/mul_const_ED25519_L0_6.svh +++ /dev/null @@ -1,36 +0,0 @@ -logic [131-1:0] out_p; -logic [131-1:0] out_n; -logic [M-1:0] p_m_o; -logic [M-1:0] n_m_o; -logic [2-1:0][M-1:0] p_m_o_p; -logic [161-1:0] p_s_1_00; -logic [ 40-1:0] p_s_0_00; -logic [ 33-1:0] p_s_0_06; -logic [ 11-1:0] p_s_0_18; -logic [ 40-1:0] p_s_0_12; -`SHADD_6_1C( 40, p_s_0_00, 0, 10, 12, 24, 31, 33, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}); -`SHADD_6_1C( 33, p_s_0_06, 0, 5, 10, 12, 15, 26, {33'b0, in0[0+: 6]}, {33'b0, in0[0+: 6]}, {33'b0, in0[0+: 6]}, {33'b0, in0[0+: 6]}, {33'b0, in0[0+: 6]}, {33'b0, in0[0+: 6]}); -`SHADD_6_1C( 40, p_s_0_12, 0, 5, 18, 21, 23, 33, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}, {40'b0, in0[0+: 6]}); -`SHADD_6_1C( 11, p_s_0_18, 0, 2, 4, 0, 0, 0, {11'b0, in0[0+: 6]}, {11'b0, in0[0+: 6]}, {11'b0, in0[0+: 6]}, {11'b0, '0}, {11'b0, '0}, {11'b0, '0}); -`SHADD_6_1C(161, p_s_1_00, 0, 37, 72, 120, 0, 0, {161'b0, p_s_0_00}, {161'b0, p_s_0_06}, {161'b0, p_s_0_12}, {161'b0, p_s_0_18}, {161'b0, '0}, {161'b0, '0}); -assign out_p = p_s_1_00 << 0; -assign p_m_o = p_m_o_p[2-1]; -always_ff@(posedge clk) p_m_o_p[0] <= m_i; -always_ff@(posedge clk) p_m_o_p[1] <= p_m_o_p[1-1]; -logic [2-1:0][M-1:0] n_m_o_p; -logic [ 39-1:0] n_s_0_06; -logic [126-1:0] n_s_1_00; -logic [ 31-1:0] n_s_0_00; -logic [ 25-1:0] n_s_0_12; -logic [ 36-1:0] n_s_0_18; -`SHADD_6_1C( 31, n_s_0_00, 0, 2, 12, 15, 17, 24, {31'b0, in0[0+: 6]}, {31'b0, in0[0+: 6]}, {31'b0, in0[0+: 6]}, {31'b0, in0[0+: 6]}, {31'b0, in0[0+: 6]}, {31'b0, in0[0+: 6]}); -`SHADD_6_1C( 39, n_s_0_06, 0, 6, 11, 16, 30, 32, {39'b0, in0[0+: 6]}, {39'b0, in0[0+: 6]}, {39'b0, in0[0+: 6]}, {39'b0, in0[0+: 6]}, {39'b0, in0[0+: 6]}, {39'b0, in0[0+: 6]}); -`SHADD_6_1C( 25, n_s_0_12, 0, 2, 4, 9, 14, 18, {25'b0, in0[0+: 6]}, {25'b0, in0[0+: 6]}, {25'b0, in0[0+: 6]}, {25'b0, in0[0+: 6]}, {25'b0, in0[0+: 6]}, {25'b0, in0[0+: 6]}); -`SHADD_6_1C( 36, n_s_0_18, 0, 9, 13, 19, 24, 29, {36'b0, in0[0+: 6]}, {36'b0, in0[0+: 6]}, {36'b0, in0[0+: 6]}, {36'b0, in0[0+: 6]}, {36'b0, in0[0+: 6]}, {36'b0, in0[0+: 6]}); -`SHADD_6_1C(126, n_s_1_00, 0, 27, 63, 86, 0, 0, {126'b0, n_s_0_00}, {126'b0, n_s_0_06}, {126'b0, n_s_0_12}, {126'b0, n_s_0_18}, {126'b0, '0}, {126'b0, '0}); -assign out_n = n_s_1_00 << 2; -assign n_m_o = n_m_o_p[2-1]; -always_ff@(posedge clk) n_m_o_p[0] <= m_i; -always_ff@(posedge clk) n_m_o_p[1] <= n_m_o_p[1-1]; -always_ff@(posedge clk) out0 <= out_p - out_n; -always_ff@(posedge clk) m_o <= p_m_o; diff --git a/src/wiredancer/rtl/mul_wide.sv b/src/wiredancer/rtl/mul_wide.sv deleted file mode 100644 index 14ab307743d..00000000000 --- a/src/wiredancer/rtl/mul_wide.sv +++ /dev/null @@ -1,324 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module mul_wide #( - W=127,W0=W,W1=W,L=4,T=32'h07FCCC, - W2=W/2, - R_I=0, - CT = T[0 +: 4], - ST = T >> 4, - M=32, - S=0 -)( - input wire clk, - input wire rst, - - input wire [W0-1:0] in0, - input wire [W1-1:0] in1, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - output logic [W0+W1-1:0] out0 -); - -logic [W0-1:0] in0_r; -logic [W1-1:0] in1_r; -logic [M-1:0] m_i_r; - -generate - - if (R_I) begin - always_ff@(posedge clk) in0_r <= in0; - always_ff@(posedge clk) in1_r <= in1; - always_ff@(posedge clk) m_i_r <= m_i; - end else begin - assign in0_r = in0; - assign in1_r = in1; - assign m_i_r = m_i; - end - - // Native - if (CT == 0) begin: T0 - - // always_ff@(posedge clk) $display("%t: %m. T = %x, W = %0d",$time, T, W); - - if (0) begin - end else if (S == 0) begin - - if (ST == 0) begin - - always_ff@(posedge clk) out0 <= in0_r * in1_r; - always_ff@(posedge clk) m_o <= m_i_r; - - end else begin - - logic [W0+W1-1:0] m; - - assign m = in0_r * in1_r; - - `WIDE_PIPE(m_o, m_i, M, ST) - `WIDE_PIPE(out0, m, W0+W1, ST) - - end - - end else begin - logic signed [W-1:0] i0; - logic signed [W-1:0] i1; - logic signed [W*2-1:0] o0; - - assign i0 = in0; - assign i1 = in1; - assign out0 = o0; - - always_ff@(posedge clk) o0 <= i0 * i1; - always_ff@(posedge clk) m_o <= m_i; - end - - // Naive - end else if (CT == 1) begin: T1 - logic [2-1:0][W2-1:0] x; - logic [2-1:0][W2-1:0] y; - logic [3-1:0][W-1:0] z; - logic [W+1-1:0] z1; - logic [4-1:0][W-1:0] m; - logic [2-1:0][M-1:0] m_o_p; - - assign x = in0; - assign y = in1; - - always_ff@(posedge clk) z[0] <= m[0]; - always_ff@(posedge clk) z1 <= m[1] + m[2]; - always_ff@(posedge clk) z[2] <= m[3]; - always_ff@(posedge clk) m_o_p[1]<= m_o_p[0]; - - always_ff@(posedge clk) out0 <= {z[2], {W{1'b0}}} + {z1, {W2{1'b0}}} + z[0]; - always_ff@(posedge clk) m_o <= m_o_p[1]; - - mul_wide #( - .W(W2),.L(L), - .T(ST) - ) mul_wide_m0_inst( - .clk(clk), - .rst(rst), - .in0(x[0]), - .in1(y[0]), - .out0(m[0]) - ); - mul_wide #( - .W(W2),.L(L), - .T(ST) - ) mul_wide_m1_inst( - .clk(clk), - .rst(rst), - .in0(x[0]), - .in1(y[1]), - .out0(m[1]) - ); - mul_wide #( - .W(W2),.L(L), - .T(ST) - ) mul_wide_m2_inst( - .clk(clk), - .rst(rst), - .in0(x[1]), - .in1(y[0]), - .out0(m[2]) - ); - mul_wide #( - .W(W2),.L(L), - .T(ST), - .M(M) - ) mul_wide_m3_inst( - .clk(clk), - .rst(rst), - .in0(x[1]), - .in1(y[1]), - .out0(m[3]), - .m_i(m_i), - .m_o(m_o_p[0]) - ); - - // W-1 - end else if (CT == 2) begin: T2 - - localparam W_M1 = W-1; - - logic [W+1-1:0] a; - logic [W+1-1:0] b; - logic [W_M1*2-1:0] m; - logic [M-1:0] m_o_p; - - assign a = (in0[W-1] ? in1[0 +: W_M1] : '0) + (in1[W-1] ? in0[0 +: W] : '0); - - assign out0 = m + (b << W_M1); - assign m_o = m_o_p; - - mul_wide #( - .W(W_M1), - .T(ST), - .M(M + W+1) - ) mul_wide_m0_inst( - .clk(clk), - .rst(rst), - .in0(in0[0 +: W_M1]), - .in1(in1[0 +: W_M1]), - .out0(m), - .m_i({m_i, a}), - .m_o({m_o_p, b}) - ); - // Karatsuba - end else if (CT == 12) begin: T12 - - // A = x[0] * y[0] - // B = x[1] * y[1] - // C = x[0] + x[1] - // D = y[0] + y[1] - // E = C * D - // F = A + (B << W) + (E << W2) - (A << W2) - (B << W2) - - localparam WR = W - W2; - localparam PA_C_W2 = 0; - localparam PA_C_WW = W >= 127 ? 1 : 0; - - logic [W2-1:0] c_0_x0, c_a_x0; - logic [W2-1:0] c_0_y0, c_a_y0; - logic [WR-1:0] c_0_x1, c_a_x1; - logic [WR-1:0] c_0_y1, c_a_y1; - - logic [WR-1:0] c_0_x0_, c_0_y0_; - - logic [WR+1-1:0] c_a_C; - logic [WR+1-1:0] c_a_D; - - logic [W2+W2-1:0] c_am_A; - logic [WR+WR-1:0] c_am_B; - logic [WR+WR+2-1:0] c_am_E; - - logic [W+W-1:0] c_am_A_; - logic [W+W-1:0] c_am_B_; - logic [W+W-1:0] c_am_E_; - - logic [W+W-1:0] c_amb_F; - - logic [3-1:0][M-1:0] m_o_p; - - assign {c_0_x1, c_0_x0} = in0_r; - assign {c_0_y1, c_0_y0} = in1_r; - - assign c_0_x0_ = {1'b0, c_0_x0}; - assign c_0_y0_ = {1'b0, c_0_y0}; - - assign c_am_A_ = {1'b0, c_am_A}; - assign c_am_B_ = {1'b0, c_am_B}; - assign c_am_E_ = {1'b0, c_am_E}; - - assign out0 = c_amb_F; - assign m_o = m_o_p[2]; - - mul_wide #(.W(W2+0), .T(ST), .M(M)) m0_inst (.clk(clk), .rst(rst), .in0(c_a_x0), .in1(c_a_y0), .out0(c_am_A), .m_i(m_o_p[0]), .m_o(m_o_p[1])); - mul_wide #(.W(WR+0), .T(ST), .M(1)) m1_inst (.clk(clk), .rst(rst), .in0(c_a_x1), .in1(c_a_y1), .out0(c_am_B), .m_i( '0), .m_o( )); - mul_wide #(.W(WR+1), .T(ST), .M(1)) m2_inst (.clk(clk), .rst(rst), .in0( c_a_C), .in1( c_a_D), .out0(c_am_E), .m_i( '0), .m_o( )); - - piped_adder #( - .W(WR), - .R(1), - .C(PA_C_W2), - .M(W+W+M) - ) a0_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_0_x0_), - .in1(c_0_x1), - .out0(c_a_C[0+:WR]), - .cout0(c_a_C[WR]), - .m_i({c_0_x1, c_0_x0, c_0_y1, c_0_y0, m_i}), - .m_o({c_a_x1, c_a_x0, c_a_y1, c_a_y0, m_o_p[0]}) - ); - piped_adder #( - .W(WR), - .R(1), - .C(PA_C_W2) - ) a1_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_0_y0_), - .in1(c_0_y1), - .out0(c_a_D[0+:WR]), - .cout0(c_a_D[WR]), - .m_i(), - .m_o() - ); - shift_adder_6 #( - .W (W+W), - .S0 (0), - .S1 (W2+W2), - .S2 (W2), - .S3 (W2), - .S4 (W2), - .S5 (W2), - .C (PA_C_WW), - .M (M), - .R (1), - .R0 (0), - .R1 (0) - ) a2_inst ( - .clk(clk), - .rst(rst), - .cin0('0), - .in0(c_am_A_), - .in1(c_am_B_), - .in2({ c_am_E_}), - .in3({~c_am_A_}), - .in4({~c_am_B_}), - .in5({{(W+W-2){1'b0}}, 2'b10}), - .out0(c_amb_F), - .m_i(m_o_p[1]), - .m_o(m_o_p[2]), - .cout0() - ); - // cascaded DSP - end else if (CT == 15 && ST[0+:4] == 7) begin: T15_7 - - localparam DEPTH = W0 < 27 ? 1 : W0 <= 26*2 ? 2 : W0 == 64 ? 2 : 1+4; - - if (W0 < 27) begin - `include "mul_wide_17nx26_dsp48e2.svh" - end else if (W0 <= 26*2) begin - - logic [26+W1-1:0] m_0; - logic [W0-26+W1-1:0] m_1; - - mul_wide #(.W0(26) , .W1(W1), .T(T)) m_0_inst (.clk(clk), .rst(rst), .in0(in0[ 0 +: 26]) , .in1(in1), .out0(m_0)); - mul_wide #(.W0(W0-26), .W1(W1), .T(T)) m_1_inst (.clk(clk), .rst(rst), .in0(in0[ 26 +: W0-26]), .in1(in1), .out0(m_1)); - - always_ff@(posedge clk) begin - out0[0 +:26] <= m_0[0+:26]; - out0[26+:W0+W1-26] <= m_1 + m_0[26+:W1]; - // out0 <= m_0 + (m_1 << 26); - end - end - - logic [DEPTH-1:0][M-1:0] m_o_p; - - always_ff@(posedge clk) begin - integer i; - m_o_p[0] <= m_i; - for (i = 1; i < DEPTH; i ++) - m_o_p[i] <= m_o_p[i-1]; - end - assign m_o = m_o_p[DEPTH-1]; - - end else if (CT == 15 && ST[0+:4] == 1) begin: T15_1 - `include "mul_const_ED25519_L0_260.svh" - end else if (CT == 15 && ST[0+:4] == 2) begin: T15_1 - `include "mul_const_ED25519_L0_133.svh" - end else if (CT == 15 && ST[0+:4] == 3) begin: T15_1 - `include "mul_const_ED25519_L0_6.svh" - end -endgenerate - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/mul_wide_17nx26_dsp48e2.svh b/src/wiredancer/rtl/mul_wide_17nx26_dsp48e2.svh deleted file mode 100644 index d8b3150b65f..00000000000 --- a/src/wiredancer/rtl/mul_wide_17nx26_dsp48e2.svh +++ /dev/null @@ -1,223 +0,0 @@ - -localparam NO_CASCADE = (W1+16)/17; - -logic [NO_CASCADE-1:0][W0+17-1:0] Ps; -logic [NO_CASCADE-1:0][48-1:0] Pcouts; - -for (genvar g_i = 0; g_i < NO_CASCADE; g_i ++) begin: G_I - -logic [30-1:0] ACOUT; -logic [18-1:0] BCOUT; -logic [1-1:0] CARRYCASCOUT; -logic [1-1:0] MULTSIGNOUT; -logic [48-1:0] PCOUT; - // Control outputs: Control Inputs/Status Bits -logic [1-1:0] OVERFLOW; -logic [1-1:0] PATTERNBDETECT; -logic [1-1:0] PATTERNDETECT; -logic [1-1:0] UNDERFLOW; - // Data outputs: Data Ports -logic [4-1:0] CARRYOUT; -logic [48-1:0] P; -logic [8-1:0] XOROUT; - // Cascade inputs: Cascade Ports -logic [30-1:0] ACIN = 0; -logic [18-1:0] BCIN = 0; -logic [1-1:0] CARRYCASCIN = 0; -logic [1-1:0] MULTSIGNIN = 0; -logic [48-1:0] PCIN; - // Control inputs: Control Inputs/Status Bits -logic [4-1:0] ALUMODE; -logic [3-1:0] CARRYINSEL = 0; -logic [1-1:0] CLK; -logic [5-1:0] INMODE = 0; -logic [9-1:0] OPMODE; - // Data inputs: Data Ports -logic [30-1:0] A; -logic [18-1:0] B; -logic [48-1:0] C = 0; -logic [1-1:0] CARRYIN = 0; -logic [27-1:0] D = 0; - // Reset/Clock Enable inputs: Reset/Clock Enable Inputs -logic [1-1:0] CEA1 = 1; -logic [1-1:0] CEA2 = 1; -logic [1-1:0] CEAD = 1; -logic [1-1:0] CEALUMODE = 1; -logic [1-1:0] CEB1 = 1; -logic [1-1:0] CEB2 = 1; -logic [1-1:0] CEC = 1; -logic [1-1:0] CECARRYIN = 1; -logic [1-1:0] CECTRL = 1; -logic [1-1:0] CED = 1; -logic [1-1:0] CEINMODE = 1; -logic [1-1:0] CEM = 1; -logic [1-1:0] CEP = 1; -logic [1-1:0] RSTA; -logic [1-1:0] RSTALLCARRYIN; -logic [1-1:0] RSTALUMODE; -logic [1-1:0] RSTB; -logic [1-1:0] RSTC; -logic [1-1:0] RSTCTRL; -logic [1-1:0] RSTD; -logic [1-1:0] RSTINMODE; -logic [1-1:0] RSTM; -logic [1-1:0] RSTP; - -// DSP48E2 : In order to incorporate this function into the design, -// Verilog : the following instance declaration needs to be placed -// instance : in the body of the design code. The instance name -// declaration : (DSP48E2_inst) and/or the port declarations within the -// code : parenthesis may be changed to properly reference and -// : connect this function to the design. All inputs -// : and outputs must be connected. - -// <-----Cut code below this line----> - - // DSP48E2: 48-bit Multi-Functional Arithmetic Block - // Virtex UltraScale+ - // Xilinx HDL Language Template, version 2020.1 - - DSP48E2 #( - // Feature Control Attributes: Data Path Selection - .AMULTSEL("A"), // Selects A input to multiplier (A, AD) - .A_INPUT("DIRECT"), // Selects A input source, "DIRECT" (A port) or "CASCADE" (ACIN port) - .BMULTSEL("B"), // Selects B input to multiplier (AD, B) - .B_INPUT("DIRECT"), // Selects B input source, "DIRECT" (B port) or "CASCADE" (BCIN port) - .PREADDINSEL("A"), // Selects input to pre-adder (A, B) - .RND(48'h000000000000), // Rounding Constant - .USE_MULT("MULTIPLY"), // Select multiplier usage (DYNAMIC, MULTIPLY, NONE) - .USE_SIMD("ONE48"), // SIMD selection (FOUR12, ONE48, TWO24) - .USE_WIDEXOR("FALSE"), // Use the Wide XOR function (FALSE, TRUE) - .XORSIMD("XOR24_48_96"), // Mode of operation for the Wide XOR (XOR12, XOR24_48_96) - // Pattern Detector Attributes: Pattern Detection Configuration - .AUTORESET_PATDET("NO_RESET"), // NO_RESET, RESET_MATCH, RESET_NOT_MATCH - .AUTORESET_PRIORITY("RESET"), // Priority of AUTORESET vs. CEP (CEP, RESET). - .MASK(48'h3fffffffffff), // 48-bit mask value for pattern detect (1=ignore) - .PATTERN(48'h000000000000), // 48-bit pattern match for pattern detect - .SEL_MASK("MASK"), // C, MASK, ROUNDING_MODE1, ROUNDING_MODE2 - .SEL_PATTERN("PATTERN"), // Select pattern value (C, PATTERN) - .USE_PATTERN_DETECT("NO_PATDET"), // Enable pattern detect (NO_PATDET, PATDET) - // Programmable Inversion Attributes: Specifies built-in programmable inversion on specific pins - .IS_ALUMODE_INVERTED(4'b0000), // Optional inversion for ALUMODE - .IS_CARRYIN_INVERTED(1'b0), // Optional inversion for CARRYIN - .IS_CLK_INVERTED(1'b0), // Optional inversion for CLK - .IS_INMODE_INVERTED(5'b00000), // Optional inversion for INMODE - .IS_OPMODE_INVERTED(9'b000000000), // Optional inversion for OPMODE - .IS_RSTALLCARRYIN_INVERTED(1'b0), // Optional inversion for RSTALLCARRYIN - .IS_RSTALUMODE_INVERTED(1'b0), // Optional inversion for RSTALUMODE - .IS_RSTA_INVERTED(1'b0), // Optional inversion for RSTA - .IS_RSTB_INVERTED(1'b0), // Optional inversion for RSTB - .IS_RSTCTRL_INVERTED(1'b0), // Optional inversion for RSTCTRL - .IS_RSTC_INVERTED(1'b0), // Optional inversion for RSTC - .IS_RSTD_INVERTED(1'b0), // Optional inversion for RSTD - .IS_RSTINMODE_INVERTED(1'b0), // Optional inversion for RSTINMODE - .IS_RSTM_INVERTED(1'b0), // Optional inversion for RSTM - .IS_RSTP_INVERTED(1'b0), // Optional inversion for RSTP - // Register Control Attributes: Pipeline Register Configuration - .ACASCREG(0), // Number of pipeline stages between A/ACIN and ACOUT (0-2) - .ADREG(0), // Pipeline stages for pre-adder (0-1) - .ALUMODEREG(0), // Pipeline stages for ALUMODE (0-1) - .AREG(0), // Pipeline stages for A (0-2) - .BCASCREG(0), // Number of pipeline stages between B/BCIN and BCOUT (0-2) - .BREG(0), // Pipeline stages for B (0-2) - .CARRYINREG(0), // Pipeline stages for CARRYIN (0-1) - .CARRYINSELREG(0), // Pipeline stages for CARRYINSEL (0-1) - .CREG(0), // Pipeline stages for C (0-1) - .DREG(0), // Pipeline stages for D (0-1) - .INMODEREG(0), // Pipeline stages for INMODE (0-1) - .MREG(1), // Multiplier pipeline stages (0-1) - .OPMODEREG(0), // Pipeline stages for OPMODE (0-1) - .PREG(0) // Number of pipeline stages for P (0-1) - ) - DSP48E2_inst ( - // Cascade outputs: Cascade Ports - .ACOUT(ACOUT), // 30-bit output: A port cascade - .BCOUT(BCOUT), // 18-bit output: B cascade - .CARRYCASCOUT(CARRYCASCOUT), // 1-bit output: Cascade carry - .MULTSIGNOUT(MULTSIGNOUT), // 1-bit output: Multiplier sign cascade - .PCOUT(PCOUT), // 48-bit output: Cascade output - // Control outputs: Control Inputs/Status Bits - .OVERFLOW(OVERFLOW), // 1-bit output: Overflow in add/acc - .PATTERNBDETECT(PATTERNBDETECT), // 1-bit output: Pattern bar detect - .PATTERNDETECT(PATTERNDETECT), // 1-bit output: Pattern detect - .UNDERFLOW(UNDERFLOW), // 1-bit output: Underflow in add/acc - // Data outputs: Data Ports - .CARRYOUT(CARRYOUT), // 4-bit output: Carry - .P(P), // 48-bit output: Primary data - .XOROUT(XOROUT), // 8-bit output: XOR data - // Cascade inputs: Cascade Ports - .ACIN(ACIN), // 30-bit input: A cascade data - .BCIN(BCIN), // 18-bit input: B cascade - .CARRYCASCIN(CARRYCASCIN), // 1-bit input: Cascade carry - .MULTSIGNIN(MULTSIGNIN), // 1-bit input: Multiplier sign cascade - .PCIN(PCIN), // 48-bit input: P cascade - // Control inputs: Control Inputs/Status Bits - .ALUMODE(ALUMODE), // 4-bit input: ALU control - .CARRYINSEL(CARRYINSEL), // 3-bit input: Carry select - .CLK(CLK), // 1-bit input: Clock - .INMODE(INMODE), // 5-bit input: INMODE control - .OPMODE(OPMODE), // 9-bit input: Operation mode - // Data inputs: Data Ports - .A(A), // 30-bit input: A data - .B(B), // 18-bit input: B data - .C(C), // 48-bit input: C data - .CARRYIN(CARRYIN), // 1-bit input: Carry-in - .D(D), // 27-bit input: D data - // Reset/Clock Enable inputs: Reset/Clock Enable Inputs - .CEA1(CEA1), // 1-bit input: Clock enable for 1st stage AREG - .CEA2(CEA2), // 1-bit input: Clock enable for 2nd stage AREG - .CEAD(CEAD), // 1-bit input: Clock enable for ADREG - .CEALUMODE(CEALUMODE), // 1-bit input: Clock enable for ALUMODE - .CEB1(CEB1), // 1-bit input: Clock enable for 1st stage BREG - .CEB2(CEB2), // 1-bit input: Clock enable for 2nd stage BREG - .CEC(CEC), // 1-bit input: Clock enable for CREG - .CECARRYIN(CECARRYIN), // 1-bit input: Clock enable for CARRYINREG - .CECTRL(CECTRL), // 1-bit input: Clock enable for OPMODEREG and CARRYINSELREG - .CED(CED), // 1-bit input: Clock enable for DREG - .CEINMODE(CEINMODE), // 1-bit input: Clock enable for INMODEREG - .CEM(CEM), // 1-bit input: Clock enable for MREG - .CEP(CEP), // 1-bit input: Clock enable for PREG - .RSTA(RSTA), // 1-bit input: Reset for AREG - .RSTALLCARRYIN(RSTALLCARRYIN), // 1-bit input: Reset for CARRYINREG - .RSTALUMODE(RSTALUMODE), // 1-bit input: Reset for ALUMODEREG - .RSTB(RSTB), // 1-bit input: Reset for BREG - .RSTC(RSTC), // 1-bit input: Reset for CREG - .RSTCTRL(RSTCTRL), // 1-bit input: Reset for OPMODEREG and CARRYINSELREG - .RSTD(RSTD), // 1-bit input: Reset for DREG and ADREG - .RSTINMODE(RSTINMODE), // 1-bit input: Reset for INMODEREG - .RSTM(RSTM), // 1-bit input: Reset for MREG - .RSTP(RSTP) // 1-bit input: Reset for PREG - ); - - // End of DSP48E2_inst instantiation - -// always_ff@(posedge clk) -// $display("%t: %m.DSP: %x %x %x %x %x",$time -// , A -// , B -// , P -// , PCOUT -// , PCIN); - -localparam _W1 = (W1 - (g_i*17)) > 17 ? 17 : W1 - (g_i*17); - -assign CLK = clk; - -assign ALUMODE = 4'b0000; -assign OPMODE = (g_i == 0 ) ? 9'b00_000_01_01 : 9'b00_101_01_01; -assign A = in0; -// assign B = in1[(NO_CASCADE-1-g_i)*17 +: 17]; -assign B = in1[g_i*17 +: _W1]; -assign Ps[g_i] = P; -assign Pcouts[g_i] = PCOUT; -if (g_i > 0) - assign PCIN = Pcouts[g_i-1]; - -assign out0[g_i*17 +: 17] = P[0 +: 17]; - -end - -localparam LEFTOVER = W0+W1-(NO_CASCADE*17); - -if (LEFTOVER > 0) - assign out0[NO_CASCADE*17 +: LEFTOVER] = Ps[NO_CASCADE-1][17 +: LEFTOVER]; diff --git a/src/wiredancer/rtl/pcie_inorder.sv b/src/wiredancer/rtl/pcie_inorder.sv deleted file mode 100644 index 6b84c671021..00000000000 --- a/src/wiredancer/rtl/pcie_inorder.sv +++ /dev/null @@ -1,135 +0,0 @@ -`default_nettype none - -module pcie_inorder #( - ADDR_MASK = 64'hffff_ffff_ffff_ffff, - ADDR_VAL = 64'h0000_0000_0000_0000, - W = 512, - D = 512, - REG_O = 0, - W2 = W/2, - W_L = $clog2(W), - D_L = $clog2(D) -) ( - input wire [2-1:0] pcie_v, - input wire [64-1:0] pcie_a, - input wire [2-1:0][W2-1:0] pcie_d, - - output logic [1-1:0] out_v, - output logic [1-1:0] out_s, - input wire [1-1:0] out_p, - output logic [64-1:0] out_a, - output logic [W-1:0] out_d, - - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -logic [64-1:0] timestamp; - -logic [1-1:0] addr_match; - -bit [64-1:0] last_ts; - -logic [1-1:0] out_iv; -logic [64-1:0] out_ia; -bit [2-1:0][64-1:0] out_a_p; -logic [2-1:0][W2-1:0] out_d_p; -logic [2-1:0][64-1:0] out_t_p; - -logic [D_L-1:0] rd_addr; -logic [D_L-1:0] rd_addr_n; - -assign addr_match = (pcie_a & ADDR_MASK) == ADDR_VAL; - -assign out_iv = (out_ia == out_a_p[0]) & (out_ia == out_a_p[1]) & (out_t_p[0] > last_ts) & (out_t_p[1] > last_ts); -assign rd_addr_n = (out_iv & out_p) ? rd_addr + 1 : rd_addr; - -generate - - if (REG_O == 0) begin - - assign out_v = out_iv; - assign out_a = out_ia; - assign out_d = out_d_p; - assign out_s = out_a == ADDR_VAL; - - end else begin - - // assuming out_p==1 - always_ff@(posedge clk) out_v <= out_iv; - always_ff@(posedge clk) out_a <= out_ia; - always_ff@(posedge clk) out_d <= out_d_p; - always_ff@(posedge clk) out_s <= out_ia == ADDR_VAL; - - end - - for (genvar g_i = 0; g_i < 2; g_i ++) begin - simple_dual_port_ram #( - .WRITE_MODE ("read_first"), - .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH (D_L), - .DATA_WIDTH ($bits({timestamp, pcie_d[g_i], pcie_a})) - ) ram_inst ( - .wr_clock (clk), - .wr_address (pcie_a[W_L-3 +: D_L]), - .wr_en (pcie_v[g_i] & addr_match), - .wr_byteenable ('1), - .data ({timestamp, pcie_d[g_i], pcie_a}), - - .rd_clock (clk), - .rd_address (rd_addr_n), - .q ({out_t_p[g_i], out_d_p[g_i], out_a_p[g_i]}), - .rd_en (1'b1) - ); - end -endgenerate - -simple_dual_port_ram #( - .WRITE_MODE ("read_first"), - .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH (D_L), - .DATA_WIDTH ($bits({timestamp})) -) ts_ram_inst ( - .wr_clock (clk), - .wr_address (rd_addr), - .wr_en (out_iv & out_p), - .wr_byteenable ('1), - .data (timestamp), - - .rd_clock (clk), - .rd_address (rd_addr_n), - .q (last_ts), - .rd_en (1'b1) -); - -always_ff@(posedge clk) begin - timestamp <= timestamp + 1; - - if (pcie_v & (pcie_a == ADDR_VAL)) begin - rd_addr <= 0; - out_ia <= ADDR_VAL; - end else begin - rd_addr <= rd_addr_n; - - if (out_iv & out_p) - out_ia <= out_ia + (1 << (W_L-3)); - end - - if (rst) begin - out_ia <= ADDR_VAL; - timestamp <= 0; - end -end - -// always_ff@(posedge clk) -// if (addr_match & |pcie_v) -// $display("%t: %m.in: %x %x %x - %b %x %x",$time, ADDR_MASK, ADDR_VAL, addr_match, pcie_v, pcie_a, pcie_d); - -// always_ff@(posedge clk) -// if (out_iv & out_p) -// $display("%t: %m.out: %x - %x %x - %x %x %x %x - %x %x",$time, rd_addr_n, out_iv, out_p, out_d, out_d_p[0], out_d_p[1], out_a_p, out_t_p, last_ts); - -endmodule - - -`default_nettype wire diff --git a/src/wiredancer/rtl/pcie_tr_ext.sv b/src/wiredancer/rtl/pcie_tr_ext.sv deleted file mode 100644 index b94e2fd7f79..00000000000 --- a/src/wiredancer/rtl/pcie_tr_ext.sv +++ /dev/null @@ -1,115 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module pcie_tr_ext #( - BUFF_SZ = 1024, - BUFF_SZ_L = $clog2(BUFF_SZ) -) ( - input wire [1-1:0] pcie_v, - input wire [512-1:0] pcie_d, - output logic [1-1:0] pcie_f, // full - output logic [BUFF_SZ_L+1-1:0] pcie_l, // fill - - output logic [1-1:0] o_v, - input wire [1-1:0] o_r, - output logic [1-1:0] o_e, - output logic [$bits(sv_meta2_t)-1:0] o_m0, - output logic [$bits(pcie_meta_t)-1:0] o_m1, - - input wire clk, - input wire rst -); - -logic [1-1:0] st_blk; - -pcie_meta_t pcie_m; -sv_meta2_t blk_m0; -pcie_meta_t blk_m1; - -logic [1-1:0] blk_v; -logic [1-1:0] blk_eop; -logic [5-1:0] blk_c; -logic [5-1:0] blk_tc; - -logic [64-1:0] tid; - -assign pcie_m = pcie_d; - -always_ff@(posedge clk) begin - case (st_blk) - 0: begin - - blk_v <= 0; - blk_eop <= 0; - blk_c <= 1; - blk_tc <= (pcie_m.size >> 6) + |pcie_m.size[0+:6]; - - blk_m1 <= pcie_m; - - blk_m0.m.m.tid <= tid; - blk_m0.m.m.src <= pcie_m.src; - blk_m0.m.sig_l <= pcie_m.sig_l; - blk_m0.size <= pcie_m.size; - blk_m0.emp <= (512/8) - pcie_m.size; - - if (pcie_v & (pcie_m.magic == PCIE_MAGIC)) begin - tid <= tid + 1; - st_blk <= 1; - end - end - 1: begin - blk_v <= pcie_v; - if (blk_c == 1) - blk_m0.data <= {pcie_d[256+:256], blk_m0.m.sig_l}; - else - blk_m0.data <= {pcie_d[256+:256], pcie_d[0+:256]}; - if (pcie_v) begin - - if (blk_c == 1) begin - blk_m0.m.sig_h <= pcie_d[ 0+:256]; - blk_m0.m.pub <= pcie_d[256+:256]; - end - - blk_c <= blk_c + 1; - blk_m0.sop <= blk_c == 1; - blk_eop <= blk_c == blk_tc; - - if (blk_c == blk_tc) begin - st_blk <= 0; - end - end - end - endcase - - if (rst) begin - st_blk <= 0; - blk_v <= 0; - tid <= 32'hABCD_0000; - end -end - -showahead_fifo #( - .WIDTH ($bits({blk_m1, blk_m0, blk_eop})), - .DEPTH (BUFF_SZ) -) o_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (blk_v), - .wr_full (pcie_f), - .wr_full_b (), - .wr_data ({blk_m1, blk_m0, blk_eop}), - .wr_count (pcie_l), - - .rd_clk (clk), - .rd_req (o_v & o_r), - .rd_empty (), - .rd_not_empty (o_v), - .rd_count (), - .rd_data ({o_m1, o_m0, o_e}) -); - -endmodule - -`default_nettype wire \ No newline at end of file diff --git a/src/wiredancer/rtl/schl_cpu.sv b/src/wiredancer/rtl/schl_cpu.sv deleted file mode 100644 index c5e97f87f93..00000000000 --- a/src/wiredancer/rtl/schl_cpu.sv +++ /dev/null @@ -1,1058 +0,0 @@ -`default_nettype none - -/* -- SigVerify Scheduler -- - -This is a Xilinx Virtex UltraScale+ (VU9P) targeted implementation for scheduling -the runtime of the Solana SigVerify algorithm as a part of the Firedancer consensus -node implementation developed by Jump Crypto - -This has been implemented as a simplified N-thread RISC CPU architecture -built around a fixed 256-bit pipeline ALU that schedules the needed runtime -math operations based on a generated fixed instruction stream. The implementation -being generic allowed for a hardware-software codesign process where, combined -with the Python reference implementation that you will find in sw/py/*, gave us -the opportunity to optimize the architecture to balance the hardware footprint, -mathematical complexity and runtime latency/throughput over the course of our -development (ending up with a very simple and efficient end result design) - -There are a few assumptions that this processor makes (after several iterations of -experiments on the architecture), the most significant being that the ALU operations -are all set to a fixed-duration (21 cycles) and that the instruction stream order is -identical for each handled SigVerify calculation. The architecture ended up -non-traditional, as the main instruction pipeline is handled virtually via a set -of parallel state machines rather than being fixed set of stages (our "logical thread" -is effectively a simple state machine managing transitions around input and output -from the ALU containing the math primitives) - -Instructions generated from our scipts use a virtual address scheme to specify -operating runtime and have independent physical scratchpad memory space within the -main memories for storing temporary values as needed by the algorithm. This allows -our reference/instruction generation scripts to logically treat the hardware as a -single-threaded process despite having the ability to configurably scale the number -of threads based on physical hardware limitations of the FPGA - -A "tag" (logical thread) consists of a metadata including an instruction counter, -support for 4 input hashes from the input bus, a shared set of constants set -during initialization, and 24-slot scratch pad for storing intermediate values -for runtime. The tags interact with the main memories through a virtual address -space that abstracts the physical memory space allowing us to treat the runtime -as single-threaded in the scripts that generates the instruction stream - -The following is the instruction and virtual address translation scheme: - Instruction: - { 1 [tern], 4 [op], 6 [INA], 6 [INB], 6 [TERN], 6 [OUT] } - Memory Range: ( Physical: 0x000 - 0x3FF | Virtual: 0x00 - 0x3B ) - PHY ADDR | NAME | VIRT ADDR | ACCESS | NOTES - 0x000-0x01F | in0 Input Data | 0x00 | read/write | initial input from bus - 0x020-0x03F | in1 Input Data | 0x01 | read/write | initial input from bus - 0x040-0x05F | in2 Input Data | 0x02 | read/write | initial input from bus - 0x060-0x07F | in3 Input Data | 0x03 | read/write | initial input from bus (unused) - 0x080-0x09F | Constant Data | 0x04-0x23 | read-only | set at synthesis - 0x0A0-0x3A0 | Scratch Memory | 0x24-0x3B | read/write | working memory for tag - -As this is a traditional (if highly simplified) RISC CPU pipeline, there is a fetch -stage prior to insertion of data into the ALU, where after the virtual -address translation there is a read from the three parallel, synchronized RAMs: - -- Two [256-bit] x 1024 dual-port RAMs for A and B input, constant, and working memory -- One [1-bit] x 1024 distributed memory for Ternary selector input into the ALU - -Selection of port A, B, and T is done via instruction and the synchronized -memories are all mirrored via write, allowing for unique-address reads while -having identical internal state (barring the Ternary memory which only stores the -least-significant bit of every write). In addition, each tag being blocking and -having independent physical address spaces allows for an simplified 3 stage -pipeline consisting of FETCH, EXEC, BLOCK - -As we are mainly physically resource-bound within the FPGA by the size of the -math primitives (mainly the mul_modp circuit) we physical restrict a max of one -of each operation per CPU and fix our pipeline duration to 21 cycles for all -instructions. With a more flexible (or a physically larger FPGA) this design -would cater to further math primitives within our ALU and allow for more active -tags to be used without having stalls - -The processor can fully fill the ALU's pipeline so we cap the number of tags -to MAX_INFLIGHT (which is our ALU's fixed pipeline depth). This prevents a tag -from "stalling" and getting blocked for 10's of microseconds waiting for -pipeline gaps to appear vs it's priority. If we add further randomness to the -tag selection at bottlenecks we can increase MAX_INFLIGHT up to NUM_TAGS, however -this will increase the mean latency as primary bottleneck for the entire processor -is the ALU and tags will still stall waiting for ALU pipeline slots to open - -Runtime for the final 1153 generated instructions takes ~101us before output starts -to stream to the DSDP block that follows this. Supported instructions can -be found in ed25519_sigverify_ecc.sv which contains the ALU logic, except for -the "output" instruction (0xC) which writes result values out to the DSDP - -Corresponding (non-exhaustive) cocoTB testbech can be found in sim/schl_cpu */ - -package schl_pkg; - parameter SCHL_SM_STATE_SIZE = 4; - typedef enum bit [SCHL_SM_STATE_SIZE-1:0] { - ST_INIT = 4'd0, - ST_IDLE = 4'd1, // Wait for new hash - ST_FETCH = 4'd2, // Read data from memory - ST_EXEC0 = 4'd3, // Start send data to primitive - ST_EXEC1 = 4'd4, // Propagate send data to primitive - ST_BLOCK = 4'd5, // Wait for result from primitive - ST_JMP = 4'd6 // Jump to another instr (unused for now) - } ssm_state_t; -endpackage - -module shcl_cpu -#( - MUL_T = 32'h007F_CCC2, - MUL_D = 15, - W_HASH = 256, - W_IN_MEM = 6, - W_T = 16, - MAX_INFLIGHT = (MUL_D+1)+6 -) -( - input var logic clk, - input var logic rst, - - input var logic [W_HASH-1:0] in_hash_data, - input var logic in_hash_valid, - input var logic [W_T-1:0] in_hash_ref, - output logic in_hash_ready, - - output logic [W_HASH-1:0] out_hash_data, - output logic [W_T-1:0] out_ref, - output logic [W_IN_MEM-1:0] out_d_addr, - output logic out_hash_valid -); - - localparam NUM_PRIMS = 11 + 1; // 11 math, 1 tern - localparam W_PRIMS = $clog2(NUM_PRIMS); - localparam NUM_TAGS = 32; // transaction handles == tag - localparam W_TAGS = $clog2(NUM_TAGS); - localparam NUM_OPS = 16; //max supported ops - localparam W_OPS = $clog2(NUM_OPS); - - localparam SZ_MEM = 1024; - localparam W_MEM = $clog2(SZ_MEM); - - localparam NUM_CONSTS = 12; - parameter bit [W_HASH-1:0] CONST_MEM [12] = { - 256'h0000000000000000000000000000000000000000000000000000000000000000, - 256'h7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed, - 256'h0000000000000000000000000000000000000000000000000000000000000001, - 256'h1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ed, - 256'h52036cee2b6ffe738cc740797779e89800700a4d4141d8ab75eb4dca135978a3, - 256'h67875f0fd78b766566ea4e8e64abe37d20f09f80775152f56dde8ab3a5b7dda3, - 256'h6666666666666666666666666666666666666666666666666666666666666658, - 256'h216936d3cd6e53fec0a4e231fdd6dc5c692cc7609525a7b2c9562d608f25d51a, - 256'h2b8324804fc1df0b2b4d00993dfbd7a72f431806ad2fe478c4ee1b274a0ea0b0, - 256'h7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec, - 256'h7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff, - 256'h00000000000000000000000000000000000000000000000000000000000000ff - }; - - localparam ROM_WIDTH = 1 + W_OPS + W_IN_MEM + W_IN_MEM + W_IN_MEM + W_IN_MEM; - localparam ROM_DEPTH = 512*4; // upper bound, real is 1153 - localparam W_ROM_DEPTH = $clog2(ROM_DEPTH); - - logic [ROM_WIDTH-1:0] instr_romA_data; - logic [W_ROM_DEPTH-1:0] instr_romA_addr; - logic [W_ROM_DEPTH-1:0] instr_romA_addr_d; - logic instr_romA_en; - logic instr_romA_en_d; - - logic [ROM_WIDTH-1:0] instr_romB_data; - logic [W_ROM_DEPTH-1:0] instr_romB_addr; - logic [W_ROM_DEPTH-1:0] instr_romB_addr_d; - logic instr_romB_en; - logic instr_romB_en_d; - - schl_cpu_instr_rom #( - .ROM_WIDTH ( ROM_WIDTH ), - .ROM_DEPTH ( ROM_DEPTH ) - ) schl_cpu_instr_rom_inst ( - .clk ( clk ), - .rst ( rst ), - - .a_addr ( instr_romA_addr ), - .a_en ( instr_romA_en ), - .a_data ( instr_romA_data ), - - .b_addr ( instr_romB_addr ), - .b_en ( instr_romB_en ), - .b_data ( instr_romB_data ) - ); - - // Next instruction fetch - logic [ROM_WIDTH-1:0] next_instr [NUM_TAGS]; // The next instruction to run - logic [NUM_TAGS-1:0] next_instr_ready; // Active high when instruction set - logic [NUM_TAGS-1:0] next_instr_req; // Request next instruction - logic [NUM_TAGS-1:0] next_instrA_check; - logic [NUM_TAGS-1:0] next_instrB_check; - logic [NUM_TAGS-1:0] next_instrA_tmp; - logic [NUM_TAGS-1:0] next_instrB_tmp; - logic [NUM_TAGS-1:0] next_instrA_tmp_next; - logic [NUM_TAGS-1:0] next_instrB_tmp_next; - logic [NUM_TAGS-1:0] next_instr_tmp_mask; - - // Next instruction fetch lookup (helpers) - logic [NUM_TAGS-1:0] prev_instrA_req; - logic [NUM_TAGS-1:0] prev_instrA_req_d; - logic [NUM_TAGS-1:0] prev_instrB_req; - logic [NUM_TAGS-1:0] prev_instrB_req_d; - logic [NUM_TAGS-1:0] upper_instr_req; - logic [NUM_TAGS-1:0] lower_instr_req; - logic [NUM_TAGS-1:0] upper_instr_req_next; - logic [NUM_TAGS-1:0] lower_instr_req_next; - - logic [W_ROM_DEPTH-1:0] last_instr; // Numerically last instruction to run in the ROM - logic [W_ROM_DEPTH-1:0] instr_addr[NUM_TAGS]; // Current running instruction address in rom (Instruction Pointer) - assign last_instr = 1153; - - logic [W_MEM-1:0] init_addr; // Initialization address to set up constants - logic [1:0] in_hash_cnt; // Sequencer for input setting - - // Instruction helpers - logic [W_OPS-1:0] in_tern [NUM_TAGS]; // Ternary selector (TODO: this isn't really necessary) - logic [W_OPS-1:0] in_op [NUM_TAGS]; // OPcode from Instruction - logic [W_MEM-1:0] in_memA_addr [NUM_TAGS]; // A physical memory address - logic [W_MEM-1:0] in_memB_addr [NUM_TAGS]; // B physical memory address - logic [W_MEM-1:0] in_tern_addr [NUM_TAGS]; // Ternary bit input memory address - logic [W_MEM-1:0] in_done_addr [NUM_TAGS]; // Instruction completion write location - logic [W_T-1:0] in_ref [NUM_TAGS]; // Reference tag set at calc start - - logic [W_IN_MEM-1:0] in_memA_addr_tmp [NUM_TAGS]; // Virtual A memory addr read from Instruction (helper) - logic [W_IN_MEM-1:0] in_memB_addr_tmp [NUM_TAGS]; // Virtual B memory addr read from Instruction (helper) - logic [W_IN_MEM-1:0] in_tern_addr_tmp [NUM_TAGS]; // Virtual T memory addr read from Instruction (helper) - logic [W_IN_MEM-1:0] in_done_addr_tmp [NUM_TAGS]; // Virtual Done memory addr read from Instruction (helper) - logic [W_IN_MEM-1:0] pr_done_addr [NUM_TAGS]; // Previous done address - logic pr_done_set [NUM_TAGS]; - logic jmp_instr [NUM_TAGS]; // INFO: this isn't used for now - localparam OP_JMP = 4'hF; - - // Primitive Input connectivity - logic [W_MEM-1:0] prim_done_addr; - logic [W_HASH-1:0] prim_inA_data; - logic [W_HASH-1:0] prim_inB_data; - logic prim_inC_data; - logic [W_TAGS-1:0] prim_tag; - logic [W_OPS:0] prim_op; // {1-in_tern, 4-in_op} - logic prim_valid; - - // Primitive Output connectivity - logic [W_HASH-1:0] prim_out_data; - logic [W_TAGS-1:0] prim_out_tag; - logic [W_MEM-1:0] prim_out_addr; - logic prim_out_valid; - - // Scratch Memory - localparam MEM_CNT = 2; // 2 identical memories for parallel reads, one 1 bit memory for tern (below) - logic [W_HASH-1:0] mem_d_wr_data; - logic [W_MEM-1:0] mem_d_wr_addr; - logic mem_d_wr_en; - logic mem_t_wr_data; - logic [W_MEM-1:0] mem_t_wr_addr; - logic mem_t_wr_en; - - logic [W_HASH-1:0] mem_d_rd_data [MEM_CNT]; - logic [W_MEM-1:0] mem_d_rd_addr [MEM_CNT]; - logic mem_d_rd_en [MEM_CNT]; - logic mem_t_rd_data; - logic [W_MEM-1:0] mem_t_rd_addr; - logic mem_t_rd_en; - - // Memory Input connectivity - logic wrote_in [NUM_TAGS]; - logic write_in; - logic write_in_d; - logic [W_TAGS-1:0] write_in_addr; - logic [W_TAGS-1:0] write_in_addr_d; - logic [W_MEM-1:0] write_in_done; - logic [W_MEM-1:0] write_in_done_tmp; - logic [W_MEM-1:0] write_in_done_tmp_d; - logic [W_MEM-1:0] write_in_done_d; - logic [W_OPS:0] write_in_op; // {1-in_tern, 4-in_op} - logic [W_OPS:0] write_in_op_d; - - logic out_done [NUM_TAGS]; - - logic [NUM_TAGS-1:0] rd_data_check; - logic [NUM_TAGS-1:0] rd_data_tmp; - logic [W_TAGS-1:0] rd_data_addr; - logic rd_data_addr_en; - logic [NUM_TAGS-1:0] rd_data_ready; - logic [NUM_TAGS-1:0] rd_data_request; - - // State Management (one per tag/calc) - import schl_pkg::*; - ssm_state_t curr_state [NUM_TAGS]; - logic calc_start [NUM_TAGS]; - - logic [NUM_TAGS-1:0] block_valid; - logic [NUM_TAGS-1:0] block_valid_d; - logic [NUM_TAGS-1:0] calc_ready; - logic [NUM_TAGS-1:0] calc_done; - logic [W_TAGS-1:0] calc_next; - logic calc_next_ready; - - logic [W_HASH-1:0] mem_man_data; - logic [W_MEM-1:0] mem_man_addr; - logic [W_TAGS-1:0] mem_man_tag; - logic mem_man_valid; - logic mem_man_ready; - logic init_done; - - assign in_hash_ready = calc_ready != '0 && !mem_man_valid && calc_next_ready; - always_comb begin - for (int i=0; i= 6'h04 && in_memA_addr_tmp[i] <= 6'h23 ) begin // constant memory - in_memA_addr[i] = 10'h080 + (in_memA_addr_tmp[i] - 6'h04); - end - else begin // scratch memory - in_memA_addr[i] = 10'h0A0 + SCRATCH_TAG_OFFSET[i] + s_memA_offset; - end - - /* VIRTUAL ADDRESS FOR B MEMORY */ - if ( in_memB_addr_tmp[i] == 6'h00) in_memB_addr[i] = 10'h000 + i[W_IN_MEM-1:0]; // in0 - else if ( in_memB_addr_tmp[i] == 6'h01) in_memB_addr[i] = 10'h020 + i[W_IN_MEM-1:0]; // in1 - else if ( in_memB_addr_tmp[i] == 6'h02) in_memB_addr[i] = 10'h040 + i[W_IN_MEM-1:0]; // in2 - else if ( in_memB_addr_tmp[i] == 6'h03) in_memB_addr[i] = 10'h060 + i[W_IN_MEM-1:0]; // in3 - else if ( in_memB_addr_tmp[i] >= 6'h04 && in_memB_addr_tmp[i] <= 6'h23) begin // constant memory - in_memB_addr[i] = 10'h080 + (in_memB_addr_tmp[i] - 6'h04); - end - else begin // scratch memory - in_memB_addr[i] = 10'h0A0 + SCRATCH_TAG_OFFSET[i] + s_memB_offset; - end - - /* VIRTUAL ADDRESS FOR TERNARY MEMORY */ - if ( in_tern_addr_tmp[i] == 6'h00) in_tern_addr[i] = 10'h000 + i[W_IN_MEM-1:0]; // in0 - else if ( in_tern_addr_tmp[i] == 6'h01) in_tern_addr[i] = 10'h020 + i[W_IN_MEM-1:0]; // in1 - else if ( in_tern_addr_tmp[i] == 6'h02) in_tern_addr[i] = 10'h040 + i[W_IN_MEM-1:0]; // in2 - else if ( in_tern_addr_tmp[i] == 6'h03) in_tern_addr[i] = 10'h060 + i[W_IN_MEM-1:0]; // in3 - else if ( in_tern_addr_tmp[i] >= 6'h04 && in_tern_addr_tmp[i] <= 6'h23) begin // constant memory - in_tern_addr[i] = 10'h080 + (in_tern_addr_tmp[i] - 6'h04); - end - else begin // scratch memory - in_tern_addr[i] = 10'h0A0 + SCRATCH_TAG_OFFSET[i] + s_tern_offset; - end - - /* VIRTUAL ADDRESS FOR OUTPUT MEMORY */ - // note: we can overwrite input memory locations if we want... - if ( in_done_addr_tmp[i] == 6'h00) in_done_addr[i] = 10'h000 + i[W_IN_MEM-1:0]; - else if ( in_done_addr_tmp[i] == 6'h01) in_done_addr[i] = 10'h020 + i[W_IN_MEM-1:0]; - else if ( in_done_addr_tmp[i] == 6'h02) in_done_addr[i] = 10'h040 + i[W_IN_MEM-1:0]; - else if ( in_done_addr_tmp[i] == 6'h03) in_done_addr[i] = 10'h060 + i[W_IN_MEM-1:0]; - //... otherwise write to scratch memory - else in_done_addr[i] = 10'h0A0 + SCRATCH_TAG_OFFSET[i] + s_done_offset; - - // TODO: plumb faults to some failure state, but these are mainly for sim testing - fault_inA_mem [i] = in_memA_addr_tmp[i] > 6'h3B; - fault_inB_mem [i] = in_memB_addr_tmp[i] > 6'h3B; - fault_tern_mem [i] = in_tern_addr_tmp[i] > 6'h3B; - fault_done_mem [i] = in_done_addr_tmp[i] > 6'h3B || in_done_addr_tmp[i] < 6'h24; - end - end - - // Save our previous done_addr for later - always_ff @ (posedge clk) begin - if (rst) begin - for(int i=0; i [explicit fixed cycle latency of the ALU] - - In this logic tags are prioritized by lsb by selecting the lowest active bit of our - one-hot request register (see $next_instrA_tmp below). This effect is compounded when - the ALU is not a fixed duration between instructions which was the case with our design - initially but was changed to improve timing in other areas of the design - - In the "next instruction" logic here we create fairness across tags by having - the two read ports in the instruction ROM split between from upper/lower - tags and allowing for "lower" reads to select "upper" values to read if they are - idle (and vice-versa). While still having implicit priority to the lsb tags - within $next_instr_req, this logic provides sufficient access bandwidth to not - lead to any blocking with our final configuration - - If increasing MAX_INFLIGHT beyond the limits explained before, it is advised to - use a psuedo-random Galios LFSR in addition to this logic to "deselect" - {upper|lower}_instr_req. Adding randomness in this manner was thoroughly - tested in a previous implementation and did well to provide further fairness beyond - what was necessary for our end result implementation */ - - assign lower_instr_req = next_instr_req & 32'h0000FFFF & ~prev_instrB_req & ~prev_instrB_req_d; - assign upper_instr_req = next_instr_req & 32'hFFFF0000 & ~prev_instrA_req & ~prev_instrA_req_d; - assign lower_instr_req_next = lower_instr_req & ~next_instrA_tmp; // "second lowest" - assign upper_instr_req_next = upper_instr_req & ~next_instrB_tmp; // "second lowest" - - assign next_instrA_tmp = lower_instr_req & ~(lower_instr_req - 1); // only Lowest req bit - assign next_instrB_tmp = upper_instr_req & ~(upper_instr_req - 1); - assign next_instrA_tmp_next = lower_instr_req_next & ~(lower_instr_req_next - 1); // only "second lowest" req bit - assign next_instrB_tmp_next = upper_instr_req_next & ~(upper_instr_req_next - 1); - assign next_instrA_check = (next_instrA_tmp) ? next_instrA_tmp : next_instrB_tmp_next; // either read lowest req from A or second lowest req from B - assign next_instrB_check = (next_instrB_tmp) ? next_instrB_tmp : next_instrA_tmp_next; // either read lowest req from B or second lowest req from A - - logic [W_TAGS-1:0] next_instrA_check_sel; - logic [W_TAGS-1:0] next_instrB_check_sel; - always_comb begin - for( int i=0; i We've fetched the next instruction for [i] and it's ready in next_instr[i] - for( int i=0; i last_instr) ? '0 : instr_romB_data; - next_instr_ready[i] <= 1'd1; - end - end - - // request the next instruction for [i]. can be a prefetch'd in ST_BLOCK - for (int i=0; i 2, then add register - stages before feeding the first round !! - (alternative solutions may exist though) */ - .i_data ( c02_i_data ), - .o_word ( t_word ), - - .clk ( clk ), - .rst ( rst ) -); - - -generate - for (g_r = 0; g_r < N_ROUNDS; g_r ++) begin: G_R - sha512_round #( - .HASH_W ( HASH_W ), - .WORD_W ( WORD_W ) - ) sha512_round_inst ( - .i_hash ( p_hash [g_r] ), - .i_word ( t_word [g_r] ), - .i_cval ( Constants [g_r] ), - - .o_hash ( t_hash [g_r] ), - - .clk ( clk ), - .rst ( rst ) - ); - end -endgenerate - - -simple_dual_port_ram #( - .ADDRESS_WIDTH (H_RAM_ADDR_W), - .DATA_WIDTH (H_RAM_DATA_W), - .REGISTER_OUTPUT (1), - .CLOCKING_MODE ("common_clock") -) p_hash_ram_inst ( - - .wr_clock ( clk ), - .wr_address ( c99_h_ram_wr_addr ), - .wr_en ( c99_h_ram_wr_en ), - .wr_byteenable ( '1 ), - .data ( c99_h_ram_wr_data ), - - .rd_clock ( clk ), - .rd_address ( c94_h_ram_rd_addr ), - .q ( c96_h_ram_rd_data ), - .rd_en ( 1'b1 ) -); - - -showahead_fifo #( - .WIDTH ( M_FIFO_WIDTH ), - .DEPTH ( M_FIFO_DEPTH ) -) meta_fifo_inst ( - .aclr ( rst ), - - .wr_clk ( clk ), - .wr_req ( c02_m_fifo_push ), - .wr_full ( ), - .wr_data ( c02_m_fifo_data ), - - .rd_clk ( clk ), - .rd_req ( c98_m_fifo_pop ), - .rd_empty ( ), /* FIXME needed? */ - .rd_not_empty ( ), /* FIXME needed? */ - .rd_count ( ), /* FIXME needed? */ - .rd_data ( c98_m_fifo_data ) -); - - -endmodule // sha512_block - -`default_nettype wire diff --git a/src/wiredancer/rtl/sha512_modq.sv b/src/wiredancer/rtl/sha512_modq.sv deleted file mode 100644 index 26531b787ad..00000000000 --- a/src/wiredancer/rtl/sha512_modq.sv +++ /dev/null @@ -1,174 +0,0 @@ -`default_nettype none - -module sha512_modq #( - META_W = 64 -) ( - input wire [1-1:0] i_v, // valid - input wire [1-1:0] i_f, // first blk - input wire [4-1:0] i_c, // number of blocks - input wire [1024-1:0] i_d, // data - input wire [META_W-1:0] i_t, // transaction id - output logic [1-1:0] i_p, // backpressure is only applied for the first block - - output logic o_v, - output logic [META_W-1:0] o_t, - output logic [256-1:0] o_d, - - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -localparam logic [253-1:0] ED25519_Q = 253'h1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ed; -localparam logic [125-1:0] ED25519_L0 = 125'h14def9dea2f79cd65812631a5cf5d3ed; - -logic [1-1:0] sha_i_v; -logic [1024-1:0] sha_i_d; -logic [META_W-1:0] sha_i_t; -logic [1-1:0] sha_i_f; -logic [1-1:0] sha_i_m; -logic [1-1:0] sha_i_l; - -logic [1-1:0] sha_o_v; -logic [META_W-1:0] sha_o_t; -logic [64-1:0][8-1:0] sha_o_d; -logic [64-1:0][8-1:0] sha_o_dl; - -logic [META_W+1-1:0] c_s_t; -logic [META_W+1-1:0] c_sm0_t; -logic [META_W+1-1:0] c_sm0m1_t; -logic [META_W+1-1:0] c_sm0m1m2_t; -logic [META_W+1-1:0] c_sm0m1m2a_t; -logic [META_W+1-1:0] c_sm0m1m2aa_t; -logic [META_W+1-1:0] c_sm0m1m2aaa_t; - -logic [512-1:0] c_s_x; -logic [252-1:0] c_s_x0; -logic [252-1:0] c_sm0_x0; -logic [252-1:0] c_sm0m1_x0; -logic [252-1:0] c_sm0m1m2_x0; -logic [260-1:0] c_s_x1; - -logic [385-1:0] c_sm0_y; -logic [252-1:0] c_sm0_y0; -logic [252-1:0] c_sm0m1_y0; -logic [252-1:0] c_sm0m1m2_y0; -logic [133-1:0] c_sm0_y1; - -logic [258-1:0] c_sm0m1_z; -logic [252-1:0] c_sm0m1_z0; -logic [252-1:0] c_sm0m1m2_z0; -logic [ 6-1:0] c_sm0m1_z1; - -logic [131-1:0] c_sm0m1m2_r; - -logic [253-1:0] c_sm0m1m2a_x0z0; -logic [253-1:0] c_sm0m1m2a_y0r; - -logic [253-1:0] c_sm0m1m2aa_m; -logic [253-1:0] c_sm0m1m2aaa_m; - -logic [252+1-1:0] x0z0; -logic [252+1-1:0] y0r; - -logic [252+1-1:0] m0; -logic [252+1-1:0] m1; - -sha512_sch #( - .W_BLK (1024), - .W_M (META_W), - .BLKS_PER_TR (11), - .RAM_D (512), - .N_CYCLES (84)//sha512_block_inst.CYCLES_BLOCK) -) sha512_sch_inst ( - .iblk_v (i_v), - .iblk_f (i_f), - .iblk_c (i_c), - .iblk_d (i_d), - .iblk_t (i_t), - .iblk_p (i_p), - - .oblk_v (sha_i_v), - .oblk_d (sha_i_d), - .oblk_t (sha_i_t), - .oblk_f (sha_i_f), - .oblk_m (sha_i_m), - .oblk_l (sha_i_l), - - .clk (clk), - .rst (rst) -); - -sha512_block #( - .DATA_W (1024), - .CTRL_W (3), - .MSGI_W (META_W), - .HASH_W (512), - .WORD_W (64) -) sha512_block_inst ( - .i_valid (sha_i_v), - .i_data (sha_i_d), - .i_ctrl ({sha_i_f, sha_i_m, sha_i_l}), - .i_msgi (sha_i_t), - - .o_valid (sha_o_v), - .o_msgi (sha_o_t), - .o_hash (sha_o_d), - - .clk (clk), - .rst (rst) -); - -mul_wide #(.W0(260), .W1(125), .T(8'h1F), .M($bits( {c_sm0_x0, c_sm0_t}))) m0_inst (.clk(clk), .rst(rst), .in0( {c_s_x1}), .in1(), .out0( c_sm0_y), .m_i( {c_s_x0, c_s_t}), .m_o( {c_sm0_x0, c_sm0_t})); -mul_wide #(.W0(133), .W1(125), .T(8'h2F), .M($bits( {c_sm0m1_y0, c_sm0m1_x0, c_sm0m1_t}))) m1_inst (.clk(clk), .rst(rst), .in0( {c_sm0_y1}), .in1(), .out0( c_sm0m1_z), .m_i( {c_sm0_y0, c_sm0_x0, c_sm0_t}), .m_o( {c_sm0m1_y0, c_sm0m1_x0, c_sm0m1_t})); -mul_wide #(.W0( 6), .W1(125), .T(8'h3F), .M($bits({c_sm0m1m2_z0, c_sm0m1m2_y0, c_sm0m1m2_x0, c_sm0m1m2_t}))) m2_inst (.clk(clk), .rst(rst), .in0({c_sm0m1_z1}), .in1(), .out0(c_sm0m1m2_r), .m_i({c_sm0m1_z0, c_sm0m1_y0, c_sm0m1_x0, c_sm0m1_t}), .m_o({c_sm0m1m2_z0, c_sm0m1m2_y0, c_sm0m1m2_x0, c_sm0m1m2_t})); - -always_ff@(posedge clk) begin - - c_sm0m1m2a_x0z0 <= c_sm0m1m2_x0 + c_sm0m1m2_z0; - c_sm0m1m2a_y0r <= c_sm0m1m2_y0 + c_sm0m1m2_r; - c_sm0m1m2a_t <= c_sm0m1m2_t; - - c_sm0m1m2aa_m <= c_sm0m1m2a_x0z0 + (c_sm0m1m2a_x0z0 < c_sm0m1m2a_y0r ? ED25519_Q : '0) - c_sm0m1m2a_y0r; - c_sm0m1m2aa_t <= c_sm0m1m2a_t; - - c_sm0m1m2aaa_m <= c_sm0m1m2aa_m - (c_sm0m1m2aa_m >= ED25519_Q ? ED25519_Q : '0); - c_sm0m1m2aaa_t <= c_sm0m1m2aa_t; -end - -always_comb begin - integer i; - for (i = 0; i < 64; i ++) - sha_o_dl[i] = sha_o_d[64-i-1]; -end - -always_comb begin - c_s_x = sha_o_dl; - c_s_t = {sha_o_t, sha_o_v}; - - c_s_x0 = c_s_x[0+:252]; - c_s_x1 = c_s_x[252+:260]; - - c_sm0_y0 = c_sm0_y[0+:252]; - c_sm0_y1 = c_sm0_y[252+:133]; - - c_sm0m1_z0 = c_sm0m1_z[0+:252]; - c_sm0m1_z1 = c_sm0m1_z[252+:6]; - - o_d = c_sm0m1m2aaa_m; - {o_t, o_v} = c_sm0m1m2aaa_t; -end - -// always_ff@(posedge clk) $display("%t: %x %x %x - %x %x %x", $time -// ,i_v // valid -// ,i_f // first blk -// ,i_c // number of blocks -// ,i_d // data -// ,i_t // transaction id -// ,i_p // pop blk -// ); -// always_ff@(posedge clk) if (sha_o_v) $display("%t: %m.sha_o: %x %x", $time, sha_o_t, sha_o_d); -// always_ff@(posedge clk) if (o_v) $display("%t: %m.o: %x %x", $time, o_t, o_d); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/sha512_modq_meta.sv b/src/wiredancer/rtl/sha512_modq_meta.sv deleted file mode 100644 index 9664646badd..00000000000 --- a/src/wiredancer/rtl/sha512_modq_meta.sv +++ /dev/null @@ -1,89 +0,0 @@ -`default_nettype none - -module sha512_modq_meta #( - parameter integer KEY_D = 512, - parameter integer KEY_D_L = $clog2(KEY_D) -) ( - output logic [1-1:0] i_r, // backpressure is only applied for the first block (i_m.f) - input wire [1-1:0] i_w, // wait - input wire [1-1:0] i_v, // valid - input wire [1-1:0] i_e, // last blk - input wire [$bits(sv_meta3_t)-1:0] i_m, // meta data - - output logic [1-1:0] o_v, - output logic [1-1:0] o_e, - output logic [$bits(sv_meta4_t)-1:0] o_m, - - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -sv_meta3_t i_mm; -sv_meta4_t o_mm; - -logic [1-1:0] sha_i_r; -logic [1-1:0] sha_i_v; -logic [KEY_D_L-1:0] sha_i_k; - -logic [1-1:0] key_i_r; -logic [1-1:0] key_i_v; - -logic [1-1:0] sha_o_v; -logic [KEY_D_L-1:0] sha_o_k; -logic [256-1:0] sha_o_d; - -assign i_r = (i_mm.f & sha_i_r & key_i_r & ~i_w) | (sha_i_r & ~i_mm.f); -assign i_mm = i_m; - -assign key_i_v = i_mm.f & sha_i_r & i_v & ~i_w; -assign sha_i_v = (i_mm.f & key_i_r & i_v & ~i_w) | (i_v & ~i_mm.f); - -assign o_e = 1; -assign o_m = o_mm; - -always_ff@(posedge clk) begin - o_v <= sha_o_v; - o_mm.h <= sha_o_d; - if (rst) - o_v <= 0; -end - -key_store #( - .D (KEY_D), - .W ($bits({i_m})) -) keystore_inst ( - .i_r (key_i_r), - .i_v (key_i_v), - .i_k (sha_i_k), - .i_d ({i_m}), - - .o_r (sha_o_v), - .o_k (sha_o_k), - .o_d ({o_mm.m}), - - .clk (clk), - .rst (rst) -); - -sha512_modq #( - .META_W (KEY_D_L) -) sha512_modq_inst ( - .i_p (sha_i_r), // backpressure is only applied for the first block (i_m.f) - .i_v (sha_i_v), - .i_t (sha_i_k), // key - .i_f (i_mm.f), // first blk - .i_c (i_mm.c), // number of blocks - .i_d (i_mm.d), // data - - .o_v (sha_o_v), - .o_t (sha_o_k), - .o_d (sha_o_d), - - .clk (clk), - .rst (rst) -); - -endmodule - -`default_nettype wire - diff --git a/src/wiredancer/rtl/sha512_msgseq.sv b/src/wiredancer/rtl/sha512_msgseq.sv deleted file mode 100644 index 96fbd563dac..00000000000 --- a/src/wiredancer/rtl/sha512_msgseq.sv +++ /dev/null @@ -1,221 +0,0 @@ -`default_nettype none - - -module aux_round_msgseq #( - DATA_W = 1024, - WORD_W = 64, - INST_I = -1 -) ( - input wire [DATA_W-1:0] i_data, - output logic [DATA_W-1:0] o_data, - output logic [WORD_W-1:0] o_word, - - input wire clk, - input wire rst -); - -assert property (@(posedge clk) (INST_I >= 0)) else begin $display("%m usupported INST_I: %d",INST_I); $error("INST_I!!"); $fatal; end -assert property (@(posedge clk) (INST_I < 80)) else begin $display("%m usupported INST_I: %d",INST_I); $error("INST_I!!"); $fatal; end -assert property (@(posedge clk) (DATA_W==1024)) else begin $display("%m usupported DATA_W: %d",DATA_W); $error("DATA_W!!"); $fatal; end -assert property (@(posedge clk) (WORD_W== 64)) else begin $display("%m usupported WORD_W: %d",WORD_W); $error("WORD_W!!"); $fatal; end - -logic [WORD_W-1:0] word; -logic [16-1:0][WORD_W-1:0] data; - -logic [WORD_W-1:0] w2; -logic [WORD_W-1:0] w7; -logic [WORD_W-1:0] w15; -logic [WORD_W-1:0] w16; - -logic [WORD_W-1:0] a; -logic [WORD_W-1:0] b; -logic [WORD_W-1:0] c; -logic [WORD_W-1:0] d; - -assign data = i_data; - -generate - if ( INST_I < 16 ) begin - assign word = data[INST_I]; - end else begin - always_comb begin - w2 = data[14]; - w7 = data[ 9]; - w15 = data[ 1]; - w16 = data[ 0]; - - a = { w2[19-1:0], w2[WORD_W-1:19]} ^ - { w2[61-1:0], w2[WORD_W-1:61]} ^ - { 6'd0, w2[WORD_W-1: 6]}; - b = w7; - c = {w15[ 1-1:0],w15[WORD_W-1: 1]} ^ - {w15[ 8-1:0],w15[WORD_W-1: 8]} ^ - { 7'd0,w15[WORD_W-1: 7]}; - d = w16; - end - shift_adder_6 #( - .W ( WORD_W ), - .S0 ( 0 ), - .S1 ( 0 ), - .S2 ( 0 ), - .S3 ( 0 ), - .S4 ( 0 ), - .S5 ( 0 ), - .C ( 0 ), - .M ( 0 ), - .R0 ( 0 ), /* output of reduction 6-3 (first layer of reduction) */ - .R1 ( 0 ), /* output of reduction 3-2 (second layer of reduction)*/ - /* reduction 2-1 just an adder */ - .R ( 0 ) /* output */ - ) shift_adder_6_inst ( - .clk ( clk ), - .rst ( rst ), - - .cin0 ( '0 ), - .in0 ( a ), - .in1 ( b ), - .in2 ( c ), - .in3 ( d ), - .in4 ( '0 ), - .in5 ( '0 ), - .out0 ( word ), - - .m_i ('0), - .m_o (), - .cout0 () - ); - end -endgenerate - - -always_ff@(posedge clk) begin - o_word <= word; - /* INST_I is guaranteed to be in range [0,80) - (see the asserts above) */ - if ( INST_I < 16 ) begin - o_data <= data; - end else begin - o_data <= {word, data[16-1:1]}; -end end - -endmodule // _round_msgseq - - - - - - -module sha512_msgseq #( - DATA_W = 1024, - WORD_W = 64, - ROUNDS = 80 -) ( - input wire [DATA_W-1:0] i_data, - output logic [WORD_W-1:0] o_word [ROUNDS-1:0], - - input wire clk, - input wire rst -); - -assert property (@(posedge clk) (DATA_W==1024)) else begin $display("%m usupported DATA_W: %d",DATA_W); $error("DATA_W!!"); $fatal; end -assert property (@(posedge clk) (WORD_W== 64)) else begin $display("%m usupported WORD_W: %d",WORD_W); $error("WORD_W!!"); $fatal; end -assert property (@(posedge clk) (ROUNDS== 80)) else begin $display("%m usupported ROUNDS: %d",ROUNDS); $error("ROUNDS!!"); $fatal; end -localparam D_WD_N = DATA_W / WORD_W; -assert property (@(posedge clk) (D_WD_N== 16)) else begin $display("%m usupported D_WD_N: %d",D_WD_N); $error("D_WD_N!!"); $fatal; end - - -localparam CYCLES_MSGSEQ = 1; - -logic [DATA_W-1:0] t_data [(ROUNDS+1)-1:0]; - -/* big-endian convention - from RFC6234: - - "Throughout this document, the "big-endian" convention is used when - expressing both 32-bit and 64-bit words, so that within each word - the most significant bit is shown in the leftmost bit position." - - "Again, the "big-endian" convention is used and the most - significant word is in the leftmost word position for values - represented by multiple-words." - - "... The - message or data file should be considered to be a bit string." - - "The words of the message schedule are labeled W0, W1, ..., W79." - - "The input message is padded as described in Section 4.2 above, then - parsed into 1024-bit blocks that are considered to be composed of - sixteen 64-bit words M(i)0, M(i)1, ..., M(i)15." - - ------------------------------------ - Example from Section 4.2 of RFC6234: - - Suppose the original message is the bit string: - 01100001 01100010 01100011 01100100 01100101 - (sz = 40bits) - after padding and size addition, the message (1 block here) becomes: - - 61626364 65800000 00000000 00000000 - 00000000 00000000 00000000 00000000 - 00000000 00000000 00000000 00000000 - 00000000 00000000 00000000 00000000 - 00000000 00000000 00000000 00000000 - 00000000 00000000 00000000 00000000 - 00000000 00000000 00000000 00000000 - 00000000 00000000 00000000 00000028 - - This means: - M(i)0 = 64'h6162636465800000 - M(i)2 = 64'h0000000000000000 - M(i)3 = 64'h0000000000000000 - M(i)4 = 64'h0000000000000000 - M(i)5 = 64'h0000000000000000 - M(i)6 = 64'h0000000000000000 - M(i)7 = 64'h0000000000000000 - M(i)8 = 64'h0000000000000000 - M(i)9 = 64'h0000000000000000 - M(i)10 = 64'h0000000000000000 - M(i)11 = 64'h0000000000000000 - M(i)12 = 64'h0000000000000000 - M(i)13 = 64'h0000000000000000 - M(i)14 = 64'h0000000000000000 - M(i)15 = 64'h0000000000000028 - - Then, as a reference, round 0 receives: - W_0 = M(i)0 = 64'h6162636465800000 - K_0 = 64'h428a2f98d728ae22 - ------------------------------------ -*/ - -integer k; -always_comb begin - /* big-endian convention (see RFC6234) */ - for (k=0; k ((1024/8) - 16 - 1); - - if (i_v & ~i_w) - next.st = i_e ? 3 : 2; - end - end - - 1: begin - next.ready = 1; - if (1) begin - next.o_v = 0; - next.o_f = 0; - next.o_d[0+:64] = i_mm.data; - - if (i_v) - next.st = i_e ? 3 : 2; - end - end - - 2: begin - next.ready = 1; - next.o_v = i_v & ((~i_e) | (i_mm.emp == 0)); - if (1) begin - next.o_d[64+:64] = i_mm.data; - if (i_v) - next.st = i_e ? (self.extra ? 4 : 3) : 1; - end - end - - 3: begin - next.ready = 0; - next.o_v = 1; - next.o_f = (self.o_v) ? 1'b0 : self.o_f; - if (1) begin - for (i = 0; i < 128-16; i ++) begin - case({ - i == self.last, - i > self.last - }) - 2'b10: next.o_d[i] = 8'h80; - 2'b01: next.o_d[i] = 8'h00; - endcase - end - next.o_d[128-16+:14]= 112'h0; - next.o_d[128-2+:1] = (self.size << 3) >> 8; - next.o_d[128-1+:1] = (self.size << 3) >> 0; - - next.o_l = 1; - - next.st = 0; - end - end - 4: begin - next.ready = 0; - next.o_v = 1; - next.o_f = (self.o_v) ? 1'b0 : self.o_f; - if (1) begin - for (i = 0; i < 128; i ++) begin - case({ - i == self.last, - i > self.last - }) - 2'b10: next.o_d[i] = 8'h80; - 2'b01: next.o_d[i] = 8'h00; - endcase - end - next.st = 5; - end - end - 5: begin - next.ready = 0; - next.o_v = 1; - next.o_f = (self.o_v) ? 1'b0 : self.o_f; - if (1) begin - for (i = 0; i < 128-16; i ++) begin - next.o_d[i] = 8'h00; - end - next.o_d[128-16+:14]= 112'h0; - next.o_d[128-2+:1] = (self.size << 3) >> 8; - next.o_d[128-1+:1] = (self.size << 3) >> 0; - - next.o_l = 1; - - next.st = 0; - end - end - endcase -end - -always_ff@(posedge clk) begin - - self <= next; - if (rst) begin - self <= '0; - end -end - -// always_ff@(posedge clk) $display("%t: %x %x %x %x %x %x %x %x - ", $time -// , i_r -// , i_v -// , i_sop -// , i_e -// , i_emp -// , i_s -// , i_m -// , i_d -// ); - -endmodule - - - -`default_nettype wire diff --git a/src/wiredancer/rtl/sha512_round.sv b/src/wiredancer/rtl/sha512_round.sv deleted file mode 100644 index a0a62aa2afd..00000000000 --- a/src/wiredancer/rtl/sha512_round.sv +++ /dev/null @@ -1,114 +0,0 @@ -`default_nettype none - - -module sha512_round #( - HASH_W = 512, - WORD_W = 64 -)( - input wire [HASH_W-1:0] i_hash, - input wire [WORD_W-1:0] i_word, - input wire [WORD_W-1:0] i_cval, - - output logic [HASH_W-1:0] o_hash, - - input wire clk, - input wire rst -); - -assert property (@(posedge clk) (HASH_W==512)) else begin $display("%m usupported HASH_W: %d",HASH_W); $error("HASH_W!!"); $fatal; end -assert property (@(posedge clk) (WORD_W== 64)) else begin $display("%m usupported WORD_W: %d",WORD_W); $error("WORD_W!!"); $fatal; end - - -localparam CYCLES_ROUND = 1; - -logic [WORD_W-1:0] a; -logic [WORD_W-1:0] b; -logic [WORD_W-1:0] c; -logic [WORD_W-1:0] d; -logic [WORD_W-1:0] e; -logic [WORD_W-1:0] f; -logic [WORD_W-1:0] g; -logic [WORD_W-1:0] h; - -logic [WORD_W-1:0] w; -logic [WORD_W-1:0] k; - -logic [WORD_W-1:0] ma; -logic [WORD_W-1:0] ea; -logic [WORD_W-1:0] ee; -logic [WORD_W-1:0] ch; - -logic [WORD_W-1:0] s0; -logic [WORD_W-1:0] s1; -logic [WORD_W-1:0] s2; -logic [WORD_W-1:0] s6; - -always_comb begin - - w = i_word; - k = i_cval; - - {a,b,c,d,e,f,g,h} = i_hash; - - ma = (a&b) ^ (a&c) ^ (b&c); - ch = (e&f) ^ ((~e)&g); - - ea = {a[28-1:0],a[WORD_W-1:28]} ^ - {a[34-1:0],a[WORD_W-1:34]} ^ - {a[39-1:0],a[WORD_W-1:39]}; - - ee = {e[14-1:0],e[WORD_W-1:14]} ^ - {e[18-1:0],e[WORD_W-1:18]} ^ - {e[41-1:0],e[WORD_W-1:41]}; - - s0 = ea + ma; - - /* keep as a reference - s3 = ch + h; - s4 = s3 + ee; - s5 = s4 + w; - s6 = s5 + k; - s1 = s0 + s6; - s2 = d + s6; */ - s1 = s0 + s6; - s2 = d + s6; -end - -shift_adder_6 #( - .W ( WORD_W ), - .S0 ( 0 ), - .S1 ( 0 ), - .S2 ( 0 ), - .S3 ( 0 ), - .S4 ( 0 ), - .S5 ( 0 ), - .C ( 0 ), - .M ( 0 ), - .R0 ( 0 ), /* output of reduction 6-3 (first layer of reduction) */ - .R1 ( 0 ), /* output of reduction 3-2 (second layer of reduction)*/ - /* reduction 2-1 just an adder */ - .R ( 0 ) /* output */ -) shift_adder_6_s1_inst ( - .clk ( clk ), - .rst ( rst ), - - .cin0 ( '0 ), - .in0 ( ch ), - .in1 ( h ), - .in2 ( ee ), - .in3 ( w ), - .in4 ( k ), - .in5 ( '0 ), - .out0 ( s6 ), - .cout0 ( ), - .m_i ( ), - .m_o ( ) -); - -always_ff@(posedge clk) begin - o_hash <= {s1,a,b,c,s2,e,f,g}; -end - -endmodule // sha512_round - -`default_nettype wire diff --git a/src/wiredancer/rtl/sha512_sch.sv b/src/wiredancer/rtl/sha512_sch.sv deleted file mode 100644 index 771fcaf30bd..00000000000 --- a/src/wiredancer/rtl/sha512_sch.sv +++ /dev/null @@ -1,258 +0,0 @@ -`default_nettype none - -module sha512_sch #( - W_BLK = 64, - W_M = 64, - BLKS_PER_TR = 10, - RAM_D = 512, - N_CYCLES = 100, - RAM_E = RAM_D / BLKS_PER_TR, - N_CYCLES_L = $clog2(N_CYCLES), - RAM_D_L = $clog2(RAM_D) -) ( - output logic [1-1:0] oblk_v, // valid - output logic [W_BLK-1:0] oblk_d, // data - output logic [W_M-1:0] oblk_t, // transaction id - output logic [1-1:0] oblk_f, // first blk - output logic [1-1:0] oblk_m, // middle blk - output logic [1-1:0] oblk_l, // last blk - - input wire [1-1:0] iblk_v, // valid - input wire [1-1:0] iblk_f, // first blk - input wire [4-1:0] iblk_c, // number of blocks - input wire [W_BLK-1:0] iblk_d, // data - input wire [W_M-1:0] iblk_t, // transaction id - output logic [1-1:0] iblk_p, // backpressure is only applied for the first block - - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -logic [N_CYCLES_L-1:0] c00_c_addr; -logic [N_CYCLES_L-1:0] c01_c_addr; -logic [N_CYCLES_L-1:0] c02_c_addr; -logic [N_CYCLES_L-1:0] c03_c_addr; - -logic [RAM_D_L-1:0] c01_a; -logic [4-1:0] c01_c; -logic [W_M-1:0] c01_t; -bit [4-1:0] c01_r; - -logic [1-1:0] c02_v; -logic [1-1:0] c02_n; -logic [RAM_D_L-1:0] c02_a; -logic [4-1:0] c02_c; -logic [4-1:0] c02_r; -logic [1-1:0] c02_f; -logic [1-1:0] c02_m; -logic [1-1:0] c02_l; -logic [W_M-1:0] c02_t; -logic [W_BLK-1:0] c02_d; -logic [1-1:0] c02_free_v; -logic [RAM_D_L-1:0] c02_free_a; -logic [1-1:0] c02_free_pop; - -logic [W_M-1:0] c03_t; -logic [RAM_D_L-1:0] c03_a; -logic [4-1:0] c03_c; -logic [4-1:0] c03_r; -logic [1-1:0] c03_f; -logic [1-1:0] c03_m; -logic [1-1:0] c03_l; - -logic [1-1:0] c03_free_push_v; -logic [RAM_D_L-1:0] c03_free_push_d; - -logic [1-1:0] c03_d_wr_en; -logic [RAM_D_L-1:0] c03_d_wr_addr; -logic [W_BLK-1:0] c03_d_wr_data; - -logic [16-1:0] free_init_cnt; - -assign oblk_v = c02_v; -assign oblk_d = c02_d; -assign oblk_t = c02_t; -assign oblk_f = c02_f & c02_v; -assign oblk_m = c02_m & c02_v; -assign oblk_l = c02_l & c02_v; - -always_comb begin - iblk_p = 0 - | (iblk_v & ~iblk_f) - | ((~c02_n) & iblk_v & c02_free_v) - ; - - c02_free_pop = iblk_v & iblk_p & iblk_f; -end - -always_ff@(posedge clk) begin - - c00_c_addr <= c00_c_addr == (N_CYCLES - 1) ? 0 : (c00_c_addr + 1); - - // c00: read cycle-meta - c01_c_addr <= c00_c_addr; - - // c01: read blk data - - c02_c_addr <= c01_c_addr; - - c02_v <= (c01_r > 0); // valid this round - c02_n <= (c01_r > 1); // valid next round - c02_t <= c01_t; - c02_a <= c01_a; - c02_c <= c01_c; - c02_r <= c01_r - 1; - c02_f <= (c01_r == c01_c); - c02_m <= (c01_r != c01_c) & (c01_r != 1); - c02_l <= (c01_r == 1); - - // c02: mux cycle-meta / send out - - c03_c_addr <= c02_c_addr; - - c03_a <= c02_n ? c02_a - 1 : c02_free_a + iblk_c - 1; - c03_c <= c02_n ? c02_c : iblk_c; - c03_r <= c02_n ? c02_r : (iblk_v & iblk_p & iblk_f) ? iblk_c : 0; - c03_t <= c02_n ? c02_t : iblk_t; - - c03_free_push_v <= ((free_init_cnt >= (1*RAM_E)) & (free_init_cnt < (2*RAM_E))) | (c02_v & ~c02_n); - if (0 - | ((free_init_cnt >= (1*RAM_E)) & (free_init_cnt < (2*RAM_E))) - | (c02_v & ~c02_n) - ) - c03_free_push_d <= ((free_init_cnt >= (1*RAM_E)) & (free_init_cnt < (2*RAM_E))) ? (c03_free_push_d + BLKS_PER_TR) : c02_a; - - c03_d_wr_en <= iblk_v & iblk_p; - c03_d_wr_addr <= iblk_v & iblk_p ? (iblk_f ? c02_free_a + iblk_c - 1 : c03_d_wr_addr - 1) : c03_d_wr_addr - 1; - c03_d_wr_data <= iblk_d; - - - // c03: write cycle-meta - - if (free_init_cnt < (2*RAM_E)) begin - free_init_cnt <= free_init_cnt + 1; - end - - if (rst) begin - c00_c_addr <= '0; - - c02_v <= '0; - c02_n <= '0; - - c03_free_push_v <= '0; - c03_free_push_d <= '0 - BLKS_PER_TR; - - free_init_cnt <= '0; - end - -end - - -simple_dual_port_ram #( - // .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH (N_CYCLES_L), - .DATA_WIDTH ($bits({c03_t, c03_r, c03_c, c03_a})), - .REGISTER_OUTPUT (0) -) cycle_ram_inst( - - .wr_clock (clk), - .wr_address (c03_c_addr), - .wr_en ('1), - .wr_byteenable ('1), - .data ({c03_t, c03_r, c03_c, c03_a}), - - .rd_clock (clk), - .rd_address (c00_c_addr), - .q ({c01_t, c01_r, c01_c, c01_a}), - .rd_en (1'b1) -); - -simple_dual_port_ram #( - // .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH (RAM_D_L), - .DATA_WIDTH (W_BLK), - .REGISTER_OUTPUT (0) -) blk_ram_inst( - - .wr_clock (clk), - .wr_address (c03_d_wr_addr), - .wr_en (c03_d_wr_en), - .wr_byteenable ('1), - .data (c03_d_wr_data), - - .rd_clock (clk), - .rd_address (c01_a), - .q (c02_d), - .rd_en (1'b1) -); - -showahead_fifo #( - .WIDTH (RAM_D_L), - .DEPTH (RAM_E <= 512 ? 512 : 1024) -) idx_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (c03_free_push_v), - .wr_full (), - .wr_data (c03_free_push_d), - - .rd_clk (clk), - .rd_req (c02_free_pop), - .rd_empty (), - .rd_not_empty (c02_free_v), - .rd_count (), - .rd_data (c02_free_a) -); - -// always_ff@(posedge clk) -// if (iblk_v) -// $display("%t: %m.iblk: %x - %x %x - %x %x",$time() -// , iblk_p -// , iblk_f -// , iblk_c -// , iblk_t -// , iblk_d -// ); - -// always_ff@(posedge clk) -// if (oblk_v) -// $display("%t: %m.oblk: %x %x %x - %x %x",$time() -// , oblk_f -// , oblk_m -// , oblk_l -// , oblk_t -// , oblk_d -// ); - -// always_ff@(posedge clk) -// if (c02_free_v | c02_free_pop | c03_free_push_v) -// $display("%t: %m.free: %x - %x %x %x - %x %x",$time() -// , idx_fifo_inst.xpm_fifo_sync_inst.wr_rst_busy -// , c02_free_pop -// , c02_free_v -// , c02_free_a - -// , c03_free_push_v -// , c03_free_push_d -// ); - -// always_ff@(posedge clk) -// // if () -// $display("%t: %m.c01: %x %x %x %x",$time() -// , c01_a -// , c01_c -// , c01_t -// , c01_r -// ); - -// always_ff@(negedge clk) -// $display("%t: -----------",$time()); - -endmodule - - - -`default_nettype wire - - diff --git a/src/wiredancer/rtl/showahead_fifo.sv b/src/wiredancer/rtl/showahead_fifo.sv deleted file mode 100644 index ed7473eda74..00000000000 --- a/src/wiredancer/rtl/showahead_fifo.sv +++ /dev/null @@ -1,528 +0,0 @@ - -`default_nettype none - -module showahead_fifo #(parameter WIDTH=32, DEPTH=32, D_L=$clog2(DEPTH), FULL_THRESH=DEPTH-6)( - input wire wr_clk, - input wire wr_req, - input wire [WIDTH-1:0] wr_data, - output wire wr_full, - output wire wr_full_b, - output wire [D_L+1-1:0] wr_count, - - input wire rd_clk, - input wire rd_req, - output wire [WIDTH-1:0] rd_data, - output wire rd_empty, - output wire rd_not_empty, - output wire [D_L+1-1:0] rd_count, - - input wire aclr -); - -assign wr_full_b = ~wr_full; -assign rd_not_empty = ~rd_empty; - -// XPM_FIFO instantiation template for Synchronous FIFO configurations -// Refer to the targeted device family architecture libraries guide for XPM_FIFO documentation -// ======================================================================================================================= - -// Parameter usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Parameter name | Data type | Restrictions, if applicable | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | DOUT_RESET_VALUE | String | Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Reset value of read data path. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | ECC_MODE | String | Allowed values: no_ecc, en_ecc. Default value = no_ecc. | -// |---------------------------------------------------------------------------------------------------------------------| -// | | -// | "no_ecc" - Disables ECC | -// | "en_ecc" - Enables both ECC Encoder and Decoder | -// | | -// | NOTE: ECC_MODE should be "no_ecc" if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | FIFO_MEMORY_TYPE | String | Allowed values: auto, block, distributed, ultra. Default value = auto. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Designate the fifo memory primitive (resource type) to use- | -// | | -// | "auto"- Allow Vivado Synthesis to choose | -// | "block"- Block RAM FIFO | -// | "distributed"- Distributed RAM FIFO | -// | "ultra"- URAM FIFO | -// | | -// | NOTE: There may be a behavior mismatch if Block RAM or Ultra RAM specific features, like ECC or Asymmetry, are selected with FIFO_MEMORY_TYPE set to "auto".| -// +---------------------------------------------------------------------------------------------------------------------+ -// | FIFO_READ_LATENCY | Integer | Range: 0 - 100. Default value = 1. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Number of output register stages in the read data path | -// | | -// | If READ_MODE = "fwft", then the only applicable value is 0 | -// +---------------------------------------------------------------------------------------------------------------------+ -// | FIFO_WRITE_DEPTH | Integer | Range: 16 - 4194304. Default value = 2048. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Defines the FIFO Write Depth, must be power of two | -// | | -// | In standard READ_MODE, the effective depth = FIFO_WRITE_DEPTH | -// | In First-Word-Fall-Through READ_MODE, the effective depth = FIFO_WRITE_DEPTH+2 | -// | | -// | NOTE: The maximum FIFO size (width x depth) is limited to 150-Megabits. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | FULL_RESET_VALUE | Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Sets full, almost_full and prog_full to FULL_RESET_VALUE during reset | -// +---------------------------------------------------------------------------------------------------------------------+ -// | PROG_EMPTY_THRESH | Integer | Range: 3 - 4194304. Default value = 10. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the minimum number of read words in the FIFO at or below which prog_empty is asserted. | -// | | -// | Min_Value = 3 + (READ_MODE_VAL*2) | -// | Max_Value = (FIFO_WRITE_DEPTH-3) - (READ_MODE_VAL*2) | -// | | -// | If READ_MODE = "std", then READ_MODE_VAL = 0; Otherwise READ_MODE_VAL = 1. | -// | NOTE: The default threshold value is dependent on default FIFO_WRITE_DEPTH value. If FIFO_WRITE_DEPTH value is | -// | changed, ensure the threshold value is within the valid range though the programmable flags are not used. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | PROG_FULL_THRESH | Integer | Range: 3 - 4194301. Default value = 10. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the maximum number of write words in the FIFO at or above which prog_full is asserted. | -// | | -// | Min_Value = 3 + (READ_MODE_VAL*2*(FIFO_WRITE_DEPTH/FIFO_READ_DEPTH)) | -// | Max_Value = (FIFO_WRITE_DEPTH-3) - (READ_MODE_VAL*2*(FIFO_WRITE_DEPTH/FIFO_READ_DEPTH)) | -// | | -// | If READ_MODE = "std", then READ_MODE_VAL = 0; Otherwise READ_MODE_VAL = 1. | -// | NOTE: The default threshold value is dependent on default FIFO_WRITE_DEPTH value. If FIFO_WRITE_DEPTH value is | -// | changed, ensure the threshold value is within the valid range though the programmable flags are not used. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | RD_DATA_COUNT_WIDTH | Integer | Range: 1 - 23. Default value = 1. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the width of rd_data_count. To reflect the correct value, the width should be log2(FIFO_READ_DEPTH)+1. | -// | | -// | FIFO_READ_DEPTH = FIFO_WRITE_DEPTH*WRITE_DATA_WIDTH/READ_DATA_WIDTH | -// +---------------------------------------------------------------------------------------------------------------------+ -// | READ_DATA_WIDTH | Integer | Range: 1 - 4096. Default value = 32. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Defines the width of the read data port, dout | -// | | -// | Write and read width aspect ratio must be 1:1, 1:2, 1:4, 1:8, 8:1, 4:1 and 2:1 | -// | For example, if WRITE_DATA_WIDTH is 32, then the READ_DATA_WIDTH must be 32, 64,128, 256, 16, 8, 4. | -// | | -// | NOTE: | -// | | -// | READ_DATA_WIDTH should be equal to WRITE_DATA_WIDTH if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior. | -// | The maximum FIFO size (width x depth) is limited to 150-Megabits. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | READ_MODE | String | Allowed values: std, fwft. Default value = std. | -// |---------------------------------------------------------------------------------------------------------------------| -// | | -// | "std"- standard read mode | -// | "fwft"- First-Word-Fall-Through read mode | -// +---------------------------------------------------------------------------------------------------------------------+ -// | SIM_ASSERT_CHK | Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | 0- Disable simulation message reporting. Messages related to potential misuse will not be reported. | -// | 1- Enable simulation message reporting. Messages related to potential misuse will be reported. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | USE_ADV_FEATURES | String | Default value = 0707. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Enables data_valid, almost_empty, rd_data_count, prog_empty, underflow, wr_ack, almost_full, wr_data_count, | -// | prog_full, overflow features. | -// | | -// | Setting USE_ADV_FEATURES[0] to 1 enables overflow flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[1] to 1 enables prog_full flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[2] to 1 enables wr_data_count; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[3] to 1 enables almost_full flag; Default value of this bit is 0 | -// | Setting USE_ADV_FEATURES[4] to 1 enables wr_ack flag; Default value of this bit is 0 | -// | Setting USE_ADV_FEATURES[8] to 1 enables underflow flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[9] to 1 enables prog_empty flag; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[10] to 1 enables rd_data_count; Default value of this bit is 1 | -// | Setting USE_ADV_FEATURES[11] to 1 enables almost_empty flag; Default value of this bit is 0 | -// | Setting USE_ADV_FEATURES[12] to 1 enables data_valid flag; Default value of this bit is 0 | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WAKEUP_TIME | Integer | Range: 0 - 2. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | | -// | 0 - Disable sleep | -// | 2 - Use Sleep Pin | -// | | -// | NOTE: WAKEUP_TIME should be 0 if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WRITE_DATA_WIDTH | Integer | Range: 1 - 4096. Default value = 32. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Defines the width of the write data port, din | -// | | -// | Write and read width aspect ratio must be 1:1, 1:2, 1:4, 1:8, 8:1, 4:1 and 2:1 | -// | For example, if WRITE_DATA_WIDTH is 32, then the READ_DATA_WIDTH must be 32, 64,128, 256, 16, 8, 4. | -// | | -// | NOTE: | -// | | -// | WRITE_DATA_WIDTH should be equal to READ_DATA_WIDTH if FIFO_MEMORY_TYPE is set to "auto". Violating this may result incorrect behavior.| -// | The maximum FIFO size (width x depth) is limited to 150-Megabits. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WR_DATA_COUNT_WIDTH | Integer | Range: 1 - 23. Default value = 1. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specifies the width of wr_data_count. To reflect the correct value, the width should be log2(FIFO_WRITE_DEPTH)+1. | -// +---------------------------------------------------------------------------------------------------------------------+ - -// Port usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Port name | Direction | Size, in bits | Domain | Sense | Handling if unused | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | almost_empty | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Almost Empty : When asserted, this signal indicates that only one more read can be performed before the FIFO goes to| -// | empty. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | almost_full | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Almost Full: When asserted, this signal indicates that only one more write can be performed before the FIFO is full.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | data_valid | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Data Valid: When asserted, this signal indicates that valid data is available on the output bus (dout). | -// +---------------------------------------------------------------------------------------------------------------------+ -// | dbiterr | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Double Bit Error: Indicates that the ECC decoder detected a double-bit error and data in the FIFO core is corrupted.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | din | Input | WRITE_DATA_WIDTH | wr_clk | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Data: The input data bus used when writing the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | dout | Output | READ_DATA_WIDTH | wr_clk | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Data: The output data bus is driven when reading the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | empty | Output | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Empty Flag: When asserted, this signal indicates that the FIFO is empty. | -// | Read requests are ignored when the FIFO is empty, initiating a read while empty is not destructive to the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | full | Output | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Full Flag: When asserted, this signal indicates that the FIFO is full. | -// | Write requests are ignored when the FIFO is full, initiating a write when the FIFO is full is not destructive | -// | to the contents of the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | injectdbiterr | Input | 1 | wr_clk | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Double Bit Error Injection: Injects a double bit error if the ECC feature is used on block RAMs or | -// | UltraRAM macros. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | injectsbiterr | Input | 1 | wr_clk | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Single Bit Error Injection: Injects a single bit error if the ECC feature is used on block RAMs or | -// | UltraRAM macros. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | overflow | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Overflow: This signal indicates that a write request (wren) during the prior clock cycle was rejected, | -// | because the FIFO is full. Overflowing the FIFO is not destructive to the contents of the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | prog_empty | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Programmable Empty: This signal is asserted when the number of words in the FIFO is less than or equal | -// | to the programmable empty threshold value. | -// | It is de-asserted when the number of words in the FIFO exceeds the programmable empty threshold value. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | prog_full | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Programmable Full: This signal is asserted when the number of words in the FIFO is greater than or equal | -// | to the programmable full threshold value. | -// | It is de-asserted when the number of words in the FIFO is less than the programmable full threshold value. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rd_data_count | Output | RD_DATA_COUNT_WIDTH | wr_clk | NA | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Data Count: This bus indicates the number of words read from the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rd_en | Input | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Enable: If the FIFO is not empty, asserting this signal causes data (on dout) to be read from the FIFO. | -// | | -// | Must be held active-low when rd_rst_busy is active high. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rd_rst_busy | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Read Reset Busy: Active-High indicator that the FIFO read domain is currently in a reset state. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rst | Input | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Reset: Must be synchronous to wr_clk. The clock(s) can be unstable at the time of applying reset, but reset must be released only after the clock(s) is/are stable.| -// +---------------------------------------------------------------------------------------------------------------------+ -// | sbiterr | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Single Bit Error: Indicates that the ECC decoder detected and fixed a single-bit error. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | sleep | Input | 1 | NA | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Dynamic power saving- If sleep is High, the memory/fifo block is in power saving mode. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | underflow | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Underflow: Indicates that the read request (rd_en) during the previous clock cycle was rejected | -// | because the FIFO is empty. Under flowing the FIFO is not destructive to the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_ack | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Acknowledge: This signal indicates that a write request (wr_en) during the prior clock cycle is succeeded. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_clk | Input | 1 | NA | Rising edge | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write clock: Used for write operation. wr_clk must be a free running clock. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_data_count | Output | WR_DATA_COUNT_WIDTH | wr_clk | NA | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Data Count: This bus indicates the number of words written into the FIFO. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_en | Input | 1 | wr_clk | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Enable: If the FIFO is not full, asserting this signal causes data (on din) to be written to the FIFO | -// | | -// | Must be held active-low when rst or wr_rst_busy or rd_rst_busy is active high | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wr_rst_busy | Output | 1 | wr_clk | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write Reset Busy: Active-High indicator that the FIFO write domain is currently in a reset state. | -// +---------------------------------------------------------------------------------------------------------------------+ - - -// xpm_fifo_sync : In order to incorporate this function into the design, -// Verilog : the following instance declaration needs to be placed -// instance : in the body of the design code. The instance name -// declaration : (xpm_fifo_sync_inst) and/or the port declarations within the -// code : parenthesis may be changed to properly reference and -// : connect this function to the design. All inputs -// : and outputs must be connected. - -// Please reference the appropriate libraries guide for additional information on the XPM modules. - -// <-----Cut code below this line----> - - // xpm_fifo_sync: Synchronous FIFO - // Xilinx Parameterized Macro, version 2019.1 - - xpm_fifo_sync #( - .DOUT_RESET_VALUE("0"), // String - .ECC_MODE("no_ecc"), // String - .FIFO_MEMORY_TYPE("auto"), // String - .FIFO_READ_LATENCY(1), // DECIMAL - .FIFO_WRITE_DEPTH(DEPTH), // DECIMAL - .FULL_RESET_VALUE(0), // DECIMAL - .PROG_EMPTY_THRESH(5), // DECIMAL - .PROG_FULL_THRESH(FULL_THRESH), // DECIMAL - .RD_DATA_COUNT_WIDTH(1+D_L), // DECIMAL - .READ_DATA_WIDTH(WIDTH), // DECIMAL - .READ_MODE("fwft"), // String - .SIM_ASSERT_CHK(0), // DECIMAL; 0=disable simulation messages, 1=enable simulation messages - .USE_ADV_FEATURES("0707"), // String - .WAKEUP_TIME(0), // DECIMAL - .WRITE_DATA_WIDTH(WIDTH), // DECIMAL - .WR_DATA_COUNT_WIDTH(1+D_L) // DECIMAL - ) - xpm_fifo_sync_inst ( - .almost_empty(), // 1-bit output: Almost Empty : When asserted, this signal indicates that - // only one more read can be performed before the FIFO goes to empty. - - .almost_full(), // 1-bit output: Almost Full: When asserted, this signal indicates that - // only one more write can be performed before the FIFO is full. - - .data_valid(), // 1-bit output: Read Data Valid: When asserted, this signal indicates - // that valid data is available on the output bus (dout). - - .dbiterr(), // 1-bit output: Double Bit Error: Indicates that the ECC decoder detected - // a double-bit error and data in the FIFO core is corrupted. - - .dout(rd_data), // READ_DATA_WIDTH-bit output: Read Data: The output data bus is driven - // when reading the FIFO. - - .empty(rd_empty), // 1-bit output: Empty Flag: When asserted, this signal indicates that the - // FIFO is empty. Read requests are ignored when the FIFO is empty, - // initiating a read while empty is not destructive to the FIFO. - - .full(), // 1-bit output: Full Flag: When asserted, this signal indicates that the - // FIFO is full. Write requests are ignored when the FIFO is full, - // initiating a write when the FIFO is full is not destructive to the - // contents of the FIFO. - - .overflow(), // 1-bit output: Overflow: This signal indicates that a write request - // (wren) during the prior clock cycle was rejected, because the FIFO is - // full. Overflowing the FIFO is not destructive to the contents of the - // FIFO. - - .prog_empty(), // 1-bit output: Programmable Empty: This signal is asserted when the - // number of words in the FIFO is less than or equal to the programmable - // empty threshold value. It is de-asserted when the number of words in - // the FIFO exceeds the programmable empty threshold value. - - .prog_full(wr_full), // 1-bit output: Programmable Full: This signal is asserted when the - // number of words in the FIFO is greater than or equal to the - // programmable full threshold value. It is de-asserted when the number of - // words in the FIFO is less than the programmable full threshold value. - - .rd_data_count(rd_count), // RD_DATA_COUNT_WIDTH-bit output: Read Data Count: This bus indicates the - // number of words read from the FIFO. - - .rd_rst_busy(), // 1-bit output: Read Reset Busy: Active-High indicator that the FIFO read - // domain is currently in a reset state. - - .sbiterr(), // 1-bit output: Single Bit Error: Indicates that the ECC decoder detected - // and fixed a single-bit error. - - .underflow(), // 1-bit output: Underflow: Indicates that the read request (rd_en) during - // the previous clock cycle was rejected because the FIFO is empty. Under - // flowing the FIFO is not destructive to the FIFO. - - .wr_ack(), // 1-bit output: Write Acknowledge: This signal indicates that a write - // request (wr_en) during the prior clock cycle is succeeded. - - .wr_data_count(wr_count), // WR_DATA_COUNT_WIDTH-bit output: Write Data Count: This bus indicates - // the number of words written into the FIFO. - - .wr_rst_busy(), // 1-bit output: Write Reset Busy: Active-High indicator that the FIFO - // write domain is currently in a reset state. - - .din(wr_data), // WRITE_DATA_WIDTH-bit input: Write Data: The input data bus used when - // writing the FIFO. - - .injectdbiterr(), // 1-bit input: Double Bit Error Injection: Injects a double bit error if - // the ECC feature is used on block RAMs or UltraRAM macros. - - .injectsbiterr(), // 1-bit input: Single Bit Error Injection: Injects a single bit error if - // the ECC feature is used on block RAMs or UltraRAM macros. - - .rd_en(rd_req), // 1-bit input: Read Enable: If the FIFO is not empty, asserting this - // signal causes data (on dout) to be read from the FIFO. Must be held - // active-low when rd_rst_busy is active high. - - .rst(aclr), // 1-bit input: Reset: Must be synchronous to wr_clk. The clock(s) can be - // unstable at the time of applying reset, but reset must be released only - // after the clock(s) is/are stable. - - .sleep(1'b0), // 1-bit input: Dynamic power saving- If sleep is High, the memory/fifo - // block is in power saving mode. - - .wr_clk(wr_clk), // 1-bit input: Write clock: Used for write operation. wr_clk must be a - // free running clock. - - .wr_en(wr_req) // 1-bit input: Write Enable: If the FIFO is not full, asserting this - // signal causes data (on din) to be written to the FIFO Must be held - // active-low when rst or wr_rst_busy or rd_rst_busy is active high - - ); - - // End of xpm_fifo_sync_inst instantiation - - -endmodule - - - - - - - - - - - - - - - - - - - - - - - - - - - - -module showahead_fifo_nx1 #( - N = 8, - N_L = $clog2(N), - WIDTH = 1, - DEPTH = 512, - D_L = $clog2(DEPTH), - FULL_THRESH = DEPTH-6 -) ( - input wire wr_clk, - input wire wr_req, - input wire [N-1:0][WIDTH-1:0] wr_data, - output wire wr_full, - output wire wr_full_b, - output wire [D_L+1-1:0] wr_count, - - input wire rd_clk, - input wire rd_req, - input wire rd_all, - output wire [WIDTH-1:0] rd_data, - output wire [N-1:0][WIDTH-1:0] rd_data_all, - output wire rd_empty, - output wire rd_not_empty, - output wire [D_L+1-1:0] rd_count, - - input wire aclr -); - -logic [N-1:0] full; - -logic [N_L-1:0] i; -logic [N-1:0] out_vs; -logic [N-1:0][WIDTH-1:0] out_ds; -logic [N-1:0][D_L+1-1:0] wr_count_p; - -assign wr_full = full[N-1]; -assign wr_full_b = ~wr_full; -assign wr_count = wr_count_p[N-1]; - -assign rd_not_empty = out_vs[i]; -assign rd_empty = ~rd_not_empty; -assign rd_data = out_ds[i]; -assign rd_data_all = out_ds; - -always_ff@(posedge rd_clk) begin - if (rd_req) - i <= rd_all ? 0 : (i + 1); - if (aclr) - i <= 0; -end - -generate - for (genvar g_i = 0; g_i < N; g_i ++) begin - showahead_fifo #( - .WIDTH (WIDTH), - .DEPTH (DEPTH), - .FULL_THRESH (FULL_THRESH) - ) fifo_inst ( - .aclr (aclr), - - .wr_clk (wr_clk), - .wr_req (wr_req), - .wr_full (full[g_i]), - .wr_data (wr_data[g_i]), - .wr_count (wr_count_p[g_i]), - - .rd_clk (rd_clk), - .rd_req (rd_req & ((i == g_i) | (rd_all & (i <= g_i)))), - .rd_empty (), - .rd_not_empty (out_vs[g_i]), - .rd_count (), - .rd_data (out_ds[g_i]) - ); - end -endgenerate - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/simple_dual_port_ram.sv b/src/wiredancer/rtl/simple_dual_port_ram.sv deleted file mode 100644 index 82ef72f1351..00000000000 --- a/src/wiredancer/rtl/simple_dual_port_ram.sv +++ /dev/null @@ -1,381 +0,0 @@ -`default_nettype none - -module simple_dual_port_ram #( - ADDRESS_WIDTH=10, - DATA_WIDTH = 32, - REGISTER_OUTPUT=0, - CLOCKING_MODE="independent_clock", - MEMORY_PRIMITIVE="block", - WRITE_MODE="read_first" -) ( - - input wire rd_clock, - input wire [ADDRESS_WIDTH-1:0] rd_address, - output logic [DATA_WIDTH-1:0] q, - input wire rd_en, - - input wire wr_clock, - input wire [ADDRESS_WIDTH-1:0] wr_address, - input wire [DATA_WIDTH/8-1:0] wr_byteenable, - input wire [DATA_WIDTH-1:0] data, - input wire wr_en -); - - - - -// XPM_MEMORY instantiation template for Simple Dual Port RAM configurations -// Refer to the targeted device family architecture libraries guide for XPM_MEMORY documentation -// ======================================================================================================================= - -// Parameter usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Parameter name | Data type | Restrictions, if applicable | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | ADDR_WIDTH_A | Integer | Range: 1 - 20. Default value = 6. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify the width of the port A address port addra, in bits. | -// | Must be large enough to access the entire memory from port A, i.e. = $clog2(MEMORY_SIZE/WRITE_DATA_WIDTH_A). | -// +---------------------------------------------------------------------------------------------------------------------+ -// | ADDR_WIDTH_B | Integer | Range: 1 - 20. Default value = 6. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify the width of the port B address port addrb, in bits. | -// | Must be large enough to access the entire memory from port B, i.e. = $clog2(MEMORY_SIZE/READ_DATA_WIDTH_B). | -// +---------------------------------------------------------------------------------------------------------------------+ -// | AUTO_SLEEP_TIME | Integer | Range: 0 - 15. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Number of clk[a|b] cycles to auto-sleep, if feature is available in architecture. | -// | | -// | 0 - Disable auto-sleep feature | -// | 3-15 - Number of auto-sleep latency cycles | -// | | -// | Do not change from the value provided in the template instantiation. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | BYTE_WRITE_WIDTH_A | Integer | Range: 1 - 4608. Default value = 32. | -// |---------------------------------------------------------------------------------------------------------------------| -// | To enable byte-wide writes on port A, specify the byte width, in bits. | -// | | -// | 8- 8-bit byte-wide writes, legal when WRITE_DATA_WIDTH_A is an integer multiple of 8 | -// | 9- 9-bit byte-wide writes, legal when WRITE_DATA_WIDTH_A is an integer multiple of 9 | -// | | -// | Or to enable word-wide writes on port A, specify the same value as for WRITE_DATA_WIDTH_A. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | CASCADE_HEIGHT | Integer | Range: 0 - 64. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | 0- No Cascade Height, Allow Vivado Synthesis to choose. | -// | 1 or more - Vivado Synthesis sets the specified value as Cascade Height. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | CLOCKING_MODE | String | Allowed values: common_clock, independent_clock. Default value = common_clock.| -// |---------------------------------------------------------------------------------------------------------------------| -// | Designate whether port A and port B are clocked with a common clock or with independent clocks. | -// | | -// | "common_clock"- Common clocking; clock both port A and port B with clka | -// | "independent_clock"- Independent clocking; clock port A with clka and port B with clkb | -// +---------------------------------------------------------------------------------------------------------------------+ -// | ECC_MODE | String | Allowed values: no_ecc, both_encode_and_decode, decode_only, encode_only. Default value = no_ecc.| -// |---------------------------------------------------------------------------------------------------------------------| -// | | -// | "no_ecc" - Disables ECC | -// | "encode_only" - Enables ECC Encoder only | -// | "decode_only" - Enables ECC Decoder only | -// | "both_encode_and_decode" - Enables both ECC Encoder and Decoder | -// +---------------------------------------------------------------------------------------------------------------------+ -// | MEMORY_INIT_FILE | String | Default value = none. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify "none" (including quotes) for no memory initialization, or specify the name of a memory initialization file.| -// | Enter only the name of the file with .mem extension, including quotes but without path (e.g. "my_file.mem"). | -// | File format must be ASCII and consist of only hexadecimal values organized into the specified depth by | -// | narrowest data width generic value of the memory. See the Memory File (MEM) section for more | -// | information on the syntax. Initialization of memory happens through the file name specified only when parameter | -// | MEMORY_INIT_PARAM value is equal to "". | -// | When using XPM_MEMORY in a project, add the specified file to the Vivado project as a design source. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | MEMORY_INIT_PARAM | String | Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify "" or "0" (including quotes) for no memory initialization through parameter, or specify the string | -// | containing the hex characters. Enter only hex characters with each location separated by delimiter (,). | -// | Parameter format must be ASCII and consist of only hexadecimal values organized into the specified depth by | -// | narrowest data width generic value of the memory.For example, if the narrowest data width is 8, and the depth of | -// | memory is 8 locations, then the parameter value should be passed as shown below. | -// | parameter MEMORY_INIT_PARAM = "AB,CD,EF,1,2,34,56,78" | -// | Where "AB" is the 0th location and "78" is the 7th location. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | MEMORY_OPTIMIZATION | String | Allowed values: true, false. Default value = true. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify "true" to enable the optimization of unused memory or bits in the memory structure. Specify "false" to | -// | disable the optimization of unused memory or bits in the memory structure. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | MEMORY_PRIMITIVE | String | Allowed values: auto, block, distributed, ultra. Default value = auto. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Designate the memory primitive (resource type) to use. | -// | | -// | "auto"- Allow Vivado Synthesis to choose | -// | "distributed"- Distributed memory | -// | "block"- Block memory | -// | "ultra"- Ultra RAM memory | -// | | -// | NOTE: There may be a behavior mismatch if Block RAM or Ultra RAM specific features, like ECC or Asymmetry, are selected with MEMORY_PRIMITIVE set to "auto".| -// +---------------------------------------------------------------------------------------------------------------------+ -// | MEMORY_SIZE | Integer | Range: 2 - 150994944. Default value = 2048. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify the total memory array size, in bits. For example, enter 65536 for a 2kx32 RAM. | -// | | -// | When ECC is enabled and set to "encode_only", then the memory size has to be multiples of READ_DATA_WIDTH_B | -// | When ECC is enabled and set to "decode_only", then the memory size has to be multiples of WRITE_DATA_WIDTH_A | -// +---------------------------------------------------------------------------------------------------------------------+ -// | MESSAGE_CONTROL | Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify 1 to enable the dynamic message reporting such as collision warnings, and 0 to disable the message reporting| -// +---------------------------------------------------------------------------------------------------------------------+ -// | READ_DATA_WIDTH_B | Integer | Range: 1 - 4608. Default value = 32. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify the width of the port B read data output port doutb, in bits. | -// | | -// | When ECC is enabled and set to "encode_only", then READ_DATA_WIDTH_B has to be multiples of 72-bits | -// | When ECC is enabled and set to "decode_only" or "both_encode_and_decode", then READ_DATA_WIDTH_B has to be | -// | multiples of 64-bits | -// +---------------------------------------------------------------------------------------------------------------------+ -// | READ_LATENCY_B | Integer | Range: 0 - 100. Default value = 2. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify the number of register stages in the port B read data pipeline. Read data output to port doutb takes this | -// | number of clkb cycles (clka when CLOCKING_MODE is "common_clock"). | -// | To target block memory, a value of 1 or larger is required- 1 causes use of memory latch only; 2 causes use of | -// | output register. To target distributed memory, a value of 0 or larger is required- 0 indicates combinatorial output.| -// | Values larger than 2 synthesize additional flip-flops that are not retimed into memory primitives. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | READ_RESET_VALUE_B | String | Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify the reset value of the port B final output register stage in response to rstb input port is assertion. | -// | As this parameter is a string, please specify the hex values inside double quotes. As an example, | -// | If the read data width is 8, then specify READ_RESET_VALUE_B = "EA"; | -// | When ECC is enabled, reset value is not supported. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | RST_MODE_A | String | Allowed values: SYNC, ASYNC. Default value = SYNC. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Describes the behaviour of the reset | -// | | -// | "SYNC" - when reset is applied, synchronously resets output port douta to the value specified by parameter READ_RESET_VALUE_A| -// | "ASYNC" - when reset is applied, asynchronously resets output port douta to zero | -// +---------------------------------------------------------------------------------------------------------------------+ -// | RST_MODE_B | String | Allowed values: SYNC, ASYNC. Default value = SYNC. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Describes the behaviour of the reset | -// | | -// | "SYNC" - when reset is applied, synchronously resets output port doutb to the value specified by parameter READ_RESET_VALUE_B| -// | "ASYNC" - when reset is applied, asynchronously resets output port doutb to zero | -// +---------------------------------------------------------------------------------------------------------------------+ -// | SIM_ASSERT_CHK | Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | 0- Disable simulation message reporting. Messages related to potential misuse will not be reported. | -// | 1- Enable simulation message reporting. Messages related to potential misuse will be reported. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | USE_EMBEDDED_CONSTRAINT| Integer | Range: 0 - 1. Default value = 0. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify 1 to enable the set_false_path constraint addition between clka of Distributed RAM and doutb_reg on clkb | -// +---------------------------------------------------------------------------------------------------------------------+ -// | USE_MEM_INIT | Integer | Range: 0 - 1. Default value = 1. | -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify 1 to enable the generation of below message and 0 to disable generation of the following message completely.| -// | "INFO - MEMORY_INIT_FILE and MEMORY_INIT_PARAM together specifies no memory initialization. | -// | Initial memory contents will be all 0s." | -// | NOTE: This message gets generated only when there is no Memory Initialization specified either through file or | -// | Parameter. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WAKEUP_TIME | String | Allowed values: disable_sleep, use_sleep_pin. Default value = disable_sleep.| -// |---------------------------------------------------------------------------------------------------------------------| -// | Specify "disable_sleep" to disable dynamic power saving option, and specify "use_sleep_pin" to enable the | -// | dynamic power saving option | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WRITE_DATA_WIDTH_A | Integer | Range: 1 - 4608. Default value = 32. | -// |---------------------------------------------------------------------------------------------------------------------| -// | multiples of 64-bits | -// | When ECC is enabled and set to "decode_only", then WRITE_DATA_WIDTH_A has to be multiples of 72-bits | -// +---------------------------------------------------------------------------------------------------------------------+ -// | WRITE_MODE_B | String | Allowed values: no_change, read_first, write_first. Default value = no_change.| -// |---------------------------------------------------------------------------------------------------------------------| -// | Write mode behavior for port B output data port, doutb. | -// +---------------------------------------------------------------------------------------------------------------------+ - -// Port usage table, organized as follows: -// +---------------------------------------------------------------------------------------------------------------------+ -// | Port name | Direction | Size, in bits | Domain | Sense | Handling if unused | -// |---------------------------------------------------------------------------------------------------------------------| -// | Description | -// +---------------------------------------------------------------------------------------------------------------------+ -// +---------------------------------------------------------------------------------------------------------------------+ -// | addra | Input | ADDR_WIDTH_A | clka | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Address for port A write operations. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | addrb | Input | ADDR_WIDTH_B | clkb | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Address for port B read operations. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | clka | Input | 1 | NA | Rising edge | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Clock signal for port A. Also clocks port B when parameter CLOCKING_MODE is "common_clock". | -// +---------------------------------------------------------------------------------------------------------------------+ -// | clkb | Input | 1 | NA | Rising edge | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Clock signal for port B when parameter CLOCKING_MODE is "independent_clock". | -// | Unused when parameter CLOCKING_MODE is "common_clock". | -// +---------------------------------------------------------------------------------------------------------------------+ -// | dbiterrb | Output | 1 | clkb | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Status signal to indicate double bit error occurrence on the data output of port B. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | dina | Input | WRITE_DATA_WIDTH_A | clka | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Data input for port A write operations. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | doutb | Output | READ_DATA_WIDTH_B | clkb | NA | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Data output for port B read operations. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | ena | Input | 1 | clka | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Memory enable signal for port A. | -// | Must be high on clock cycles when write operations are initiated. Pipelined internally. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | enb | Input | 1 | clkb | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Memory enable signal for port B. | -// | Must be high on clock cycles when read operations are initiated. Pipelined internally. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | injectdbiterra | Input | 1 | clka | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Controls double bit error injection on input data when ECC enabled (Error injection capability is not available in | -// | "decode_only" mode). | -// +---------------------------------------------------------------------------------------------------------------------+ -// | injectsbiterra | Input | 1 | clka | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Controls single bit error injection on input data when ECC enabled (Error injection capability is not available in | -// | "decode_only" mode). | -// +---------------------------------------------------------------------------------------------------------------------+ -// | regceb | Input | 1 | clkb | Active-high | Tie to 1'b1 | -// |---------------------------------------------------------------------------------------------------------------------| -// | Clock Enable for the last register stage on the output data path. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | rstb | Input | 1 | clkb | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Reset signal for the final port B output register stage. | -// | Synchronously resets output port doutb to the value specified by parameter READ_RESET_VALUE_B. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | sbiterrb | Output | 1 | clkb | Active-high | DoNotCare | -// |---------------------------------------------------------------------------------------------------------------------| -// | Status signal to indicate single bit error occurrence on the data output of port B. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | sleep | Input | 1 | NA | Active-high | Tie to 1'b0 | -// |---------------------------------------------------------------------------------------------------------------------| -// | sleep signal to enable the dynamic power saving feature. | -// +---------------------------------------------------------------------------------------------------------------------+ -// | wea | Input | WRITE_DATA_WIDTH_A | clka | Active-high | Required | -// |---------------------------------------------------------------------------------------------------------------------| -// | Write enable vector for port A input data port dina. 1 bit wide when word-wide writes are used. | -// | In byte-wide write configurations, each bit controls the writing one byte of dina to address addra. | -// | For example, to synchronously write only bits [15-8] of dina when WRITE_DATA_WIDTH_A is 32, wea would be 4'b0010. | -// +---------------------------------------------------------------------------------------------------------------------+ - - -// xpm_memory_sdpram : In order to incorporate this function into the design, -// Verilog : the following instance declaration needs to be placed -// instance : in the body of the design code. The instance name -// declaration : (xpm_memory_sdpram_inst) and/or the port declarations within the -// code : parenthesis may be changed to properly reference and -// : connect this function to the design. All inputs -// : and outputs must be connected. - -// Please reference the appropriate libraries guide for additional information on the XPM modules. - -// <-----Cut code below this line----> - - // xpm_memory_sdpram: Simple Dual Port RAM - // Xilinx Parameterized Macro, version 2019.1 - - xpm_memory_sdpram #( - .ADDR_WIDTH_A(ADDRESS_WIDTH), // DECIMAL - .ADDR_WIDTH_B(ADDRESS_WIDTH), // DECIMAL - .AUTO_SLEEP_TIME(0), // DECIMAL - .BYTE_WRITE_WIDTH_A(DATA_WIDTH), // DECIMAL - .CASCADE_HEIGHT(0), // DECIMAL - .CLOCKING_MODE(CLOCKING_MODE), // String - .ECC_MODE("no_ecc"), // String - .MEMORY_INIT_FILE("none"), // String - .MEMORY_INIT_PARAM("0"), // String - .MEMORY_OPTIMIZATION("true"), // String - .MEMORY_PRIMITIVE(MEMORY_PRIMITIVE), // String - .MEMORY_SIZE((1<= D-1; -assign oo_v = (oo_t > last_ts); - -simple_dual_port_ram #( - .WRITE_MODE ("read_first"), - .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH (D_L), - .DATA_WIDTH ($bits({timestamp, i_d})) -) ram_inst ( - .wr_clock (clk), - .wr_address (i_a), - .wr_en (i_v), - .wr_byteenable ('1), - .data ({timestamp, i_d}), - - .rd_clock (clk), - .rd_address (oo_a_n), - .q ({oo_t, oo_d}), - .rd_en (1'b1) -); - -simple_dual_port_ram #( - .WRITE_MODE ("read_first"), - .CLOCKING_MODE ("common_clock"), - .ADDRESS_WIDTH (D_L), - .DATA_WIDTH ($bits({timestamp})) -) ts_ram_inst ( - .wr_clock (clk), - .wr_address (oo_a), - .wr_en (oo_v & oo_r), - .wr_byteenable ('1), - .data (timestamp), - - .rd_clock (clk), - .rd_address (oo_a_n), - .q (last_ts), - .rd_en (1'b1) -); - -always_ff@(posedge clk) begin - timestamp <= timestamp + 1; - oo_a <= oo_a_n; - - if (oo_r) begin - o_v <= oo_v; - o_d <= oo_d; - end - - case({ - i_v, - o_v & o_r - }) - 2'b10: i_c <= i_c + 1; - 2'b01: i_c <= i_c - 1; - endcase - - if (rst) begin - i_c <= 0; - o_v <= 0; - oo_a <= 0; - timestamp <= 0; - end -end - -always_ff@(posedge clk) if (i_v | o_v) $display("%t: %x %x %b %b %x - %x %x %x", $time - , i_v - , i_a - , i_c - , i_f - , i_d - - , o_v - , o_r - , o_d -); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/rtl/top_f1.sv b/src/wiredancer/rtl/top_f1.sv deleted file mode 100644 index a7235856cbe..00000000000 --- a/src/wiredancer/rtl/top_f1.sv +++ /dev/null @@ -1,981 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module top_f1 #( - // fast sim - // MUL_T = 32'h0000_0802, // 8-cycle mock mul_wide - // MUL_D = 15, - // N_SCH = 2, - // DSDP_WS = 256, - // TH_PRE = {12'h0, 12'd2, 12'd2}, - // TH_SHA = {12'h0, 12'd2, 12'd2}, - // TH_SV0 = {12'h0, 12'd2, 12'd2}, - // TH_SV1 = {12'h0, 12'd2, 12'd2}, - // TH_SV2 = {12'h0, 12'd2, 12'd2}, - - // small - // KEY_D = 32, - // MUL_T = 32'h007F_CCC2, - // MUL_D = 15, - // N_SCH = 2, - // DSDP_WS = 256, - // TH_PRE = {12'h0, 12'd2, 12'd2}, - // TH_SHA = {12'h0, 12'd2, 12'd2}, - // TH_SV0 = {12'h0, 12'd2, 12'd2}, - // TH_SV1 = {12'h0, 12'd2, 12'd2}, - // TH_SV2 = {12'h0, 12'd2, 12'd2}, - - // full - KEY_D = 512, - MUL_T = 32'h07F_CCC2, - MUL_D = 15, - N_SCH = 5, - DSDP_WS = 256, - TH_PRE = {12'h0, 12'd10, 12'd10}, - TH_SHA = {12'h0, 12'd200, 12'd200}, - TH_SV0 = {12'h0, 12'd200, 12'd200}, - TH_SV1 = {12'h0, 12'd200, 12'd200}, - TH_SV2 = {12'h0, 12'd200, 12'd200}, - - DBG_WIDTH = 1024, - NO_DDR = 4, - DMA_N = 2, - DDR_BUFF_W = 20 -) ( - - input wire [1-1:0] avmm_read, - input wire [1-1:0] avmm_write, - input wire [32-1:0] avmm_address, - input wire [32-1:0] avmm_writedata, - output logic [32-1:0] avmm_readdata, - output logic [1-1:0] avmm_readdatavalid, - output logic [1-1:0] avmm_waitrequest, - - input wire [16-1:0][8-1:0] priv_bytes, - - input wire [2-1:0] pcie_v, - input wire [64-1:0] pcie_a, - input wire [2-1:0][256-1:0] pcie_d, - - input wire [1-1:0] dma_r, - output logic [1-1:0] dma_v, - output logic [64-1:0] dma_a, - output logic [64-1:0] dma_b, - input wire [1-1:0] dma_f, - output logic [256-1:0] dma_d, - - output logic [NO_DDR-1:0][1-1:0] ddr_rd_en, - input wire [NO_DDR-1:0][1-1:0] ddr_rd_pop, - output logic [NO_DDR-1:0][64-1:0] ddr_rd_addr, - output logic [NO_DDR-1:0][9-1:0] ddr_rd_sz, - input wire [NO_DDR-1:0][1-1:0] ddr_rd_v, - input wire [NO_DDR-1:0][512-1:0] ddr_rd_data, - - output logic [NO_DDR-1:0][1-1:0] ddr_wr_en, - input wire [NO_DDR-1:0][1-1:0] ddr_wr_pop, - input wire [NO_DDR-1:0][1-1:0] ddr_wr_res, - output logic [NO_DDR-1:0][64-1:0] ddr_wr_addr, - output wire [NO_DDR-1:0][512-1:0] ddr_wr_data, - - output logic [DBG_WIDTH-1:0] dbg_wire, - - input wire clk_f, - input wire rst_f, - - input wire clk, - input wire rst -); - -logic [64-1:0] timestamp = 0; - -logic [32-1:0] tr_pending; - -(* dont_touch = "yes" *) logic [10-1:0] rst_r; -(* dont_touch = "yes" *) logic [10-1:0] rst_f_r; - -logic [1-1:0] send_fails = 0; - -logic [4-1:0] ths_msb; -logic [4-1:0][36-1:0] ths = { - TH_SV1, - TH_SV0, - TH_SHA, - TH_PRE -}; - -logic [1-1:0] pcie_iv [N_PCIE-1:0]; -logic [1-1:0] pcie_if [N_PCIE-1:0]; -logic [16-1:0] pcie_il [N_PCIE-1:0]; -logic [512-1:0] pcie_id [N_PCIE-1:0]; - -logic [N_PCIE-1:0][1-1:0] ext_r; -logic [N_PCIE-1:0][1-1:0] ext_v; -logic [N_PCIE-1:0][1-1:0] ext_e; -sv_meta2_t [N_PCIE-1:0] ext_m0; -pcie_meta_t [N_PCIE-1:0] ext_m1; - -logic [1-1:0] pad_i_r; -logic [1-1:0] pad_i_w; -logic [1-1:0] pad_i_v; -logic [1-1:0] pad_i_e; -sv_meta2_t pad_i_m; - -logic [1-1:0] pad_o_v; -logic [1-1:0] pad_o_e; -sv_meta3_t pad_o_m; -logic [10-1:0] pad_o_f; - -logic [1-1:0] sha_f_v; -logic [1-1:0] sha_f_e; -sv_meta3_t sha_f_m; -logic [10-1:0] sha_f_f; - -logic [1-1:0] sha_i_r; -logic [1-1:0] sha_i_w; -logic [1-1:0] sha_i_v; -logic [1-1:0] sha_i_e; -sv_meta3_t sha_i_m; - -logic [1-1:0] sha_o_v; -sv_meta4_t sha_o_m; -logic [10-1:0] sha_o_f; - -logic [1-1:0] sv0_f_v; -sv_meta4_t sv0_f_m; -logic [10-1:0] sv0_f_f; - -logic [1-1:0] sv0_i_r; -logic [1-1:0] sv0_i_w; -logic [1-1:0] sv0_i_v; -sv_meta4_t sv0_i_m; - -logic [1-1:0] sv0_o_v; -sv_meta5_t sv0_o_m; -logic [10-1:0] sv0_o_f; - -logic [1-1:0] sv1_f_v; -sv_meta5_t sv1_f_m; -logic [10-1:0] sv1_f_f; - -logic [1-1:0] sv1_i_r; -logic [1-1:0] sv1_i_w; -logic [1-1:0] sv1_i_v; -sv_meta5_t sv1_i_m; - -logic [1-1:0] sv1_o_v; -sv_meta6_t sv1_o_m; -logic [10-1:0] sv1_o_f; - -logic [1-1:0] sv2_f_v; -sv_meta6_t sv2_f_m; -logic [10-1:0] sv2_f_f; - -logic [1-1:0] sv2_i_r; -logic [1-1:0] sv2_i_v; -sv_meta6_t sv2_i_m; - -logic [1-1:0] sv2_o_v; -sv_meta7_t sv2_o_m; - -logic [1-1:0] ecc_o_v; -sv_meta7_t ecc_o_m; - -logic [N_PCIE-1:0][1-1:0] res_o_v; -logic [N_PCIE-1:0][64-1:0] res_o_t; -logic [N_PCIE-1:0][1-1:0] res_o_d; -logic [N_PCIE-1:0][16-1:0] res_o_c; -logic [N_PCIE-1:0][1-1:0] res_o_f; -logic [N_PCIE-1:0][1-1:0] res_o_p; - -logic [N_PCIE-1:0][1-1:0] dma_p_r; -logic [N_PCIE-1:0][1-1:0] dma_p_v; -logic [N_PCIE-1:0][1-1:0] dma_p_vv; -logic [N_PCIE-1:0][1-1:0] dma_p_f; -logic [N_PCIE-1:0][16-1:0] dma_p_c; -mcache_pcim_t [N_PCIE-1:0] dma_p_dab; - - - - - - - -// AAA VVVVVVVV VVVVVVVVMMMMMMMM MMMMMMMMMMMMMMMM MMMMMMMM -// A:::A V::::::V V::::::VM:::::::M M:::::::MM:::::::M M:::::::M -// A:::::A V::::::V V::::::VM::::::::M M::::::::MM::::::::M M::::::::M -// A:::::::A V::::::V V::::::VM:::::::::M M:::::::::MM:::::::::M M:::::::::M -// A:::::::::A V:::::V V:::::V M::::::::::M M::::::::::MM::::::::::M M::::::::::M -// A:::::A:::::A V:::::V V:::::V M:::::::::::M M:::::::::::MM:::::::::::M M:::::::::::M -// A:::::A A:::::A V:::::V V:::::V M:::::::M::::M M::::M:::::::MM:::::::M::::M M::::M:::::::M -// A:::::A A:::::A V:::::V V:::::V M::::::M M::::M M::::M M::::::MM::::::M M::::M M::::M M::::::M -// A:::::A A:::::A V:::::V V:::::V M::::::M M::::M::::M M::::::MM::::::M M::::M::::M M::::::M -// A:::::AAAAAAAAA:::::A V:::::V V:::::V M::::::M M:::::::M M::::::MM::::::M M:::::::M M::::::M -// A:::::::::::::::::::::A V:::::V:::::V M::::::M M:::::M M::::::MM::::::M M:::::M M::::::M -// A:::::AAAAAAAAAAAAA:::::A V:::::::::V M::::::M MMMMM M::::::MM::::::M MMMMM M::::::M -// A:::::A A:::::A V:::::::V M::::::M M::::::MM::::::M M::::::M -// A:::::A A:::::A V:::::V M::::::M M::::::MM::::::M M::::::M -// A:::::A A:::::A V:::V M::::::M M::::::MM::::::M M::::::M -// AAAAAAA AAAAAAA VVV MMMMMMMM MMMMMMMMMMMMMMMM MMMMMMMM - -logic [8-1:0] w_i; -logic [1-1:0] cnt_rst; -logic [1-1:0] cnt_snp; -logic [32-1:0][32-1:0] cnt = 0; -logic [32-1:0][32-1:0] cnt_s; - -assign avmm_waitrequest = '0; - -always_ff@(posedge clk) begin - - timestamp <= timestamp + 1; - - avmm_readdatavalid <= avmm_read; - - case (avmm_address[2+:8]) - 8'h00: avmm_readdata <= 32'h5000_0000; - 8'h01: avmm_readdata <= 32'h0002_0006; - - 8'h10: avmm_readdata <= w_i; - 8'h11: avmm_readdata <= timestamp[0 +:32]; - 8'h12: avmm_readdata <= timestamp[32+:32]; - - 8'h20: avmm_readdata <= cnt_s[w_i]; - 8'h21: avmm_readdata <= {tr_pending[0+:10], res_o_c[0][0+:10], pcie_il[0][0+:12]}; - 8'h22: avmm_readdata <= {tr_pending[0+:10], res_o_c[1][0+:10], pcie_il[1][0+:12]}; - endcase - - if (avmm_write) begin - case (avmm_address[2+:8]) - - 8'h10: w_i <= avmm_writedata; - 8'h11: send_fails <= avmm_writedata; - 8'h13: ths_msb <= avmm_writedata; - 8'h14: ths[w_i] <= {ths_msb, avmm_writedata}; - - 8'h20: begin - cnt_snp <= avmm_writedata[1]; - cnt_rst <= avmm_writedata[0]; - end - - endcase - end else begin - cnt_rst <= 0; - cnt_snp <= 0; - end -end - -`define CNT(ci, expr) piped_counter #(.D(2),.W(32)) cntr_``ci`` (.clk(clk), .rst(rst), .c(cnt_s[ci]), .p(expr), .r(cnt_rst), .s(cnt_snp)); -`define CNM(ci, expr, c) always_ff@(posedge clk) begin if (cnt_snp) cnt_s[ci] <= cnt[ci]; cnt[ci] <= (cnt_rst) ? '0 : cnt[ci] + ((expr) ? c : 0); end -`define MON(ci, expr) always_ff@(posedge clk) begin if (cnt_snp) cnt_s[ci] <= cnt[ci]; cnt[ci] <= (expr); end - -`CNT( 0, pad_i_v & pad_i_r & pad_i_e) -`CNT( 1, pad_o_v & pad_o_e) -`CNT( 2, sha_o_v) -`CNT( 3, sv0_o_v) -`CNT( 4, sv2_f_v) // sv1_o_v is in clk_f domain -`CNT( 5, sv2_o_v) -`CNT( 6, ecc_o_v) - -`MON( 7, tr_pending) - -`CNT(10, pcie_iv[0]) // input count -`MON(11, pcie_il[0]) // input fill -`CNT(12, pcie_iv[0] & pcie_if[0]) // input drops -`CNT(13, res_o_v[0]) // result count -`MON(14, res_o_c[0]) // result fill -`CNT(15, res_o_v[0] & res_o_f[0]) // result drops -`CNT(16, res_o_p[0]) // result dma count - -`CNT(20, pcie_iv[1]) // input count -`MON(21, pcie_il[1]) // input fill -`CNT(22, pcie_iv[1] & pcie_if[1]) // input drops -`CNT(23, res_o_v[1]) // result count -`MON(24, res_o_c[1]) // result fill -`CNT(25, res_o_v[1] & res_o_f[1]) // result drops -`CNT(26, res_o_p[1]) // result dma count - -`undef CNT -`undef CNM -`undef MON - -piped_pending #( - .W(32), - .D(2) -) tr_pending_pending ( - .u(pad_i_v & pad_i_r & pad_i_e), - .d(ecc_o_v), - .p(tr_pending), - .clk(clk), - .rst(rst) -); - - -(* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({rst_r})), - .DEPTH (2) -) rst_pipe_inst ( - .in ({10{rst}}), - .out (rst_r), - - .clk (clk), - .reset (rst) -); - -(* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({rst_f_r})), - .DEPTH (2) -) rst_f_pipe_inst ( - .in ({10{rst_f}}), - .out (rst_f_r), - - .clk (clk_f), - .reset (rst_f) -); - -// PPPPPPPPPPPPPPPPP CCCCCCCCCCCCCIIIIIIIIIIEEEEEEEEEEEEEEEEEEEEEE -// P::::::::::::::::P CCC::::::::::::CI::::::::IE::::::::::::::::::::E -// P::::::PPPPPP:::::P CC:::::::::::::::CI::::::::IE::::::::::::::::::::E -// PP:::::P P:::::P C:::::CCCCCCCC::::CII::::::IIEE::::::EEEEEEEEE::::E -// P::::P P:::::P C:::::C CCCCCC I::::I E:::::E EEEEEE -// P::::P P:::::PC:::::C I::::I E:::::E -// P::::PPPPPP:::::P C:::::C I::::I E::::::EEEEEEEEEE -// P:::::::::::::PP C:::::C I::::I E:::::::::::::::E -// P::::PPPPPPPPP C:::::C I::::I E:::::::::::::::E -// P::::P C:::::C I::::I E::::::EEEEEEEEEE -// P::::P C:::::C I::::I E:::::E -// P::::P C:::::C CCCCCC I::::I E:::::E EEEEEE -// PP::::::PP C:::::CCCCCCCC::::CII::::::IIEE::::::EEEEEEEE:::::E -// P::::::::P CC:::::::::::::::CI::::::::IE::::::::::::::::::::E -// P::::::::P CCC::::::::::::CI::::::::IE::::::::::::::::::::E -// PPPPPPPPPP CCCCCCCCCCCCCIIIIIIIIIIEEEEEEEEEEEEEEEEEEEEEE - -generate - - for (genvar g_i = 0; g_i < N_PCIE; g_i ++) begin: P_IN - - localparam logic [64-1:0] PCIE_OFF = {32'h0000_0001 + g_i, 32'h0000_0000}; - - pcie_inorder #( - .W (512), - .D (512), - .REG_O (1), // assumes no backpressure - .ADDR_MASK (64'hFFFF_FFFF_0000_0000), - .ADDR_VAL (PCIE_OFF) - ) pcie_inorder_inst ( - .pcie_v (pcie_v), - .pcie_a (pcie_a), - .pcie_d (pcie_d), - - .out_v (pcie_iv[g_i]), - .out_p ('1), - .out_a (), - .out_d (pcie_id[g_i]), - .out_s (), - - .clk (clk), - .rst (rst_r[0]) - ); - - pcie_tr_ext #( - .BUFF_SZ (EXT_BUFFER_SZ) - ) tr_ext_inst ( - .pcie_v (pcie_iv [g_i]), - .pcie_d (pcie_id [g_i]), - .pcie_f (pcie_if [g_i]), // full - .pcie_l (pcie_il [g_i]), // fill - - .o_v (ext_v [g_i]), - .o_r (ext_r [g_i]), - .o_e (ext_e [g_i]), - .o_m0 (ext_m0 [g_i]), - .o_m1 (ext_m1 [g_i]), - - .clk (clk), - .rst (rst_r[1]) - ); - - always_ff@(posedge clk) begin - res_o_v [g_i] <= ecc_o_v & (ecc_o_m.m.src[0+:4] == g_i); - res_o_t [g_i] <= ecc_o_m.m.tid; - res_o_d [g_i] <= ecc_o_m.res; - end - - end -endgenerate - -// DDDDDDDDDDDDD MMMMMMMM MMMMMMMM AAA -// D::::::::::::DDD M:::::::M M:::::::M A:::A -// D:::::::::::::::DD M::::::::M M::::::::M A:::::A -// DDD:::::DDDDD:::::D M:::::::::M M:::::::::M A:::::::A -// D:::::D D:::::D M::::::::::M M::::::::::M A:::::::::A -// D:::::D D:::::DM:::::::::::M M:::::::::::M A:::::A:::::A -// D:::::D D:::::DM:::::::M::::M M::::M:::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M::::M M::::M M::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M::::M::::M M::::::M A:::::A A:::::A -// D:::::D D:::::DM::::::M M:::::::M M::::::M A:::::AAAAAAAAA:::::A -// D:::::D D:::::DM::::::M M:::::M M::::::M A:::::::::::::::::::::A -// D:::::D D:::::D M::::::M MMMMM M::::::M A:::::AAAAAAAAAAAAA:::::A -// DDD:::::DDDDD:::::D M::::::M M::::::M A:::::A A:::::A -// D:::::::::::::::DD M::::::M M::::::M A:::::A A:::::A -// D::::::::::::DDD M::::::M M::::::M A:::::A A:::::A -// DDDDDDDDDDDDD MMMMMMMM MMMMMMMMAAAAAAA AAAAAAA - -dma_result #( - .N_PCIE (N_PCIE) -) dma_result_inst ( - - .dma_r (dma_r), - .dma_v (dma_v), - .dma_a (dma_a), - .dma_b (dma_b), - .dma_f (dma_f), - .dma_d (dma_d), - - .ext_v (ext_v), - .ext_r (ext_r), - .ext_e (ext_e), - .ext_m (ext_m1), - - .res_v (res_o_v), - .res_t (res_o_t), - .res_d (res_o_d), - .res_c (res_o_c), - .res_f (res_o_f), - .res_p (res_o_p), - - .send_fails (send_fails), - - .priv_base (priv_bytes[0+:8]), - .priv_mask (priv_bytes[8+:8]), - - .clk (clk), - .rst (rst_r[2]) -); - -// PPPPPPPPPPPPPPPPP AAA DDDDDDDDDDDDD -// P::::::::::::::::P A:::A D::::::::::::DDD -// P::::::PPPPPP:::::P A:::::A D:::::::::::::::DD -// PP:::::P P:::::P A:::::::A DDD:::::DDDDD:::::D -// P::::P P:::::P A:::::::::A D:::::D D:::::D -// P::::P P:::::P A:::::A:::::A D:::::D D:::::D -// P::::PPPPPP:::::P A:::::A A:::::A D:::::D D:::::D -// P:::::::::::::PP A:::::A A:::::A D:::::D D:::::D -// P::::PPPPPPPPP A:::::A A:::::A D:::::D D:::::D -// P::::P A:::::AAAAAAAAA:::::A D:::::D D:::::D -// P::::P A:::::::::::::::::::::A D:::::D D:::::D -// P::::P A:::::AAAAAAAAAAAAA:::::A D:::::D D:::::D -// PP::::::PP A:::::A A:::::A DDD:::::DDDDD:::::D -// P::::::::P A:::::A A:::::A D:::::::::::::::DD -// P::::::::P A:::::A A:::::A D::::::::::::DDD -// PPPPPPPPPP AAAAAAA AAAAAAADDDDDDDDDDDDD - -rrb_merge #( - .W ($bits(ext_m0[0])), - .N (N_PCIE) -) ext_merge_inst ( - .i_r (ext_r), - .i_v (ext_v), - .i_e (ext_e), - .i_m (ext_m0), - - .o_r (pad_i_r), - .o_v (pad_i_v), - .o_e (pad_i_e), - .o_m (pad_i_m), - - .clk (clk), - .rst (rst) -); - -sha512_pre #( -) sha512_pre_inst ( - - .i_r (pad_i_r), - .i_w (pad_i_w), - .i_v (pad_i_v), - .i_e (pad_i_e), - .i_m (pad_i_m), - - .o_v (pad_o_v), - .o_e (pad_o_e), - .o_m (pad_o_m), - - .clk (clk), - .rst (rst_r[4]) -); - -(* keep_hierarchy = "yes" *) throttle pad_th_inst ( - .i (pad_i_r & pad_i_v & pad_i_e), - .o (pad_o_v & pad_o_e), - .f (pad_o_f), - .w (pad_i_w), - .ths ({ths[0]}), - .clk (clk), - .rst (rst) -); - -// SSSSSSSSSSSSSSS HHHHHHHHH HHHHHHHHH AAA -// SS:::::::::::::::SH:::::::H H:::::::H A:::A -// S:::::SSSSSS::::::SH:::::::H H:::::::H A:::::A -// S:::::S SSSSSSSHH::::::H H::::::HH A:::::::A -// S:::::S H:::::H H:::::H A:::::::::A -// S:::::S H:::::H H:::::H A:::::A:::::A -// S::::SSSS H::::::HHHHH::::::H A:::::A A:::::A -// SS::::::SSSSS H:::::::::::::::::H A:::::A A:::::A -// SSS::::::::SS H:::::::::::::::::H A:::::A A:::::A -// SSSSSS::::S H::::::HHHHH::::::H A:::::AAAAAAAAA:::::A -// S:::::S H:::::H H:::::H A:::::::::::::::::::::A -// S:::::S H:::::H H:::::H A:::::AAAAAAAAAAAAA:::::A -// SSSSSSS S:::::SHH::::::H H::::::HH A:::::A A:::::A -// S::::::SSSSSS:::::SH:::::::H H:::::::H A:::::A A:::::A -// S:::::::::::::::SS H:::::::H H:::::::H A:::::A A:::::A -// SSSSSSSSSSSSSSS HHHHHHHHH HHHHHHHHHAAAAAAA AAAAAAA - -(* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({sha_f_f, pad_o_m, pad_o_e, pad_o_v})), - .DEPTH (2) -) sha_i_pipe_inst ( - .in ({sha_f_f, pad_o_m, pad_o_e, pad_o_v}), - .out ({pad_o_f, sha_f_m, sha_f_e, sha_f_v}), - - .clk (clk), - .reset (rst) -); - -(* keep_hierarchy = "yes" *) showahead_pkt_fifo #( - .WIDTH ($bits({sha_f_m})), - .DEPTH (512) -) sha_f_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (sha_f_v), - .wr_full (), - .wr_full_b (), - .wr_data (sha_f_m), - .wr_eop (sha_f_e), - .wr_count (), - .wr_count_pkt (sha_f_f), - - .rd_clk (clk), - .rd_req (sha_i_v & sha_i_r), - .rd_empty (), - .rd_not_empty (sha_i_v), - .rd_count (), - .rd_data (sha_i_m), - .rd_eop (sha_i_e) -); - -(* keep_hierarchy = "yes" *) sha512_modq_meta #( - .KEY_D (KEY_D) -) sha512_modq_meta_inst ( - .i_r (sha_i_r), - .i_w (sha_i_w), - .i_v (sha_i_v), - .i_e (sha_i_e), - .i_m (sha_i_m), - - .o_v (sha_o_v), - .o_e (), - .o_m (sha_o_m), - - .clk (clk), - .rst (rst_r[5]) -); - -(* keep_hierarchy = "yes" *) throttle #( -) sha_th_inst ( - .i (sha_i_r & sha_i_v & sha_i_e), - .o (sha_o_v), - .f (sha_o_f), - .w (sha_i_w), - .ths ({ths[1]}), - .clk (clk), - .rst (rst) -); - -// SSSSSSSSSSSSSSS VVVVVVVV VVVVVVVV 000000000 -// SS:::::::::::::::SV::::::V V::::::V 00:::::::::00 -// S:::::SSSSSS::::::SV::::::V V::::::V 00:::::::::::::00 -// S:::::S SSSSSSSV::::::V V::::::V0:::::::000:::::::0 -// S:::::S V:::::V V:::::V 0::::::0 0::::::0 -// S:::::S V:::::V V:::::V 0:::::0 0:::::0 -// S::::SSSS V:::::V V:::::V 0:::::0 0:::::0 -// SS::::::SSSSS V:::::V V:::::V 0:::::0 000 0:::::0 -// SSS::::::::SS V:::::V V:::::V 0:::::0 000 0:::::0 -// SSSSSS::::S V:::::V V:::::V 0:::::0 0:::::0 -// S:::::S V:::::V:::::V 0:::::0 0:::::0 -// S:::::S V:::::::::V 0::::::0 0::::::0 -// SSSSSSS S:::::S V:::::::V 0:::::::000:::::::0 -// S::::::SSSSSS:::::S V:::::V 00:::::::::::::00 -// S:::::::::::::::SS V:::V 00:::::::::00 -// SSSSSSSSSSSSSSS VVV 000000000 - -(* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({sv0_f_f, sha_o_m, sha_o_v})), - .DEPTH (2) -) sv_0_i_pipe_inst ( - .in ({sv0_f_f, sha_o_m, sha_o_v}), - .out ({sha_o_f, sv0_f_m, sv0_f_v}), - - .clk (clk), - .reset (rst) -); - -(* keep_hierarchy = "yes" *) showahead_fifo #( - .WIDTH ($bits({sv0_f_m})), - .DEPTH (512) -) sv0_f_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (sv0_f_v), - .wr_full (), - .wr_data (sv0_f_m), - .wr_count (sv0_f_f), - - .rd_clk (clk), - .rd_req (sv0_i_v & sv0_i_r), - .rd_empty (), - .rd_not_empty (sv0_i_v), - .rd_count (), - .rd_data (sv0_i_m) -); - -(* keep_hierarchy = "yes" *) ed25519_sigverify_0 #( - .MUL_T (MUL_T), - .MUL_D (MUL_D), - .N_SCH (N_SCH), - .KEY_D (KEY_D) -) ed25519_sigverify_0_inst ( - .i_r (sv0_i_r), - .i_w (sv0_i_w), - .i_v (sv0_i_v), - .i_m (sv0_i_m), - - .o_v (sv0_o_v), - .o_m (sv0_o_m), - - .clk (clk), - .rst (rst_r[6]) -); - -(* keep_hierarchy = "yes" *) throttle #( -) sv0_th_inst ( - .i (sv0_i_r & sv0_i_v), - .o (sv0_o_v), - .f (sv0_o_f), - .w (sv0_i_w), - .ths ({ths[2]}), - .clk (clk), - .rst (rst) -); - -// SSSSSSSSSSSSSSS VVVVVVVV VVVVVVVV 1111111 -// SS:::::::::::::::SV::::::V V::::::V 1::::::1 -// S:::::SSSSSS::::::SV::::::V V::::::V1:::::::1 -// S:::::S SSSSSSSV::::::V V::::::V111:::::1 -// S:::::S V:::::V V:::::V 1::::1 -// S:::::S V:::::V V:::::V 1::::1 -// S::::SSSS V:::::V V:::::V 1::::1 -// SS::::::SSSSS V:::::V V:::::V 1::::l -// SSS::::::::SS V:::::V V:::::V 1::::l -// SSSSSS::::S V:::::V V:::::V 1::::l -// S:::::S V:::::V:::::V 1::::l -// S:::::S V:::::::::V 1::::l -// SSSSSSS S:::::S V:::::::V 111::::::111 -// S::::::SSSSSS:::::S V:::::V 1::::::::::1 -// S:::::::::::::::SS V:::V 1::::::::::1 -// SSSSSSSSSSSSSSS VVV 111111111111 - -(* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({sv1_f_f, sv0_o_m, sv0_o_v})), - .DEPTH (2) -) sv1_i_pipe_inst ( - .in ({sv1_f_f, sv0_o_m, sv0_o_v}), - .out ({sv0_o_f, sv1_f_m, sv1_f_v}), - - .clk (clk), - .reset (rst) -); - -(* keep_hierarchy = "yes" *) dual_clock_showahead_fifo #( - .WIDTH ($bits({sv1_f_m})), - .DEPTH (512) -) sv1_f_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (sv1_f_v), - .wr_full (), - .wr_data (sv1_f_m), - .wr_count (sv1_f_f), - - .rd_clk (clk_f), - .rd_req (sv1_i_v & sv1_i_r), - .rd_empty (), - .rd_not_empty (sv1_i_v), - .rd_count (), - .rd_data (sv1_i_m) -); - -(* keep_hierarchy = "yes" *) ed25519_sigverify_1 #( - .DSDP_WS (DSDP_WS), - .MUL_T (MUL_T), - .MUL_D (MUL_D), - .KEY_D (KEY_D) -) ed25519_sigverify_1_inst ( - .i_r (sv1_i_r), - .i_w (sv1_i_w), - .i_v (sv1_i_v), - .i_m (sv1_i_m), - - .o_v (sv1_o_v), - .o_m (sv1_o_m), - - .clk (clk_f), - .rst (rst_f_r[0]) -); - -(* keep_hierarchy = "yes" *) throttle #( -) sv1_th_inst ( - .i (sv1_i_r & sv1_i_v), - .o (sv1_o_v), - .f (sv1_o_f), - .w (sv1_i_w), - .ths ({ths[3]}), - .clk (clk_f), - .rst (rst_f_r[1]) -); - -// SSSSSSSSSSSSSSS VVVVVVVV VVVVVVVV 222222222222222 -// SS:::::::::::::::SV::::::V V::::::V2:::::::::::::::22 -// S:::::SSSSSS::::::SV::::::V V::::::V2::::::222222:::::2 -// S:::::S SSSSSSSV::::::V V::::::V2222222 2:::::2 -// S:::::S V:::::V V:::::V 2:::::2 -// S:::::S V:::::V V:::::V 2:::::2 -// S::::SSSS V:::::V V:::::V 2222::::2 -// SS::::::SSSSS V:::::V V:::::V 22222::::::22 -// SSS::::::::SS V:::::V V:::::V 22::::::::222 -// SSSSSS::::S V:::::V V:::::V 2:::::22222 -// S:::::S V:::::V:::::V 2:::::2 -// S:::::S V:::::::::V 2:::::2 -// SSSSSSS S:::::S V:::::::V 2:::::2 222222 -// S::::::SSSSSS:::::S V:::::V 2::::::2222222:::::2 -// S:::::::::::::::SS V:::V 2::::::::::::::::::2 -// SSSSSSSSSSSSSSS VVV 22222222222222222222 - -(* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({sv2_f_f, sv1_o_m, sv1_o_v})), - .DEPTH (2) -) sv2_i_pipe_inst ( - .in ({sv2_f_f, sv1_o_m, sv1_o_v}), - .out ({sv1_o_f, sv2_f_m, sv2_f_v}), - - .clk (clk_f), - .reset (rst_f_r[2]) -); - -(* keep_hierarchy = "yes" *) dual_clock_showahead_fifo #( - .WIDTH ($bits({sv2_f_m})), - .DEPTH (512) -) sv2_f_inst ( - .aclr (rst_f_r[3]), - - .wr_clk (clk_f), - .wr_req (sv2_f_v), - .wr_full (), - .wr_data (sv2_f_m), - .wr_count (sv2_f_f), - - .rd_clk (clk), - .rd_req (sv2_i_v & sv2_i_r), - .rd_empty (), - .rd_not_empty (sv2_i_v), - .rd_count (), - .rd_data (sv2_i_m) -); - -(* keep_hierarchy = "yes" *) ed25519_sigverify_2 #( - .MUL_T (MUL_T) -) ed25519_sigverify_2_inst ( - .i_r (sv2_i_r), - .i_w ('0), - .i_v (sv2_i_v), - .i_m (sv2_i_m), - - .o_v (sv2_o_v), - .o_m (sv2_o_m), - - .clk (clk), - .rst (rst_r[8]) -); - -(* dont_touch = "yes" *) piped_wire #( - .WIDTH ($bits({sv2_o_m, sv2_o_v})), - .DEPTH (4) -) ecc_o_pipe_inst ( - .in ({sv2_o_m, sv2_o_v}), - .out ({ecc_o_m, ecc_o_v}), - - .clk (clk), - .reset (rst) -); - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -always_ff@(posedge clk) dbg_wire[DBG_WIDTH-1:2] <= { - - ecc_o_m.m.tid, - ecc_o_v, - - sv2_o_m.m.tid, - sv2_o_v, - - sv2_i_m.m.tid, - sv2_i_v, - - sv0_o_m.m.m.m.tid, - sv0_o_v, - - sha_o_m.m.m.tid, - sha_o_v, - - pad_o_m.m.m.tid, - pad_o_v, - - |{ - ecc_o_v, - sv2_o_v, - sv2_i_v, - sv0_o_v, - sha_o_v, - pad_o_v - } -}; - -assign dbg_wire[0+:2] = { - rst, - clk -}; - -always_ff@(posedge clk) if (pad_i_v & pad_i_r & pad_i_e) $display("%t: pad_i :", $time); -always_ff@(posedge clk) if (sha_i_v & sha_i_r & sha_i_e) $display("%t: sha_i : %x", $time, sha_i_m.m.m.tid); -always_ff@(posedge clk) if (sv0_i_v & sv0_i_r) $display("%t: sv0_i :", $time); -always_ff@(posedge clk_f) if (sv1_i_v & sv1_i_r) $display("%t: sv1_i :", $time); -always_ff@(posedge clk) if (sv2_i_v & sv2_i_r) $display("%t: sv2_i :", $time); - -// always_ff@(posedge clk) if (pad_i_w) $display("%t: pad_i_w: %0d %0d %0d - %0d - %0d", $time, ths[0][0+:12], ths[0][12+:12], ths[0][24+:12], pad_o_f, pad_th_inst.cnt); -// always_ff@(posedge clk) if (sha_i_w) $display("%t: sha_i_w: %0d %0d %0d - %0d - %0d", $time, ths[1][0+:12], ths[1][12+:12], ths[1][24+:12], sha_o_f, sha_th_inst.cnt); -// always_ff@(posedge clk) if (sv0_i_w) $display("%t: sv0_i_w: %0d %0d %0d - %0d - %0d", $time, ths[2][0+:12], ths[2][12+:12], ths[2][24+:12], sv0_o_f, sv0_th_inst.cnt); -// always_ff@(posedge clk_f) if (sv1_i_w) $display("%t: sv1_i_w: %0d %0d %0d - %0d - %0d", $time, ths[3][0+:12], ths[3][12+:12], ths[3][24+:12], sv1_o_f, sv1_th_inst.cnt); - -always_ff@(posedge clk) if (pad_o_v & pad_o_e) $display("%t: o_pad_o: %x", $time, pad_o_m.m.m.tid); -always_ff@(posedge clk) if (sha_o_v) $display("%t: o_sha_o: %x", $time, sha_o_m.m.m.tid); -always_ff@(posedge clk) if (sv0_o_v) $display("%t: o_sv0_o: %x", $time, sv0_o_m.m.m.m.tid); -always_ff@(posedge clk_f) if (sv1_o_v) $display("%t: o_sv1_o: %x", $time, sv1_o_m.m.tid); -always_ff@(posedge clk) if (sv2_o_v) $display("%t: o_sv2_o: %x", $time, sv2_o_m.m.tid); -always_ff@(posedge clk) if (ecc_o_v) $display("%t: o_ecc_o: %x", $time, ecc_o_m.m.tid); -always_ff@(posedge clk) if (res_o_v[0]) $display("%t: o_re[0]: %x", $time, res_o_d[0]); - -always_ff@(negedge clk) $display("%t: -----------",$time); - - -endmodule - - -`default_nettype wire diff --git a/src/wiredancer/rtl/wd_pkg.sv b/src/wiredancer/rtl/wd_pkg.sv deleted file mode 100644 index 591531e3b2e..00000000000 --- a/src/wiredancer/rtl/wd_pkg.sv +++ /dev/null @@ -1,867 +0,0 @@ - -package wd_sigverify; - -localparam logic[32-1:0] PCIE_MAGIC = 32'hACE0_FBAC; -localparam integer N_PCIE = 2; -localparam integer EXT_BUFFER_SZ = 4*1024; - -localparam logic [255-1:0] ED25519_P = 255'h7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed; -localparam logic [256-1:0] ED25519_P_N = 256'h1 + ~256'h7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed; -localparam logic [255-1:0] ED25519_D = 255'h52036cee2b6ffe738cc740797779e89800700a4d4141d8ab75eb4dca135978a3; -localparam logic [255-1:0] ED25519_2D = 255'h2406d9dc56dffce7198e80f2eef3d13000e0149a8283b156ebd69b9426b2f159; -localparam logic [255-1:0] ED25519_Ix = 255'h0000000000000000000000000000000000000000000000000000000000000000; -localparam logic [255-1:0] ED25519_Iy = 255'h0000000000000000000000000000000000000000000000000000000000000001; -localparam logic [255-1:0] ED25519_Iz = 255'h0000000000000000000000000000000000000000000000000000000000000001; -localparam logic [255-1:0] ED25519_It = 255'h0000000000000000000000000000000000000000000000000000000000000000; -localparam logic [255-1:0] ED25519_Gx = 255'h216936d3cd6e53fec0a4e231fdd6dc5c692cc7609525a7b2c9562d608f25d51a; -localparam logic [255-1:0] ED25519_Gy = 255'h6666666666666666666666666666666666666666666666666666666666666658; -localparam logic [255-1:0] ED25519_Gz = 255'h0000000000000000000000000000000000000000000000000000000000000001; -localparam logic [255-1:0] ED25519_Gt = 255'h67875f0fd78b766566ea4e8e64abe37d20f09f80775152f56dde8ab3a5b7dda3; - -localparam integer N_SCH_O = 8; - -localparam integer SCH_O_RES = 0; -localparam integer SCH_O_AX = 1; -localparam integer SCH_O_AT = 2; -localparam integer SCH_O_RX = 3; -localparam integer SCH_O_TX = 4; -localparam integer SCH_O_TY = 5; -localparam integer SCH_O_TZ = 6; -localparam integer SCH_O_TT = 7; - -typedef struct packed { - logic [256-1:0] sig_l; - - logic [32-1:0] dma_chunk; - logic [64-1:0] dma_seq; - logic [64-1:0] dma_addr; - logic [16-1:0] dma_ctrl; - logic [16-1:0] dma_size; - - logic [16-1:0] size; - logic [16-1:0] src; - logic [32-1:0] magic; -} pcie_meta_t; - -typedef struct packed { - logic [64-1:0] tid; - logic [16-1:0] src; -} sv_meta0_t; - -typedef struct packed { - logic [256-1:0] pub; - logic [256-1:0] sig_h; - logic [256-1:0] sig_l; - sv_meta0_t m; -} sv_meta1_t; - -typedef struct packed { - logic [512-1:0] data; - logic [1-1:0] sop; - logic [$clog2(512/8)-1:0] emp; - logic [16-1:0] size; - sv_meta1_t m; -} sv_meta2_t; - -typedef struct packed { - logic [1024-1:0] d; - logic [4-1:0] c; - logic [1-1:0] f; - sv_meta1_t m; -} sv_meta3_t; - -typedef struct packed { - logic [256-1:0] h; - sv_meta1_t m; -} sv_meta4_t; - -typedef struct packed { - logic [8-1:0][256-1:0] os; - sv_meta4_t m; -} sv_meta5_t; - -typedef struct packed { - logic [1-1:0] res; - logic [256-1:0] Rx; - logic [256-1:0] Zx; - logic [256-1:0] Zy; - logic [256-1:0] Zz; - logic [256-1:0] sig_l; - sv_meta0_t m; -} sv_meta6_t; - -typedef struct packed { - logic [1-1:0] res; - sv_meta0_t m; -} sv_meta7_t; - -typedef struct packed { - logic [32-1:0] tspub; - logic [32-1:0] tsorig; - logic [16-1:0] ctrl; - logic [16-1:0] sz; - logic [32-1:0] chunk; - logic [64-1:0] sig; - logic [64-1:0] seq; - - logic [64-1:0] pcim_strb; - logic [64-1:0] pcim_addr; -} mcache_pcim_t; - -endpackage : wd_sigverify - - - - - - - - - - - - - - - - - - - - - - - - -`default_nettype none - -`define WIDE_PIPE(___, _, _W, __) piped_wire #(.WIDTH(_W), .DEPTH(__)) piped_wire_``___`` (.clk(clk), .reset(rst), .in(_), .out(___)); -`define SHADD_6_1C(_W, _O, _S0, _S1, _S2, _S3, _S4, _S5, _I0, _I1, _I2, _I3, _I4, _I5) shift_adder_6 #(.W(_W),.S0(_S0),.S1(_S1),.S2(_S2),.S3(_S3),.S4(_S4),.S5(_S5),.R0(0),.R1(0),.R(1),.C(0)) ``_O``_shadd6_inst (.cin0('0), .in0(_I0),.in1(_I1),.in2(_I2),.in3(_I3),.in4(_I4),.in5(_I5),.out0(_O),.clk(clk),.rst(rst), .m_i('0), .m_o(), .cout0()); -`define SHADD_6_2C(_W, _O, _S0, _S1, _S2, _S3, _S4, _S5, _I0, _I1, _I2, _I3, _I4, _I5) shift_adder_6 #(.W(_W),.S0(_S0),.S1(_S1),.S2(_S2),.S3(_S3),.S4(_S4),.S5(_S5),.R0(0),.R1(0),.R(1),.C(1)) ``_O``_shadd6_inst (.cin0('0), .in0(_I0),.in1(_I1),.in2(_I2),.in3(_I3),.in4(_I4),.in5(_I5),.out0(_O),.clk(clk),.rst(rst), .m_i('0), .m_o(), .cout0()); - -(* dont_touch = "yes" *) module piped_wire #( - parameter integer WIDTH=32, - parameter integer DEPTH=1 -)( - input wire [WIDTH-1:0] in, - output bit [WIDTH-1:0] out, - input wire clk, - input wire reset -); - -generate - if (DEPTH == 0) begin - assign out = in; - end - else - if (DEPTH == 1) begin - always_ff @(posedge clk) begin - out <= in; - end - end - else - begin - bit [DEPTH-1-1:0] pipe [WIDTH-1:0]; - for (genvar g_i = 0; g_i < WIDTH; g_i ++) begin: G_I - always_ff @(posedge clk) begin - {out[g_i], pipe[g_i]} <= {pipe[g_i], in[g_i]}; - end - end - end -endgenerate - -endmodule // piped_wire - -module piped_pending #( - parameter integer W=32, - parameter integer D=2 -)( - input wire [1-1:0] u, - input wire [1-1:0] d, - output logic [W-1:0] p, - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -logic [1-1:0] uu; -logic [1-1:0] dd; - -piped_wire #( - .WIDTH (2), - .DEPTH (D) -) ecc_o_pipe_inst ( - .in ({u, d}), - .out ({uu, dd}), - - .clk (clk), - .reset (rst) -); - -always_ff@(posedge clk) begin - case({ - uu, - dd - }) - 2'b10: p <= p + 1; - 2'b01: p <= p - 1; - endcase - if (rst) p <= 0; -end - -endmodule // piped_pending - -module piped_counter #( - parameter integer D=2, - parameter integer W=32 -)( - output logic [W-1:0] c, - input wire [1-1:0] p, - input wire [1-1:0] s, - input wire [1-1:0] r, - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -logic [1-1:0] pp, ss, rr; -logic [W-1:0] cnt; - -piped_wire #( - .WIDTH (3), - .DEPTH (D) -) ecc_o_pipe_inst ( - .in ({p, s, r}), - .out ({pp, ss, rr}), - - .clk (clk), - .reset (rst) -); - -always_ff@(posedge clk) begin - cnt <= rr ? '0 : cnt + pp; - if (ss) - c <= cnt; -end - -endmodule // piped_counter - -module piped_adder #( - parameter integer W=384, - parameter integer C=1, - parameter integer M=1, - parameter integer R=0 -)( - input wire clk, - input wire rst, - - input wire [1-1:0] cin0, - input wire [W-1:0] in0, - input wire [W-1:0] in1, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - output logic [W-1:0] out0, - output logic [1-1:0] cout0 -); - -generate - if (C == 0) begin - logic [W+1-1:0] s; - assign s = in0 + in1 + cin0; - if (R == 0) begin - assign out0 = s[0+:W]; - assign cout0 = s[W]; - assign m_o = m_i; - end else begin - always_ff@(posedge clk) out0 <= s[0+:W]; - always_ff@(posedge clk) cout0 <= s[W]; - always_ff@(posedge clk) m_o <= m_i; - end - end else begin - localparam W1 = W / 2; - localparam W2 = W - W1; - - logic [2-1:0] c; - logic [W1-1:0] i00; - logic [W2-1:0] i01; - logic [W1-1:0] i10; - logic [W2-1:0] i11; - logic [W2-1:0] i0_p; - logic [W2-1:0] i1_p; - logic [W1-1:0] s0; - logic [W1-1:0] s1; - logic [W2-1:0] s2; - logic [2-1:0][M-1:0] m_o_p; - - assign {i01, i00} = in0; - assign {i11, i10} = in1; - - assign out0 = {s2, s1}; - assign m_o = m_o_p[1]; - assign cout0 = c[1]; - - piped_adder #( - .W(W1), - .C(C-1), - .R(1), - .M(M+W2+W2) - ) piped_adder_inst_0 ( - .clk (clk), - .rst (rst), - - .cin0 (cin0), - .in0 (i00), - .in1 (i10), - .out0 (s0), - .cout0 (c[0]), - .m_i ({m_i, i11, i01}), - .m_o ({m_o_p[0], i1_p, i0_p}) - ); - - piped_adder #( - .W(W2), - .C(C-1), - .R(1), - .M(M+W1) - ) piped_adder_inst_1 ( - .clk (clk), - .rst (rst), - - .cin0 (c[0]), - .in0 (i0_p), - .in1 (i1_p), - .out0 (s2), - .cout0 (c[1]), - .m_i ({m_o_p[0], s0}), - .m_o ({m_o_p[1], s1}) - ); - - end -endgenerate - -endmodule // piped_adder - -module shift_adder_6 #( - parameter integer W = 384, - parameter integer S0 = 0, - parameter integer S1 = 1, - parameter integer S2 = 2, - parameter integer S3 = 3, - parameter integer S4 = 4, - parameter integer S5 = 5, - parameter integer C = 0, - parameter integer M = 1, - parameter integer R = 1, - parameter integer R0 = 0, - parameter integer R1 = 1 -)( - input wire clk, - input wire rst, - - input wire [1-1:0] cin0, - input wire [W-1:0] in0, - input wire [W-1:0] in1, - input wire [W-1:0] in2, - input wire [W-1:0] in3, - input wire [W-1:0] in4, - input wire [W-1:0] in5, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - output logic [W-1:0] out0, - output logic [1-1:0] cout0 -); - -logic [1-1:0] cin0_, cin0__; - -logic [2-1:0][M-1:0] m_o_p; - -wire [W-1:0] i0, i1, i2, i3, i4, i5; - -wire [W-1:0] c01_s; -wire [W-1:0] c01_c0; -wire [W-1:0] c01_c1; -wire [W-1:0] c02_s; -wire [W-1:0] c02_c; - -assign i0 = in0 << S0; -assign i1 = in1 << S1; -assign i2 = in2 << S2; -assign i3 = in3 << S3; -assign i4 = in4 << S4; -assign i5 = in5 << S5; - -red_6_3 #( - .W(W), - .R(R0), - .M($bits({m_i, cin0})) -) red_6_3_inst ( - .in0(i0), - .in1(i1), - .in2(i2), - .in3(i3), - .in4(i4), - .in5(i5), - .sout(c01_s), - .cout0(c01_c0), - .cout1(c01_c1), - .m_i({m_i, cin0}), - .m_o({m_o_p[0], cin0_}), - .clk(clk) -); - -red_3_2 #( - .W(W), - .R(R1), - .M($bits({m_i, cin0})) -) red_3_2_inst ( - .i0(c01_s), - .i1(c01_c0 << 1), - .i2(c01_c1 << 2), - .s(c02_s), - .c(c02_c), - .m_i({m_o_p[0], cin0_}), - .m_o({m_o_p[1], cin0__}), - .clk(clk) -); - -piped_adder #( - .W(W), - .R(R), - .C(C), - .M(M) -) a0_inst ( - .clk(clk), - .rst(rst), - .cin0(cin0__), - .in0(c02_s), - .in1(c02_c << 1), - .out0(out0), - .cout0(cout0), - .m_i(m_o_p[1]), - .m_o(m_o) -); - -endmodule // shift_adder_6 - -module shift_adder_3 #( - parameter integer W = 384, - parameter integer S0 = 0, - parameter integer S1 = 1, - parameter integer S2 = 2, - parameter integer C = 0, - parameter integer M = 1, - parameter integer R = 1, - parameter integer R0 = 0 -)( - input wire clk, - input wire rst, - - input wire [1-1:0] cin0, - input wire [W-1:0] in0, - input wire [W-1:0] in1, - input wire [W-1:0] in2, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - output logic [W-1:0] out0, - output logic [1-1:0] cout0 -); - -logic [1-1:0][M-1:0] m_o_p; - -wire [W-1:0] i0, i1, i2; - -wire [W-1:0] c01_s; -wire [W-1:0] c01_c; - -assign i0 = in0 << S0; -assign i1 = in1 << S1; -assign i2 = in2 << S2; - -red_3_2 #( - .W(W), - .R(R0), - .M(M) -) red_3_2_inst ( - .i0(i0), - .i1(i1), - .i2(i2), - .s(c01_s), - .c(c01_c), - .m_i(m_i), - .m_o(m_o_p[0]), - .clk(clk) -); - -piped_adder #( - .W(W), - .R(R), - .C(C), - .M(M) -) a0_inst ( - .clk(clk), - .rst(rst), - .cin0(cin0), - .in0(c01_s), - .in1(c01_c << 1), - .out0(out0), - .cout0(cout0), - .m_i(m_o_p[0]), - .m_o(m_o) -); - -endmodule // shift_adder_3 - -module red_6_3 #( - parameter integer W = 1, - parameter integer R = 0, - parameter integer M = 1 -)( - input wire [W-1:0] in0, - input wire [W-1:0] in1, - input wire [W-1:0] in2, - input wire [W-1:0] in3, - input wire [W-1:0] in4, - input wire [W-1:0] in5, - output logic [W-1:0][1-1:0] sout, - output logic [W-1:0][1-1:0] cout0, - output logic [W-1:0][1-1:0] cout1, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - input wire [1-1:0] clk -); - -generate - for (genvar g_i = 0; g_i < W; g_i ++) begin - logic [3-1:0] ss; - - assign ss = in0[g_i]+in1[g_i]+in2[g_i]+in3[g_i]+in4[g_i]+in5[g_i]; - - if (R == 0) begin - assign sout[g_i] = ss[0]; - assign cout0[g_i] = ss[1]; - assign cout1[g_i] = ss[2]; - end else begin - always_ff@(posedge clk) sout[g_i] <= ss[0]; - always_ff@(posedge clk) cout0[g_i] <= ss[1]; - always_ff@(posedge clk) cout1[g_i] <= ss[2]; - always_ff@(posedge clk) cout1[g_i] <= ss[2]; - end - - end - - if (R == 0) begin - assign m_o = m_i; - end else begin - always_ff@(posedge clk) m_o <= m_i; - end -endgenerate - -endmodule // red_6_3 - -module red_3_2 #( - parameter integer W = 1, - parameter integer R = 1, - parameter integer M = 1 -)( - input wire [W-1:0] i0, - input wire [W-1:0] i1, - input wire [W-1:0] i2, - output logic [W-1:0] s, - output logic [W-1:0] c, - input wire [M-1:0] m_i, - output logic [M-1:0] m_o, - input wire [1-1:0] clk -); - -generate - for (genvar g_i = 0; g_i < W; g_i ++) begin - logic [2-1:0] ss; - - assign ss = i0[g_i] + i1[g_i] + i2[g_i]; - - if (R == 0) begin - assign s[g_i] = ss[0]; - assign c[g_i] = ss[1]; - end else begin - always_ff@(posedge clk) s[g_i] <= ss[0]; - always_ff@(posedge clk) c[g_i] <= ss[1]; - end - end - - if (R == 0) begin - assign m_o = m_i; - end else begin - always_ff@(posedge clk) m_o <= m_i; - end -endgenerate - -endmodule // red_3_2 - -module throttle #( -) ( - input wire [1-1:0] i, - input wire [1-1:0] o, - input wire [10-1:0] f, - input wire [3-1:0][12-1:0] ths, - output logic [1-1:0] w, - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -logic [12-1:0] th0_r; -logic [12-1:0] th1_r; -logic [12-1:0] th2_r; -logic [12-1:0] cnt; -logic [12-1:0] cnt_n; -logic [12-1:0] gap; - -piped_wire #( - .WIDTH ($bits({ths})), - .DEPTH (2) -) th_pipe_inst ( - .in ({ths}), - .out ({th2_r, th1_r, th0_r}), - - .clk (clk), - .reset (rst) -); - -always_comb begin - case ({i, o}) - 2'b10 : cnt_n = cnt + 1; - 2'b01 : cnt_n = cnt - 1; - default : cnt_n = cnt; - endcase -end - -always_ff@(posedge clk) begin - gap <= i ? 0 : (gap + ~&gap); - cnt <= cnt_n; - w <= (0 - | (cnt_n + f >= th0_r) - | (f >= th1_r) - | (gap < th2_r) - ); - - if (rst) begin - gap <= 0; - cnt <= 0; - w <= 0; - end -end - -endmodule // throttle - -module showahead_pkt_fifo #( - parameter integer WIDTH = 32, - parameter integer DEPTH = 32, - parameter integer D_L = $clog2(DEPTH), - parameter integer FULL_THRESH = DEPTH-6 -)( - input wire wr_clk, - input wire wr_req, - input wire [WIDTH-1:0] wr_data, - input wire [1-1:0] wr_eop, - output wire wr_full, - output wire wr_full_b, - output wire [D_L+1-1:0] wr_count, - output wire [D_L+1-1:0] wr_count_pkt, - - input wire rd_clk, - input wire rd_req, - output wire [WIDTH-1:0] rd_data, - output wire [1-1:0] rd_eop, - output wire rd_empty, - output wire rd_not_empty, - output wire [D_L+1-1:0] rd_count, - output wire [D_L+1-1:0] rd_count_pkt, - - input wire aclr -); - -logic [2-1:0] rd_not_empty_; - -assign rd_not_empty = &rd_not_empty_; -assign rd_empty = ~rd_not_empty; - -showahead_fifo #( - .WIDTH (1), - .DEPTH (DEPTH) -) f0_inst ( - .aclr (aclr), - - .wr_clk (wr_clk), - .wr_req (wr_req & wr_eop), - .wr_full (), - .wr_data ('0), - .wr_count (wr_count_pkt), - - .rd_clk (rd_clk), - .rd_req (rd_req & rd_eop), - .rd_empty (), - .rd_not_empty (rd_not_empty_[0]), - .rd_count (rd_count_pkt), - .rd_data () -); - -showahead_fifo #( - .WIDTH ($bits({wr_data, wr_eop})), - .DEPTH (DEPTH) -) f1_inst ( - .aclr (aclr), - - .wr_clk (wr_clk), - .wr_req (wr_req), - .wr_full (wr_full), - .wr_full_b (wr_full_b), - .wr_data ({wr_data, wr_eop}), - .wr_count (wr_count), - - .rd_clk (rd_clk), - .rd_req (rd_req), - .rd_empty (), - .rd_not_empty (rd_not_empty_[1]), - .rd_count (rd_count), - .rd_data ({rd_data, rd_eop}) -); - -endmodule // showahead_pkt_fifo - -module rrb_merge #( - parameter integer W = 1, - parameter integer N = 2 -) ( - output logic [N-1:0][1-1:0] i_r, - input wire [N-1:0][1-1:0] i_v, - input wire [N-1:0][1-1:0] i_e, - input wire [N-1:0][W-1:0] i_m, - - input wire [1-1:0] o_r, - output logic [1-1:0] o_v, - output logic [1-1:0] o_e, - output logic [W-1:0] o_m, - - input wire [1-1:0] clk, - input wire [1-1:0] rst - -); - -localparam integer N_L = $clog2(N); - -logic [1-1:0] st; -logic [N_L-1:0] rrb; - -assign o_v = i_v[rrb]; -assign o_e = i_e[rrb]; -assign o_m = i_m[rrb]; - -always_comb begin - i_r = 0; - i_r[rrb] = o_r; -end - -always_ff@(posedge clk) begin - case(st) - 0: begin - casez({o_v, o_e, o_r}) - 3'b0zz, 3'b111 : rrb <= (rrb + 1 == N) ? 0 : rrb + 1; - 3'b10z, 3'b110 : st <= 1; - endcase - end - 1: begin - if (o_v & o_e & o_r) begin - rrb <= (rrb + 1 == N) ? 0 : rrb + 1; - st <= 0; - end - end - endcase - if (rst) begin - st <= 0; - rrb <= 0; - end -end - -endmodule // rrb_merge - -module var_pipe #( - parameter integer W_D = 512, - parameter integer D = 512 -) ( - output logic [1-1:0] i_r, - input wire [1-1:0] i_v, - input wire [1-1:0] i_e, - input wire [1-1:0] i_w, - input wire [W_D-1:0] i_m, - - output logic [1-1:0] o_v, - output logic [1-1:0] o_e, - output logic [W_D-1:0] o_m, - - input wire [1-1:0] clk, - input wire [1-1:0] rst -); - -logic [64-1:0] timestamp = 0; - -logic [1-1:0] ii_r; -logic [1-1:0] ii_v; -logic [16-1:0] ii_w; -logic [W_D-1:0] ii_d; - -logic [1-1:0] push_v; -logic [1-1:0] push_e; -logic [64-1:0] push_t; -logic [W_D-1:0] push_m; - -logic [1-1:0] pop_v; -logic [64-1:0] pop_t; - -always_ff@(posedge clk) timestamp <= timestamp + 1; - -always_comb begin - - i_r = ~i_w; - - push_v = i_v & ~i_w; - push_e = i_e; - push_m = i_m; - push_t = i_m[0+:16] + timestamp; - - o_v = 0; - - if (pop_v) begin - if (pop_t < timestamp) begin - i_r = 0; - - push_v = 1; - push_e = o_e; - push_m = o_m; - push_t = pop_t; - end else begin - o_v = 1; - end - end -end - -showahead_fifo #( - .WIDTH ($bits({push_m, push_e, push_t})), - .DEPTH (D) -) w_fifo_inst ( - .aclr (rst), - - .wr_clk (clk), - .wr_req (push_v), - .wr_full (), - .wr_full_b (), - .wr_count (), - .wr_data ({push_m, push_e, push_t}), - - .rd_clk (clk), - .rd_req (pop_v), - .rd_empty (), - .rd_not_empty (pop_v), - .rd_count (), - .rd_data ({o_m, o_e, pop_t}) -); - -endmodule // var_pipe - -`default_nettype wire diff --git a/src/wiredancer/sim/README.md b/src/wiredancer/sim/README.md deleted file mode 100644 index b48a47b0a2f..00000000000 --- a/src/wiredancer/sim/README.md +++ /dev/null @@ -1,3 +0,0 @@ -This folder includes all simulations for verification and testing of the RTL. Cocotb v1.7.2 is the verification platform used, and all tests were run with Questa v2019.4. - -To run a test, simply run `make` inside the test folder. diff --git a/src/wiredancer/sim/ed25519_point_add/Makefile b/src/wiredancer/sim/ed25519_point_add/Makefile deleted file mode 100755 index 7c95d4e8be4..00000000000 --- a/src/wiredancer/sim/ed25519_point_add/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = ed25519_point_add -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_add_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sub_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_mul_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_point_add.sv - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/ed25519_point_add/test.py b/src/wiredancer/sim/ed25519_point_add/test.py deleted file mode 100755 index 3c914a88af8..00000000000 --- a/src/wiredancer/sim/ed25519_point_add/test.py +++ /dev/null @@ -1,98 +0,0 @@ - -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import wd_cocotil -import ref_ed25519 - -@cocotb.test() -async def test(dut): - - clk = dut.clk - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - - for i in range(1024): - await RisingEdge(clk) - - W_M = int(dut.M) - es = list() - D = 0 - - for i in range(1024): - - m_i = wd_cocotil.random_int(W_M-1) - - P0 = random.choice([ - (0, 1, 1, 0), - ref_ed25519.point_mul(wd_cocotil.random_int(100), ref_ed25519.G) - ]) - P1 = random.choice([ - P0, - (0, 1, 1, 0), - ref_ed25519.point_mul(wd_cocotil.random_int(100), ref_ed25519.G) - ]) - - P2 = ref_ed25519.point_add(P0, P1) - - es.append((P2, m_i)) - - p0_x = BinaryValue(bits=255, bigEndian=False) - p0_y = BinaryValue(bits=255, bigEndian=False) - p0_z = BinaryValue(bits=255, bigEndian=False) - p0_t = BinaryValue(bits=255, bigEndian=False) - p1_x = BinaryValue(bits=255, bigEndian=False) - p1_y = BinaryValue(bits=255, bigEndian=False) - p1_z = BinaryValue(bits=255, bigEndian=False) - p1_t = BinaryValue(bits=255, bigEndian=False) - - p0_x[255-1:0] = P0[0] - p0_y[255-1:0] = P0[1] - p0_z[255-1:0] = P0[2] - p0_t[255-1:0] = P0[3] - p1_x[255-1:0] = P1[0] - p1_y[255-1:0] = P1[1] - p1_z[255-1:0] = P1[2] - p1_t[255-1:0] = P1[3] - - dut.in0_x = p0_x - dut.in0_y = p0_y - dut.in0_z = p0_z - dut.in0_t = p0_t - dut.in1_x = p1_x - dut.in1_y = p1_y - dut.in1_z = p1_z - dut.in1_t = p1_t - - dut.m_i = 1 | (m_i << 1) - - await RisingEdge(dut.clk) - - if str(dut.m_o[0]) != '1': - D += 1 - assert D < 100 - continue - - P2, m_i = es.pop(0) - o_x = int(dut.out0_x) - o_y = int(dut.out0_y) - o_z = int(dut.out0_z) - o_t = int(dut.out0_t) - m_o = int(dut.m_o) >> 1 - - dut._log.info ('D: {}, {}'.format(i, D)) - dut._log.info ('m: \n{:x} =?= \n{:x}'.format(m_i, m_o)) - dut._log.info ('x: \n{:x} =?= \n{:x}'.format(P2[0], o_x)) - dut._log.info ('y: \n{:x} =?= \n{:x}'.format(P2[1], o_y)) - dut._log.info ('z: \n{:x} =?= \n{:x}'.format(P2[2], o_z)) - dut._log.info ('t: \n{:x} =?= \n{:x}'.format(P2[3], o_t)) - - assert P2[0] == o_x - assert P2[1] == o_y - assert P2[2] == o_z - assert P2[3] == o_t - assert m_i == m_o diff --git a/src/wiredancer/sim/ed25519_point_dbl/Makefile b/src/wiredancer/sim/ed25519_point_dbl/Makefile deleted file mode 100755 index c98326bc0e6..00000000000 --- a/src/wiredancer/sim/ed25519_point_dbl/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = ed25519_point_dbl -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_add_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sub_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_mul_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_point_dbl.sv - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/ed25519_point_dbl/test.py b/src/wiredancer/sim/ed25519_point_dbl/test.py deleted file mode 100755 index 40b262f688f..00000000000 --- a/src/wiredancer/sim/ed25519_point_dbl/test.py +++ /dev/null @@ -1,81 +0,0 @@ - -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import wd_cocotil -import ref_ed25519 - -@cocotb.test() -async def test(dut): - - clk = dut.clk - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - - for i in range(1024): - await RisingEdge(clk) - - W_M = int(dut.M) - es = list() - D = 0 - - for i in range(1024): - - m_i = wd_cocotil.random_int(W_M-1) - - P0 = random.choice([ - (0, 1, 1, 0), - ref_ed25519.point_mul(wd_cocotil.random_int(100), ref_ed25519.G) - ]) - - P2 = ref_ed25519.point_add(P0, P0) - - es.append((P2, m_i)) - - p0_x = BinaryValue(bits=255, bigEndian=False) - p0_y = BinaryValue(bits=255, bigEndian=False) - p0_z = BinaryValue(bits=255, bigEndian=False) - p0_t = BinaryValue(bits=255, bigEndian=False) - - p0_x[255-1:0] = P0[0] - p0_y[255-1:0] = P0[1] - p0_z[255-1:0] = P0[2] - p0_t[255-1:0] = P0[3] - - dut.in0_x = p0_x - dut.in0_y = p0_y - dut.in0_z = p0_z - dut.in0_t = p0_t - - dut.m_i = 1 | (m_i << 1) - - await RisingEdge(dut.clk) - - if str(dut.m_o[0]) != '1': - D += 1 - assert D < 100 - continue - - P2, m_i = es.pop(0) - o_x = int(dut.out0_x) - o_y = int(dut.out0_y) - o_z = int(dut.out0_z) - o_t = int(dut.out0_t) - m_o = int(dut.m_o) >> 1 - - dut._log.info ('D: {}, {}'.format(i, D)) - dut._log.info ('m: \n{:x} =?= \n{:x}'.format(m_i, m_o)) - dut._log.info ('x: \n{:x} =?= \n{:x}'.format(P2[0], o_x)) - dut._log.info ('y: \n{:x} =?= \n{:x}'.format(P2[1], o_y)) - dut._log.info ('z: \n{:x} =?= \n{:x}'.format(P2[2], o_z)) - dut._log.info ('t: \n{:x} =?= \n{:x}'.format(P2[3], o_t)) - - assert P2[0] == o_x - assert P2[1] == o_y - assert P2[2] == o_z - assert P2[3] == o_t - assert m_i == m_o diff --git a/src/wiredancer/sim/ed25519_sigverify_0/Makefile b/src/wiredancer/sim/ed25519_sigverify_0/Makefile deleted file mode 100755 index ec874ea4b4c..00000000000 --- a/src/wiredancer/sim/ed25519_sigverify_0/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = ed25519_sigverify_0 -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_memory/hdl/xpm_memory.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_fifo/hdl/xpm_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/simple_dual_port_ram.sv -VERILOG_SOURCES += $(RTL_DIR)/showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/schl_cpu_instr_rom.sv -VERILOG_SOURCES += $(RTL_DIR)/schl_cpu.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_mul_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_ecc.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_0.sv - -COMPILE_ARGS += +define+PATH_TO_INSTR_ROM_MIF=\"$(RTL_DIR)/schl_cpu_instr_rom.mif\" - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/ed25519_sigverify_0/test.py b/src/wiredancer/sim/ed25519_sigverify_0/test.py deleted file mode 100755 index cd65ef43e1b..00000000000 --- a/src/wiredancer/sim/ed25519_sigverify_0/test.py +++ /dev/null @@ -1,82 +0,0 @@ - -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import wd_cocotil - -@cocotb.test() -async def test(dut): - - dut.i_v.value = 0 - dut.i_m.value = 0 - dut.o_r.value = 0 - dut.max_pending.value = 2 - - q_o_ed25519_sigverify_0 = dict() - - clk = dut.clk - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - await cocotb.start(wd_cocotil.random_toggle(dut.clk, dut.o_r, 50)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_0(dut, dut, clk, q_o=q_o_ed25519_sigverify_0, do_log=True)) - - # wait for xpm post reset - for i in range(1024): - await RisingEdge(clk) - - W_M = int(dut.W_M) - tid = wd_cocotil.random_int(64) - - for i in range(4): - - # backpressured - while str(dut.i_v) == '1' and str(dut.i_r) != '1': - await RisingEdge(clk) - - # random gap - while random.randint(0, 100) > 50: - dut.i_v = 0 - await RisingEdge(clk) - - tid += 1 - - tr = wd_cocotil.random_tr( - src = wd_cocotil.random_int(32), - tid = tid, - ) - - q_o_ed25519_sigverify_0[tr['tid']] = tr - - b_i_m = BinaryValue(bits=W_M, bigEndian=False) - b_i_pub = BinaryValue(bits=256, bigEndian=False) - b_i_sig_l = BinaryValue(bits=256, bigEndian=False) - b_i_sig_h = BinaryValue(bits=256, bigEndian=False) - b_i_h = BinaryValue(bits=256, bigEndian=False) - - b_i_m [W_M-1:0] = wd_cocotil.build_ecc_meta(tr) - b_i_pub [256-1:0] = tr['pub'] - b_i_sig_l [256-1:0] = tr['sig_l'] - b_i_sig_h [256-1:0] = tr['sig_h'] - b_i_h [256-1:0] = tr['sha_modq'] - - dut.i_v = 1 - dut.i_t = tid - dut.i_m = b_i_m - dut.i_pub = b_i_pub - dut.i_sig_l = b_i_sig_l - dut.i_sig_h = b_i_sig_h - dut.i_h = b_i_h - - await RisingEdge(clk) - - # backpressured - while str(dut.i_v) == '1' and str(dut.i_r) != '1': - await RisingEdge(clk) - dut.i_v = 0 - - while len(q_o_ed25519_sigverify_0) > 0: - await RisingEdge(clk) diff --git a/src/wiredancer/sim/ed25519_sigverify_1/Makefile b/src/wiredancer/sim/ed25519_sigverify_1/Makefile deleted file mode 100755 index 0923a004662..00000000000 --- a/src/wiredancer/sim/ed25519_sigverify_1/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = ed25519_sigverify_1 -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_memory/hdl/xpm_memory.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_fifo/hdl/xpm_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/simple_dual_port_ram.sv -VERILOG_SOURCES += $(RTL_DIR)/showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_mul_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_dsdp_mul.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_1.sv - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/ed25519_sigverify_1/test.py b/src/wiredancer/sim/ed25519_sigverify_1/test.py deleted file mode 100755 index 9ea02d57410..00000000000 --- a/src/wiredancer/sim/ed25519_sigverify_1/test.py +++ /dev/null @@ -1,114 +0,0 @@ - -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import wd_cocotil - -@cocotb.test() -async def test(dut): - - dut.i_v.value = 0 - dut.i_m.value = 0 - dut.o_r.value = 0 - dut.max_pending.value = 2 - - q_o_ed25519_sigverify_1 = dict() - - clk = dut.clk - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - await cocotb.start(wd_cocotil.random_toggle(dut.clk, dut.o_r, 50)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_1(dut, dut, clk, q_o=q_o_ed25519_sigverify_1, do_log=True)) - - # wait for xpm post reset - for i in range(1024): - await RisingEdge(clk) - - W_M = int(dut.W_M) - tid = wd_cocotil.random_int(64) - - for i in range(4): - - # backpressured - while str(dut.i_v) == '1' and str(dut.i_r) != '1': - await RisingEdge(clk) - - # random gap - while random.randint(0, 100) > 50: - dut.i_v = 0 - await RisingEdge(clk) - - tid += 1 - - tr = wd_cocotil.random_tr( - src = wd_cocotil.random_int(32), - tid = tid, - ) - - q_o_ed25519_sigverify_1[tr['tid']] = tr - - b_i_m = BinaryValue(bits=W_M, bigEndian=False) - b_i_pub = BinaryValue(bits=256, bigEndian=False) - b_i_sig_l = BinaryValue(bits=256, bigEndian=False) - b_i_sig_h = BinaryValue(bits=256, bigEndian=False) - b_i_h = BinaryValue(bits=256, bigEndian=False) - - b_i_t = BinaryValue(bits= 64, bigEndian=False) - b_i_m = BinaryValue(bits=W_M, bigEndian=False) - b_i_pub = BinaryValue(bits=256, bigEndian=False) - b_i_sig_l = BinaryValue(bits=256, bigEndian=False) - b_i_sig_h = BinaryValue(bits=256, bigEndian=False) - b_i_h = BinaryValue(bits=256, bigEndian=False) - b_i_res = BinaryValue(bits= 1, bigEndian=False) - b_i_Rx = BinaryValue(bits=256, bigEndian=False) - b_i_Ax = BinaryValue(bits=256, bigEndian=False) - b_i_At = BinaryValue(bits=256, bigEndian=False) - b_i_Tx = BinaryValue(bits=256, bigEndian=False) - b_i_Ty = BinaryValue(bits=256, bigEndian=False) - b_i_Tz = BinaryValue(bits=256, bigEndian=False) - b_i_Tt = BinaryValue(bits=256, bigEndian=False) - - b_i_t [64-1:0] = tid - b_i_m [W_M-1:0] = i_m - b_i_pub [255:0] = i_pub - b_i_sig_l [255:0] = i_sig_l - b_i_sig_h [255:0] = i_sig_h - b_i_h [255:0] = i_h - b_i_res [ 0:0] = i_res - b_i_Rx [255:0] = i_Rx - b_i_Ax [255:0] = i_Ax - b_i_At [255:0] = i_At - b_i_Tx [255:0] = i_Tx - b_i_Ty [255:0] = i_Ty - b_i_Tz [255:0] = i_Tz - b_i_Tt [255:0] = i_Tt - - dut.i_v = 1 - dut.i_t = b_i_t - dut.i_m = b_i_m - dut.i_pub = b_i_pub - dut.i_sig_l = b_i_sig_l - dut.i_sig_h = b_i_sig_h - dut.i_h = b_i_h - dut.i_res = b_i_res - dut.i_Rx = b_i_Rx - dut.i_Ax = b_i_Ax - dut.i_At = b_i_At - dut.i_Tx = b_i_Tx - dut.i_Ty = b_i_Ty - dut.i_Tz = b_i_Tz - dut.i_Tt = b_i_Tt - - await RisingEdge(clk) - - # backpressured - while str(dut.i_v) == '1' and str(dut.i_r) != '1': - await RisingEdge(clk) - dut.i_v = 0 - - while len(q_o_ed25519_sigverify_1) > 0: - await RisingEdge(clk) diff --git a/src/wiredancer/sim/mul_wide/Makefile b/src/wiredancer/sim/mul_wide/Makefile deleted file mode 100755 index cd2b9c9d551..00000000000 --- a/src/wiredancer/sim/mul_wide/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = mul_wide -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/mul_wide/test.py b/src/wiredancer/sim/mul_wide/test.py deleted file mode 100755 index fa804692e80..00000000000 --- a/src/wiredancer/sim/mul_wide/test.py +++ /dev/null @@ -1,65 +0,0 @@ - -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly - -from cocotb.binary import BinaryValue - -import wd_cocotil - -@cocotb.test() -async def test(dut): - - c = Clock(dut.clk, 1, 'ns') - await cocotb.start(c.start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - clk = dut.clk - - W = int(dut.W) - if hasattr(dut, 'W0'): - W0 = int(dut.W0) - else: - W0 = W - if hasattr(dut, 'W1'): - W1 = int(dut.W1) - else: - W1 = W - T = int(dut.T) & 0xff - - o_rs = list() - - D = 0 - - for i in range(1024): - await RisingEdge(dut.clk) - - for i in range(1024): - - m_i = wd_cocotil.random_int(31) - i0 = wd_cocotil.random_int(b=W0) - if T in [0x1F, 0x2F, 0x3F]: - i1 = L0 - else: - i1 = wd_cocotil.random_int(b=W1) - o_rs.append((i0, i1, i0 * i1, m_i)) - - dut.in0.value = i0 - dut.in1.value = i1 - dut.m_i.value = 1 | (m_i << 1) - - await RisingEdge(dut.clk) - - if str(dut.m_o[0]) != '1': - D += 1 - assert D < 100 - continue - - i0, i1, e_d, e_m = o_rs.pop(0) - o_d = int(dut.out0) - o_m = int(dut.m_o) >> 1 - - dut._log.info ('{:x} x {:x} = \n{:x} =?= \n{:x}\nD.{}, {:x} =?= {:x}'.format(i0, i1, e_d, o_d, D, e_m, o_m)) - assert e_d == o_d - assert e_m == o_m diff --git a/src/wiredancer/sim/schl_cpu/Makefile b/src/wiredancer/sim/schl_cpu/Makefile deleted file mode 100644 index cbc50b11389..00000000000 --- a/src/wiredancer/sim/schl_cpu/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = schl -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_memory/hdl/xpm_memory.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_fifo/hdl/xpm_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/simple_dual_port_ram.sv -VERILOG_SOURCES += $(RTL_DIR)/showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/schl_cpu_instr_rom.sv -VERILOG_SOURCES += $(RTL_DIR)/schl_cpu.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_mul_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_ecc.sv - -COMPILE_ARGS += +define+PATH_TO_INSTR_ROM_MIF=\"$(RTL_DIR)/schl_cpu_instr_rom.mif\" - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/schl_cpu/test.py b/src/wiredancer/sim/schl_cpu/test.py deleted file mode 100644 index 68310a43380..00000000000 --- a/src/wiredancer/sim/schl_cpu/test.py +++ /dev/null @@ -1,305 +0,0 @@ -import cocotb -import os -import copy -import sys -import struct - -import hashlib -import random - -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.regression import TestFactory -from cocotb.binary import BinaryValue - -import ref_ed25519 -from ref_ed25519 import point_decompress -from ed25519_lib import Expr - -sent_in = [] - -def FAIL( ): - assert False - -def clamp(val): - return int(val) & ((1<<256)-1) - -def getTern(val): - return int(val >> 28) - -def getOP(val): - return int(val >> 24 & 0xF) - -def getMemAAddr(val): - return int(val >> 18 & 0x3F) - -def getMemBAddr(val): - return int(val >> 12 & 0x3F) - -def getMemTAddr(val): - return int(val >> 6 & 0x3F) - -def getMemOAddr(val): - return int(val & 0x3F) - -def scratch_offset(tag): - a = [0x000, 0x018, 0x030, 0x048, 0x060, 0x078, 0x090, 0x0A8, 0x0C0, 0x0D8, 0x0F0, 0x108, 0x120, 0x138, 0x150, 0x168, - 0x180, 0x198, 0x1B0, 0x1C8, 0x1E0, 0x1F8, 0x210, 0x228, 0x240, 0x258, 0x270, 0x288, 0x2A0, 0x2B8, 0x2D0, 0x2E8] - return a[tag] - -def get_const(addr): - const = [ - 0x0000000000000000000000000000000000000000000000000000000000000000, - 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed, - 0x0000000000000000000000000000000000000000000000000000000000000001, - 0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ed, - 0x52036cee2b6ffe738cc740797779e89800700a4d4141d8ab75eb4dca135978a3, - 0x67875f0fd78b766566ea4e8e64abe37d20f09f80775152f56dde8ab3a5b7dda3, - 0x6666666666666666666666666666666666666666666666666666666666666658, - 0x216936d3cd6e53fec0a4e231fdd6dc5c692cc7609525a7b2c9562d608f25d51a, - 0x2b8324804fc1df0b2b4d00993dfbd7a72f431806ad2fe478c4ee1b274a0ea0b0, - 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec, - 0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff, - 0x00000000000000000000000000000000000000000000000000000000000000ff ] - # print("LOOKUP: {}".format(addr)) - return const[addr] - -def evalOp(op, valA, valB, valT): - if (op == 0): return clamp(valA & valB ) - if (op == 1): return clamp(valA == valB) - if (op == 2): return clamp(valA != valB) - if (op == 3): return clamp(valA >= valB) - if (op == 4): return clamp(valA << 1) - if (op == 5): return clamp((valA >> 255) & 0x1) - if (op == 6): return clamp(valA + valB ) - if (op == 7): return clamp(valA - valB ) - if (op == 8): return clamp( (valA + valB) % ref_ed25519.p ) - if (op == 9): return clamp( (valA - valB) % ref_ed25519.p ) - if (op == 10): return clamp( (valA * valB) % ref_ed25519.p ) - if (op == 11): - if( valT ): return valA - else: return valB - if (op == 12): - return 0 - return 0 - -class MathMonitor: - - def __init__(self, dut): - self.dut = dut - self.total_tags = int(self.dut.cpu0.NUM_TAGS) - self.mem = {} - self.verbose = True - for i in range(0x3FF): - self.mem[i] = 0 - for i in range(12): - self.setMem(0, i+0x04, get_const(i)) - - def check( self, tag, errstr, expected, observed ): - if( int(expected) == int(observed) ): - #print("SUCCESS! [{}]".format(errstr)) - return - self.dut._log.info("[{}] FAILED! {}- exp: {:X} vs obs: {:X}".format(tag, errstr, expected, observed)) - FAIL() - - def setIn(self, tag, addr, val): - wr_addr = self.getPhyAddr(tag, addr) - self.mem[wr_addr] = val - if self.verbose: self.dut._log.info("[{}] Writing Input to Mem [{:X}] = {:X}".format(tag, wr_addr, val)) - - - def setMem(self, tag, addr, val): - wr_addr = self.getPhyAddr(tag, addr) - self.mem[wr_addr] = int(val) - if self.verbose: self.dut._log.info("[{}] Writing to Mem [{:X}] = {:X}".format(tag, wr_addr, val)) - - def getPhyAddr(self, tag, addr): - if (addr == 0x00): return int(tag + 0x000) - elif (addr == 0x01): return int(tag + 0x020) - elif (addr == 0x02): return int(tag + 0x040) - elif (addr == 0x03): return int(tag + 0x060) - elif (addr >= 0x04 and addr <= 0x23 ): return int(0x080 + addr - 0x04) - else: return int(0x0A0 + scratch_offset(tag) + addr - 0x24) - - def getMem(self, tag, addr): - rd_addr = self.getPhyAddr(tag, addr) - res = self.mem[ rd_addr ] - if self.verbose: self.dut._log.info("[{}] Reading from Mem [{:X}] = {:X}".format(tag, rd_addr, res)) - return res - - async def run(self): - next_tag = 0 - init_vals = 0 - instr = [] - addrA = [] - addrB = [] - addrT = [] - valA = [] - valB = [] - valT = [] - addrO = [] - expRslt = [] - resultChecked = [] - for _ in range(self.total_tags): - instr.append(0) - addrA.append(0) - addrB.append(0) - addrT.append(0) - valA.append(0) - valB.append(0) - valT.append(0) - addrO.append(0) - expRslt.append(0) - resultChecked.append(False) - while True: - inserted = False - await RisingEdge(self.dut.clk) - for t in range(self.total_tags): - cpu = self.dut.cpu0 - state = int(cpu.curr_state[t]) - if (int(cpu.calc_next) == t and state == 1 and int(cpu.in_hash_valid) == 1 and not inserted): - self.setIn( int(cpu.calc_next), init_vals, int(cpu.in_hash_data) ) - inserted = True - if init_vals == 2: - init_vals = 0 - if (next_tag == 31): - next_tag = 0 - else: - next_tag += 1 - else: - init_vals += 1 - if (state == 3): # SEND0 - resultChecked[t] = False - instr[t] = int(cpu.next_instr[t]) - addrA[t] = getMemAAddr(instr[t]) - addrB[t] = getMemBAddr(instr[t]) - addrT[t] = getMemTAddr(instr[t]) - addrO[t] = getMemOAddr(instr[t]) - valA[t] = self.getMem(t, addrA[t]) - valB[t] = self.getMem(t, addrB[t]) - valT[t] = self.getMem(t, addrT[t]) - valT[t] = valT[t] & 0x1 - expRslt[t] = evalOp(getOP(instr[t]), valA[t], valB[t], valT[t]) - if self.verbose: self.dut._log.info("[{}] EXPECTED: A[{:X}:{:X}] OP: {} B[{:X}:{:X}] T[{:X}:{:X}] OUT:{:X}".format(t, addrA[t], valA[t], getOP(instr[t]), addrB[t], valB[t], addrT[t], valT[t], expRslt[t])) - if (state == 5): # BLOCK - if int(cpu.mem_man_tag) == t and int(cpu.block_valid[t]) == 1 and not resultChecked[t]: - if getOP(instr[t]) != 12: - self.check(t, "Result", expRslt[t], int(cpu.mem_man_data)) - self.check(t, "OutAddr", self.getPhyAddr(t, addrO[t]), int(cpu.mem_man_addr)) - self.setMem(t, getMemOAddr(instr[t]), expRslt[t]) - resultChecked[t] = True - # print("Out Addr- a: {:X} vs e: {:X}".format(, )) - -class OutMonitor: - def __init__(self, dut): - self.dut = dut - self.done = False - - async def is_done(self): - while not self.done: - await RisingEdge(self.dut.clk) - - async def run(self, total=3): - tic = 0 - start = 0 - self.in_vals = [] - self.out_vals = [] - self.expected = [] - - self.inout_match = {} - start = [] - i_cnt = 0 - - while True: - await RisingEdge(self.dut.clk) - tic += 1 - if (int(self.dut.i_valid) == 1): - if (i_cnt == 0): - in_val = int(self.dut.i_hash) - self.in_vals.append(in_val) - start.append(tic) - - if (i_cnt == 2): i_cnt = 0 - else: i_cnt += 1 - - if (int(self.dut.o_valid) == 1): - self.out_vals.append(int(self.dut.o_hash)) - t = len(self.out_vals)-1 - u = int(t/8) - - self.dut._log.info("RESULT Output: [{}/{}] {:x}".format( len(self.out_vals), (total*8), self.out_vals[t] ) ) - self.dut._log.info("-- start tic: {}".format(start[u])) - self.dut._log.info("-- stop tick: {}".format(tic)) - self.dut._log.info("-- elapsed: {}ns".format( ((tic-start[u])*4))) - if len(self.out_vals) >= total*8: break #8 outs per input set - self.done = True - -async def send_rand_input(dut, i=0): - #in_vals = ( (random.getrandbits(int(dut.W_HASH)) % p), (random.getrandbits(int(dut.W_HASH)) % p), (random.getrandbits(int(dut.W_HASH)) % p), (random.getrandbits(int(dut.W_HASH)) % p )) - from ref_ed25519 import p - - in_tmp = [0xf94f19db1a86d8574bc885068848db78a886b6a96537c1b442e8b9402c4a341d, - 0xc9e8131bf9cf7c67479f7589949c8241be3581537f9724b611464bcbe2b1f30b, - 0x013e4a41676acaed06ac560dfe46283e77a1ccbc919cb8a08ddbd36134ad9a4d, ] - in_vals = ( in_tmp[0], in_tmp[1], in_tmp[2]) - - total = 3 - i=0 - while i < total: - if int(dut.i_ready) == 1: - dut.i_hash .value = in_vals[i] - dut.i_valid .value = 1 - i+=1 - else: - dut.i_hash .value = 0 - dut.i_valid .value = 0 - await RisingEdge(dut.clk) - dut.i_hash .value = 0 - dut.i_valid .value = 0 - - sent_in.append( in_vals ) - -@cocotb.test() -async def run_test(dut): - num_sent = 0 - num_inputs = 64 - - om = OutMonitor(dut) - im = MathMonitor(dut) - await cocotb.start(Clock(dut.clk, 4000).start()) - await cocotb.start(om.run(num_inputs)) - await cocotb.start(im.run()) - dut.i_hash .value = 0 - dut.i_valid .value = 0 - await RisingEdge(dut.clk) - dut.rst .value = 1 - await RisingEdge(dut.clk) - await RisingEdge(dut.clk) - dut.rst .value = 0 - await RisingEdge(dut.clk) - while( int(dut.i_ready) == 0): - await RisingEdge(dut.clk) - - # allow DSPs to warm up - for _ in range(2048): - await RisingEdge(dut.clk) - - - while num_sent < num_inputs: - if ( int(dut.i_ready) == 1 ): - await send_rand_input(dut, num_sent) - num_sent += 1 - for _ in range(random.randint(0,10)): - await RisingEdge(dut.clk) - else: - await RisingEdge(dut.clk) - - await RisingEdge(dut.clk) - dut.i_hash .value = 0 - dut.i_valid .value = 0 - for _ in range(100): - await RisingEdge(dut.clk) - - await om.is_done() - - await RisingEdge(dut.clk) diff --git a/src/wiredancer/sim/sha512_modq_meta/Makefile b/src/wiredancer/sim/sha512_modq_meta/Makefile deleted file mode 100755 index a620d1d0151..00000000000 --- a/src/wiredancer/sim/sha512_modq_meta/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = sha512_modq_meta -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_memory/hdl/xpm_memory.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_fifo/hdl/xpm_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/simple_dual_port_ram.sv -VERILOG_SOURCES += $(RTL_DIR)/showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_sch.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_msgseq.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_round.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_block.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_modq.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_modq_meta.sv - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/sha512_modq_meta/test.py b/src/wiredancer/sim/sha512_modq_meta/test.py deleted file mode 100755 index 1183ad782e2..00000000000 --- a/src/wiredancer/sim/sha512_modq_meta/test.py +++ /dev/null @@ -1,91 +0,0 @@ - -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import wd_cocotil - -@cocotb.test() -async def test(dut): - - dut.i_v = 0 - dut.o_r = 0 - dut.max_pending = 100 - - q_i_sha_modq_meta = list() - q_o_sha_modq_meta = dict() - - clk = dut.clk - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - await cocotb.start(wd_cocotil.random_toggle(dut.clk, dut.o_r, 50)) - await cocotb.start(wd_cocotil.mon_sha_modq_meta(dut, dut, clk, q_i=q_i_sha_modq_meta, q_o=q_o_sha_modq_meta, do_log=True)) - - M = int(dut.META_W) - - # wait for xpm post reset - # then wait more for key-store to initialize - for i in range(1024+1024): - await RisingEdge(clk) - - min_l = 0 - max_l = 1280 - mlens = list() - - tid = wd_cocotil.random_int(64) - - for _ in range(max_l * 2): - - # backpressured - while str(dut.i_v) == '1' and str(dut.i_r) != '1': - await RisingEdge(clk) - - tid += 1 - - # random gap - while random.randint(0, 100) > 50: - dut.i_v = 0 - await RisingEdge(clk) - - if len(mlens) == 0: - mlens = list(range(min_l, max_l+1)) - mlen = random.choice(mlens) - mlens.remove(mlen) - - tr = wd_cocotil.random_tr( - tid = tid, - mlen = mlen, - sha_modq_meta = wd_cocotil.random_int(M) - ) - - q_i_sha_modq_meta.append(tr) - q_o_sha_modq_meta[tr['tid']] = tr - - blks = wd_cocotil.build_sha_modq_meta_i(tr) - for f, l, c, t, m, d in blks: - - # backpressured - while str(dut.i_v) == '1' and str(dut.i_r) != '1': - await RisingEdge(clk) - - b_m = BinaryValue(bits=M, bigEndian=False) - b_d = BinaryValue(bits=1024, bigEndian=False) - b_m[M-1:0] = m - b_d[1023:0] = d - dut.i_v = 1 - dut.i_f = f - dut.i_l = l - dut.i_c = c - dut.i_t = t - dut.i_m = b_m - dut.i_d = b_d - - await RisingEdge(clk) - - while True: - if len(q_o_sha_modq_meta) == 0: - break - await RisingEdge(clk) diff --git a/src/wiredancer/sim/sha512_pre/Makefile b/src/wiredancer/sim/sha512_pre/Makefile deleted file mode 100755 index 49566b4fd80..00000000000 --- a/src/wiredancer/sim/sha512_pre/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = sha512_pre -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_memory/hdl/xpm_memory.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_fifo/hdl/xpm_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/simple_dual_port_ram.sv -VERILOG_SOURCES += $(RTL_DIR)/showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_pre.sv - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/sha512_pre/test.py b/src/wiredancer/sim/sha512_pre/test.py deleted file mode 100755 index 979ca036141..00000000000 --- a/src/wiredancer/sim/sha512_pre/test.py +++ /dev/null @@ -1,94 +0,0 @@ - -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import wd_cocotil - -@cocotb.test() -async def test(dut): - - dut.i_w.value = 0 - dut.i_v.value = 0 - - q_o_sha_pre = list() - - clk = dut.clk - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - await cocotb.start(wd_cocotil.mon_sha_pre(dut, dut, clk, q_i=None, q_o=q_o_sha_pre, do_log=True)) - - # wait for xpm post reset - for i in range(1024): - await RisingEdge(clk) - - W = int(dut.W_D) - M = sum(wd_cocotil.meta2) - min_l = 32+32 - max_l = 128*3 - mlens = list() - - for _ in range(max_l * 2): - - if len(mlens) == 0: - mlens = list(range(min_l, max_l+1)) - mlen = random.choice(mlens) - mlens.remove(mlen) - - tr = wd_cocotil.random_tr( - mlen = mlen, - src = wd_cocotil.random_int(16), - ) - - q_o_sha_pre.append(tr) - - msg = tr['sha_msg'] - sop = 1 - size = len(msg) - - while len(msg) > 0: - - # backpressured - while str(dut.i_v.value) == '1' and str(dut.i_r.value) != '1': - await RisingEdge(clk) - - # random gap - while random.randint(0, 100) > 80: - dut.i_v.value = 0 - await RisingEdge(clk) - - eop = len(msg) <= (W//8) - e = (W//8) - len(msg) - e = 0 if e < 0 else e - b_m = BinaryValue(n_bits=M, bigEndian=False) - b_m[16 -1:0 ] = tr['src'] - b_m[16+64 -1:16 ] = tr['tid'] - b_m[16+64+256 -1:16+64 ] = tr['sig_l'] - b_m[16+64+256+256 -1:16+64+256 ] = tr['sig_h'] - b_m[16+64+256+256+256 -1:16+64+256+256 ] = tr['pub'] - b_m[16+64+256+256+256+16 -1:16+64+256+256+256 ] = size - b_m[16+64+256+256+256+16+6 -1:16+64+256+256+256+16 ] = e - b_m[16+64+256+256+256+16+6+1 -1:16+64+256+256+256+16+6 ] = sop - off = M-W - for i in range(W//8): - if len(msg) == 0: - b_m[off+i*8+8-1:off+i*8] = 0xff#random.randint(0, 255) - else: - b_m[off+i*8+8-1:off+i*8] = msg[0] - msg = msg[1:] - - dut.i_v.value = 1 - dut.i_e.value = eop - dut.i_m.value = b_m - - sop = 0 - - await RisingEdge(clk) - - dut.i_v.value = 0 - - while len(q_o_sha_pre) > 0: - await RisingEdge(clk) diff --git a/src/wiredancer/sim/top_f1/Makefile b/src/wiredancer/sim/top_f1/Makefile deleted file mode 100755 index b96679b436c..00000000000 --- a/src/wiredancer/sim/top_f1/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(pwd)../../rtl -TOPLEVEL = top_f1 -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_memory/hdl/xpm_memory.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_fifo/hdl/xpm_fifo.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_cdc/hdl/xpm_cdc.sv -VERILOG_SOURCES += $(RTL_DIR)/simple_dual_port_ram.sv -VERILOG_SOURCES += $(RTL_DIR)/showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/dual_clock_showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/pcie_inorder.sv -VERILOG_SOURCES += $(RTL_DIR)/pcie_tr_ext.sv -VERILOG_SOURCES += $(RTL_DIR)/tid_inorder.sv -VERILOG_SOURCES += $(RTL_DIR)/dma_result.sv -VERILOG_SOURCES += $(RTL_DIR)/key_store.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_pre.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_sch.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_msgseq.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_round.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_block.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_modq.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_modq_meta.sv -VERILOG_SOURCES += $(RTL_DIR)/schl_cpu_instr_rom.sv -VERILOG_SOURCES += $(RTL_DIR)/schl_cpu.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_add_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sub_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_mul_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_point_add.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_point_dbl.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_ecc.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_dsdp_mul.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_0.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_1.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_2.sv -VERILOG_SOURCES += $(RTL_DIR)/top_f1.sv - -COMPILE_ARGS += +define+PATH_TO_INSTR_ROM_MIF=\"$(RTL_DIR)/schl_cpu_instr_rom.mif\" - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/top_f1/test.py b/src/wiredancer/sim/top_f1/test.py deleted file mode 100755 index 952bda06812..00000000000 --- a/src/wiredancer/sim/top_f1/test.py +++ /dev/null @@ -1,127 +0,0 @@ - -import os -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import ref_ed25519 -import wd_cocotil -import pcap - -@cocotb.test() -async def test(dut): - - dut.avmm_read.value = 0 - dut.avmm_write.value = 0 - dut.dma_f.value = 0 - dut.pcie_v.value = 0 - dut.send_fails = 1 - - q_i_pcie_tr_ext = list() - q_o_pcie_tr_ext = list() - q_o_sha_pre = list() - q_o_sha_modq_meta = dict() - q_o_ed25519_sigverify_0 = dict() - q_o_ed25519_sigverify_1 = dict() - q_o_ed25519_sigverify_2 = dict() - q_o_res_dma = list() - - clk = dut.clk - clk_f = dut.clk_f - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - await cocotb.start(Clock(dut.clk_f, .9, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk_f, dut.rst_f, 32, active_high=True)) - await cocotb.start(wd_cocotil.random_toggle(dut.clk, dut.dma_r, 50)) - - await cocotb.start(wd_cocotil.mon_pcie_tr_ext(dut, dut.P_IN[0].tr_ext_inst, clk, q_i=q_i_pcie_tr_ext, q_o=q_o_pcie_tr_ext, do_log=True)) - await cocotb.start(wd_cocotil.mon_sha_pre(dut, dut.sha512_pre_inst, clk, q_o=q_o_sha_pre, do_log=True)) - await cocotb.start(wd_cocotil.mon_sha_modq_meta(dut, dut.sha512_modq_meta_inst, clk, q_o=q_o_sha_modq_meta, do_log=True)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_0(dut, dut.ed25519_sigverify_0_inst, clk, q_o=q_o_ed25519_sigverify_0, do_log=True)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_1(dut, dut.ed25519_sigverify_1_inst, clk_f, q_o=q_o_ed25519_sigverify_1, do_log=True)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_2(dut, dut.ed25519_sigverify_2_inst, clk, q_o=q_o_ed25519_sigverify_2, do_log=True)) - await cocotb.start(wd_cocotil.mon_dma(dut, dut, clk, q_o=q_o_res_dma, do_log=True)) - - # wait for xpm post reset - for i in range(1024): - await RisingEdge(clk) - # wait for key_stores post reset - for i in range(1024): - await RisingEdge(clk) - - tid = 0xabcd0000 - 1 - pcie_a = 0 - pcie_b = 1 << 32 - min_sz = 32+32 - max_sz = 1280 - mls = list() - - pc = pcap.kpcap(r='../../pcaps/val4ni_flood-1653425338.pcap') - - for i in range(32): - - for i in range(random.randint(10, 250)): - await RisingEdge(clk) - - # backpressure for PCIe fifo - while int(dut.pcie_il[0]) > 2: - await RisingEdge(clk) - - if True: - # if random.randint(0, 100) > 50: - S, P, M = pc.read() - - sig = S[0] - pub = P[0] - msg = M - err = random.randint(0, 100) - else: - if len(mls) == 0: - mls = list(range(min_sz, max_sz+1)) - ml = random.choice(mls) - mls.remove(ml) - - sec = ''.join([random.choice(['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']) for i in range(32)]) - pub = ref_ed25519.secret_to_public(sec) - msg = [random.randint(0, 255) for i in range(ml)] - sig = ref_ed25519.sign(sec, msg) - err = random.randint(0, 100) - - if err < 10: - msg = wd_cocotil.random_byte_error(msg) - elif err < 20: - sig = wd_cocotil.random_byte_error(sig) - elif err < 30: - pub = wd_cocotil.random_byte_error(pub) - - tid += 1 - - tr = wd_cocotil.random_tr( - src = wd_cocotil.random_int(16-4) << 4, # LSBs == src - tid = tid, - sig = wd_cocotil.bytes_to_little(sig), - pub = wd_cocotil.bytes_to_little(pub), - msg = msg, - ) - - q_i_pcie_tr_ext.append(tr) - q_o_pcie_tr_ext.append(tr) - q_o_sha_pre.append(tr) - q_o_sha_modq_meta[tr['tid']] = tr - q_o_ed25519_sigverify_0[tr['tid']] = tr - q_o_ed25519_sigverify_1[tr['tid']] = tr - q_o_ed25519_sigverify_2[tr['tid']] = tr - q_o_res_dma.append(tr) - - for blk in tr['pcie_tr']: - await wd_cocotil.f1_write_32x16(dut, clk, pcie_b | pcie_a, blk) - pcie_a += 64 - pcie_a &= (1 << wd_cocotil.PCIE_ADDR_W)-1 - - await RisingEdge(clk) - - while len(q_o_res_dma) > 0: - await RisingEdge(clk) diff --git a/src/wiredancer/sim/top_f1_models/Makefile b/src/wiredancer/sim/top_f1_models/Makefile deleted file mode 100755 index 6144053b55d..00000000000 --- a/src/wiredancer/sim/top_f1_models/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -# Makefile - -SIM ?= questa -MODULE = test -RTL_DIR = $(shell pwd)/../../rtl -SIM_DIR = $(shell pwd) -TOPLEVEL = top_f1 -SIM_ARGS += sim_build/work.glbl -TOPLEVEL_LANG ?= verilog - -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/glbl.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/verilog/src/unisims/DSP48E2.v -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_memory/hdl/xpm_memory.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_fifo/hdl/xpm_fifo.sv -VERILOG_SOURCES += $(XILINX_VIVADO)/data/ip/xpm/xpm_cdc/hdl/xpm_cdc.sv -VERILOG_SOURCES += $(RTL_DIR)/simple_dual_port_ram.sv -VERILOG_SOURCES += $(RTL_DIR)/showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/dual_clock_showahead_fifo.sv -VERILOG_SOURCES += $(RTL_DIR)/wd_pkg.sv -VERILOG_SOURCES += $(RTL_DIR)/pcie_inorder.sv -VERILOG_SOURCES += $(RTL_DIR)/pcie_tr_ext.sv -VERILOG_SOURCES += $(RTL_DIR)/tid_inorder.sv -VERILOG_SOURCES += $(RTL_DIR)/dma_result.sv -VERILOG_SOURCES += $(RTL_DIR)/key_store.sv -VERILOG_SOURCES += $(RTL_DIR)/mul_wide.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_pre.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_sch.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_msgseq.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_round.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_block.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_modq.sv -VERILOG_SOURCES += $(RTL_DIR)/sha512_modq_meta.sv -VERILOG_SOURCES += $(RTL_DIR)/schl_cpu_instr_rom.sv -VERILOG_SOURCES += $(SIM_DIR)/schl_cpu.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_add_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sub_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_mul_modp.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_point_add.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_point_dbl.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_ecc.sv -VERILOG_SOURCES += $(SIM_DIR)/ed25519_sigverify_dsdp_mul.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_0.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_1.sv -VERILOG_SOURCES += $(RTL_DIR)/ed25519_sigverify_2.sv -VERILOG_SOURCES += $(RTL_DIR)/top_f1.sv - -COMPILE_ARGS += +define+PATH_TO_INSTR_ROM_MIF=\"$(RTL_DIR)/schl_cpu_instr_rom.mif\" - -include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/src/wiredancer/sim/top_f1_models/ed25519_sigverify_dsdp_mul.sv b/src/wiredancer/sim/top_f1_models/ed25519_sigverify_dsdp_mul.sv deleted file mode 100755 index 2a052687445..00000000000 --- a/src/wiredancer/sim/top_f1_models/ed25519_sigverify_dsdp_mul.sv +++ /dev/null @@ -1,45 +0,0 @@ -`default_nettype none - -import wd_sigverify::*; - -module ed25519_sigverify_dsdp_mul #( - MUL_T = 32'h007F_CCC2, - MUL_D = 15, - ADD_D = 4, - PIPE_D = (MUL_D*3) + ADD_D, - N_TH = 2+PIPE_D+PIPE_D, - W_M = 64, - W_S = 2 -) ( - output logic [1-1:0] i_r, - input wire [1-1:0] i_v, - input wire [W_M-1:0] i_m, - - input wire [255-1:0] i_Ax, - input wire [255-1:0] i_Ay, - input wire [255-1:0] i_Az, - input wire [255-1:0] i_At, - - input wire [255-1:0] i_ApGx, // A+G - input wire [255-1:0] i_ApGy, - input wire [255-1:0] i_ApGz, - input wire [255-1:0] i_ApGt, - - input wire [256-1:0] i_As, - input wire [256-1:0] i_Gs, - - output logic [1-1:0] o_v, - output logic [W_M-1:0] o_m, - - output logic [255-1:0] o_Cx, - output logic [255-1:0] o_Cy, - output logic [255-1:0] o_Cz, - output logic [255-1:0] o_Ct, - - input wire clk, - input wire rst -); - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/sim/top_f1_models/schl_cpu.sv b/src/wiredancer/sim/top_f1_models/schl_cpu.sv deleted file mode 100755 index 4c095ff7e65..00000000000 --- a/src/wiredancer/sim/top_f1_models/schl_cpu.sv +++ /dev/null @@ -1,30 +0,0 @@ -`default_nettype none - -module shcl_cpu -#( - MUL_T = 32'h007F_CCC2, - MUL_D = 15, - W_HASH = 256, - W_IN_MEM = 6, - W_T = 16, - MAX_INFLIGHT = (MUL_D+1)+6 -) -( - input var logic clk, - input var logic rst, - - input var logic [W_HASH-1:0] in_hash_data, - input var logic in_hash_valid, - input var logic [W_T-1:0] in_hash_ref, - output logic in_hash_ready, - - output logic [W_HASH-1:0] out_hash_data, - output logic [W_T-1:0] out_ref, - output logic [W_IN_MEM-1:0] out_d_addr, - output logic out_hash_valid -); - - -endmodule - -`default_nettype wire diff --git a/src/wiredancer/sim/top_f1_models/test.py b/src/wiredancer/sim/top_f1_models/test.py deleted file mode 100755 index 8e8d1e95f7e..00000000000 --- a/src/wiredancer/sim/top_f1_models/test.py +++ /dev/null @@ -1,132 +0,0 @@ - -import os -import random - -import cocotb -from cocotb.clock import Clock -from cocotb.triggers import Timer, RisingEdge, ReadOnly -from cocotb.binary import BinaryValue - -import ref_ed25519 -import wd_cocotil -import pcap - -@cocotb.test() -async def test(dut): - - dut.avmm_read.value = 0 - dut.avmm_write.value = 0 - dut.dma_f.value = 0 - dut.pcie_v.value = 0 - dut.send_fails = 1 - - q_i_pcie_tr_ext = list() - q_o_pcie_tr_ext = list() - q_o_sha_pre = list() - q_o_sha_modq_meta = dict() - q_o_ed25519_sigverify_0 = dict() - q_o_ed25519_sigverify_1 = dict() - q_o_ed25519_sigverify_2 = dict() - q_o_res_dma = list() - - clk = dut.clk - clk_f = dut.clk_f - await cocotb.start(Clock(dut.clk, 1, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk, dut.rst, 32, active_high=True)) - await cocotb.start(Clock(dut.clk_f, .9, 'ns').start()) - await cocotb.start(wd_cocotil.toggle_reset(dut.clk_f, dut.rst_f, 32, active_high=True)) - await cocotb.start(wd_cocotil.random_toggle(dut.clk, dut.dma_r, 50)) - - await cocotb.start(wd_cocotil.mon_pcie_tr_ext(dut, dut.P_IN[0].tr_ext_inst, clk, q_i=q_i_pcie_tr_ext, q_o=q_o_pcie_tr_ext, do_log=True)) - await cocotb.start(wd_cocotil.mon_sha_pre(dut, dut.sha512_pre_inst, clk, q_o=q_o_sha_pre, do_log=True)) - await cocotb.start(wd_cocotil.mon_sha_modq_meta(dut, dut.sha512_modq_meta_inst, clk, q_o=q_o_sha_modq_meta, do_log=True)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_0(dut, dut.ed25519_sigverify_0_inst, clk, q_o=q_o_ed25519_sigverify_0, do_log=True)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_1(dut, dut.ed25519_sigverify_1_inst, clk_f, q_o=q_o_ed25519_sigverify_1, do_log=True)) - await cocotb.start(wd_cocotil.mon_ed25519_sigverify_2(dut, dut.ed25519_sigverify_2_inst, clk, q_o=q_o_ed25519_sigverify_2, do_log=True)) - await cocotb.start(wd_cocotil.mon_dma(dut, dut, clk, q_o=q_o_res_dma, do_log=True)) - - for i in range(int(dut.N_SCH)): - await cocotb.start(wd_cocotil.model_schl_cpu(dut, dut.ed25519_sigverify_0_inst.G_SCH[i].schl_cpu_inst, clk, do_log=True)) - await cocotb.start(wd_cocotil.model_dsdp(dut, dut.ed25519_sigverify_1_inst.ed25519_sigverify_dsdp_mul_inst, clk_f, do_log=True)) - - # wait for xpm post reset - for i in range(1024): - await RisingEdge(clk) - # wait for key_stores post reset - for i in range(1024): - await RisingEdge(clk) - - tid = 0xabcd0000 - 1 - pcie_a = 0 - pcie_b = 1 << 32 - min_sz = 32+32 - max_sz = 1280 - mls = list() - - pc = pcap.kpcap(r='../../pcaps/val4ni_flood-1653425338.pcap') - - for i in range(1024): - - for i in range(random.randint(10, 250)): - await RisingEdge(clk) - - # backpressure for PCIe fifo - while int(dut.pcie_il[0]) > 2: - await RisingEdge(clk) - - if False: - # if True: - # if random.randint(0, 100) > 50: - S, P, M = pc.read() - - sig = S[0] - pub = P[0] - msg = M - err = random.randint(0, 100) - else: - if len(mls) == 0: - mls = list(range(min_sz, max_sz+1)) - ml = random.choice(mls) - mls.remove(ml) - - sec = bytes([random.randint(0, 255) for i in range(32)]) - pub = ref_ed25519.secret_to_public(sec) - msg = bytes([random.randint(0, 255) for i in range(ml)]) - sig = ref_ed25519.sign(sec, msg) - err = random.randint(0, 100) - - if err < 10: - msg = wd_cocotil.random_byte_error(msg) - elif err < 20: - sig = wd_cocotil.random_byte_error(sig) - elif err < 30: - pub = wd_cocotil.random_byte_error(pub) - - tid += 1 - - tr = wd_cocotil.random_tr( - src = wd_cocotil.random_int(16-4) << 4, # LSBs == src - tid = tid, - sig = wd_cocotil.bytes_to_little(sig), - pub = wd_cocotil.bytes_to_little(pub), - msg = msg, - ) - - q_i_pcie_tr_ext.append(tr) - q_o_pcie_tr_ext.append(tr) - q_o_sha_pre.append(tr) - q_o_sha_modq_meta[tr['tid']] = tr - q_o_ed25519_sigverify_0[tr['tid']] = tr - q_o_ed25519_sigverify_1[tr['tid']] = tr - q_o_ed25519_sigverify_2[tr['tid']] = tr - q_o_res_dma.append(tr) - - for blk in tr['pcie_tr']: - await wd_cocotil.f1_write_32x16(dut, clk, pcie_b | pcie_a, blk) - pcie_a += 64 - pcie_a &= (1 << wd_cocotil.PCIE_ADDR_W)-1 - - await RisingEdge(clk) - - while len(q_o_res_dma) > 0: - await RisingEdge(clk) diff --git a/src/wiredancer/test/Local.mk b/src/wiredancer/test/Local.mk deleted file mode 100644 index 6e380305631..00000000000 --- a/src/wiredancer/test/Local.mk +++ /dev/null @@ -1,6 +0,0 @@ -ifdef FD_HAS_WIREDANCER -$(call make-lib,fd_wiredancer_test) -$(call add-hdrs,wd_f1_mon.h fd_replay_loop.h) -$(call add-objs,fd_replay_loop wd_f1_mon,fd_wiredancer_test) -$(call make-unit-test,test_wiredancer_demo,test_wiredancer_demo wd_f1_mon fd_replay_loop,fd_wiredancer fd_ballet fd_disco fd_tango fd_util) -endif diff --git a/src/wiredancer/test/fd_replay_loop.c b/src/wiredancer/test/fd_replay_loop.c deleted file mode 100644 index d5e2d0782d8..00000000000 --- a/src/wiredancer/test/fd_replay_loop.c +++ /dev/null @@ -1,635 +0,0 @@ -#include "fd_replay_loop.h" - -#if FD_HAS_HOSTED && FD_HAS_X86 - -#include "../../util/net/fd_pcap.h" -#include -#include -#include /* FIXME remove when ready */ - -FD_STATIC_ASSERT( FD_FCTL_ALIGN<=FD_REPLAY_TILE_SCRATCH_ALIGN, packing ); - -ulong -fd_replay_tile_scratch_align( void ) { - return FD_REPLAY_TILE_SCRATCH_ALIGN; -} - -ulong -fd_replay_tile_scratch_footprint( ulong out_cnt ) { - if( FD_UNLIKELY( out_cnt>FD_REPLAY_TILE_OUT_MAX ) ) return 0UL; - ulong l = FD_LAYOUT_INIT; - l = FD_LAYOUT_APPEND( l, fd_fctl_align(), fd_fctl_footprint( out_cnt ) ); /* fctl */ - return FD_LAYOUT_FINI( l, fd_replay_tile_scratch_align() ); -} - -int -fd_replay_tile( fd_cnc_t * cnc, - char const * pcap_path, - ulong pkt_max, - ulong orig, - fd_frag_meta_t * mcache, - uchar * dcache, - ulong out_cnt, - ulong ** out_fseq, - ulong cr_max, - long lazy, - fd_rng_t * rng, - void * scratch ) { - - /* cnc state */ - ulong * cnc_diag; /* ==fd_cnc_app_laddr( cnc ), local address of the replay tile cnc diagnostic region */ - ulong cnc_diag_in_backp; /* is the run loop currently backpressured by one or more of the outs, in [0,1] */ - ulong cnc_diag_backp_cnt; /* Accumulates number of transitions of tile to backpressured between housekeeping events */ - ulong cnc_diag_pcap_done; /* is the pcap file stream replay done */ - ulong cnc_diag_pcap_pub_cnt; /* Accumulates number of pcap packets published between housekeeping events */ - ulong cnc_diag_pcap_pub_sz; /* Accumulates pcap payload bytes published between housekeeping events */ - ulong cnc_diag_pcap_filt_cnt; /* Accumulates number of pcap packets filtered between housekeeping events */ - ulong cnc_diag_pcap_filt_sz; /* Accumulates pcap payload bytes filtered between housekeeping events */ - - /* in pcap stream state */ - FILE * pcap_file; /* handle of pcap file stream */ - fd_pcap_iter_t * pcap_iter; /* iterator for the pcap file stream */ - - /* out frag stream state */ - ulong depth; /* ==fd_mcache_depth( mcache ), depth of the mcache / positive integer power of 2 */ - ulong * sync; /* ==fd_mcache_seq_laddr( mcache ), local addr where replay mcache sync info is published */ - ulong seq; /* seq replay frag sequence number to publish */ - - void * base; /* ==fd_wksp_containing( dcache ), chunk reference address in the tile's local address space */ - ulong chunk0; /* ==fd_dcache_compact_chunk0( base, dcache, pkt_max ) */ - ulong wmark; /* ==fd_dcache_compact_wmark ( base, dcache, _pkt_max ), packets chunks start in [chunk0,wmark] */ - ulong chunk; /* Chunk where next packet will be written, in [chunk0,wmark] */ - - /* flow control state */ - fd_fctl_t * fctl; /* output flow control */ - ulong cr_avail; /* number of flow control credits available to publish downstream, in [0,cr_max] */ - - /* housekeeping state */ - ulong async_min; /* minimum number of ticks between processing a housekeeping event, positive integer power of 2 */ - - do { - - FD_LOG_INFO(( "Booting replay (out-cnt %lu)", out_cnt )); - if( FD_UNLIKELY( out_cnt>FD_REPLAY_TILE_OUT_MAX ) ) { FD_LOG_WARNING(( "out_cnt too large" )); return 1; } - - if( FD_UNLIKELY( !scratch ) ) { - FD_LOG_WARNING(( "NULL scratch" )); - return 1; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)scratch, fd_replay_tile_scratch_align() ) ) ) { - FD_LOG_WARNING(( "misaligned scratch" )); - return 1; - } - - FD_SCRATCH_ALLOC_INIT( l, scratch ); - - /* cnc state init */ - - if( FD_UNLIKELY( !cnc ) ) { FD_LOG_WARNING(( "NULL cnc" )); return 1; } - if( FD_UNLIKELY( fd_cnc_app_sz( cnc )<64UL ) ) { FD_LOG_WARNING(( "cnc app sz must be at least 64" )); return 1; } - if( FD_UNLIKELY( fd_cnc_signal_query( cnc )!=FD_CNC_SIGNAL_BOOT ) ) { FD_LOG_WARNING(( "already booted" )); return 1; } - - cnc_diag = (ulong *)fd_cnc_app_laddr( cnc ); - - /* in_backp==1, backp_cnt==0 indicates waiting for initial credits, - cleared during first housekeeping if credits available */ - cnc_diag_in_backp = 1UL; - cnc_diag_backp_cnt = 0UL; - cnc_diag_pcap_done = 0UL; - cnc_diag_pcap_pub_cnt = 0UL; - cnc_diag_pcap_pub_sz = 0UL; - cnc_diag_pcap_filt_cnt = 0UL; - cnc_diag_pcap_filt_sz = 0UL; - - /* in pcap stream init */ - - if( FD_UNLIKELY( !pkt_max ) ) { FD_LOG_WARNING(( "pkt_max must be positive" )); return 1; } - if( FD_UNLIKELY( !pcap_path ) ) { FD_LOG_WARNING(( "NULL pcap path" )); return 1; } - FD_LOG_INFO(( "Opening pcap %s (pkt_max %lu)", pcap_path, pkt_max )); - pcap_file = fopen( pcap_path, "r" ); - if( FD_UNLIKELY( !pcap_file ) ) { FD_LOG_WARNING(( "fopen failed" )); return 1; } - - pcap_iter = fd_pcap_iter_new( pcap_file ); - if( FD_UNLIKELY( !pcap_iter ) ) { FD_LOG_WARNING(( "fd_pcap_iter_new failed" )); return 1; } - FD_COMPILER_MFENCE(); - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_DONE ] = 0UL; /* Clear before entering running state */ - FD_COMPILER_MFENCE(); - - /* out frag stream init */ - - if( FD_UNLIKELY( !mcache ) ) { FD_LOG_WARNING(( "NULL mcache" )); return 1; } - depth = fd_mcache_depth ( mcache ); - sync = fd_mcache_seq_laddr( mcache ); - - seq = fd_mcache_seq_query( sync ); /* FIXME: ALLOW OPTION FOR MANUAL SPECIFICATION */ - - if( FD_UNLIKELY( !dcache ) ) { FD_LOG_WARNING(( "NULL dcache" )); return 1; } - - base = fd_wksp_containing( dcache ); - if( FD_UNLIKELY( !base ) ) { FD_LOG_WARNING(( "fd_wksp_containing failed" )); return 1; } - - if( FD_UNLIKELY( !fd_dcache_compact_is_safe( base, dcache, pkt_max, depth ) ) ) { - FD_LOG_WARNING(( "--dcache not compatible with wksp base, --pkt-max and --mcache depth" )); - return 1; - } - - chunk0 = fd_dcache_compact_chunk0( base, dcache ); - wmark = fd_dcache_compact_wmark ( base, dcache, pkt_max ); - chunk = FD_VOLATILE_CONST( cnc_diag[ FD_REPLAY_CNC_DIAG_CHUNK_IDX ] ); - if( FD_UNLIKELY( !((chunk0<=chunk) & (chunk<=wmark)) ) ) chunk = chunk0; - FD_LOG_INFO(( "out of bounds cnc chunk index; overriding initial chunk to chunk0" )); - FD_LOG_INFO(( "chunk %lu", chunk )); - - /* out flow control init */ - - if( FD_UNLIKELY( !!out_cnt && !out_fseq ) ) { FD_LOG_WARNING(( "NULL out_fseq" )); return 1; } - - fctl = fd_fctl_join( fd_fctl_new( FD_SCRATCH_ALLOC_APPEND( l, fd_fctl_align(), fd_fctl_footprint( out_cnt ) ), out_cnt ) ); - if( FD_UNLIKELY( !fctl ) ) { FD_LOG_WARNING(( "join failed" )); return 1; } - - for( ulong out_idx=0UL; out_idx=0L ) ) { - - /* Send synchronization info */ - fd_mcache_seq_update( sync, seq ); - - /* Send diagnostic info */ - /* When we drain, we don't do a fully atomic update of the - diagnostics as it is only diagnostic and it will still be - correct the usual case where individual diagnostic counters - aren't used by multiple writers spread over different threads - of execution. */ - fd_cnc_heartbeat( cnc, now ); - FD_COMPILER_MFENCE(); - cnc_diag[ FD_CNC_DIAG_IN_BACKP ] = cnc_diag_in_backp; - cnc_diag[ FD_CNC_DIAG_BACKP_CNT ] += cnc_diag_backp_cnt; - cnc_diag[ FD_REPLAY_CNC_DIAG_CHUNK_IDX ] = chunk; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_DONE ] = cnc_diag_pcap_done; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_PUB_CNT ] += cnc_diag_pcap_pub_cnt; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_PUB_SZ ] += cnc_diag_pcap_pub_sz; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_FILT_CNT ] += cnc_diag_pcap_filt_cnt; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_FILT_SZ ] += cnc_diag_pcap_filt_sz; - FD_COMPILER_MFENCE(); - cnc_diag_backp_cnt = 0UL; - cnc_diag_pcap_pub_cnt = 0UL; - cnc_diag_pcap_pub_sz = 0UL; - cnc_diag_pcap_filt_cnt = 0UL; - cnc_diag_pcap_filt_sz = 0UL; - - /* Receive command-and-control signals */ - ulong s = fd_cnc_signal_query( cnc ); - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_RUN ) ) { - if( FD_LIKELY( s==FD_CNC_SIGNAL_HALT ) ) break; - if( FD_UNLIKELY( s!=FD_REPLAY_CNC_SIGNAL_ACK ) ) { - char buf[ FD_CNC_SIGNAL_CSTR_BUF_MAX ]; - FD_LOG_WARNING(( "Unexpected signal %s (%lu) received; trying to resume", fd_cnc_signal_cstr( s, buf ), s )); - } - fd_cnc_signal( cnc, FD_CNC_SIGNAL_RUN ); - } - - /* Receive flow control credits */ - cr_avail = fd_fctl_tx_cr_update( fctl, cr_avail, seq ); - - /* Reload housekeeping timer */ - then = now + (long)fd_tempo_async_reload( rng, async_min ); - } - - /* Check if we are backpressured. If so, count any transition into - a backpressured regime and spin to wait for flow control credits - to return. We don't do a fully atomic update here as it is only - diagnostic and it will still be correct the usual case where - individual diagnostic counters aren't used by writers in - different threads of execution. We only count the transition - from not backpressured to backpressured. */ - - if( FD_UNLIKELY( !cr_avail ) ) { - cnc_diag_backp_cnt += (ulong)!cnc_diag_in_backp; - cnc_diag_in_backp = 1UL; - FD_SPIN_PAUSE(); - now = fd_tickcount(); - continue; - } - cnc_diag_in_backp = 0UL; - - /* Try to load the next packet directly into the dcache at chunk */ - - if( FD_UNLIKELY( cnc_diag_pcap_done ) ) { - FD_SPIN_PAUSE(); - now = fd_tickcount(); - continue; - } - - long ts; - ulong sz = fd_pcap_iter_next( pcap_iter, fd_chunk_to_laddr( base, chunk ), pkt_max, &ts ); - if( FD_UNLIKELY( !sz ) ) { - cnc_diag_pcap_done = 1UL; - now = fd_tickcount(); - continue; - } - - int should_filter = 0; /* FIXME: filter logic goes here */ - - if( FD_UNLIKELY( should_filter ) ) { - cnc_diag_pcap_filt_cnt++; - cnc_diag_pcap_filt_sz += sz; - now = fd_tickcount(); - continue; - } - - ulong sig = (ulong)ts; /* FIXME: TEMPORARY HACK */ - ulong ctl = fd_frag_meta_ctl( orig, 1 /*som*/, 1 /*eom*/, 0 /*err*/ ); - - now = fd_tickcount(); - ulong tsorig = fd_frag_meta_ts_comp( now ); - ulong tspub = tsorig; - fd_mcache_publish( mcache, depth, seq, sig, chunk, sz, ctl, tsorig, tspub ); - - /* Windup for the next iteration and accumulate diagnostics */ - - chunk = fd_dcache_compact_next( chunk, sz, chunk0, wmark ); - seq = fd_seq_inc( seq, 1UL ); - cr_avail--; - cnc_diag_pcap_pub_cnt++; - cnc_diag_pcap_pub_sz += sz; - } - - do { - - FD_LOG_INFO(( "Halting replay" )); - - FD_LOG_INFO(( "Destroying fctl" )); - fd_fctl_delete( fd_fctl_leave( fctl ) ); - - FD_LOG_INFO(( "Closing pcap" )); - if( FD_UNLIKELY( fclose( fd_pcap_iter_delete( pcap_iter ) ) ) ) - FD_LOG_WARNING(( "fclose failed (%i-%s)", errno, strerror( errno ) )); - - FD_LOG_INFO(( "Halted replay" )); - fd_cnc_signal( cnc, FD_CNC_SIGNAL_BOOT ); - - } while(0); - - return 0; -} - -int -fd_replay_tile_loop( fd_cnc_t * cnc, - char const * pcap_path, - ulong pkt_max, - ulong orig, - fd_frag_meta_t * mcache, - uchar * dcache, - ulong out_cnt, - ulong ** out_fseq, - ulong cr_max, - long lazy, - fd_rng_t * rng, - void * scratch ) { - - /* cnc state */ - ulong * cnc_diag; /* ==fd_cnc_app_laddr( cnc ), local address of the replay tile cnc diagnostic region */ - ulong cnc_diag_in_backp; /* is the run loop currently backpressured by one or more of the outs, in [0,1] */ - ulong cnc_diag_backp_cnt; /* Accumulates number of transitions of tile to backpressured between housekeeping events */ - ulong cnc_diag_pcap_done; /* is the pcap file stream replay done */ - ulong cnc_diag_pcap_pub_cnt; /* Accumulates number of pcap packets published between housekeeping events */ - ulong cnc_diag_pcap_pub_sz; /* Accumulates pcap payload bytes published between housekeeping events */ - ulong cnc_diag_pcap_filt_cnt; /* Accumulates number of pcap packets filtered between housekeeping events */ - ulong cnc_diag_pcap_filt_sz; /* Accumulates pcap payload bytes filtered between housekeeping events */ - - /* in pcap stream state */ - FILE * pcap_file; /* handle of pcap file stream */ - fd_pcap_iter_t * pcap_iter; /* iterator for the pcap file stream */ - - /* out frag stream state */ - ulong depth; /* ==fd_mcache_depth( mcache ), depth of the mcache / positive integer power of 2 */ - ulong * sync; /* ==fd_mcache_seq_laddr( mcache ), local addr where replay mcache sync info is published */ - ulong seq; /* seq replay frag sequence number to publish */ - - void * base; /* ==fd_wksp_containing( dcache ), chunk reference address in the tile's local address space */ - ulong chunk0; /* ==fd_dcache_compact_chunk0( base, dcache, pkt_max ) */ - ulong wmark; /* ==fd_dcache_compact_wmark ( base, dcache, _pkt_max ), packets chunks start in [chunk0,wmark] */ - ulong chunk; /* Chunk where next packet will be written, in [chunk0,wmark] */ - - /* flow control state */ - fd_fctl_t * fctl; /* output flow control */ - ulong cr_avail; /* number of flow control credits available to publish downstream, in [0,cr_max] */ - - /* housekeeping state */ - ulong async_min; /* minimum number of ticks between processing a housekeeping event, positive integer power of 2 */ - - do { - - FD_LOG_INFO(( "Booting replay (out-cnt %lu)", out_cnt )); - if( FD_UNLIKELY( out_cnt>FD_REPLAY_TILE_OUT_MAX ) ) { FD_LOG_WARNING(( "out_cnt too large" )); return 1; } - - if( FD_UNLIKELY( !scratch ) ) { - FD_LOG_WARNING(( "NULL scratch" )); - return 1; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)scratch, fd_replay_tile_scratch_align() ) ) ) { - FD_LOG_WARNING(( "misaligned scratch" )); - return 1; - } - - FD_SCRATCH_ALLOC_INIT( l, scratch ); - - /* cnc state init */ - - if( FD_UNLIKELY( !cnc ) ) { FD_LOG_WARNING(( "NULL cnc" )); return 1; } - if( FD_UNLIKELY( fd_cnc_app_sz( cnc )<64UL ) ) { FD_LOG_WARNING(( "cnc app sz must be at least 64" )); return 1; } - if( FD_UNLIKELY( fd_cnc_signal_query( cnc )!=FD_CNC_SIGNAL_BOOT ) ) { FD_LOG_WARNING(( "already booted" )); return 1; } - - cnc_diag = (ulong *)fd_cnc_app_laddr( cnc ); - - /* in_backp==1, backp_cnt==0 indicates waiting for initial credits, - cleared during first housekeeping if credits available */ - cnc_diag_in_backp = 1UL; - cnc_diag_backp_cnt = 0UL; - cnc_diag_pcap_done = 0UL; - cnc_diag_pcap_pub_cnt = 0UL; - cnc_diag_pcap_pub_sz = 0UL; - cnc_diag_pcap_filt_cnt = 0UL; - cnc_diag_pcap_filt_sz = 0UL; - - /* in pcap stream init */ - - if( FD_UNLIKELY( !pkt_max ) ) { FD_LOG_WARNING(( "pkt_max must be positive" )); return 1; } - if( FD_UNLIKELY( !pcap_path ) ) { FD_LOG_WARNING(( "NULL pcap path" )); return 1; } - FD_LOG_INFO(( "Opening pcap %s (pkt_max %lu)", pcap_path, pkt_max )); - pcap_file = fopen( pcap_path, "r" ); - if( FD_UNLIKELY( !pcap_file ) ) { FD_LOG_WARNING(( "fopen failed" )); return 1; } - - pcap_iter = fd_pcap_iter_new( pcap_file ); - if( FD_UNLIKELY( !pcap_iter ) ) { FD_LOG_WARNING(( "fd_pcap_iter_new failed" )); return 1; } - FD_COMPILER_MFENCE(); - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_DONE ] = 0UL; /* Clear before entering running state */ - FD_COMPILER_MFENCE(); - - /* out frag stream init */ - - if( FD_UNLIKELY( !mcache ) ) { FD_LOG_WARNING(( "NULL mcache" )); return 1; } - depth = fd_mcache_depth ( mcache ); - sync = fd_mcache_seq_laddr( mcache ); - - seq = fd_mcache_seq_query( sync ); /* FIXME: ALLOW OPTION FOR MANUAL SPECIFICATION */ - - if( FD_UNLIKELY( !dcache ) ) { FD_LOG_WARNING(( "NULL dcache" )); return 1; } - - base = fd_wksp_containing( dcache ); - if( FD_UNLIKELY( !base ) ) { FD_LOG_WARNING(( "fd_wksp_containing failed" )); return 1; } - - if( FD_UNLIKELY( !fd_dcache_compact_is_safe( base, dcache, pkt_max, depth ) ) ) { - FD_LOG_WARNING(( "--dcache not compatible with wksp base, --pkt-max and --mcache depth" )); - return 1; - } - - chunk0 = fd_dcache_compact_chunk0( base, dcache ); - wmark = fd_dcache_compact_wmark ( base, dcache, pkt_max ); - chunk = FD_VOLATILE_CONST( cnc_diag[ FD_REPLAY_CNC_DIAG_CHUNK_IDX ] ); - if( FD_UNLIKELY( !((chunk0<=chunk) & (chunk<=wmark)) ) ) chunk = chunk0; - FD_LOG_INFO(( "out of bounds cnc chunk index; overriding initial chunk to chunk0" )); - FD_LOG_INFO(( "chunk %lu", chunk )); - - /* out flow control init */ - - if( FD_UNLIKELY( !!out_cnt && !out_fseq ) ) { FD_LOG_WARNING(( "NULL out_fseq" )); return 1; } - - fctl = fd_fctl_join( fd_fctl_new( FD_SCRATCH_ALLOC_APPEND( l, fd_fctl_align(), fd_fctl_footprint( out_cnt ) ), out_cnt ) ); - if( FD_UNLIKELY( !fctl ) ) { FD_LOG_WARNING(( "join failed" )); return 1; } - - for( ulong out_idx=0UL; out_idx=0L ) ) { - - /* Send synchronization info */ - fd_mcache_seq_update( sync, seq ); - - /* Send diagnostic info */ - /* When we drain, we don't do a fully atomic update of the - diagnostics as it is only diagnostic and it will still be - correct the usual case where individual diagnostic counters - aren't used by multiple writers spread over different threads - of execution. */ - fd_cnc_heartbeat( cnc, now ); - FD_COMPILER_MFENCE(); - cnc_diag[ FD_CNC_DIAG_IN_BACKP ] = cnc_diag_in_backp; - cnc_diag[ FD_CNC_DIAG_BACKP_CNT ] += cnc_diag_backp_cnt; - cnc_diag[ FD_REPLAY_CNC_DIAG_CHUNK_IDX ] = chunk; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_DONE ] = cnc_diag_pcap_done; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_PUB_CNT ] += cnc_diag_pcap_pub_cnt; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_PUB_SZ ] += cnc_diag_pcap_pub_sz; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_FILT_CNT ] += cnc_diag_pcap_filt_cnt; - cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_FILT_SZ ] += cnc_diag_pcap_filt_sz; - FD_COMPILER_MFENCE(); - cnc_diag_backp_cnt = 0UL; - cnc_diag_pcap_pub_cnt = 0UL; - cnc_diag_pcap_pub_sz = 0UL; - cnc_diag_pcap_filt_cnt = 0UL; - cnc_diag_pcap_filt_sz = 0UL; - - /* Receive command-and-control signals */ - ulong s = fd_cnc_signal_query( cnc ); - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_RUN ) ) { - if( FD_LIKELY( s==FD_CNC_SIGNAL_HALT ) ) break; - if( FD_UNLIKELY( s!=FD_REPLAY_CNC_SIGNAL_ACK ) ) { - char buf[ FD_CNC_SIGNAL_CSTR_BUF_MAX ]; - FD_LOG_WARNING(( "Unexpected signal %s (%lu) received; trying to resume", fd_cnc_signal_cstr( s, buf ), s )); - } - fd_cnc_signal( cnc, FD_CNC_SIGNAL_RUN ); - } - - /* Receive flow control credits */ - cr_avail = fd_fctl_tx_cr_update( fctl, cr_avail, seq ); - - /* Reload housekeeping timer */ - then = now + (long)fd_tempo_async_reload( rng, async_min ); - } - - /* Check if we are backpressured. If so, count any transition into - a backpressured regime and spin to wait for flow control credits - to return. We don't do a fully atomic update here as it is only - diagnostic and it will still be correct the usual case where - individual diagnostic counters aren't used by writers in - different threads of execution. We only count the transition - from not backpressured to backpressured. */ - - if( FD_UNLIKELY( !cr_avail ) ) { - cnc_diag_backp_cnt += (ulong)!cnc_diag_in_backp; - cnc_diag_in_backp = 1UL; - FD_SPIN_PAUSE(); - now = fd_tickcount(); - continue; - } - cnc_diag_in_backp = 0UL; - - /* Try to load the next packet directly into the dcache at chunk */ - - if( FD_UNLIKELY( cnc_diag_pcap_done ) ) { - FD_SPIN_PAUSE(); - now = fd_tickcount(); - continue; - } - - long ts; - ulong sz = fd_pcap_iter_next( pcap_iter, fd_chunk_to_laddr( base, chunk ), pkt_max, &ts ); - if( FD_UNLIKELY( !sz ) ) { - // cnc_diag_pcap_done = 1UL; - // -------------------------------------- - // pcap_file = freopen( pcap_path, "r", pcap_file ); /* Note: freopen */ - rewind( fd_pcap_iter_delete( pcap_iter ) ); - pcap_iter = fd_pcap_iter_new( pcap_file ); - if( FD_UNLIKELY( !pcap_iter ) ) { FD_LOG_WARNING(( "fd_pcap_iter_new failed" )); return 1; } - FD_COMPILER_MFENCE(); - // -------------------------------------- - now = fd_tickcount(); - continue; - } - - int should_filter = 0; /* FIXME: filter logic goes here */ - - if( FD_UNLIKELY( should_filter ) ) { - cnc_diag_pcap_filt_cnt++; - cnc_diag_pcap_filt_sz += sz; - now = fd_tickcount(); - continue; - } - - ulong sig = (ulong)ts; /* FIXME: TEMPORARY HACK */ - ulong ctl = fd_frag_meta_ctl( orig, 1 /*som*/, 1 /*eom*/, 0 /*err*/ ); - - now = fd_tickcount(); - ulong tsorig = fd_frag_meta_ts_comp( now ); - ulong tspub = tsorig; - fd_mcache_publish( mcache, depth, seq, sig, chunk, sz, ctl, tsorig, tspub ); - - // FD_LOG_WARNING(("replay-loop published seq %016lx", seq)); - - /* Windup for the next iteration and accumulate diagnostics */ - - chunk = fd_dcache_compact_next( chunk, sz, chunk0, wmark ); - seq = fd_seq_inc( seq, 1UL ); - cr_avail--; - cnc_diag_pcap_pub_cnt++; - cnc_diag_pcap_pub_sz += sz; - } - - do { - - FD_LOG_INFO(( "Halting replay" )); - - FD_LOG_INFO(( "Destroying fctl" )); - fd_fctl_delete( fd_fctl_leave( fctl ) ); - - FD_LOG_INFO(( "Closing pcap" )); - if( FD_UNLIKELY( fclose( fd_pcap_iter_delete( pcap_iter ) ) ) ) - FD_LOG_WARNING(( "fclose failed (%i-%s)", errno, strerror( errno ) )); - - FD_LOG_INFO(( "Halted replay" )); - fd_cnc_signal( cnc, FD_CNC_SIGNAL_BOOT ); - - } while(0); - - return 0; -} - -#endif diff --git a/src/wiredancer/test/fd_replay_loop.h b/src/wiredancer/test/fd_replay_loop.h deleted file mode 100644 index 610e3bbe74a..00000000000 --- a/src/wiredancer/test/fd_replay_loop.h +++ /dev/null @@ -1,177 +0,0 @@ -#ifndef HEADER_fd_src_wiredancer_test_fd_replay_loop_h -#define HEADER_fd_src_wiredancer_test_fd_replay_loop_h - -/* fd_replay provides services to replay data from a pcap file into a - tango frag stream. */ - -#include "../../disco/fd_disco_base.h" - -#if FD_HAS_HOSTED && FD_HAS_X86 - -/* Beyond the standard FD_CNC_SIGNAL_HALT, FD_REPLAY_CNC_SIGNAL_ACK can - be raised by a cnc thread with an open command session while the - replay is in the RUN state. The replay will transition from ACK->RUN - the next time it processes cnc signals to indicate it is running - normally. If a signal other than ACK, HALT, or RUN is raised, it - will be logged as unexpected and transitioned by back to RUN. */ - -#define FD_REPLAY_CNC_SIGNAL_ACK (4UL) - -/* A fd_replay_tile will use the fseq and cnc application regions - to accumulate flow control diagnostics in the standard ways. It - additionally will accumulate to the cnc application region the - following tile specific counters: - - CHUNK_IDX is the chunk idx where reply tile should start publishing payloads on boot (ignored if not valid on boot) - PCAP_DONE is cleared before the tile starts processing the pcap and is set when the pcap processing is done - PCAP_PUB_CNT is the number of pcap packets published by the replay - PCAP_PUB_SZ is the number of pcap packet payload bytes published by the replay - PCAP_FILT_CNT is the number of pcap packets filtered by the replay - PCAP_FILT_SZ is the number of pcap packet payload bytes filtered by the replay - - As such, the cnc app region must be at least 64B in size. - - Except for IN_BACKP, none of the diagnostics are cleared at - tile startup (as such that they can be accumulated over multiple - runs). Clearing is up to monitoring scripts. */ - -#define FD_REPLAY_CNC_DIAG_CHUNK_IDX (2UL) /* On 1st cache line of app region, updated by producer, frequently */ -#define FD_REPLAY_CNC_DIAG_PCAP_DONE (3UL) /* ", rarely */ -#define FD_REPLAY_CNC_DIAG_PCAP_PUB_CNT (4UL) /* ", frequently */ -#define FD_REPLAY_CNC_DIAG_PCAP_PUB_SZ (5UL) /* ", frequently */ -#define FD_REPLAY_CNC_DIAG_PCAP_FILT_CNT (6UL) /* ", frequently */ -#define FD_REPLAY_CNC_DIAG_PCAP_FILT_SZ (7UL) /* ", frequently */ - -/* FD_REPLAY_TILE_OUT_MAX are the maximum number of outputs a replay - tile can have. These limits are more or less arbitrary from a - functional correctness POV. They mostly exist to set some practical - upper bounds for things like scratch footprint. */ - -#define FD_REPLAY_TILE_OUT_MAX FD_FRAG_META_ORIG_MAX - -/* FD_REPLAY_TILE_SCRATCH_{ALIGN,FOOTPRINT} specify the alignment and - footprint needed for a replay tile scratch region that can support - out_cnt outputs. ALIGN is an integer power of 2 of at least double - cache line to mitigate various kinds of false sharing. FOOTPRINT - will be an integer multiple of ALIGN. out_cnt is assumed to be valid - (i.e. at most FD_REPLAY_TILE_OUT_MAX). These are provided to - facilitate compile time declarations. */ - -#define FD_REPLAY_TILE_SCRATCH_ALIGN (128UL) -#define FD_REPLAY_TILE_SCRATCH_FOOTPRINT( out_cnt ) \ - FD_LAYOUT_FINI( FD_LAYOUT_APPEND( FD_LAYOUT_INIT, \ - FD_FCTL_ALIGN, FD_FCTL_FOOTPRINT( (out_cnt) ) ), \ - FD_REPLAY_TILE_SCRATCH_ALIGN ) - -FD_PROTOTYPES_BEGIN - -/* fd_replay_tile replays a packets in a pcap file as a tango fragment - stream from origin orig into the given mcache and dcache. The tile - can send to out_cnt reliable consumers and an arbitrary number of - unreliable consumers. (While reliable consumers are simple to reason - about, they have especially high demands on their implementation as a - single slow reliable consumer can backpressure the replay and _all_ - other consumers using the replay.) - - When this is called, the cnc should be in the BOOT state. Returns 0 - on a successful run of the replay tile. That is, the tile booted - successfully (transitioning the cnc from BOOT->RUN), ran (handling - any application specific cnc signals while running), and (after - receiving a HALT signal) halted successfully (transitioning the cnc - from HALT->BOOT before return). Returns a non-zero error code if the - tile fails to boot up (logs details ... the cnc will not be - transitioned from its original state and thus is likely bootable - again if its original state was BOOT). For maximally robust - operation in the current implementation, all reliable consumers - should be halted and/or caught up before this tile is halted. - - There are no theoretical restrictions on the mcache depth. - Practically, it is recommend it be as large as possible, especially - for bursty streams and/or a large number of reliable consumers. This - implementation indexes chunks relative to the workspace used by the - mcache to facilitate easy muxing. The dcache size should be adequate - for compact writing. - - cr_max is the maximum number of flow control credits the replay tile - is allowed for publishing frags. It represents the maximum number of - frags a reliable out can lag behind the output stream. In the - general case, the optimal value is usually - min(mcache.depth,out[*].lag_max). If cr_max is zero, mcache.depth - will be used as a default for cr_max. This is equivalent to - assuming, as is typically the case, outs are allowed to lag the - replay by up mcache.depth frags. - - lazy is the ballpark interval in ns for how often to receive credits - from consumers. Too small a lazy will drown the system in cache - coherence traffic. Too large a lazy will degrade system throughput - because of producers stalled, waiting for credits. lazy should be - roughly proportional to cr_max and the constant of proportionality - should be less than the smaller of how fast a producer can generate - frags / how fast a consumer can process frags typically. <=0 - indicates to pick a conservative default. - - scratch points to tile scratch memory. fd_replay_tile_scratch_align - and fd_replay_tile_scratch_footprint return the required alignment - and footprint needed for this region. This memory region is - exclusively owned by the replay tile while the tile is running and is - ideally near the core running the replay tile. - fd_replay_tile_scratch_align will return the same value as - FD_REPLAY_TILE_SCRATCH_ALIGN. If out_cnt is not valid, - fd_replay_tile_scratch_footprint silently returns 0 so callers can - diagnose configuration issues. Otherwise, - fd_replay_tile_scratch_footprint will return the same value as - FD_REPLAY_TILE_SCRATCH_FOOTPRINT. - - The lifetime of the cnc, mcache, dcache, out_fseq[*], rng and scratch - used by this tile should be a superset of this tile's lifetime. - While this tile is running, no other tile should use cnc for its - command and control, publish into mcache or dcache, use the rng for - anything (and the rng should be seeded distinctly from all other rngs - in the system), or use scratch for anything. This tile uses the - fseqs passed to it in the usual producer ways (e.g. discovering the - location of reliable consumers in the mcache's sequence space and - updating producer oriented diagnostics). The out_fseq array and - pcap_path cstr will not be used the after the tile has successfully - booted (transitioned the cnc from BOOT to RUN) or returned (e.g. - failed to boot), whichever comes first. */ - -FD_FN_CONST ulong -fd_replay_tile_scratch_align( void ); - -FD_FN_CONST ulong -fd_replay_tile_scratch_footprint( ulong out_cnt ); - -int -fd_replay_tile( fd_cnc_t * cnc, /* Local join to the replay's command-and-control */ - char const * pcap_path, /* Points to first byte of cstr with the path to the pcap to use */ - ulong pkt_max, /* Upper bound of a size of packet in the pcap */ - ulong orig, /* Origin for this pcap fragment stream, in [0,FD_FRAG_META_ORIG_MAX) */ - fd_frag_meta_t * mcache, /* Local join to the replay's frag stream output mcache */ - uchar * dcache, /* Local join to the replay's frag stream output dcache */ - ulong out_cnt, /* Number of reliable consumers, reliable consumers are indexed [0,out_cnt) */ - ulong ** out_fseq, /* out_fseq[out_idx] is the local join to reliable consumer out_idx's fseq */ - ulong cr_max, /* Maximum number of flow control credits, 0 means use a reasonable default */ - long lazy, /* Lazyiness, <=0 means use a reasonable default */ - fd_rng_t * rng, /* Local join to the rng this replay should use */ - void * scratch ); /* Tile scratch memory */ - -int -fd_replay_tile_loop( fd_cnc_t * cnc, /* Local join to the replay's command-and-control */ - char const * pcap_path, /* Points to first byte of cstr with the path to the pcap to use */ - ulong pkt_max, /* Upper bound of a size of packet in the pcap */ - ulong orig, /* Origin for this pcap fragment stream, in [0,FD_FRAG_META_ORIG_MAX) */ - fd_frag_meta_t * mcache, /* Local join to the replay's frag stream output mcache */ - uchar * dcache, /* Local join to the replay's frag stream output dcache */ - ulong out_cnt, /* Number of reliable consumers, reliable consumers are indexed [0,out_cnt) */ - ulong ** out_fseq, /* out_fseq[out_idx] is the local join to reliable consumer out_idx's fseq */ - ulong cr_max, /* Maximum number of flow control credits, 0 means use a reasonable default */ - long lazy, /* Lazyiness, <=0 means use a reasonable default */ - fd_rng_t * rng, /* Local join to the rng this replay should use */ - void * scratch ); /* Tile scratch memory */ - -FD_PROTOTYPES_END - -#endif - -#endif /* HEADER_fd_src_wiredancer_test_fd_replay_loop_h */ - diff --git a/src/wiredancer/test/test_wiredancer_demo.c b/src/wiredancer/test/test_wiredancer_demo.c deleted file mode 100644 index 886717ecfa3..00000000000 --- a/src/wiredancer/test/test_wiredancer_demo.c +++ /dev/null @@ -1,1318 +0,0 @@ -/* Wiredancer Unit Test. - This test is intended to be run in AWS-F1. Its main goal is to check Wiredancer - against x86. A "replay" tile feeds packets to a "parser" tile. The latter parses - the transaction (it assumes 1 txn per packet for this test), and sends sigverify - workload downstream (as many requests as needed per txn). This can be fed into - an x86-based "verify" tile and/or Wiredancer. Due to the (substantial) difference - in throughput, the x86-based "verify" tile will constantly try to catch up by - skipping sequence numbers (i.e. signature verification requests). However, the - "checker" tile will check every output from x86 against the corresponding output - from Wiredancer (by matching the sequence numbers). - - This unit test can also be run with either Wiredancer or x86 separately (in which - case the "checker" will just consume either output). - - General notes: - It is assumed that all FPGAs have already been programmed with the Wiredancer - image. - - The test replays a pcap in a loop. The packets in that pcap are expected to be - network packets ( i.e. they must contain eth/ip4/udp headers + txn ). - - The FPGA slots in AWS-F1 need to be passed using 1-hot encoding: - * 1x fpga in slot 0: --wd-slots 1 - * 1x fpga in slot 1: --wd-slots 2 - * 2x fpga(s) in slots 1 and 2: --wd-slots 3 - - The duration of the test is set in ns. If "--duration 0" is passed, then the - test will continue indefinitely (unit Ctrl+c). - - Transactions can be randomly corrupted (approx 50% of the time) by setting - "--rand-txn-corrupt 1". - - Sample tests: - # Check Wiredancer vs x86 ( 1 fpga in slot 0, with random txn corruption enabled ) # - sudo ${BUILD_FOLDER}/linux/gcc/x86_64/unit-test/test_wiredancer_demo --replay-pcap ${PCAP_PATH} --tile-cpus 1,2,3,4,5 --duration 0 --v-x86-enabled 1 --v--wd-enabled 1 --wd-slots 1 --rand-txn-corrupt 1 - - # Check Wiredancer vs x86 ( 2 fpgas in slot 0 and 1, with random txn corruption enabled ) # - sudo ${BUILD_FOLDER}/linux/gcc/x86_64/unit-test/test_wiredancer_demo --replay-pcap ${PCAP_PATH} --tile-cpus 1,2,3,4,5 --duration 0 --v-x86-enabled 1 --v--wd-enabled 1 --wd-slots 3 --rand-txn-corrupt 1 - - # Wiredancer only ( 2 fpgas in slot 0 and 1 ) # - sudo ${BUILD_FOLDER}/linux/gcc/x86_64/unit-test/test_wiredancer_demo --replay-pcap ${PCAP_PATH} --tile-cpus 1,2,3,4,5 --duration 0 --v-x86-enabled 0 --v--wd-enabled 1 --wd-slots 3 --rand-txn-corrupt 0 - - # x86 only # - sudo ${BUILD_FOLDER}/linux/gcc/x86_64/unit-test/test_wiredancer_demo --replay-pcap ${PCAP_PATH} --tile-cpus 1,2,3,4,5 --duration 0 --v-x86-enabled 1 --v--wd-enabled 0 --wd-slots 0 --rand-txn-corrupt 0 - - * x86 only (5 seconds) # - sudo ${BUILD_FOLDER}/linux/gcc/x86_64/unit-test/test_wiredancer_demo --replay-pcap ${PCAP_PATH} --tile-cpus 1,2,3,4,5 --duration 5000000000 --v-x86-enabled 1 --v--wd-enabled 0 --wd-slots 0 --rand-txn-corrupt 0 -*/ - - -#include "fd_replay_loop.h" -#include "../../util/net/fd_eth.h" -#include "../../util/net/fd_ip4.h" -#include "../../util/net/fd_udp.h" -#include "../../ballet/txn/fd_txn.h" -#include "../../ballet/ed25519/fd_ed25519_private.h" -#include "../../ballet/ed25519/fd_ed25519.h" -#include "../../ballet/sha512/fd_sha512.h" -#include "../c/wd_f1.h" -#include "wd_f1_mon.h" -#include - -/* Wiredancer unit test */ - -#if FD_HAS_HOSTED && FD_HAS_X86 && FD_HAS_WIREDANCER - -#define DEMO_REPLAY_FSEQ_CNT_MAX 1 /* max number of replay consumers*/ - -#define DEMO_PARSER_CNC_PKT_CNT_IDX 0 -#define DEMO_PARSER_CNC_TXN_CNT_IDX 1 -#define DEMO_PARSER_CNC_SIG_CNT_IDX 2 -#define DEMO_PARSER_CNC_OUT_SEQ_IDX 3 -#define DEMO_PARSER_CNC_OUT_SIG_IDX 4 - -#define DEMO_V_X86_CNC_TXN_CNT_IDX 0 -#define DEMO_V_X86_CNC_SIG_CNT_IDX 1 - -#define DEMO_VCHECK_CNC_SIG__X86_CNT_IDX 0 -#define DEMO_VCHECK_CNC_SIG_FPGA_CNT_IDX 1 -#define DEMO_VCHECK_CNC_VALIDATE_CNT_IDX 2 -#define DEMO_VCHECK_CNC_SIG_PASS_CNT_IDX 3 -#define DEMO_VCHECK_CNC_SIG_FAIL_CNT_IDX 4 - -#define DEMO_TEST_VERSION_SIGVERIFY 0 -#define DEMO_TEST_VERSION_SHA512MODQ 1 - -FD_STATIC_ASSERT( FD_REPLAY_CNC_SIGNAL_ACK==4UL, unit_test ); - -FD_STATIC_ASSERT( FD_REPLAY_CNC_DIAG_CHUNK_IDX ==2UL, unit_test ); -FD_STATIC_ASSERT( FD_REPLAY_CNC_DIAG_PCAP_DONE ==3UL, unit_test ); -FD_STATIC_ASSERT( FD_REPLAY_CNC_DIAG_PCAP_PUB_CNT ==4UL, unit_test ); -FD_STATIC_ASSERT( FD_REPLAY_CNC_DIAG_PCAP_PUB_SZ ==5UL, unit_test ); -FD_STATIC_ASSERT( FD_REPLAY_CNC_DIAG_PCAP_FILT_CNT==6UL, unit_test ); -FD_STATIC_ASSERT( FD_REPLAY_CNC_DIAG_PCAP_FILT_SZ ==7UL, unit_test ); - -FD_STATIC_ASSERT( FD_REPLAY_TILE_OUT_MAX==8192UL, unit_test ); - -FD_STATIC_ASSERT( FD_REPLAY_TILE_SCRATCH_ALIGN==128UL, unit_test ); - -union parsed_txn_compressed_meta { - ulong all; - struct { - ushort msg_sz; /* msessage size */ - ushort msg_off; /* message offset from the start of the pkt */ - ushort signature_off; /* signature offset from the start of the pkt */ - ushort public_key_off; /* public_key offset from the start of the pkt */ - } value; -}; -typedef union parsed_txn_compressed_meta parsed_txn_compressed_meta_t; -FD_STATIC_ASSERT( sizeof(parsed_txn_compressed_meta_t) == sizeof(ulong), unit_test ); - - -struct test_cfg { - fd_wksp_t * wksp; - - fd_cnc_t * replay_cnc; - char const * replay_pcap; - ulong replay_mtu; - ulong replay_orig; - fd_frag_meta_t * replay_mcache; - uchar * replay_dcache; - ulong replay_cr_max; - long replay_lazy; - uint replay_seed; - ulong ** replay_fseq; - ulong replay_fseq_cnt; - - fd_cnc_t * parser_cnc; - fd_frag_meta_t * parser_mcache; - int parser_lazy; - uint parser_seed; - int parser_enabled; - ulong * parser_replay_fseq; - int parser_rand_txn_corrupt; - - fd_cnc_t * v_x86_cnc; - fd_frag_meta_t * v_x86_mcache; - int v_x86_lazy; - uint v_x86_seed; - int v_x86_enabled; - - int v__wd_enabled; - fd_frag_meta_t * v__wd_mcache; - - fd_cnc_t * vcheck_cnc; - uint vcheck_seed; - int vcheck_lazy; - - int test_version; - - /* Wiredancer */ - ulong wd_slots; - int wd_split; -}; -typedef struct test_cfg test_cfg_t; - - - - - -// SSSSSSSSSSSSSSS HHHHHHHHH HHHHHHHHH AAA 555555555555555555 1111111 222222222222222 -// SS:::::::::::::::SH:::::::H H:::::::H A:::A 5::::::::::::::::5 1::::::1 2:::::::::::::::22 -// S:::::SSSSSS::::::SH:::::::H H:::::::H A:::::A 5::::::::::::::::5 1:::::::1 2::::::222222:::::2 -// S:::::S SSSSSSSHH::::::H H::::::HH A:::::::A 5:::::555555555555 111:::::1 2222222 2:::::2 -// S:::::S H:::::H H:::::H A:::::::::A 5:::::5 1::::1 2:::::2 -// S:::::S H:::::H H:::::H A:::::A:::::A 5:::::5 1::::1 2:::::2 -// S::::SSSS H::::::HHHHH::::::H A:::::A A:::::A 5:::::5555555555 1::::1 2222::::2 -// SS::::::SSSSS H:::::::::::::::::H A:::::A A:::::A 5:::::::::::::::5 1::::l 22222::::::22 -// SSS::::::::SS H:::::::::::::::::H A:::::A A:::::A 555555555555:::::5 1::::l 22::::::::222 -// SSSSSS::::S H::::::HHHHH::::::H A:::::AAAAAAAAA:::::A 5:::::5 1::::l 2:::::22222 -// S:::::S H:::::H H:::::H A:::::::::::::::::::::A 5:::::5 1::::l 2:::::2 -// S:::::S H:::::H H:::::H A:::::AAAAAAAAAAAAA:::::A 5555555 5:::::5 1::::l 2:::::2 -// SSSSSSS S:::::SHH::::::H H::::::HH A:::::A A:::::A5::::::55555::::::5111::::::1112:::::2 222222 -// S::::::SSSSSS:::::SH:::::::H H:::::::H A:::::A A:::::A55:::::::::::::55 1::::::::::12::::::2222222:::::2 -// S:::::::::::::::SS H:::::::H H:::::::H A:::::A A:::::A 55:::::::::55 1::::::::::12::::::::::::::::::2 -// SSSSSSSSSSSSSSS HHHHHHHHH HHHHHHHHHAAAAAAA AAAAAAA 555555555 11111111111122222222222222222222 - - -int -sha512_modq_lsB( void const * msg, - ulong sz, - void const * sig, - void const * public_key, - fd_sha512_t * sha ) { - uchar const * r = (uchar const *)sig; - - uchar h[64]; - fd_sha512_fini( fd_sha512_append( fd_sha512_append( fd_sha512_append( fd_sha512_init( sha ), - r, 32UL ), public_key, 32UL ), msg, sz ), h ); - fd_ed25519_sc_reduce( h, h ); - return (int)h[0]; -} - - - - - - - - - - -// RRRRRRRRRRRRRRRRR EEEEEEEEEEEEEEEEEEEEEEPPPPPPPPPPPPPPPPP LLLLLLLLLLL AAA YYYYYYY YYYYYYY -// R::::::::::::::::R E::::::::::::::::::::EP::::::::::::::::P L:::::::::L A:::A Y:::::Y Y:::::Y -// R::::::RRRRRR:::::R E::::::::::::::::::::EP::::::PPPPPP:::::P L:::::::::L A:::::A Y:::::Y Y:::::Y -// RR:::::R R:::::REE::::::EEEEEEEEE::::EPP:::::P P:::::PLL:::::::LL A:::::::A Y::::::Y Y::::::Y -// R::::R R:::::R E:::::E EEEEEE P::::P P:::::P L:::::L A:::::::::A YYY:::::Y Y:::::YYY -// R::::R R:::::R E:::::E P::::P P:::::P L:::::L A:::::A:::::A Y:::::Y Y:::::Y -// R::::RRRRRR:::::R E::::::EEEEEEEEEE P::::PPPPPP:::::P L:::::L A:::::A A:::::A Y:::::Y:::::Y -// R:::::::::::::RR E:::::::::::::::E P:::::::::::::PP L:::::L A:::::A A:::::A Y:::::::::Y -// R::::RRRRRR:::::R E:::::::::::::::E P::::PPPPPPPPP L:::::L A:::::A A:::::A Y:::::::Y -// R::::R R:::::R E::::::EEEEEEEEEE P::::P L:::::L A:::::AAAAAAAAA:::::A Y:::::Y -// R::::R R:::::R E:::::E P::::P L:::::L A:::::::::::::::::::::A Y:::::Y -// R::::R R:::::R E:::::E EEEEEE P::::P L:::::L LLLLLL A:::::AAAAAAAAAAAAA:::::A Y:::::Y -// RR:::::R R:::::REE::::::EEEEEEEE:::::EPP::::::PP LL:::::::LLLLLLLLL:::::L A:::::A A:::::A Y:::::Y -// R::::::R R:::::RE::::::::::::::::::::EP::::::::P L::::::::::::::::::::::L A:::::A A:::::A YYYY:::::YYYY -// R::::::R R:::::RE::::::::::::::::::::EP::::::::P L::::::::::::::::::::::L A:::::A A:::::A Y:::::::::::Y -// RRRRRRRR RRRRRRREEEEEEEEEEEEEEEEEEEEEEPPPPPPPPPP LLLLLLLLLLLLLLLLLLLLLLLLAAAAAAA AAAAAAAYYYYYYYYYYYYY - - -/* REPLAY tile ************************************************************/ - -static int -replay_tile_main( int argc, - char ** argv ) { - (void)argc; - test_cfg_t * cfg = (test_cfg_t *)argv; - - FD_LOG_NOTICE(( "active: replay_tile_main" )); - - fd_rng_t _rng[1]; - fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, cfg->replay_seed, 0UL ) ); - - uchar scratch[ FD_REPLAY_TILE_SCRATCH_FOOTPRINT( 1UL ) ] __attribute__((aligned( FD_REPLAY_TILE_SCRATCH_ALIGN ))); - - FD_TEST( !fd_replay_tile_loop( cfg->replay_cnc, cfg->replay_pcap, cfg->replay_mtu, cfg->replay_orig, cfg->replay_mcache, cfg->replay_dcache, - cfg->replay_fseq_cnt, cfg->replay_fseq, cfg->replay_cr_max, cfg->replay_lazy, rng, scratch ) ); - - fd_rng_delete( fd_rng_leave( rng ) ); - return 0; -} - - - -// PPPPPPPPPPPPPPPPP AAA RRRRRRRRRRRRRRRRR SSSSSSSSSSSSSSS EEEEEEEEEEEEEEEEEEEEEERRRRRRRRRRRRRRRRR -// P::::::::::::::::P A:::A R::::::::::::::::R SS:::::::::::::::SE::::::::::::::::::::ER::::::::::::::::R -// P::::::PPPPPP:::::P A:::::A R::::::RRRRRR:::::R S:::::SSSSSS::::::SE::::::::::::::::::::ER::::::RRRRRR:::::R -// PP:::::P P:::::P A:::::::A RR:::::R R:::::RS:::::S SSSSSSSEE::::::EEEEEEEEE::::ERR:::::R R:::::R -// P::::P P:::::P A:::::::::A R::::R R:::::RS:::::S E:::::E EEEEEE R::::R R:::::R -// P::::P P:::::PA:::::A:::::A R::::R R:::::RS:::::S E:::::E R::::R R:::::R -// P::::PPPPPP:::::PA:::::A A:::::A R::::RRRRRR:::::R S::::SSSS E::::::EEEEEEEEEE R::::RRRRRR:::::R -// P:::::::::::::PPA:::::A A:::::A R:::::::::::::RR SS::::::SSSSS E:::::::::::::::E R:::::::::::::RR -// P::::PPPPPPPPP A:::::A A:::::A R::::RRRRRR:::::R SSS::::::::SS E:::::::::::::::E R::::RRRRRR:::::R -// P::::P A:::::AAAAAAAAA:::::A R::::R R:::::R SSSSSS::::S E::::::EEEEEEEEEE R::::R R:::::R -// P::::P A:::::::::::::::::::::A R::::R R:::::R S:::::S E:::::E R::::R R:::::R -// P::::P A:::::AAAAAAAAAAAAA:::::A R::::R R:::::R S:::::S E:::::E EEEEEE R::::R R:::::R -// PP::::::PP A:::::A A:::::A RR:::::R R:::::RSSSSSSS S:::::SEE::::::EEEEEEEE:::::ERR:::::R R:::::R -// P::::::::P A:::::A A:::::A R::::::R R:::::RS::::::SSSSSS:::::SE::::::::::::::::::::ER::::::R R:::::R -// P::::::::P A:::::A A:::::A R::::::R R:::::RS:::::::::::::::SS E::::::::::::::::::::ER::::::R R:::::R -// PPPPPPPPPPAAAAAAA AAAAAAARRRRRRRR RRRRRRR SSSSSSSSSSSSSSS EEEEEEEEEEEEEEEEEEEEEERRRRRRRR RRRRRRR - -/* PARSER tile ************************************************************/ - -static int -parser_tile_main( int argc, - char ** argv ) { - ulong parser_idx = (ulong)argc; - test_cfg_t * cfg = (test_cfg_t *)argv; - fd_wksp_t * wksp = cfg->wksp; - (void)parser_idx; - - FD_LOG_NOTICE(( "active: parser_tile_main" )); - - /* Hook up to parser cnc */ - fd_cnc_t * cnc = cfg->parser_cnc; - - /* Command and control */ - ulong * cnc_diag = (ulong *)fd_cnc_app_laddr( cnc ); - ulong cnc_diag_pkt_cnt = 0ULL; - ulong cnc_diag_txn_cnt = 0ULL; - ulong cnc_diag_sig_cnt = 0ULL; - - /* Hook up to replay mcache */ - fd_frag_meta_t const * mcache = cfg->replay_mcache; - ulong depth = fd_mcache_depth( mcache ); - ulong const * sync = fd_mcache_seq_laddr_const( mcache ); - ulong seq = fd_mcache_seq_query( sync ); - - /* Hook up to replay flow control */ - ulong * fseq = cfg->parser_replay_fseq; - - /* Hook up to the output mcache */ - fd_frag_meta_t * out_mcache = cfg->parser_mcache; - ulong out_depth = fd_mcache_depth( out_mcache ); - ulong * out_sync = fd_mcache_seq_laddr( out_mcache ); - ulong out_seq = fd_mcache_seq_query( out_sync ); - ulong out_sig = 0UL; /* this works as a counter (in this demo) */ - cnc_diag[ DEMO_PARSER_CNC_OUT_SEQ_IDX ] = out_seq; - cnc_diag[ DEMO_PARSER_CNC_OUT_SIG_IDX ] = out_sig; - - /* Enabled consumers */ - int v_x86_enabled = cfg->v_x86_enabled; - int v__wd_enabled = cfg->v__wd_enabled; - - /* Hook up wd mcache */ - fd_frag_meta_t * v__wd_mcache = ( !!v__wd_enabled )? cfg->v__wd_mcache : NULL; - ulong v__wd_depth = ( !!v__wd_enabled )? fd_mcache_depth( v__wd_mcache ) : 0UL; - ulong * v__wd_sync = ( !!v__wd_enabled )? fd_mcache_seq_laddr( v__wd_mcache ) : NULL; - ulong v__wd_seq = ( !!v__wd_enabled )? fd_mcache_seq_query( v__wd_sync ) : 0UL; - - /* Hook up to the random number generator */ - fd_rng_t _rng[1]; - fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, cfg->parser_seed, 0UL ) ); - - /* Random txn corruption */ - int rand_txn_corrupt = cfg->parser_rand_txn_corrupt; - - /* Configure housekeeping */ - ulong async_min = 1UL << cfg->parser_lazy; - ulong async_rem = 1UL; /* Do housekeeping on first iteration */ - - /* Txn parsing counters */ - fd_txn_parse_counters_t counters_opt[1]; - - /* Wiredancer init */ - wd_wksp_t wd; - int wd_split = cfg->wd_split; - uint64_t wd_slots = cfg->wd_slots; - FD_TEST( !wd_init_pci( &wd, wd_slots ) ); - wd_ed25519_verify_init_req( &wd, 1, v__wd_depth, v__wd_mcache ); - FD_TEST( !!wd_split ); - - /* Main loop */ - fd_cnc_signal( cnc, FD_CNC_SIGNAL_RUN ); - for(;;) { - - /* Wait for frag seq while doing housekeeping in the background */ - - fd_frag_meta_t const * mline; - ulong seq_found; - long diff; - - ulong sig; - ulong chunk; - ulong sz; - ulong ctl; - ulong tsorig; - ulong tspub; - FD_MCACHE_WAIT_REG( sig, chunk, sz, ctl, tsorig, tspub, mline, seq_found, diff, async_rem, mcache, depth, seq ); - - /* Housekeeping */ - if( FD_UNLIKELY( !async_rem ) ) { - /* Send flow control credits */ - fd_fctl_rx_cr_return( fseq, seq ); - /* Send diagnostic info */ - fd_cnc_heartbeat( cnc, fd_tickcount() ); - /* Receive command-and-control signals */ - ulong s = fd_cnc_signal_query( cnc ); - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_RUN ) ) { - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_HALT ) ) FD_LOG_ERR(( "Unexpected signal" )); - break; - } - /* Reload housekeeping timer */ - async_rem = fd_tempo_async_reload( rng, async_min ); - continue; - } - - if( FD_UNLIKELY( diff ) ) FD_LOG_ERR(( "Overrun while polling" )); - - /* Process the received fragment */ - ulong p0 = (ulong) fd_chunk_to_laddr_const( wksp, chunk ); - uchar const * p = (uchar const *)p0; - (void)ctl; (void)sz; (void)sig; (void)tsorig; (void)tspub; - - /* Process eth header */ - fd_eth_hdr_t * p_eth = (fd_eth_hdr_t*)p; p = p + sizeof(fd_eth_hdr_t); - ushort net_type = fd_ushort_bswap( p_eth->net_type ); - if( FD_UNLIKELY( net_type == FD_ETH_HDR_TYPE_VLAN ) ) { - do { - fd_vlan_tag_t * p_vlan = (fd_vlan_tag_t*)p; p = p + sizeof(fd_vlan_tag_t); - net_type = fd_ushort_bswap( p_vlan->net_type ); - } while( net_type == FD_ETH_HDR_TYPE_VLAN ); - } - FD_TEST( net_type == FD_ETH_HDR_TYPE_IP ); - - /* Process ipv4 header */ - fd_ip4_hdr_t * p_ip4 = (fd_ip4_hdr_t*)p; p = p + p_ip4->ihl*sizeof(uint); - FD_TEST( p_ip4->protocol == FD_IP4_HDR_PROTOCOL_UDP ); - /* FIXME remove checksum verification for performance? */ - /* FIXME assume ihl==5 and use fd_ip4_hdr_check_fast? */ - FD_TEST( !fd_ip4_hdr_check( p_ip4 ) ); - - /* Process udp header */ - fd_udp_hdr_t * p_udp = (fd_udp_hdr_t*)p; p = p + sizeof(fd_udp_hdr_t); - uchar const * dgram = p; - ulong dgram_sz = fd_ushort_bswap(p_udp->net_len) - sizeof(fd_udp_hdr_t); - /* FIXME remove checksum verification for performance? */ - FD_TEST( !fd_ip4_udp_check(p_ip4->saddr, p_ip4->daddr, p_udp, dgram) ); - - /* Process txn sigverify */ - unsigned char parsed[FD_TXN_MAX_SZ]; - ulong txn_sz = fd_txn_parse( dgram, dgram_sz, parsed, counters_opt ); - - /* Count processed packets */ - cnc_diag_pkt_cnt += 1ULL; - - if( FD_LIKELY( txn_sz ) ) { - - /* count processed transactions */ - cnc_diag_txn_cnt += 1ULL; - - fd_txn_t * txn = (fd_txn_t*)parsed; - /* FIXME any checks necessary here? */ - uchar const * msg = dgram + txn->message_off; - ulong msg_sz0 = dgram_sz - txn->message_off; - uchar const * signature = dgram + txn->signature_off; - uchar const * public_key = dgram + txn->acct_addr_off; - ulong signature_cnt = txn->signature_cnt; - - for( ulong i=0; iwksp; - (void)v_x86_idx; - - FD_LOG_NOTICE(( "active: v_x86_tile_main" )); - - /* Test version */ - int test_version = cfg->test_version; - - /* Hook up to v_x86 cnc */ - fd_cnc_t * cnc = cfg->v_x86_cnc; - - /* Hook up to parser cnc */ - fd_cnc_t * parser_cnc = cfg->parser_cnc; - ulong * parser_cnc_diag = (ulong *)fd_cnc_app_laddr( parser_cnc ); - - /* Command and control */ - ulong * cnc_diag = (ulong *)fd_cnc_app_laddr( cnc ); - cnc_diag[ DEMO_V_X86_CNC_TXN_CNT_IDX ] = 0UL; - cnc_diag[ DEMO_V_X86_CNC_SIG_CNT_IDX ] = 0UL; - - /* Hook up to parser mcache */ - fd_frag_meta_t const * mcache = cfg->parser_mcache; - ulong depth = fd_mcache_depth( mcache ); - ulong const * sync = fd_mcache_seq_laddr_const( mcache ); - ulong seq = fd_mcache_seq_query( sync ); - - /* Hook up to parser flow control */ - - /* Hook up to the output mcache */ - fd_frag_meta_t * out_mcache = cfg->v_x86_mcache; - ulong out_depth = fd_mcache_depth( out_mcache ); - ulong * out_sync = fd_mcache_seq_laddr( out_mcache ); - ulong out_seq = fd_mcache_seq_query( out_sync ); - ulong out_sig = 0UL; /* this works as a counter (in this demo) */ - - /* There is no output flow control (in this demo) - The consumer will - get overrun if it starts lagging behind */ - - /* Hook up to the random number generator */ - fd_rng_t _rng[1]; - fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, cfg->v_x86_seed, 0UL ) ); - - /* Debug only */ - ulong parser_seq_unavailable_cnt = 0UL; - - /* Main loop */ - fd_cnc_signal( cnc, FD_CNC_SIGNAL_RUN ); - while( parser_cnc_diag[ DEMO_PARSER_CNC_OUT_SEQ_IDX ] <= seq ) { FD_SPIN_PAUSE(); } - for(;;) { - - /* Housekeeping */ - if( 1 ) { - /* Send diagnostic info */ - fd_cnc_heartbeat( cnc, fd_tickcount() ); - /* Receive command-and-control signals */ - ulong s = fd_cnc_signal_query( cnc ); - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_RUN ) ) { - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_HALT ) ) FD_LOG_ERR(( "Unexpected signal" )); - break; - } - } - - /* Auto throttle */ - ulong parser_seq = parser_cnc_diag[ DEMO_PARSER_CNC_OUT_SEQ_IDX ]; - if( parser_seq < seq ) { parser_seq_unavailable_cnt += 1UL; continue; } - ulong parser_seq_diff = parser_seq - seq; - const ulong parser_seq_diff_max = 10UL; - ulong seq_delta = 0UL; - if( parser_seq_diff > parser_seq_diff_max ) { seq_delta = parser_seq_diff - parser_seq_diff_max; } - seq += seq_delta; - out_sig += seq_delta; - - fd_frag_meta_t const * mline = NULL; - ulong seq_found = 0UL; - long diff; - - ulong sig; - ulong chunk; - ulong sz; - ulong ctl; - ulong tsorig; - ulong tspub; - ulong async_rem = 2UL; - FD_MCACHE_WAIT_REG( sig, chunk, sz, ctl, tsorig, tspub, mline, seq_found, diff, async_rem, mcache, depth, seq ); - FD_TEST( async_rem == 1UL ); - - if( FD_UNLIKELY( diff ) ) FD_LOG_ERR(( "Overrun while polling" )); - - /* Process the received fragment */ - (void)ctl; (void)sz; (void)sig; (void)tsorig; (void)tspub; - uchar const * p = (uchar const*)fd_chunk_to_laddr_const( wksp, chunk ); - parsed_txn_compressed_meta_t meta; - meta.all = sig; - uchar const * msg = p + meta.value.msg_off; - ulong msg_sz = (ulong) meta.value.msg_sz; - uchar const * signature = p + meta.value.signature_off; - uchar const * public_key = p + meta.value.public_key_off; - - do { - fd_sha512_t sha[1]; - int verif = -1; - if( test_version == DEMO_TEST_VERSION_SIGVERIFY ) { - verif = fd_ed25519_verify( msg, - msg_sz, - signature, - public_key, - sha ); - } else if( test_version == DEMO_TEST_VERSION_SHA512MODQ ) { - verif = sha512_modq_lsB( msg, - msg_sz, - signature, - public_key, - sha ); - } else { - FD_TEST( 0 ); - } - - /* publish in output mcache */ - uint out_chunk = (uint)(verif & 0x1); - ulong out_sz = 0UL; - ulong out_ctl = fd_frag_meta_ctl( v_x86_idx, 1 /*som*/, 1 /*eom*/, 0 /*err*/ ); - long out_now = fd_tickcount(); - ulong out_tsorig = fd_frag_meta_ts_comp( out_now ); - ulong out_tspub = out_tsorig; - fd_mcache_publish( out_mcache, out_depth, out_seq, out_sig, out_chunk, out_sz, out_ctl, out_tsorig, out_tspub ); - out_sig += 1UL; - - /* Windup for the next iteration */ - out_seq = fd_seq_inc( out_seq, 1UL ); - } while( 0 ); - - /* Check that we weren't overrun while processing */ - seq_found = fd_frag_meta_seq_query( mline ); - // seq_found = fd_frag_meta_seq_query( input_mline ); - if( FD_UNLIKELY( fd_seq_ne( seq_found, seq ) ) ) { - FD_LOG_ERR(( "Overrun while reading" )); - FD_TEST( 0 ); - } - - /* Wind up for the next iteration */ - seq = fd_seq_inc( seq, 1UL ); - cnc_diag[ DEMO_V_X86_CNC_TXN_CNT_IDX ] += 1ULL; - cnc_diag[ DEMO_V_X86_CNC_SIG_CNT_IDX ] = out_sig; - } - - /* Debug only */ - // FD_TEST( !parser_seq_unavailable_cnt ); - - fd_rng_delete( fd_rng_leave( rng ) ); - fd_cnc_signal( cnc, FD_CNC_SIGNAL_BOOT ); - return 0; -} - - - -// VVVVVVVV VVVVVVVV CCCCCCCCCCCCCHHHHHHHHH HHHHHHHHHEEEEEEEEEEEEEEEEEEEEEE CCCCCCCCCCCCCKKKKKKKKK KKKKKKK -// V::::::V V::::::V CCC::::::::::::CH:::::::H H:::::::HE::::::::::::::::::::E CCC::::::::::::CK:::::::K K:::::K -// V::::::V V::::::V CC:::::::::::::::CH:::::::H H:::::::HE::::::::::::::::::::E CC:::::::::::::::CK:::::::K K:::::K -// V::::::V V::::::VC:::::CCCCCCCC::::CHH::::::H H::::::HHEE::::::EEEEEEEEE::::E C:::::CCCCCCCC::::CK:::::::K K::::::K -// V:::::V V:::::VC:::::C CCCCCC H:::::H H:::::H E:::::E EEEEEEC:::::C CCCCCCKK::::::K K:::::KKK -// V:::::V V:::::VC:::::C H:::::H H:::::H E:::::E C:::::C K:::::K K:::::K -// V:::::V V:::::V C:::::C H::::::HHHHH::::::H E::::::EEEEEEEEEE C:::::C K::::::K:::::K -// V:::::V V:::::V C:::::C H:::::::::::::::::H E:::::::::::::::E C:::::C K:::::::::::K -// V:::::V V:::::V C:::::C H:::::::::::::::::H E:::::::::::::::E C:::::C K:::::::::::K -// V:::::V V:::::V C:::::C H::::::HHHHH::::::H E::::::EEEEEEEEEE C:::::C K::::::K:::::K -// V:::::V:::::V C:::::C H:::::H H:::::H E:::::E C:::::C K:::::K K:::::K -// V:::::::::V C:::::C CCCCCC H:::::H H:::::H E:::::E EEEEEEC:::::C CCCCCCKK::::::K K:::::KKK -// V:::::::V C:::::CCCCCCCC::::CHH::::::H H::::::HHEE::::::EEEEEEEE:::::E C:::::CCCCCCCC::::CK:::::::K K::::::K -// V:::::V CC:::::::::::::::CH:::::::H H:::::::HE::::::::::::::::::::E CC:::::::::::::::CK:::::::K K:::::K -// V:::V CCC::::::::::::CH:::::::H H:::::::HE::::::::::::::::::::E CCC::::::::::::CK:::::::K K:::::K -// VVV CCCCCCCCCCCCCHHHHHHHHH HHHHHHHHHEEEEEEEEEEEEEEEEEEEEEE CCCCCCCCCCCCCKKKKKKKKK KKKKKKK - -/* VCHECK tile ************************************************************/ - -static int -vcheck_tile_main( int argc, - char ** argv ) { - ulong vcheck_idx = (ulong)argc; - test_cfg_t * cfg = (test_cfg_t *)argv; - // fd_wksp_t * wksp = cfg->wksp; - (void)vcheck_idx; - - FD_LOG_NOTICE(( "active: vcheck_tile_main" )); - - /* Hook up to consum cnc */ - fd_cnc_t * cnc = cfg->vcheck_cnc; - - /* Command and control */ - ulong * cnc_diag = (ulong *)fd_cnc_app_laddr( cnc ); - cnc_diag[ DEMO_VCHECK_CNC_SIG__X86_CNT_IDX ] = 0UL; - cnc_diag[ DEMO_VCHECK_CNC_SIG_FPGA_CNT_IDX ] = 0UL; - cnc_diag[ DEMO_VCHECK_CNC_VALIDATE_CNT_IDX ] = 0UL; - cnc_diag[ DEMO_VCHECK_CNC_SIG_PASS_CNT_IDX ] = 0UL; - cnc_diag[ DEMO_VCHECK_CNC_SIG_FAIL_CNT_IDX ] = 0UL; - - /* Enabled producers */ - int v_x86_enabled = cfg->v_x86_enabled; - int v__wd_enabled = cfg->v__wd_enabled; - - /* Hook up to v_x86 mcache */ - fd_frag_meta_t const * v_x86_mcache = ( !!v_x86_enabled )? cfg->v_x86_mcache : NULL; - ulong v_x86_depth = ( !!v_x86_enabled )? fd_mcache_depth( v_x86_mcache ) : 0UL; - ulong const * v_x86_sync = ( !!v_x86_enabled )? fd_mcache_seq_laddr_const( v_x86_mcache ) : NULL; - ulong v_x86_seq = ( !!v_x86_enabled )? fd_mcache_seq_query( v_x86_sync ) : 0UL; - - /* Hook up wd mcache */ - fd_frag_meta_t * v__wd_mcache = ( !!v__wd_enabled )? cfg->v__wd_mcache : NULL; - ulong v__wd_depth = ( !!v__wd_enabled )? fd_mcache_depth( v__wd_mcache ) : 0UL; - ulong * v__wd_sync = ( !!v__wd_enabled )? fd_mcache_seq_laddr( v__wd_mcache ) : NULL; - ulong v__wd_seq = ( !!v__wd_enabled )? fd_mcache_seq_query( v__wd_sync ) : 0UL; - - /* Hook up to the random number generator */ - fd_rng_t _rng[1]; - fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, cfg->vcheck_seed, 0UL ) ); - - /* Verification */ - ulong exp_sig = 0UL; - ulong sig_delta = 0UL; - - /* Wiredancer */ - wd_wksp_t wd; - // int wd_split = cfg->wd_split; - uint64_t wd_slots = cfg->wd_slots; - FD_TEST( !wd_init_pci( &wd, wd_slots ) ); - wd_ed25519_verify_init_resp( &wd ); - - /* debug */ - ulong cnt_sig_ok = 0UL; - ulong cnt_sig_ng = 0UL; - - /* Main loop */ - fd_cnc_signal( cnc, FD_CNC_SIGNAL_RUN ); - for(;;) { - - /* Wait for frag seq while doing housekeeping in the background */ - - fd_frag_meta_t const * v_x86_mline = NULL; - ulong v_x86_seq_found = 0UL; - long v_x86_diff = 0L; - ulong v_x86_chunk = 0UL; - ulong v_x86_sig = exp_sig; /* by default */ - - ulong do_halt = 0ULL; - - /* Poll v_x86 */ - if( FD_LIKELY( !!v_x86_enabled ) ) { - ulong sig; - ulong chunk; - ulong sz; - ulong ctl; - ulong tsorig; - ulong tspub; - ulong v_x86_async_rem = 0UL; - do { - v_x86_async_rem = 2UL; - FD_MCACHE_WAIT_REG( sig, chunk, sz, ctl, tsorig, tspub, v_x86_mline, v_x86_seq_found, v_x86_diff, v_x86_async_rem, v_x86_mcache, v_x86_depth, v_x86_seq ); - (void)ctl; (void)sz; (void)tsorig; (void)tspub; - - if( !v_x86_async_rem ) { - /* Send diagnostic info */ - fd_cnc_heartbeat( cnc, fd_tickcount() ); - /* Receive command-and-control signals */ - ulong s = fd_cnc_signal_query( cnc ); - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_RUN ) ) { - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_HALT ) ) FD_LOG_ERR(( "Unexpected signal" )); - do_halt = 1ULL; - break; - } - } else { - if( FD_UNLIKELY( v_x86_diff ) ) FD_LOG_ERR(( "Overrun while polling v_x86" )); - } - } while( (!do_halt) & (!v_x86_async_rem) ); - - /* Extract the required values */ - sig_delta = sig - exp_sig; - exp_sig = sig; /* override expected sig */ - v_x86_sig = sig; - v_x86_chunk = chunk; - cnc_diag[ DEMO_VCHECK_CNC_SIG__X86_CNT_IDX ] += 1UL; - /* halt if instructed to do so */ - if( FD_UNLIKELY( !!do_halt )) { break; } - } - - /* Poll wiredancer */ - fd_frag_meta_t const * v__wd_mline = NULL; - ulong v__wd_seq_found = 0UL; - long v__wd_diff = 0L; - ulong v__wd_chunk = 0UL; - ulong v__wd_sig = exp_sig; /* by default */ - - if( FD_LIKELY( !!v__wd_enabled ) ) { - ulong sig; - ulong chunk; - ulong sz; - ulong ctl; - ulong tsorig; - ulong tspub; - ulong v__wd_async_rem = 0UL; - /* Jump directly to where is x86 is currently at */ - v__wd_seq += sig_delta; - do { - v__wd_async_rem = 2UL; - FD_MCACHE_WAIT_REG( sig, chunk, sz, ctl, tsorig, tspub, v__wd_mline, v__wd_seq_found, v__wd_diff, v__wd_async_rem, v__wd_mcache, v__wd_depth, v__wd_seq ); - (void)chunk; (void)sz; (void)tsorig; (void)tspub; - - if( !v__wd_async_rem ) { - /* Send diagnostic info */ - fd_cnc_heartbeat( cnc, fd_tickcount() ); - /* Receive command-and-control signals */ - ulong s = fd_cnc_signal_query( cnc ); - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_RUN ) ) { - if( FD_UNLIKELY( s!=FD_CNC_SIGNAL_HALT ) ) FD_LOG_ERR(( "Unexpected signal" )); - do_halt = 1ULL; - break; - } - } - else { - if( FD_UNLIKELY( v__wd_diff ) ) FD_LOG_ERR(( "Overrun while polling v__wd" )); - } - } while( (!do_halt) & (!v__wd_async_rem) ); - - /* Extract the required values */ - v__wd_sig = sig; - v__wd_chunk = (ctl>>2)&0x1; - /* This is essentially the latest sig that has been processed */ - cnc_diag[ DEMO_VCHECK_CNC_SIG_FPGA_CNT_IDX ] = exp_sig; - /* halt if instructed to do so */ - if( FD_UNLIKELY( !!do_halt )) { break; } - } - - - /* Process x86 */ - if( FD_LIKELY( !!v_x86_enabled )) { - /* Check that we weren't overrun while processing */ - v_x86_seq_found = fd_frag_meta_seq_query( v_x86_mline ); - if( FD_UNLIKELY( fd_seq_ne( v_x86_seq_found, v_x86_seq ) ) ) { - FD_LOG_ERR(( "Overrun while processing v_x86" )); - FD_TEST( 0 ); - } - /* Wind up for the next iteration */ - v_x86_seq = fd_seq_inc( v_x86_seq, 1UL ); - } - - /* Process wiredancer */ - if( FD_LIKELY( !!v__wd_enabled )) { - /* Check that we weren't overrun while processing */ - v__wd_seq_found = fd_frag_meta_seq_query( v__wd_mline ); - if( FD_UNLIKELY( fd_seq_ne( v__wd_seq_found, v__wd_seq ) ) ) { - FD_LOG_ERR(( "Overrun while processing v__wd (exp_sig: %lu)", exp_sig )); - ulong * p = (ulong*)v__wd_mline; - FD_LOG_NOTICE(("mcache line %016lx_%016lx_%016lx_%016lx",*(p+0),*(p+1),*(p+2),*(p+3) )); - FD_LOG_NOTICE(("v__wd_seq_found %016lx", v__wd_seq_found)); - FD_LOG_NOTICE(("v__wd_seq %016lx", v__wd_seq)); - FD_TEST( 0 ); - } - /* Wind up for the next iteration */ - v__wd_seq = fd_seq_inc( v__wd_seq, 1UL ); - /* Validate */ - // FD_TEST( v__wd_sig == exp_sig ); - cnc_diag[ DEMO_VCHECK_CNC_SIG_PASS_CNT_IDX ] += ((v__wd_chunk & 0x01U) == FD_ED25519_SUCCESS)? 1UL : 0; - cnc_diag[ DEMO_VCHECK_CNC_SIG_FAIL_CNT_IDX ] += ((v__wd_chunk & 0x01U) != FD_ED25519_SUCCESS)? 1UL : 0; - } - - /* Compare x86 and wiredancer (if both enabled) */ - if( FD_LIKELY( ( !!v_x86_enabled ) & ( !!v__wd_enabled ) ) ) { - if( v_x86_chunk != v__wd_chunk ) { FD_LOG_NOTICE(( "(0x%016lx) verif x86 0x%lx vs 0x%lx wd verif (0x%016lx) - v__wd_seq 0x%016lx", - v_x86_sig, v_x86_chunk, v__wd_chunk, v__wd_sig, v__wd_seq )); } - // FD_TEST( v_x86_sig == v__wd_sig ); - FD_TEST( v_x86_chunk == v__wd_chunk ); - cnc_diag[ DEMO_VCHECK_CNC_VALIDATE_CNT_IDX ] += 1UL; - if( (v__wd_chunk & 0x01U) == FD_ED25519_SUCCESS ) { - cnt_sig_ok +=1UL; - } else { - cnt_sig_ng +=1UL; - } - } - - /* Wind up for the next iteration */ - exp_sig += 1UL; - } - - FD_TEST( !wd_free_pci( &wd ) ); - - fd_rng_delete( fd_rng_leave( rng ) ); - fd_cnc_signal( cnc, FD_CNC_SIGNAL_BOOT ); - return 0; -} - - - - - -// MMMMMMMM MMMMMMMM AAA IIIIIIIIIINNNNNNNN NNNNNNNN -// M:::::::M M:::::::M A:::A I::::::::IN:::::::N N::::::N -// M::::::::M M::::::::M A:::::A I::::::::IN::::::::N N::::::N -// M:::::::::M M:::::::::M A:::::::A II::::::IIN:::::::::N N::::::N -// M::::::::::M M::::::::::M A:::::::::A I::::I N::::::::::N N::::::N -// M:::::::::::M M:::::::::::M A:::::A:::::A I::::I N:::::::::::N N::::::N -// M:::::::M::::M M::::M:::::::M A:::::A A:::::A I::::I N:::::::N::::N N::::::N -// M::::::M M::::M M::::M M::::::M A:::::A A:::::A I::::I N::::::N N::::N N::::::N -// M::::::M M::::M::::M M::::::M A:::::A A:::::A I::::I N::::::N N::::N:::::::N -// M::::::M M:::::::M M::::::M A:::::AAAAAAAAA:::::A I::::I N::::::N N:::::::::::N -// M::::::M M:::::M M::::::M A:::::::::::::::::::::A I::::I N::::::N N::::::::::N -// M::::::M MMMMM M::::::M A:::::AAAAAAAAAAAAA:::::A I::::I N::::::N N:::::::::N -// M::::::M M::::::M A:::::A A:::::A II::::::IIN::::::N N::::::::N -// M::::::M M::::::M A:::::A A:::::A I::::::::IN::::::N N:::::::N -// M::::::M M::::::M A:::::A A:::::A I::::::::IN::::::N N::::::N -// MMMMMMMM MMMMMMMMAAAAAAA AAAAAAAIIIIIIIIIINNNNNNNN NNNNNNN - - -/* MAIN tile **********************************************************/ - -#include -#include - -ulong test_halt = 0UL; - -static void -test_sigaction( int sig, - siginfo_t * info, - void * context ) { - (void)info; - (void)context; - FD_LOG_NOTICE(( "received POSIX signal %i; sending halt to main", sig )); - test_halt = 1UL; -} - -static void -test_signal_trap( int sig ) { - struct sigaction act[1]; - act->sa_sigaction = test_sigaction; - if( FD_UNLIKELY( sigemptyset( &act->sa_mask ) ) ) FD_LOG_ERR(( "sigempty set failed" )); - act->sa_flags = (int)(SA_SIGINFO | SA_RESETHAND); - if( FD_UNLIKELY( sigaction( sig, act, NULL ) ) ) FD_LOG_ERR(( "unable to override signal %i", sig )); -} - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - uint rng_seq = 0U; - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, rng_seq++, 0UL ) ); - - FD_TEST( fd_replay_tile_scratch_align()==FD_REPLAY_TILE_SCRATCH_ALIGN ); - FD_TEST( !fd_replay_tile_scratch_footprint( FD_REPLAY_TILE_OUT_MAX+1UL ) ); - for( ulong iter_rem=10000000UL; iter_rem; iter_rem-- ) { - ulong out_cnt = fd_rng_ulong_roll( rng, FD_REPLAY_TILE_OUT_MAX+1UL ); - FD_TEST( fd_replay_tile_scratch_footprint( out_cnt )==FD_REPLAY_TILE_SCRATCH_FOOTPRINT( out_cnt ) ); - } - - ulong cpu_idx = fd_tile_cpu_id( fd_tile_idx() ); - if( cpu_idx>fd_shmem_cpu_cnt() ) cpu_idx = 0UL; - - char const * _page_sz = fd_env_strip_cmdline_cstr ( &argc, &argv, "--page-sz", NULL, "gigantic" ); - ulong page_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--page-cnt", NULL, 1UL ); - ulong numa_idx = fd_env_strip_cmdline_ulong( &argc, &argv, "--numa-idx", NULL, fd_shmem_numa_idx( cpu_idx ) ); - char const * replay_pcap = fd_env_strip_cmdline_cstr ( &argc, &argv, "--replay-pcap", NULL, NULL ); - ulong replay_mtu = fd_env_strip_cmdline_ulong( &argc, &argv, "--replay-mtu", NULL, 1542UL ); - ulong replay_orig = fd_env_strip_cmdline_ulong( &argc, &argv, "--replay-orig", NULL, 0UL ); - ulong replay_depth = fd_env_strip_cmdline_ulong( &argc, &argv, "--replay-depth", NULL, 131072UL ); - ulong replay_cr_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--replay-cr-max", NULL, 0UL /* use default */ ); - long replay_lazy = fd_env_strip_cmdline_long ( &argc, &argv, "--replay-lazy", NULL, 0L /* use default */ ); - ulong parser_depth = fd_env_strip_cmdline_ulong( &argc, &argv, "--parser-depth", NULL, 131072UL ); - int parser_lazy = fd_env_strip_cmdline_int ( &argc, &argv, "--parser-lazy", NULL, 7 ); - int v_x86_enabled = fd_env_strip_cmdline_int ( &argc, &argv, "--v-x86-enabled", NULL, 0 ); - ulong v_x86_depth = fd_env_strip_cmdline_ulong( &argc, &argv, "--v-x86-depth", NULL, 131072UL /* test req*/ ); - int v_x86_lazy = fd_env_strip_cmdline_int ( &argc, &argv, "--v-x86-lazy", NULL, 7 ); - int v__wd_enabled = fd_env_strip_cmdline_int ( &argc, &argv, "--v--wd-enabled", NULL, 0 ); - ulong v__wd_depth = fd_env_strip_cmdline_ulong( &argc, &argv, "--v--wd-depth", NULL, 131072UL /* test req*/ ); - int vcheck_lazy = fd_env_strip_cmdline_int ( &argc, &argv, "--vcheck-lazy", NULL, 7 ); - int test_version = fd_env_strip_cmdline_int ( &argc, &argv, "--test-version", NULL, DEMO_TEST_VERSION_SIGVERIFY ); - long duration = fd_env_strip_cmdline_long ( &argc, &argv, "--duration", NULL, (long)10e9 ); - ulong wd_slots = fd_env_strip_cmdline_ulong( &argc, &argv, "--wd-slots", NULL, 1UL ); - int rand_txn_corrupt = fd_env_strip_cmdline_int ( &argc, &argv, "--rand-txn-corrupt", NULL, 1UL ); - int replay_enabled = 1; - int parser_enabled = 1; - int vcheck_enabled = 1; - - ulong page_sz = fd_cstr_to_shmem_page_sz( _page_sz ); - if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "unsupported --page-sz" )); - if( FD_UNLIKELY( !replay_pcap ) ) FD_LOG_ERR(( "--replay-pcap not specified" )); - - /* Check minimum number of tiles */ - ulong req_tile_cnt = 1 /* main */ + (( !!replay_enabled )? 1UL : 0UL) + - (( !!parser_enabled )? 1UL : 0UL) + - (( !!v_x86_enabled )? 1UL : 0UL) + - (( !!vcheck_enabled )? 1UL : 0UL); - if( FD_UNLIKELY( fd_tile_cnt()= 0 ); - FD_TEST( test_version <= 1 ); - if( test_version == DEMO_TEST_VERSION_SIGVERIFY ) { FD_LOG_NOTICE(( "TEST VERSION: full sigveriy" )); } - if( test_version == DEMO_TEST_VERSION_SHA512MODQ ) { FD_LOG_NOTICE(( "TEST VERSION: only sha512_modq" )); } - cfg->test_version = test_version; - - /* Workspace */ - FD_LOG_NOTICE(( "Creating workspace (--page-cnt %lu, --page-sz %s, --numa-idx %lu)", page_cnt, _page_sz, numa_idx )); - cfg->wksp = fd_wksp_new_anonymous( page_sz, page_cnt, fd_shmem_cpu_idx( numa_idx ), "wksp", 0UL ); - FD_TEST( cfg->wksp ); - - /* Configure replay fseq */ - ulong * replay_fseq[ DEMO_REPLAY_FSEQ_CNT_MAX ]; - for( int i=0; iwksp, fd_fseq_align(), fd_fseq_footprint(), 1UL ), seq0 ) ); - if( FD_UNLIKELY( !replay_fseq[ i ] ) ) FD_LOG_ERR(( "fd_fseq_join failed" )); - } - ulong replay_fseq_cnt = DEMO_REPLAY_FSEQ_CNT_MAX; - ulong parser_replay_fseq_idx = 0UL; - - /* default cnc_app_sz: 64UL minimum + 64UL extra margin (future use) */ - ulong default_cnc_app_sz = 128UL; - - /* Configure replay */ - if( 1 ) { - FD_LOG_NOTICE(( "Creating replay cnc (app_sz 64, type 0, heartbeat0 %li)", hb0 )); - cfg->replay_cnc = fd_cnc_join( fd_cnc_new( fd_wksp_alloc_laddr( cfg->wksp, fd_cnc_align(), fd_cnc_footprint( default_cnc_app_sz ), 1UL ), - default_cnc_app_sz, 0UL, hb0 ) ); - FD_TEST( cfg->replay_cnc ); - - cfg->replay_pcap = replay_pcap; - cfg->replay_mtu = replay_mtu; - cfg->replay_orig = replay_orig; - - FD_LOG_NOTICE(( "Creating replay mcache (--replay-depth %lu, app_sz 0, seq0 %lu)", replay_depth, seq0 )); - cfg->replay_mcache = fd_mcache_join( fd_mcache_new( fd_wksp_alloc_laddr( cfg->wksp, - fd_mcache_align(), fd_mcache_footprint( replay_depth, 0UL ), - 1UL ), - replay_depth, 0UL, seq0 ) ); - FD_TEST( cfg->replay_mcache ); - FD_LOG_NOTICE(( "Creating replay dcache (--replay-mtu %lu, burst 1, compact 1, app_sz 0)", replay_mtu )); - ulong replay_data_sz = fd_dcache_req_data_sz( replay_mtu, replay_depth, 1UL, 1 ); FD_TEST( replay_data_sz ); - cfg->replay_dcache = fd_dcache_join( fd_dcache_new( fd_wksp_alloc_laddr( cfg->wksp, - fd_dcache_align(), fd_dcache_footprint( replay_data_sz, 0UL ), - 1UL ), - replay_data_sz, 0UL ) ); - FD_TEST( cfg->replay_dcache ); - cfg->replay_cr_max = replay_cr_max; - cfg->replay_lazy = replay_lazy; - cfg->replay_seed = rng_seq++; - cfg->replay_fseq = replay_fseq; - FD_TEST( cfg->replay_fseq ); - cfg->replay_fseq_cnt = replay_fseq_cnt; - } - - /* Configure parser */ - cfg->parser_enabled = parser_enabled; - if( !!parser_enabled ) { - FD_LOG_NOTICE(( "Creating parser cnc (app_sz 64, type 1, heartbeat0 %li)", hb0 )); - cfg->parser_cnc = fd_cnc_join( fd_cnc_new( fd_wksp_alloc_laddr( cfg->wksp, fd_cnc_align(), fd_cnc_footprint( default_cnc_app_sz ), 1UL ), - default_cnc_app_sz, 1UL, hb0 ) ); - FD_TEST( cfg->parser_cnc ); - FD_LOG_NOTICE(( "Creating parser mcache (--replay-depth %lu, app_sz 0, seq0 %lu)", parser_depth, seq0 )); - cfg->parser_mcache = fd_mcache_join( fd_mcache_new( fd_wksp_alloc_laddr( cfg->wksp, - fd_mcache_align(), fd_mcache_footprint( parser_depth, 0UL ), - 1UL ), - parser_depth, 0UL, seq0 ) ); - FD_TEST( cfg->parser_mcache ); - cfg->parser_seed = rng_seq++; - cfg->parser_lazy = parser_lazy; - cfg->parser_enabled = parser_enabled; - cfg->parser_replay_fseq = replay_fseq[parser_replay_fseq_idx]; - cfg->parser_rand_txn_corrupt = rand_txn_corrupt; - } - - /* Configure v_x86 */ - cfg->v_x86_enabled = v_x86_enabled; - if( !!v_x86_enabled ) { - FD_LOG_NOTICE(( "Creating v_x86 cnc (app_sz 64, type 1, heartbeat0 %li)", hb0 )); - cfg->v_x86_cnc = fd_cnc_join( fd_cnc_new( fd_wksp_alloc_laddr( cfg->wksp, fd_cnc_align(), fd_cnc_footprint( default_cnc_app_sz ), 1UL ), - default_cnc_app_sz, 1UL, hb0 ) ); - FD_TEST( cfg->v_x86_cnc ); - FD_LOG_NOTICE(( "Creating v_x86 mcache (--v--x86-depth %lu, app_sz 0, seq0 %lu)", v_x86_depth, seq0 )); - cfg->v_x86_mcache = fd_mcache_join( fd_mcache_new( fd_wksp_alloc_laddr( cfg->wksp, - fd_mcache_align(), fd_mcache_footprint( v_x86_depth, 0UL ), - 1UL ), - v_x86_depth, 0UL, seq0 ) ); - FD_TEST( cfg->v_x86_mcache ); - cfg->v_x86_seed = rng_seq++; - cfg->v_x86_lazy = v_x86_lazy; - cfg->v_x86_enabled = v_x86_enabled; - } - - /* v__wd */ - cfg->v__wd_enabled = v__wd_enabled; - if (!!v__wd_enabled) { - FD_LOG_NOTICE(( "Creating v__wd mcache (--v--x86-depth %lu, app_sz 0, seq0 %lu)", v__wd_depth, seq0 )); - cfg->v__wd_mcache = fd_mcache_join( fd_mcache_new( fd_wksp_alloc_laddr( cfg->wksp, - fd_mcache_align(), fd_mcache_footprint( v__wd_depth, 0UL ), - 1UL ), - v__wd_depth, 0UL, seq0 ) ); - FD_TEST( cfg->v__wd_mcache ); - } - cfg->wd_split = 1UL; /* fixed */ - cfg->wd_slots = wd_slots; - - /* consumer */ - if( 1 ) { - FD_LOG_NOTICE(( "Creating consum cnc (app_sz 64, type 1, heartbeat0 %li)", hb0 )); - cfg->vcheck_cnc = fd_cnc_join( fd_cnc_new( fd_wksp_alloc_laddr( cfg->wksp, fd_cnc_align(), fd_cnc_footprint( default_cnc_app_sz ), 1UL ), - default_cnc_app_sz, 1UL, hb0 ) ); - FD_TEST( cfg->vcheck_cnc ); - cfg->vcheck_seed = rng_seq++; - cfg->vcheck_lazy = vcheck_lazy; - } - - FD_LOG_NOTICE(( "Booting" )); - - fd_tile_exec_t * v_x86_exec = ( !!v_x86_enabled )? fd_tile_exec_new( 3UL, v_x86_tile_main, 0, (char **)fd_type_pun( cfg ) ) : NULL; - // fd_tile_exec_t * v__wd_exec = ( !!v__wd_enabled )? fd_tile_exec_new( 4UL, v__wd_tile_main, 0, (char **)fd_type_pun( cfg ) ) : NULL; - fd_tile_exec_t * vcheck_exec = ( !!vcheck_enabled )? fd_tile_exec_new( 4UL, vcheck_tile_main, 0, (char **)fd_type_pun( cfg ) ) : NULL; - fd_tile_exec_t * parser_exec = ( !!parser_enabled )? fd_tile_exec_new( 2UL, parser_tile_main, 0, (char **)fd_type_pun( cfg ) ) : NULL; - fd_tile_exec_t * replay_exec = ( !!replay_enabled )? fd_tile_exec_new( 1UL, replay_tile_main, 0, (char **)fd_type_pun( cfg ) ) : NULL; - - if( !!v_x86_enabled ) { FD_TEST( v_x86_exec ); } - if( !!vcheck_enabled ) { FD_TEST( vcheck_exec ); } - if( !!parser_enabled ) { FD_TEST( parser_exec ); } - if( !!replay_enabled ) { FD_TEST( replay_exec ); } - - if( !!replay_enabled ) { FD_TEST( fd_cnc_wait( cfg->replay_cnc, FD_CNC_SIGNAL_BOOT, (long)5e9, NULL )==FD_CNC_SIGNAL_RUN ); } - if( !!parser_enabled ) { FD_TEST( fd_cnc_wait( cfg->parser_cnc, FD_CNC_SIGNAL_BOOT, (long)5e9, NULL )==FD_CNC_SIGNAL_RUN ); } - if( !!v_x86_enabled ) { FD_TEST( fd_cnc_wait( cfg->v_x86_cnc, FD_CNC_SIGNAL_BOOT, (long)5e9, NULL )==FD_CNC_SIGNAL_RUN ); } - if( !!vcheck_enabled ) { FD_TEST( fd_cnc_wait( cfg->vcheck_cnc, FD_CNC_SIGNAL_BOOT, (long)5e9, NULL )==FD_CNC_SIGNAL_RUN ); } - - /* FIXME improve notice */ - FD_LOG_NOTICE(( "Running (--duration %li ns, --replay-lazy %li ns, --replay-cr-max %lu, replay_seed %u, --parser-lazy %i, --v--x86-lazy %i, --vcheck-lazy %i)", - duration, replay_lazy, replay_cr_max, cfg->replay_seed, parser_lazy, v_x86_lazy, vcheck_lazy )); - - ulong const * replay_cnc_diag = ( !!replay_enabled )? (ulong const *)fd_cnc_app_laddr( cfg->replay_cnc ) : NULL; - ulong const * parser_cnc_diag = ( !!parser_enabled )? (ulong const *)fd_cnc_app_laddr( cfg->parser_cnc ) : NULL; - ulong const * vcheck_cnc_diag = ( !!vcheck_enabled )? (ulong const *)fd_cnc_app_laddr( cfg->vcheck_cnc ) : NULL; - - /* Wiredancer Monitor */ -#ifdef FD_HAS_WIREDANCER - wd_mon_state_t wd_mon_state; - wd_mon_state.recv_cnt[0]= 0UL; - wd_mon_state.recv_cnt[1]= 0UL; - wd_mon_state.send_cnt = 0UL; - wd_mon_state.cnt_replay = 0UL; - wd_mon_state.cnt_parser = 0UL; - wd_mon_state.cnt_x86 = 0UL; - wd_mon_state.cnt__wd = 0UL; - wd_mon_state.rate_replay= 0UL; - wd_mon_state.rate_parser= 0UL; - wd_mon_state.rate_x86 = 0UL; - wd_mon_state.rate__wd = 0UL; - wd_mon_state.sig_pass = 0UL; - wd_mon_state.sig_fail = 0UL; - wd_mon_state.cnt_checked= 0UL; - wd_mon_state.running = 1; - FD_TEST( !wd_init_pci( &wd_mon_state.wd, wd_slots ) ); - pthread_t wd_mon_thread; - FD_TEST( !pthread_create( &wd_mon_thread, NULL, mon_thread, &wd_mon_state) ); -#endif - - /* signal trap */ - test_signal_trap( SIGTERM ); - test_signal_trap( SIGINT ); - - /* main loop */ - long now = fd_log_wallclock(); - long next = now; - long done = (!!duration)? now + duration : (long)((~(ulong)0UL)>>1); - for(;;) { - long now = fd_log_wallclock(); - if( FD_UNLIKELY( (now-done) >= 0L ) | ( !!test_halt )) { - break; - } - if( FD_UNLIKELY( (now-next) >= 0L ) ) { - FD_COMPILER_MFENCE(); - if( FD_UNLIKELY( replay_cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_DONE ] ) ) { - break; - } -#ifdef FD_HAS_WIREDANCER - /* compute the rates first */ - wd_mon_state.rate_replay = replay_cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_PUB_CNT ] - wd_mon_state.cnt_replay; /* per second */ - wd_mon_state.rate_parser = parser_cnc_diag[ DEMO_PARSER_CNC_SIG_CNT_IDX ] - wd_mon_state.cnt_parser; /* per second */ - wd_mon_state.rate_x86 = vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG__X86_CNT_IDX ] - wd_mon_state.cnt_x86; /* per second */ - wd_mon_state.rate__wd = vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG_FPGA_CNT_IDX ] - wd_mon_state.cnt__wd; /* per second */ - /* update the counts second */ - wd_mon_state.cnt_replay = replay_cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_PUB_CNT ]; - wd_mon_state.cnt_parser = parser_cnc_diag[ DEMO_PARSER_CNC_SIG_CNT_IDX ]; - wd_mon_state.cnt_x86 = vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG__X86_CNT_IDX ]; - wd_mon_state.cnt__wd = vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG_FPGA_CNT_IDX ]; - wd_mon_state.cnt_checked = vcheck_cnc_diag[ DEMO_VCHECK_CNC_VALIDATE_CNT_IDX ]; - wd_mon_state.sig_pass = vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG_PASS_CNT_IDX ]; - wd_mon_state.sig_fail = vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG_FAIL_CNT_IDX ]; -#endif - /* time increment must be 1 second (for the above rates to be computed easily) */ - next += (long)1e9; /* 1 second */ - } - FD_YIELD(); - } - - /* halt: monitor */ -#ifdef FD_HAS_WIREDANCER - wd_mon_state.running = 0; - pthread_join( wd_mon_thread, NULL); -#endif - - /* keep pcap_done for logging purposes */ - if( FD_UNLIKELY( !replay_cnc_diag[ FD_REPLAY_CNC_DIAG_PCAP_DONE ] ) ) { FD_LOG_NOTICE(( "pcap replay finished before duration" )); } - - /* halt */ - FD_LOG_NOTICE(( "Halting" )); - - if( !!replay_enabled ) { FD_TEST( !fd_cnc_open( cfg->replay_cnc ) ); } - if( !!parser_enabled ) { FD_TEST( !fd_cnc_open( cfg->parser_cnc ) ); } - if( !!v_x86_enabled ) { FD_TEST( !fd_cnc_open( cfg->v_x86_cnc ) ); } - if( !!vcheck_enabled ) { FD_TEST( !fd_cnc_open( cfg->vcheck_cnc ) ); } - - /* halt: replay */ - if( !!replay_enabled ) { fd_cnc_signal( cfg->replay_cnc, FD_CNC_SIGNAL_HALT ); } - if( !!replay_enabled ) { FD_TEST( fd_cnc_wait( cfg->replay_cnc, FD_CNC_SIGNAL_HALT, (long)5e9, NULL )==FD_CNC_SIGNAL_BOOT ); } - if( !!parser_enabled ) { FD_LOG_NOTICE(( "successfully stopped replay")); } - - /* halt: parser */ - if( !!parser_enabled ) { fd_cnc_signal( cfg->parser_cnc, FD_CNC_SIGNAL_HALT ); } - if( !!parser_enabled ) { FD_TEST( fd_cnc_wait( cfg->parser_cnc, FD_CNC_SIGNAL_HALT, (long)5e9, NULL )==FD_CNC_SIGNAL_BOOT ); } - if( !!parser_enabled ) { FD_LOG_NOTICE(( "successfully stopped parser")); } - - /* halt: vcheck (part 1) */ - if( !!vcheck_enabled ) { fd_cnc_signal( cfg->vcheck_cnc, FD_CNC_SIGNAL_HALT ); } - - /* halt: x86 */ - if( !!v_x86_enabled ) { fd_cnc_signal( cfg->v_x86_cnc, FD_CNC_SIGNAL_HALT ); } - if( !!v_x86_enabled ) { FD_TEST( fd_cnc_wait( cfg->v_x86_cnc, FD_CNC_SIGNAL_HALT, (long)5e9, NULL )==FD_CNC_SIGNAL_BOOT ); } - if( !!v_x86_enabled ) { FD_LOG_NOTICE(( "successfully stopped x86")); } - - /* halt: vcheck (part 2) */ - if( !!vcheck_enabled ) { FD_LOG_NOTICE(( "successfully checked wiredancer vs x86 on %lu sigverify(s) (sig_ok %lu vs sig_err %lu, --rand_txn_corrupt %s)", - vcheck_cnc_diag[ DEMO_VCHECK_CNC_VALIDATE_CNT_IDX ], vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG_PASS_CNT_IDX ], vcheck_cnc_diag[ DEMO_VCHECK_CNC_SIG_FAIL_CNT_IDX ], rand_txn_corrupt?"enabled":"disabled" )); } - - if( !!vcheck_enabled ) { FD_TEST( fd_cnc_wait( cfg->vcheck_cnc, FD_CNC_SIGNAL_HALT, (long)5e9, NULL )==FD_CNC_SIGNAL_BOOT ); } - if( !!vcheck_enabled ) { FD_LOG_NOTICE(( "successfully stopped vcheck")); } - - if( !!replay_enabled ) { fd_cnc_close( cfg->replay_cnc ); } - if( !!parser_enabled ) { fd_cnc_close( cfg->parser_cnc ); } - if( !!v_x86_enabled ) { fd_cnc_close( cfg->v_x86_cnc ); } - if( !!vcheck_enabled ) { fd_cnc_close( cfg->vcheck_cnc ); } - - int ret; - if( !!replay_enabled ) { FD_TEST( !fd_tile_exec_delete( replay_exec, &ret ) ); FD_TEST( !ret ); } - if( !!parser_enabled ) { FD_TEST( !fd_tile_exec_delete( parser_exec, &ret ) ); FD_TEST( !ret ); } - if( !!v_x86_enabled ) { FD_TEST( !fd_tile_exec_delete( v_x86_exec, &ret ) ); FD_TEST( !ret ); } - if( !!vcheck_enabled ) { FD_TEST( !fd_tile_exec_delete( vcheck_exec, &ret ) ); FD_TEST( !ret ); } - - FD_LOG_NOTICE(( "Cleaning up" )); - - for( int i=0; ireplay_fseq[i] ) ) ); } } - if( !!replay_enabled ) { fd_wksp_free_laddr( fd_dcache_delete( fd_dcache_leave( cfg->replay_dcache ) ) ); } - if( !!replay_enabled ) { fd_wksp_free_laddr( fd_mcache_delete( fd_mcache_leave( cfg->replay_mcache ) ) ); } - if( !!replay_enabled ) { fd_wksp_free_laddr( fd_cnc_delete ( fd_cnc_leave ( cfg->replay_cnc ) ) ); } - if( !!parser_enabled ) { fd_wksp_free_laddr( fd_mcache_delete( fd_mcache_leave( cfg->parser_mcache ) ) ); } - if( !!parser_enabled ) { fd_wksp_free_laddr( fd_cnc_delete ( fd_cnc_leave ( cfg->parser_cnc ) ) ); } - if( !!v_x86_enabled ) { fd_wksp_free_laddr( fd_cnc_delete ( fd_cnc_leave ( cfg->v_x86_cnc ) ) ); } - if( !!v_x86_enabled ) { fd_wksp_free_laddr( fd_mcache_delete( fd_mcache_leave( cfg->v_x86_mcache ) ) ); } - if( !!v__wd_enabled ) { fd_wksp_free_laddr( fd_mcache_delete( fd_mcache_leave( cfg->v__wd_mcache ) ) ); } - if( !!vcheck_enabled ) { fd_wksp_free_laddr( fd_cnc_delete ( fd_cnc_leave ( cfg->vcheck_cnc ) ) ); } - - fd_wksp_delete_anonymous( cfg->wksp ); - - fd_rng_delete( fd_rng_leave( rng ) ); - - FD_LOG_NOTICE(( "pass" )); - fd_halt(); - return 0; -} - -#undef DEMO_SCRATCH_ALLOC - -#else - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - FD_LOG_WARNING(( "skip: unit test requires FD_HAS_HOSTED, FD_HAS_X86 and FD_HAS_WIREDANCER capabilities" )); - fd_halt(); - return 0; -} - -#endif diff --git a/src/wiredancer/test/wd_f1_mon.c b/src/wiredancer/test/wd_f1_mon.c deleted file mode 100644 index 4fddeac3f45..00000000000 --- a/src/wiredancer/test/wd_f1_mon.c +++ /dev/null @@ -1,443 +0,0 @@ -#ifndef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE -#endif - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "wd_f1_mon.h" - -static volatile int keepRunning = 1; - -extern uint32_t _wd_read_32 (wd_pci_t* pci, uint32_t addr); - -typedef struct { - - uint64_t recv_cnt [2]; - uint64_t send_cnt; - uint32_t running; - uint32_t running_recv; - uint32_t slot; - - wd_wksp_t wd; - - - -} state_t; - -state_t* _state_p; - - - - - - - - -double tiff(struct timespec t0, struct timespec t1) -{ - long seconds = t1.tv_sec - t0.tv_sec; - long nanoseconds = t1.tv_nsec - t0.tv_nsec; - return (double)seconds + (double)nanoseconds*1e-9; -} - -void print_counters(wd_wksp_t* wd) -{ - for (uint32_t i = 0; i < 16; i ++) - wd_zprintf ("cnt[%2d] = %08x\n", i, wd_rd_cntr(wd, _state_p->slot, i)); -} -void intHandler(int dummy) { - (void)dummy; - // _state_p->running_recv = 1; - // for (int i = 0; i < 4096/4; i ++) - // printf ("DMA[%d]: %x\n", i, dma_buf[i]); - // printf ("DMA: %x %x %x\n", dma_seq, dma_off, dma_buf[dma_off>>2]); - // printf ("send_cnt: %lx, recv_cnt: %lx\n", _state_p->send_cnt, _state_p->recv_cnt[0]); - printf ("\n"); - print_counters(&_state_p->wd); - keepRunning = 0; - - _state_p->running = 0; - -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -char* ascii_chart[] = { -" +------------------------------------+", -" | AWS-F1 x86 |", -"+-----------------------------------------+ | | +-----------------------------------------+", -"| AWS-F1 FPGA | | | | AWS-F1 FPGA |", -"| | | | | |", -"| --------- --------- | | +=============+ | | --------- --------- |", -"| | | | | | | | Replay | | | | | | | |", -"| | <------| SHA-MOD | | | | | | | | SHA-MOD |------> | |", -"| |Scheduler| | | | | +=============+ | | | | |Scheduler| |", -"| | | ----^---- | | | | | ----^---- | | |", -"| | | | | | | | | | | | |", -"| --------- | | | v | | | --------- |", -"| | ----^---- | | +====== ======+ | | ----^---- | |", -"| | | | - - | Parser | - - | | | |", -"| | | Extract <----------<--PCIe--<----------- ------------>--PCIe-->----------> Extract | | |", -"| | | | - - +=============+ - - | | | |", -"| | --------- | | | | | --------- | |", -"| ----V---- | | v | | ----V---- |", -"| | | | | +====== ======+ | | | | |", -"| | | | | | Sigverify | | | | | |", -"| | | | | | | | | | | |", -"| | | | | +=============+ | | | | |", -"| | DS-DP | | | | | | | DS-DP | |", -"| | | --------- | | v | | --------- | | |", -"| | | | | - - +====== ======+ - - | | | | |", -"| | |------> Reorder |---------->--PCIe-->----------> Checker <-----------<--PCIe--<----------| Reorder <------| | |", -"| | | | | - - | | - - | | | | |", -"| --------- --------- | | +=============+ | | --------- --------- |", -"| | | checked | | |", -"| Wiredancer | | sig_ok | | Wiredancer |", -"+-----------------------------------------+ | sig_err | +-----------------------------------------+", -" | Firedancer |", -" +------------------------------------+", -"", -}; - -uint32_t cnt_data[][7] = { - // line, col, cnt_type, cnt_color, print_width, hw_cnt_idx, pcie_slot - {125-105, 71-5, 5, 0, 0, 256, 0}, // x86 rate - {131-105, 71-5, 5, 0, 0, 258, 0}, // fpga rate - {133-105, 70-5, 4, 0, 0, 260, 0}, // cnt checked - {134-105, 70-5, 4, 0, 0, 261, 0}, // sig_pass - {135-105, 70-5, 4, 0, 0, 262, 0}, // sig_fail - {112-105, 71-5, 7, 0, 0, 263, 0}, // replay rate - {119-105, 71-5, 5, 0, 0, 264, 0}, // parser rate - - {118-105, 34-1, 0, 0, 0, 10, 0}, // sw[0] to fpga-0 - {118-105, 101-1, 0, 0, 0, 10, 1}, // sw[0] to fpga-1 - {113-105, 24-1, 2, 0, 0, 0, 0}, // ext to sha_pre - {113-105, 111-1, 2, 0, 0, 0, 1}, // ext to sha_pre - {114-105, 7-1, 2, 0, 0, 2, 0}, // sha to sv0 - {114-105, 128-1, 2, 0, 0, 2, 1}, // sha to sv0 - {128-105, 7-1, 2, 0, 0, 3, 0}, // sv0 to sv1 - {128-105, 128-1, 2, 0, 0, 3, 1}, // sv0 to sv1 - {129-105, 34-1, 2, 0, 0, 13, 0}, // fpga-0 to sw[0] - {129-105, 101-1, 2, 0, 0, 13, 1}, // fpga-0 to sw[0] - {131-105, 33-1, 4, 0, 0, 13, 0}, // fpga-0 to sw[0] - {131-105, 99-1, 4, 0, 0, 13, 1}, // fpga-0 to sw[0] - {0, 0, 0, 0, 0, 0}, // end marker -}; - -void ascii_move_to(uint32_t from[2], uint32_t to[2]) -{ - if (from[0] < to[0]) - printf ("\033[%dB", to[0]-from[0]); - else - if (from[0] > to[0]) - printf ("\033[%dA", from[0]-to[0]); - if (from[1] < to[1]) - printf ("\033[%dC", to[1]-from[1]); - else - if (from[1] > to[1]) - printf ("\033[%dD", from[1]-to[1]); -} - -void ascii_color(uint32_t col) -{ - switch(col) - { - case 0: printf ("\033[0m"); break; - case 1: printf ("\033[32m"); break; - case 2: printf ("\033[33m"); break; - case 3: printf ("\033[31m"); break; - case 4: printf ("\033[35m"); break; - } -} - -#define THOUSAND (1000L) -#define MILLION (1000L*1000L) -#define MILLION_F (1000.0*1000.0) -#define TENMILLION (1000L*1000L*10L) -#define BILLION (1000L*1000L*1000L) -#define BILLION_F (1000.0*1000.0*1000.0) -#define TENBILLION (1000L*1000L*1000L*10L) -#define TRILLION (1000L*1000L*1000L*1000L) - -int pretty_num(char* st, uint64_t cnt, char* suffix) -{ - int sel = 0; - if (cnt == 0) - { - sel = 0; - sprintf(st, "<0> %s", suffix); - } - else if (cnt < THOUSAND) - { - sel = 0; - sprintf(st, "%4lu%s", cnt, suffix); - } - else if (cnt < MILLION) - { - sel = 0; - sprintf(st, "%3luK%s", cnt / (1000L), suffix); - } - else if (cnt < TENMILLION) - { - sel = 1; - sprintf(st, "%1.1fM%s", ((double)cnt) / MILLION_F, suffix); - } - else if (cnt < BILLION) - { - sel = 2; - sprintf(st, "%3luM%s", cnt / (MILLION), suffix); - } - else if (cnt < TENBILLION) - { - sel = 3; - sprintf(st, "%1.1fG%s", ((double)cnt) / BILLION_F, suffix); - } - else - { - sel = 4; - sprintf(st, "%3luG%s", cnt / BILLION, suffix); - } - return sel; -} - -void* mon_thread(void* arg) -{ - // printf("mon_thread\n"); - - - uint32_t n_lines = 1000; - uint64_t cnts[2][32]; - char cnt_st[32][512]; - uint32_t n_pcnt = 32; - wd_mon_state_t* state = (wd_mon_state_t*)arg; - - uint64_t ts0 = 0; - uint64_t ts1 = 0; - - memset(cnts, 0, sizeof(cnts)); - - int first = 1; - uint32_t from[2] = {n_lines, 0}; - uint32_t to[2] = {0, 0}; - // uint32_t millis = 200; - uint32_t millis = 400; - volatile uint32_t millis_inv = 1000 / millis; - - float ticks_per_ms = (float)get_tsc_ticks_ns()*(float)1e6; - float rcp_ticks_per_ms = (float)1.0f/ticks_per_ms; - long last_cycle = fd_tickcount(); - - while (state->running && !state->running_recv) - { - // every 200ms - usleep(1000*millis); - if (!state->running) - break; - - long current_cycle = fd_tickcount(); - long delta_cycle = current_cycle - last_cycle; - last_cycle = current_cycle; - uint32_t t_loop_ms = (uint32_t)(((float)delta_cycle) * rcp_ticks_per_ms); - millis_inv = 32000 / t_loop_ms; - - if (!first) - { - to[0] = 0; - to[1] = 0; - ascii_move_to(from, to); - } - first = 0; - - // snapshot hw counters - wd_snp_cntrs(&state->wd, 0); - wd_snp_cntrs(&state->wd, 1); - - // read the counters - for (int i = 0;; i ++) - { - if (cnt_data[i][0] == 0) - break; - uint64_t cnt = 0; - if (cnt_data[i][5] < 100) - cnt = wd_rd_cntr(&state->wd, cnt_data[i][6], (uint32_t)cnt_data[i][5]); - if (cnt_data[i][5] ==256 ) - cnt = (uint)state->rate_x86; - if (cnt_data[i][5] ==257 ) - cnt = (uint)state->cnt_x86; - if (cnt_data[i][5] ==258 ) - cnt = (uint)state->rate__wd; - if (cnt_data[i][5] ==259 ) - cnt = (uint)state->cnt__wd; - if (cnt_data[i][5] ==260 ) - cnt = (uint)state->cnt_checked; - if (cnt_data[i][5] ==261 ) - cnt = (uint)state->sig_pass; - if (cnt_data[i][5] ==262 ) - cnt = (uint)state->sig_fail; - if (cnt_data[i][5] ==263 ) - cnt = (uint)state->rate_replay; - if (cnt_data[i][5] ==264 ) - cnt = (uint)state->rate_parser; - cnts[1][i] = cnt; - } - - // update deltas - for (int i = 0;; i ++) - { - if (cnt_data[i][0] == 0) - break; - uint64_t cnt = 0; - cnt = cnts[1][i]; - if (cnt < cnts[0][i]) - cnt += (1UL<<32); - cnts[1][i] = cnt - cnts[0][i]; - cnts[0][i] = cnt & 0xFFFFFFFF; - } - - for (int ci = 0;; ci ++) - { - if (cnt_data[ci][0] == 0) - break; - uint32_t color = 0; - - if (cnt_data[ci][2] == 0) - { - uint32_t cols[] = {3, 2, 1, 1, 1}; - uint64_t cnt = cnts[1][ci] * (512) * millis_inv; - cnt /= 32; - int sel = pretty_num(cnt_st[ci], cnt, "bps"); - color = cols[sel]; - } - else if (cnt_data[ci][2] == 2) - { - uint32_t cols[] = {3, 1, 1, 1, 1}; - uint64_t cnt = cnts[1][ci] * millis_inv; - cnt /= 32; - int sel = pretty_num(cnt_st[ci], cnt, "Sps"); - color = cols[sel]; - } - else if (cnt_data[ci][2] == 3) - { - uint32_t cols[] = {1, 3, 1, 1, 1}; - uint64_t cnt = cnts[0][ci]; - int sel = pretty_num(cnt_st[ci], cnt, "drp"); - color = cols[sel]; - } - else if (cnt_data[ci][2] == 4) - { - uint64_t cnt = cnts[0][ci]; - sprintf(cnt_st[ci], "[%08lx]", cnt); - color = 4; - } - else if (cnt_data[ci][2] == 5) - { - uint32_t cols[] = {3, 1, 1, 1, 1}; - uint64_t cnt = cnts[0][ci]; - int sel = pretty_num(cnt_st[ci], cnt, "Sps"); - color = cols[sel]; - } - else if (cnt_data[ci][2] == 6) - { - uint32_t cols[] = {3, 2, 1, 1, 1}; - uint64_t cnt = cnts[0][ci] * 8; /* convert to bits per second */ - int sel = pretty_num(cnt_st[ci], cnt, "bps"); - color = cols[sel]; - } - else if (cnt_data[ci][2] == 7) - { - uint32_t cols[] = {4, 4, 4, 4, 4}; /* basically a counter */ - uint64_t cnt = cnts[0][ci]; - int sel = pretty_num(cnt_st[ci], cnt, "Pps"); - color = cols[sel]; - } - cnt_data[ci][3] = color; - cnt_data[ci][4] = (uint32_t)strlen(cnt_st[ci]); - } - - for (uint32_t li = 0; li < n_lines; li ++) - { - if (ascii_chart[li][0] == 0) - { - n_lines = li; - from[0] = n_lines;// + n_pcnt + 1; - (void)n_pcnt; - break; - } - for (uint32_t ci = 0;; ci ++) - { - if (ascii_chart[li][ci] == 0) - break; - - int found = 0; - for (int cnt_i = 0;; cnt_i ++) - { - if (cnt_data[cnt_i][0] == 0) - break; - if (cnt_data[cnt_i][0] != li) - continue; - if (cnt_data[cnt_i][1] != ci) - continue; - - found = 1; - - ascii_color(cnt_data[cnt_i][3]); - printf ("%s", cnt_st[cnt_i]); - ascii_color(0); - ci += cnt_data[cnt_i][4] - 1; - } - - if (!found) - { - printf ("%c", ascii_chart[li][ci]); - } - } - printf ("\n"); - } - (void)ts0; - (void)ts1; - - fflush(stdout); - } - return 0; -} - - diff --git a/src/wiredancer/test/wd_f1_mon.h b/src/wiredancer/test/wd_f1_mon.h deleted file mode 100644 index df75f60fa14..00000000000 --- a/src/wiredancer/test/wd_f1_mon.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef HEADER_fd_src_wiredancer_test_wd_f1_mon_h -#define HEADER_fd_src_wiredancer_test_wd_f1_mon_h - -// TODO remove unnecessary ones -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../c/wd_f1.h" -#include "../../util/fd_util_base.h" - -typedef struct { - uint64_t recv_cnt [2]; - uint64_t send_cnt; - uint64_t cnt_replay; - uint64_t cnt_parser; - uint64_t cnt_x86; - uint64_t cnt__wd; - uint64_t rate_replay; - uint64_t rate_parser; - uint64_t rate_x86; - uint64_t rate__wd; - uint64_t sig_pass; - uint64_t sig_fail; - uint64_t cnt_checked; - uint32_t running; - uint32_t running_recv; - uint32_t slot; - wd_wksp_t wd; -} wd_mon_state_t; - -void* mon_thread(void* arg); - -/* TSC simple "calibration" method */ - -static inline double get_tsc_ticks_ns() -{ - struct timespec ts_start, ts_end; - volatile long rdtsc_start = 0LL; - volatile long rdtsc_end = 0LL; - volatile long i = 0; - - clock_gettime(CLOCK_MONOTONIC, &ts_start); - rdtsc_start = fd_tickcount(); - /* Compute intensive - arbitrary count */ - for (i = 0; i < 100000000LL; i++); - rdtsc_end = fd_tickcount(); - clock_gettime(CLOCK_MONOTONIC, &ts_end); - - static struct timespec ts_diff; - do { /* compute the differences */ - ts_diff.tv_sec = ts_end.tv_sec - ts_start.tv_sec ; - ts_diff.tv_nsec = ts_end.tv_nsec - ts_start.tv_nsec; - if (ts_diff.tv_nsec < 0) { ts_diff.tv_sec--; ts_diff.tv_nsec += 1000000000LL; /* ns per second */} - } while(0); - uint64_t ns = (uint64_t)(ts_diff.tv_sec * 1000000000LL + ts_diff.tv_nsec); - return (double)(rdtsc_end - rdtsc_start)/(double)ns; -} - -#endif From de0f690a75da190ffa0c81bd2b5530bffaebd7fe Mon Sep 17 00:00:00 2001 From: Richard Patel Date: Fri, 23 Oct 2026 21:44:09 +0000 Subject: [PATCH 2/5] Remove groove --- src/groove/Local.mk | 14 - src/groove/fd_groove.h | 12 - src/groove/fd_groove_base.c | 16 - src/groove/fd_groove_base.h | 137 ---- src/groove/fd_groove_data.c | 1002 --------------------------- src/groove/fd_groove_data.h | 509 -------------- src/groove/fd_groove_data_szc_cfg.c | 38 - src/groove/fd_groove_meta.c | 19 - src/groove/fd_groove_meta.h | 78 --- src/groove/fd_groove_volume.c | 106 --- src/groove/fd_groove_volume.h | 149 ---- src/groove/test_groove_base.c | 88 --- src/groove/test_groove_data.c | 498 ------------- src/groove/test_groove_meta.c | 530 -------------- src/groove/test_groove_volume.c | 136 ---- 15 files changed, 3332 deletions(-) delete mode 100644 src/groove/Local.mk delete mode 100644 src/groove/fd_groove.h delete mode 100644 src/groove/fd_groove_base.c delete mode 100644 src/groove/fd_groove_base.h delete mode 100644 src/groove/fd_groove_data.c delete mode 100644 src/groove/fd_groove_data.h delete mode 100644 src/groove/fd_groove_data_szc_cfg.c delete mode 100644 src/groove/fd_groove_meta.c delete mode 100644 src/groove/fd_groove_meta.h delete mode 100644 src/groove/fd_groove_volume.c delete mode 100644 src/groove/fd_groove_volume.h delete mode 100644 src/groove/test_groove_base.c delete mode 100644 src/groove/test_groove_data.c delete mode 100644 src/groove/test_groove_meta.c delete mode 100644 src/groove/test_groove_volume.c diff --git a/src/groove/Local.mk b/src/groove/Local.mk deleted file mode 100644 index f4bee6f7ec3..00000000000 --- a/src/groove/Local.mk +++ /dev/null @@ -1,14 +0,0 @@ -$(call make-lib,fd_groove) -$(call add-hdrs,fd_groove_base.h fd_groove_meta.h fd_groove_volume.h fd_groove_data.h fd_groove.h) -$(call add-objs,fd_groove_base fd_groove_meta fd_groove_volume fd_groove_data fd_groove_data_szc_cfg,fd_groove) -$(call make-unit-test,test_groove_base,test_groove_base,fd_groove fd_util) -$(call make-unit-test,test_groove_meta,test_groove_meta,fd_groove fd_util) -$(call run-unit-test,test_groove_base) -$(call run-unit-test,test_groove_meta) - -ifdef FD_HAS_HOSTED -$(call make-unit-test,test_groove_volume,test_groove_volume,fd_groove fd_util) -$(call make-unit-test,test_groove_data,test_groove_data,fd_groove fd_util) -$(call run-unit-test,test_groove_volume) -$(call run-unit-test,test_groove_data) -endif diff --git a/src/groove/fd_groove.h b/src/groove/fd_groove.h deleted file mode 100644 index 06229276b3f..00000000000 --- a/src/groove/fd_groove.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef HEADER_fd_src_groove_fd_groove_h -#define HEADER_fd_src_groove_fd_groove_h - -/* Note: will still function without FD_HAS_ATOMIC but will not be safe - to use concurrently. */ - -//#include "fd_groove_base.h" /* includes ../util/fd_util.h */ -//#include "fd_groove_meta.h" /* includes fd_groove_base.h */ -//#include "fd_groove_volume.h" /* includes fd_groove_base.h */ -#include "fd_groove_data.h" /* includes fd_groove_meta.h, fd_groove_volume.h */ - -#endif /* HEADER_fd_src_groove_fd_groove_h */ diff --git a/src/groove/fd_groove_base.c b/src/groove/fd_groove_base.c deleted file mode 100644 index e809baaaa49..00000000000 --- a/src/groove/fd_groove_base.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "fd_groove_base.h" - -char const * -fd_groove_strerror( int err ) { - switch( err ) { - case FD_GROOVE_SUCCESS: return "success"; - case FD_GROOVE_ERR_INVAL: return "bad input"; - case FD_GROOVE_ERR_AGAIN: return "try again later"; - case FD_GROOVE_ERR_CORRUPT: return "corrupt"; - case FD_GROOVE_ERR_EMPTY: return "empty"; - case FD_GROOVE_ERR_FULL: return "full"; - case FD_GROOVE_ERR_KEY: return "key not found"; - default: break; - } - return "unknown"; -} diff --git a/src/groove/fd_groove_base.h b/src/groove/fd_groove_base.h deleted file mode 100644 index a8773bc02d1..00000000000 --- a/src/groove/fd_groove_base.h +++ /dev/null @@ -1,137 +0,0 @@ -#ifndef HEADER_fd_src_groove_fd_groove_base_h -#define HEADER_fd_src_groove_fd_groove_base_h - -#include "../util/fd_util.h" - -/* FD_GROOVE_PARANOID enables extra integrity checking in various - operations. */ - -#ifndef FD_GROOVE_PARANOID -#define FD_GROOVE_PARANOID 1 -#endif - -/* fd_groove error code API *******************************************/ - -/* Note: Harmonized with fd_*_para error codes */ - -#define FD_GROOVE_SUCCESS (0) -#define FD_GROOVE_ERR_INVAL (-1) -#define FD_GROOVE_ERR_AGAIN (-2) -#define FD_GROOVE_ERR_CORRUPT (-3) -#define FD_GROOVE_ERR_EMPTY (-4) -#define FD_GROOVE_ERR_FULL (-5) -#define FD_GROOVE_ERR_KEY (-6) - -FD_PROTOTYPES_BEGIN - -/* fd_groove_strerror converts an FD_GROOVE_SUCCESS / FD_GROOVE_ERR_* - code into a human readable cstr. The lifetime of the returned - pointer is infinite. The returned pointer is always to a non-NULL - cstr. */ - -FD_FN_CONST char const * -fd_groove_strerror( int err ); - -FD_PROTOTYPES_END - -/* fd_groove_key API **************************************************/ - -/* A fd_groove_key_t identifies a groove record. Compact binary keys - are encouraged but a cstr can be used so long as it has - strlen(cstr)c; - ulong csz = fd_ulong_min( src_sz, FD_GROOVE_KEY_FOOTPRINT ); /* typically compile time */ - ulong zsz = FD_GROOVE_KEY_FOOTPRINT - csz; /* " */ - if( zsz ) memset( dst, 0, FD_GROOVE_KEY_FOOTPRINT ); /* " */ - if( csz ) memcpy( dst, src, csz ); /* " */ - return k; -} - -/* fd_groove_key_ulong initializes the key pointed to by k with the - ulongs k0, k1, k2, and k3. Assumes k points in the caller's address - space to the location to store the key. Retains no interest in k. - Returns k. */ - -static inline fd_groove_key_t * -fd_groove_key_init_ulong( fd_groove_key_t * k, - ulong k0, - ulong k1, - ulong k2, - ulong k3 ) { - k->ul[0] = k0; k->ul[1] = k1; k->ul[2] = k2; k->ul[3] = k3; - return k; -} - -/* fd_groove_key_eq tests keys ka and kb for equality. Assumes ka and - kb point in the caller's address space to valid keys for the duration - of the call. Retains no interest in ka or kb. Returns 1 if the keys - are equal and 0 otherwise. */ - -FD_FN_PURE static inline int -fd_groove_key_eq( fd_groove_key_t const * ka, - fd_groove_key_t const * kb ) { - ulong const * a = ka->ul; - ulong const * b = kb->ul; - return !(((a[0]^b[0]) | (a[1]^b[1])) | ((a[2]^b[2]) | (a[3]^b[3]))); /* tons of ILP and vectorizable */ -} - -/* fd_groove_key_hash provides a family of hashes that hash the key - pointed to by k to a uniform quasi-random 64-bit integer. seed - selects the particular hash function to use and can be an arbitrary - 64-bit value. The hash functions are high quality but not - cryptographically secure. Assumes ka points in the caller's address - space to a valid key for the duration of the call. Retains no - interest in ka. Returns the hash. */ - -FD_FN_UNUSED FD_FN_PURE static ulong /* Workaround -Winline */ -fd_groove_key_hash( fd_groove_key_t const * ka, - ulong seed ) { - ulong const * a = ka->ul; - return (fd_ulong_hash( a[0] ^ seed ) ^ fd_ulong_hash( a[1] ^ seed )) ^ - (fd_ulong_hash( a[2] ^ seed ) ^ fd_ulong_hash( a[3] ^ seed )); /* tons of ILP and vectorizable */ -} - -FD_PROTOTYPES_END - -/* fd_groove_block API ************************************************/ - -/* Groove data objects are backed by FD_GROOVE_BLOCK_ALIGN aligned - FD_GROOVE_BLOCK_FOOTPRINT byte blocks. ALIGN==FOOTPRINT and are a - power of 2. 512 is used for tight coupling to typical HPC I/O API, - operating system, driver and hardware limits. */ - -#define FD_GROOVE_BLOCK_ALIGN (512UL) -#define FD_GROOVE_BLOCK_FOOTPRINT (512UL) - -#endif /* HEADER_fd_src_groove_fd_groove_base_h */ diff --git a/src/groove/fd_groove_data.c b/src/groove/fd_groove_data.c deleted file mode 100644 index a0de89f1128..00000000000 --- a/src/groove/fd_groove_data.c +++ /dev/null @@ -1,1002 +0,0 @@ -#include "fd_groove_data.h" - -/* fd_groove_data_private_active_displace lockfree atomically sets the - active superblock for (szc,cgroup) to the superblock at offset - superblock_off and returns the offset of the previously active - superblock. Offsets are relative to volume0. - - Assumes active_slot points in the caller's address space to the - active superblock offset for (szc,cgroup). If superblock_off is - non-zero, further assumes the input superblock is for sizeclass szc, - has at least one free block and is not in circulation (i.e. neither - in any active slot nor on any inactive stack such that nobody can - concurrently allocate from it). - - If this returns zero, there was no active superblock for (szc,cgroup) - just before it was set to the input superblock. - - If this returns non-zero, the output superblock was the previously - active superblock. The output superblock will have at least one free - block (and possibly growing over time to all blocks free due to - concurrent frees) and will not be in circulation. - - If superblock_off is non-zero, the input superblock will be in - circulation as the active superblock for (szc,cgroup) on return. - - If superblock_off is zero, there will be no active superblock for - (szc,cgroup) on return. - - This is a compiler fence. */ - -static inline ulong -fd_groove_data_private_active_displace( ulong volatile * _active_slot, - fd_groove_volume_t * volume0, - ulong superblock_off ) { - (void)volume0; - FD_COMPILER_MFENCE(); -# if FD_HAS_ATOMIC - superblock_off = FD_ATOMIC_XCHG( _active_slot, superblock_off ); -# else - ulong old = *_active_slot; - *_active_slot = superblock_off; - superblock_off = old; -# endif - FD_COMPILER_MFENCE(); - return superblock_off; -} - -/* fd_groove_data_private_inactive_push does a lockfree atomic push of - the superblock at superblock_off relative to volume0 onto the given - inactive stack. Assumes all inputs are valid, the inactive stack and - superblock have the same sizeclass, the superblock is not in - circulation, and superblock contains at least one free block. On - return, superblock will be the top of the inactive stack. This is a - compiler fence. */ - -static inline void -fd_groove_data_private_inactive_push( ulong volatile * _inactive_stack, - fd_groove_volume_t * volume0, - ulong superblock_off ) { - FD_COMPILER_MFENCE(); - - fd_groove_data_hdr_t * superblock = (fd_groove_data_hdr_t *)(((ulong)volume0) + superblock_off); - - for(;;) { - ulong ver_off = *_inactive_stack; - - ulong ver = ver_off & (FD_GROOVE_BLOCK_FOOTPRINT-1UL); - ulong next_off = ver_off & ~(FD_GROOVE_BLOCK_FOOTPRINT-1UL); - - superblock->info = next_off; - - ulong next_ver = (ver+1UL) & (FD_GROOVE_BLOCK_FOOTPRINT-1UL); - -# if FD_HAS_ATOMIC - ulong old = FD_ATOMIC_CAS( _inactive_stack, ver_off, next_ver | superblock_off ); -# else - ulong old = *_inactive_stack; - *_inactive_stack = fd_ulong_if( old==ver_off, next_ver | superblock_off, old ); -# endif - - if( FD_LIKELY( old==ver_off ) ) break; - - FD_SPIN_PAUSE(); - } - - FD_COMPILER_MFENCE(); -} - -/* fd_groove_data_private_inactive_pop does a lockfree atomic pop of the - given inactive stack. Assumes all inputs are valid. Returns the - offset relative to volume0 of the superblock. The superblock will be - for the same sizeclass as the inactive stack, will not be in - circulation and will have at least 1 block free. If the stack was - empty when observed, returns 0. This is a compiler fence. */ - -static inline ulong -fd_groove_data_private_inactive_pop( ulong volatile * _inactive_stack, - fd_groove_volume_t * volume0 ) { - - ulong off; - - FD_COMPILER_MFENCE(); - - for(;;) { - ulong ver_off = *_inactive_stack; - - ulong ver = ver_off & (FD_GROOVE_BLOCK_FOOTPRINT-1UL); - /**/ off = ver_off & ~(FD_GROOVE_BLOCK_FOOTPRINT-1UL); - - if( FD_UNLIKELY( !off ) ) break; - - fd_groove_data_hdr_t * superblock = (fd_groove_data_hdr_t *)(((ulong)volume0) + off); - - ulong next_ver = (ver+1UL) & (FD_GROOVE_BLOCK_FOOTPRINT-1UL); - ulong next_off = superblock->info; - -# if FD_HAS_ATOMIC - ulong old = FD_ATOMIC_CAS( _inactive_stack, ver_off, next_ver | next_off ); -# else - ulong old = *_inactive_stack; - *_inactive_stack = fd_ulong_if( old==ver_off, next_ver | next_off, old ); -# endif - - if( FD_LIKELY( old==ver_off ) ) break; - - FD_SPIN_PAUSE(); - } - - FD_COMPILER_MFENCE(); - - return off; -} - -void * -fd_groove_data_new( void * shmem ) { - fd_groove_data_shmem_t * shdata = (fd_groove_data_shmem_t *)shmem; - - if( FD_UNLIKELY( !shdata ) ) { - FD_LOG_WARNING(( "NULL shmem" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shdata, fd_groove_data_align() ) ) ) { - FD_LOG_WARNING(( "misaligned shmem" )); - return NULL; - } - - ulong footprint = fd_groove_data_footprint(); - - if( FD_UNLIKELY( !footprint ) ) { /* currently not possible */ - FD_LOG_WARNING(( "bad configuration" )); - return NULL; - } - - memset( shdata, 0, footprint ); - - if( FD_UNLIKELY( !fd_groove_volume_pool_new( shdata->volume_pool ) ) ) return NULL; /* logs details (currently not possible) */ - - FD_COMPILER_MFENCE(); - shdata->magic = FD_GROOVE_DATA_MAGIC; - FD_COMPILER_MFENCE(); - - return shmem; -} - -fd_groove_data_t * -fd_groove_data_join( void * ljoin, - void * shdata, - void * volume0, - ulong volume_max, - ulong cgroup_hint ) { - volume_max = fd_ulong_if( !!volume_max, volume_max, fd_groove_volume_pool_ele_max_max() ); - - fd_groove_data_t * join = (fd_groove_data_t *)ljoin; - fd_groove_data_shmem_t * data = (fd_groove_data_shmem_t *)shdata; - - if( FD_UNLIKELY( !join ) ) { - FD_LOG_WARNING(( "NULL ljoin" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)join, alignof(fd_groove_data_t) ) ) ) { - FD_LOG_WARNING(( "misaligned ljoin" )); - return NULL; - } - - if( FD_UNLIKELY( !data ) ) { - FD_LOG_WARNING(( "NULL shdata" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)data, fd_groove_data_align() ) ) ) { - FD_LOG_WARNING(( "misaligned shdata" )); - return NULL; - } - - if( FD_UNLIKELY( data->magic!=FD_GROOVE_DATA_MAGIC ) ) { - FD_LOG_WARNING(( "bad magic" )); - return NULL; - } - - if( FD_UNLIKELY( !volume0 ) ) { - FD_LOG_WARNING(( "NULL volume0" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)volume0, alignof(fd_groove_volume_t) ) ) ) { - FD_LOG_WARNING(( "misaligned volume0" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_groove_volume_pool_join( join->volume_pool, data->volume_pool, volume0, volume_max ) ) ) /* logs details */ - return NULL; - - join->active_slot = data->active_slot; - join->inactive_stack = data->inactive_stack; - join->cgroup_hint = cgroup_hint; - - return join; -} - -void * -fd_groove_data_leave( fd_groove_data_t * join ) { - if( FD_UNLIKELY( !join ) ) { - FD_LOG_WARNING(( "NULL join" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_groove_volume_pool_leave( join->volume_pool ) ) ) { /* currently not possible */ - FD_LOG_WARNING(( "fd_groove_volume_pool_leave failed" )); - return NULL; - } - - return join; -} - -void * -fd_groove_data_delete( void * shdata ) { - fd_groove_data_shmem_t * data = (fd_groove_data_shmem_t *)shdata; - - if( FD_UNLIKELY( !data ) ) { - FD_LOG_WARNING(( "NULL shdata" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)data, fd_groove_data_align() ) ) ) { - FD_LOG_WARNING(( "misaligned shdata" )); - return NULL; - } - - if( FD_UNLIKELY( data->magic!=FD_GROOVE_DATA_MAGIC ) ) { - FD_LOG_WARNING(( "bad magic" )); - return NULL; - } - - FD_COMPILER_MFENCE(); - data->magic = 0UL; - FD_COMPILER_MFENCE(); - - return shdata; -} - -/* FIXME: ideally would update the free_objs bit field after writing - an alloc / superblock data hdr to give non-invasive concurrent - real-time inspection / diagnostics a strong guarantee all objects - marked as allocated have valid data hdrs. */ - -static int -fd_groove_data_private_alloc_obj( fd_groove_data_t * data, - ulong obj_szc, - ulong * _obj_off, - ulong * _obj_idx ) { - - fd_groove_volume_t * _volume0 = (fd_groove_volume_t *)fd_groove_data_volume0( data ); - -# if FD_GROOVE_PARANOID - fd_groove_volume_t * _volume1 = (fd_groove_volume_t *)fd_groove_data_volume1( data ); -# endif - - ulong obj_cnt = (ulong)fd_groove_data_szc_cfg[ obj_szc ].obj_cnt; - ulong obj_footprint = (ulong)fd_groove_data_szc_cfg[ obj_szc ].obj_footprint; - ulong cgroup_mask = (ulong)fd_groove_data_szc_cfg[ obj_szc ].cgroup_mask; - ulong parent_szc = (ulong)fd_groove_data_szc_cfg[ obj_szc ].parent_szc; - - /* At this point, we are allocating an object from a sizeclass obj_szc - superblock. Get the locations of the active slot and inactive - stack for this sizeclass and our concurrency group. */ - - ulong cgroup = data->cgroup_hint & cgroup_mask; - - ulong volatile * _active_slot = data->active_slot + obj_szc + FD_GROOVE_DATA_SZC_CNT*cgroup; - ulong volatile * _inactive_stack = data->inactive_stack + obj_szc; - - ulong superblock_off; - - /* Try to get exclusive access to the active superblock. Note that - active superblocks have at least one free obj. We do this - test-and-test-and-set style to avoid atomic operations if there is - no current active_superblock for this cgroup. */ - - FD_COMPILER_MFENCE(); - superblock_off = *_active_slot; - FD_COMPILER_MFENCE(); - - if( FD_LIKELY( superblock_off ) ) superblock_off = fd_groove_data_private_active_displace( _active_slot, _volume0, 0UL ); - - if( FD_UNLIKELY( !superblock_off ) ) { - - /* At this point, there was no active superblock for our cgroup when - we observed it. Try to pop the inactive superblock stack for - this sizeclass instead. Note that inactive superblocks also have - at least one free obj. */ - - superblock_off = fd_groove_data_private_inactive_pop( _inactive_stack, _volume0 ); - - if( FD_UNLIKELY( !superblock_off ) ) { - - /* At this point, there were no inactive superblocks for this - sizeclass when we observed the inactive stack. Try to create a - new superblock for this sizeclass */ - - ulong parent_idx = 0UL; /* reduce risk of uninitialized variable false positives from code analysis tools */ - - if( FD_UNLIKELY( parent_szc==FD_GROOVE_DATA_SZC_CNT ) ) { /* Acquire a volume to use for the new superblock */ - - fd_groove_volume_t * _volume = fd_groove_volume_pool_acquire( data->volume_pool ); - if( FD_UNLIKELY( !_volume ) ) return FD_GROOVE_ERR_FULL; - -# if FD_GROOVE_PARANOID - ulong volume_off = (ulong)_volume - (ulong)_volume0; - - if( FD_UNLIKELY( !( (_volume0<=_volume) & (_volume<_volume1) & - fd_ulong_is_aligned( volume_off, FD_GROOVE_VOLUME_FOOTPRINT ) ) ) ) { - FD_LOG_WARNING(( "volume not at a valid groove data local address" )); - return FD_GROOVE_ERR_CORRUPT; - } - - if( FD_UNLIKELY( !( (_volume->magic ==~FD_GROOVE_VOLUME_MAGIC ) & - (_volume->idx*FD_GROOVE_VOLUME_FOOTPRINT==volume_off ) & - (_volume->info_sz <=FD_GROOVE_VOLUME_INFO_MAX) ) ) ) { - FD_LOG_WARNING(( "unexpected volume header" )); - return FD_GROOVE_ERR_CORRUPT; - } -# endif - - FD_COMPILER_MFENCE(); - _volume->magic = FD_GROOVE_VOLUME_MAGIC; /* mark volume as potentially containing groove data allocations */ - FD_COMPILER_MFENCE(); - - //parent_idx = 0UL; /* See note above about initialization */ - superblock_off = (ulong)_volume->data - (ulong)_volume0; - - } else { /* Acquire a parent_szc object to use for the new superblock */ - - int err = fd_groove_data_private_alloc_obj( data, parent_szc, &superblock_off, &parent_idx ); /* logs details */ - if( FD_UNLIKELY( err ) ) return err; - - } - - ulong superblock_align = FD_GROOVE_DATA_HDR_ALIGN; - ulong superblock_sz = FD_GROOVE_BLOCK_FOOTPRINT - FD_GROOVE_DATA_HDR_FOOTPRINT + obj_footprint*obj_cnt; - - fd_groove_data_hdr_t * _superblock_hdr = (fd_groove_data_hdr_t *)(((ulong)_volume0) + superblock_off); - -# if FD_GROOVE_PARANOID - if( FD_UNLIKELY( !( ((ulong)_volume0<(ulong)_superblock_hdr) & ((ulong)_superblock_hdr<(ulong)_volume1) & - fd_ulong_is_aligned( (ulong)_superblock_hdr, FD_GROOVE_BLOCK_ALIGN ) ) ) ) { - FD_LOG_WARNING(( "superblock not at a valid groove data local address" )); - return FD_GROOVE_ERR_CORRUPT; - } -# endif - - *_superblock_hdr = fd_groove_data_hdr( FD_GROOVE_DATA_HDR_TYPE_SUPERBLOCK, parent_idx, obj_szc, - superblock_align, fd_ulong_min( superblock_sz, (1UL<<25)-1UL ), 0UL /* no next */ ); - *(ulong *)(_superblock_hdr+1) = (1UL<volume_pool ) + obj_off); - - *obj_hdr = fd_groove_data_hdr( FD_GROOVE_DATA_HDR_TYPE_ALLOC, obj_idx, obj_szc, align, sz, tag ); - - *_err = FD_GROOVE_SUCCESS; - return (void *)((ulong)obj_hdr + off_obj); -} - -int -fd_groove_data_private_free( fd_groove_data_t * data, - void * _obj, - ulong exp_type ) { - -# if !FD_GROOVE_PARANOID - (void)exp_type; /* Suppress unused warning if running without paranoia */ -# endif - - if( FD_UNLIKELY( !data ) ) { - FD_LOG_WARNING(( "NULL data" )); - return FD_GROOVE_ERR_INVAL; - } - - if( FD_UNLIKELY( !_obj ) ) return FD_GROOVE_ERR_INVAL; - - fd_groove_data_hdr_t * _obj_hdr = fd_groove_data_object_hdr( _obj ); - - fd_groove_volume_t * _volume0 = (fd_groove_volume_t *)fd_groove_data_volume0( data ); - -# if FD_GROOVE_PARANOID - fd_groove_volume_t * _volume1 = (fd_groove_volume_t *)fd_groove_data_volume1( data ); - if( FD_UNLIKELY( !( ((ulong)_volume0<=(ulong)_obj_hdr ) & - ((ulong)_obj_hdr< (ulong)_volume1 ) & - (fd_ulong_is_aligned( (ulong)_obj_hdr, FD_GROOVE_BLOCK_ALIGN )) ) ) ) { - FD_LOG_WARNING(( "object not at a valid groove data local address" )); - return FD_GROOVE_ERR_INVAL; - } -# endif - - fd_groove_data_hdr_t obj_hdr = *_obj_hdr; - - ulong obj_type = fd_groove_data_hdr_type( obj_hdr ); - ulong obj_idx = fd_groove_data_hdr_idx ( obj_hdr ); - ulong obj_szc = fd_groove_data_hdr_szc ( obj_hdr ); - -#if FD_GROOVE_PARANOID - if( FD_UNLIKELY( !((obj_type==exp_type) & (obj_szcbits = 0UL; /* sets object type to an invalid value */ - FD_COMPILER_MFENCE(); -# endif - - FD_COMPILER_MFENCE(); -# if FD_HAS_ATOMIC - free_objs = FD_ATOMIC_FETCH_AND_ADD( _free_objs, obj ); /* Marginally better asm than FETCH_AND_OR */ -# else - free_objs = *_free_objs; - *_free_objs = free_objs | obj; -# endif - FD_COMPILER_MFENCE(); - -# if FD_GROOVE_PARANOID - if( FD_UNLIKELY( (free_objs & obj) | fd_ulong_shift_right( free_objs, (int)obj_cnt ) ) ) { - FD_LOG_WARNING(( "%s", (free_objs & obj) ? "possible concurrent double free" : "invalid free_objs bit field" )); - return FD_GROOVE_ERR_CORRUPT; - } -# endif - - /* At this point, we've freed the object. We might need to get the - object's superblock back into circulation and/or free up excess - empty superblocks for this sizeclass. */ - - ulong free_cnt = (ulong)fd_ulong_popcnt( free_objs ); - - if( FD_UNLIKELY( !free_cnt ) ) { - - /* At this point, the superblock was full before we freed it and - thus not in circulation for use by fd_alloc. We need to get the - superblock back into circulation. There are options for this - with various subtle tradeoffs. See fd_alloc.c for details. - (FIXME: amongst then, consider using allocation cgroup instead of - the groove data instance's cgroup?) */ - - ulong cgroup = fd_groove_data_cgroup_hint( data ) & (ulong)fd_groove_data_szc_cfg[ obj_szc ].cgroup_mask; - - ulong displaced_superblock_off = - fd_groove_data_private_active_displace( data->active_slot + obj_szc + FD_GROOVE_DATA_SZC_CNT*cgroup, - _volume0, (ulong)_superblock_hdr - (ulong)_volume0 ); - - if( FD_UNLIKELY( displaced_superblock_off ) ) - fd_groove_data_private_inactive_push( data->inactive_stack + obj_szc, _volume0, displaced_superblock_off ); - - } else if( FD_UNLIKELY( (free_cnt+1UL)==obj_cnt ) ) { - - /* At this point, the superblock was completely empty after we freed - from it, hence it is still in circulation. If there is also a - completely empty superblock on top of the inactive stack, we free - that one for general reuse. This is more subtle than it looks, - see fd_alloc.c for details. */ - - ulong volatile * _inactive_stack = data->inactive_stack + obj_szc; - - ulong superblock_off = fd_groove_data_private_inactive_pop( _inactive_stack, _volume0 ); - - if( FD_LIKELY( superblock_off ) ) { - - _superblock_hdr = (fd_groove_data_hdr_t *)((ulong)_volume0 + superblock_off); - - _free_objs = (ulong volatile *)(_superblock_hdr+1); - -# if FD_GROOVE_PARANOID - if( FD_UNLIKELY( !( ((ulong)_volume0<(ulong)_superblock_hdr) & ((ulong)_superblock_hdr<(ulong)_volume1) & - fd_ulong_is_aligned( (ulong)_superblock_hdr, FD_GROOVE_BLOCK_ALIGN ) ) ) ) { - FD_LOG_WARNING(( "superblock not at a valid groove data local address" )); - return FD_GROOVE_ERR_CORRUPT; - } - - superblock_type = fd_groove_data_hdr_type( superblock_hdr ); - superblock_szc = fd_groove_data_hdr_szc ( superblock_hdr ); - - if( FD_UNLIKELY( !((superblock_type==FD_GROOVE_DATA_HDR_TYPE_SUPERBLOCK) & (superblock_szc==obj_szc)) ) ) { - FD_LOG_WARNING(( "unexpected superblock header" )); - return FD_GROOVE_ERR_CORRUPT; - } -# endif - - FD_COMPILER_MFENCE(); - free_objs = *_free_objs; - FD_COMPILER_MFENCE(); - -# if FD_GROOVE_PARANOID - if( FD_UNLIKELY( fd_ulong_shift_right( free_objs, (int)obj_cnt ) ) ) { - FD_LOG_WARNING(( "invalid free_objs bit field" )); - return FD_GROOVE_ERR_CORRUPT; - } -# endif - - free_cnt = (ulong)fd_ulong_popcnt( free_objs ); - - if( FD_LIKELY( free_cntmagic ==FD_GROOVE_VOLUME_MAGIC ) & - (_volume->idx*FD_GROOVE_VOLUME_FOOTPRINT==volume_off ) & - (_volume->info_sz <=FD_GROOVE_VOLUME_INFO_MAX) ) ) ) { - FD_LOG_WARNING(( "unexpected volume header" )); - return FD_GROOVE_ERR_CORRUPT; - } -# endif - - FD_COMPILER_MFENCE(); - _volume->magic = ~FD_GROOVE_VOLUME_MAGIC; /* mark volume as containing no groove data allocations */ - FD_COMPILER_MFENCE(); - - fd_groove_volume_pool_release( data->volume_pool, _volume ); - - } - } - } - - return FD_GROOVE_SUCCESS; -} - -#define TEST(c) do { \ - if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "FAIL: %s", #c )); return FD_GROOVE_ERR_CORRUPT; } \ - } while(0) - -/* fd_groove_data_private_verify_superblock verifies the location - superblock_off (relative to _volume0) seems to contain a valid - superblock. groove data is located in the caller's address space at - [_volume0,_volume1). exp_szc gives the expected sizeclass for the - superblock. If in_circulation is non-zero, the superblock is known - to be in circulation (i.e. contains at least one free object / is - either any active or an inactive superblock / is available to alloc - for allocation). Assumes _volume0, _volume1, exp_szc and the - sizeclass configuration are valid. */ - -static int -fd_groove_data_private_verify_superblock( ulong superblock_off, - ulong exp_szc, - int in_circulation, - int verify_descendents, - fd_groove_volume_t const * _volume0, - fd_groove_volume_t const * _volume1 ) { - - /* Verify superblock_off */ - - fd_groove_data_hdr_t const * _superblock_hdr = (fd_groove_data_hdr_t const *)(((ulong)_volume0) + superblock_off); - - TEST( ((ulong)_volume0<(ulong)_superblock_hdr) & ((ulong)_superblock_hdr<(ulong)_volume1) ); - TEST( fd_ulong_is_aligned( (ulong)_superblock_hdr, FD_GROOVE_BLOCK_ALIGN ) ); - - /* Verify superblock header */ - - fd_groove_data_hdr_t hdr = *_superblock_hdr; - - TEST( fd_groove_data_hdr_type( hdr )==FD_GROOVE_DATA_HDR_TYPE_SUPERBLOCK ); - - ulong szc = fd_groove_data_hdr_szc( hdr ); - - TEST( szc==exp_szc ); - - ulong obj_cnt = (ulong)fd_groove_data_szc_cfg[ szc ].obj_cnt; - ulong obj_footprint = (ulong)fd_groove_data_szc_cfg[ szc ].obj_footprint; - ulong parent_szc = (ulong)fd_groove_data_szc_cfg[ szc ].parent_szc; - - ulong parent_obj_idx = fd_groove_data_hdr_idx( hdr ); - ulong parent_obj_cnt = (parent_szcvolume_pool; - ulong const * active_slot = data->active_slot; - ulong const * inactive_stack = data->inactive_stack; - /* cgroup_hint is arbitrary */ - - /* Verify volume pool */ - - TEST( !fd_groove_volume_pool_verify( pool ) ); - - fd_groove_volume_pool_shmem_t const * shpool = (fd_groove_volume_pool_shmem_t const *)fd_groove_volume_pool_shpool_const( pool ); - - fd_groove_volume_t const * _volume0 = (fd_groove_volume_t const *)fd_groove_volume_pool_shele_const ( pool ); - ulong volume_max = fd_groove_volume_pool_ele_max ( pool ); - fd_groove_volume_t const * _volume1 = _volume0 + volume_max; - - TEST( (!!_volume0) | (!volume_max) ); - TEST( _volume0<=_volume1 ); - TEST( fd_ulong_is_aligned( (ulong)_volume0, FD_GROOVE_VOLUME_ALIGN ) ); - - ulong volume_idx = fd_groove_volume_pool_private_vidx_idx( shpool->ver_top ); - while( volume_idxmagic ==FD_GROOVE_DATA_MAGIC ); - TEST( shdata->volume_pool ==shpool ); - TEST( shdata->active_slot ==active_slot ); - TEST( shdata->inactive_stack==inactive_stack ); - - /* Verify sizeclass configuration */ - - for( ulong szc_idx=0UL; szc_idxmagic; - ulong idx = _volume->idx; - ulong info_sz = _volume->info_sz; - - TEST( (magic==FD_GROOVE_VOLUME_MAGIC) | (magic==~FD_GROOVE_VOLUME_MAGIC) ); - TEST( idx*FD_GROOVE_VOLUME_FOOTPRINT==volume_off ); - TEST( info_sz <=FD_GROOVE_VOLUME_INFO_MAX ); - - if( magic==FD_GROOVE_VOLUME_MAGIC ) { - ulong superblock_off = volume_off + FD_GROOVE_BLOCK_FOOTPRINT; - ulong superblock_szc = FD_GROOVE_DATA_SZC_CNT-1UL; - TEST( !fd_groove_data_private_verify_superblock( superblock_off, superblock_szc, 0 /* don't know if in circulation */, - 1 /* verify children */, _volume0, _volume1 ) ); - } - - return FD_GROOVE_SUCCESS; -} - -#undef TEST diff --git a/src/groove/fd_groove_data.h b/src/groove/fd_groove_data.h deleted file mode 100644 index 3b887f4cd10..00000000000 --- a/src/groove/fd_groove_data.h +++ /dev/null @@ -1,509 +0,0 @@ -#ifndef HEADER_fd_src_groove_fd_groove_data_h -#define HEADER_fd_src_groove_fd_groove_data_h - -//#include "fd_groove_base.h" /* includes ../util/fd_util.h */ -#include "fd_groove_meta.h" /* includes fd_groove_base.h */ -#include "fd_groove_volume.h" /* includes fd_groove_base.h */ - -/* A groove data object has a worst case footprint (e.g. 10MiB). This - footprint should be much larger than FD_GROOVE_BLOCK_FOOTPRINT and - much smaller than FD_GROOVE_VOLUME_FOOTPRINT. A groove data object - can have a zero footprint. - - A groove data object has a worst case alignment of at most - FD_GROOVE_BLOCK_ALIGN. - - A groove data object is stored contiguously in a compact range of - groove data blocks. Conversely, a groove data block can hold - information about at most 1 groove data object. - - A groove data object is never split across multiple groove volumes. - - A groove data object has a header describing the object at the - beginning of their first data block. - - Groove data objects of similar size (i.e. in the same "sizeclass") - are grouped together into superblocks. Superblocks of a similar size - are grouped together into larger superblocks. And so on until the - superblock is the size of an entire groove volume. - - Volume sized superblocks are acquired from the lockfree volume pool. - - The lockfree algorithms are viritual identically to fd_alloc but the - superblock nesting has been optimized for HPC memory mapped I/O, - bounded size objects and using the (lockfree) volume pool instead of - the (locking) wksp as the allocator of last resort. */ - -/* fd_groove_data_szc_cfg[ szc ] specifies the configuration of - sizeclass szc. */ - -struct fd_groove_data_szc_cfg { - uint obj_footprint; /* FD_GROOVE_BLOCK_FOOTPRINT multiple, - superblock_footprint = FD_GROOVE_BLOCK_FOOTPRINT + obj_cnt*obj_footprint */ - uchar obj_cnt; /* ==number of objects in the superblock for this sizeclass, in [2,64] */ - uchar cgroup_mask; /* Number of concurrency groups to use for this sizeclass superblock (power-of-2) minus 1 */ - uchar parent_szc; /* Parent size class, SZC_CNT indicates to use an entire volume data region */ -}; - -typedef struct fd_groove_data_szc_cfg fd_groove_data_szc_cfg_t; - -#define FD_GROOVE_DATA_ALLOC_ALIGN_DEFAULT FD_GROOVE_DATA_HDR_ALIGN -#define FD_GROOVE_DATA_ALLOC_ALIGN_MAX FD_GROOVE_BLOCK_ALIGN - -#define FD_GROOVE_DATA_ALLOC_FOOTPRINT_MAX (10486272UL) - -#define FD_GROOVE_DATA_SZC_CNT (32UL) -#define FD_GROOVE_DATA_SZC_CGROUP_MAX (64UL) - -extern fd_groove_data_szc_cfg_t const fd_groove_data_szc_cfg[32]; - -/* A fd_groove_data_hdr encodes groove data object details: - - bits[ 0:15] 16 -> magic type - bits[16:21] 6 -> object idx in parent (0 if parent is a volume) - bits[22:28] 7 -> object sizeclass (type ALLOC), sizeclass of objects in superblock (type SUPERBLOCK) - bits[29:38] 10 -> object align (type ALLOC), DATA_HDR_ALIGN (type SUPERBLOCK) - bits[39:63] 25 -> object sz (type ALLOC), superblock sz (sat to 25-bits) (type SUPERBLOCK) - info 64 -> object tag (type ALLOC), next superblock (type SUPERBLOCK) - - This layout assumes: - - - SZC_CNT <= 128 (such that a size class index fits in 7 bits) - - superblocks have at most 64 objects (such that a object index fits - in 6 bits) - - groove data objects sizes are less than 32 MiB (such that sz fits - in 25 bits) - - groove data objects alignments are at most 512 (such that - align fits in 10 bits) - - Note: The next superblock is given as an offset relative to volume0. - This could be up to machine address width (64 bits). Since this - offset is aligned 512, we could encode additional info there as the - lower 9 bits are zero. - - Note: since alignments are powers of 2, we could use the log2 of the - alignment to compact align further (it would be nice to store - superblock offset in the header but this doesn't save enough space as - the superblock offset needs ~30 bits ... would have to lose details - like object sz and align or use a bigger header). - - Note: Consider an alternate design where these headers are separated - out (e.g. a mirror header for each data block at the start of the - volume similar to wksp or a separate mirror header region entirely). - Advantages would be better isolation / protection of headers, more - advanced analytics / repair / etc. Disadvantages are ~5-6% storage - overhead, more complexity for user to manage. Performance - implications are mixed (less implicit prefetching and more hopping - around but may be faster if header storage region runs on separate - optimized media ... e.g. headers in fast NVMe while data in slow - spinning rust). */ - -#define FD_GROOVE_DATA_HDR_ALIGN (16UL) -#define FD_GROOVE_DATA_HDR_FOOTPRINT (16UL) - -#define FD_GROOVE_DATA_HDR_TYPE_ALLOC (0xfd67UL) -#define FD_GROOVE_DATA_HDR_TYPE_SUPERBLOCK (0x0298UL) - -struct __attribute__((aligned(FD_GROOVE_DATA_HDR_ALIGN))) fd_groove_data_hdr { - - /* Top of a FD_GROOVE_BLOCK */ - - ulong bits; - ulong info; - - /* Lots of room for other hdr type dependent data here. E.g. - FD_GROOVE_DATA_HDR_TYPE_SUPERBLOCK has a ulong free_objs bit field - here (consider moving free_objs and next to different cache line?) */ - -}; - -typedef struct fd_groove_data_hdr fd_groove_data_hdr_t; - -#define FD_GROOVE_DATA_ALIGN (128UL) -#define FD_GROOVE_DATA_FOOTPRINT() sizeof(fd_groove_data_shmem_t) -#define FD_GROOVE_DATA_MAGIC (0xfd67007eda7a36c0UL) /* fd groove data mgc version 0 */ - -struct __attribute__((aligned(FD_GROOVE_DATA_ALIGN))) fd_groove_data_shmem { - - /* This point is FD_GROOVE_DATA_ALIGN aligned */ - - ulong magic; /* ==FD_GROOVE_DATA_MAGIC */ - - /* Padding to FD_GROOVE_DATA_ALIGN alignment */ - - /* active_slot indexed szc+SZC_CNT*cgroup */ - - ulong active_slot[ FD_GROOVE_DATA_SZC_CNT*FD_GROOVE_DATA_SZC_CGROUP_MAX ] __attribute__((aligned(FD_GROOVE_DATA_ALIGN))); - - /* Padding to FD_GROOVE_DATA_ALIGN alignment */ - - /* Since superblocks offsets are all aligned 512, we use the least - significant 9 bits of inactive_stack as an ABA tag. We could also - restrict the maximum size data to like 50B * 10MiB and move to most - significant bits for a ~14 bit wide tag. We could use 128-bit wide - inactive_stack, falling back on a weakly locking implementation if - the target doesn't have FD_HAS_INT128). */ - - ulong inactive_stack[ FD_GROOVE_DATA_SZC_CNT ] __attribute__((aligned(FD_GROOVE_DATA_ALIGN))); - - /* Padding to FD_GROOVE_DATA_ALIGN==FD_GROOVE_VOLUME_POOL_ALIGN alignment */ - - fd_groove_volume_pool_shmem_t volume_pool[1]; - - /* Padding to FD_GROOVE_DATA_ALIGN alignment */ - -}; - -typedef struct fd_groove_data_shmem fd_groove_data_shmem_t; - -struct fd_groove_data { - fd_groove_volume_pool_t volume_pool[1]; /* volume_pool local join (shele is volume0) */ - ulong * active_slot; /* active slot for sizeclass szc and concurrency group cgroup at - active_slot + szc + SZC_CNT*cgroup in the local address */ - ulong * inactive_stack; /* inactive stack for sizeclass szc at inactive_stack + szc in local address space */ - ulong cgroup_hint; /* cgroup_hint for this join */ -}; - -typedef struct fd_groove_data fd_groove_data_t; - -FD_PROTOTYPES_BEGIN - -/* fd_groove_data_{align,footprint} returns the alignment and footprint - needed for a memory region to hold a fd_groove_data's state. align - will be an integer power-of-two and footprint will be a multiple of - align. - - fd_groove_data_new formats a memory region with the appropriate - alignment and footprint into a fd_groove_data. shmem points in the - the caller's address space of the memory region to format. Returns - shmem on success (fd_groove_data has ownership of the memory region) - and NULL on failure (no changes, logs details). Caller is not joined - on return. The fd_groove_data will contain no volumes and have no - data allocations. - - fd_groove_data_join joins a fd_groove_data. ljoin points to a - fd_groove_data_t compatible memory region in the caller's address - space used to hold the local join's state, shdata points in the - caller's address space to the memory region containing the - fd_groove_data, volume0 points in the caller's address space reserved - for mapping groove volumes, volume_max is the maximum number of - volumes can be mapped in the caller's address space starting at - volume0 (0 indicates to use as maximal default), and cgroup_hint is - the concurrency group hint (see fd_alloc for details). Returns a - handle to the caller's local join on success (join has ownership of - the ljoin region) and NULL on failure (no changes, logs details). - - fd_groove_data_leave leaves a fd_groove_data. join points to a - current local join. Returns the memory used for the local join - (caller has ownership on return and caller is no longer joined) on - success and NULL on failure (no changes, logs details). Use the join - accessors before leaving to get shdata, volume0 and cgroup_hint used - by the join if needed. - - fd_groove_data_delete unformats a memory region used as a - fd_groove_data. Assumes shdata points in the caller's address space - to the memory region containing the fd_groove_data and that there are - no current joins globally. Returns shdata on success (caller has - ownership of the memory region, any volumes in the groove and any - groove data objects in these volumes) and NULL on failure (no - ownership changes, logs details). */ - -/* FIXME: SHOULD HAVE A WAY FOR A JOIN TO TELL THE CALLER WHICH VOLUMES - NEED TO BE MAPPED INTO THE CALLER'S ADDRESS SPACE (E.G. A VERSIONED - PMAP AND HAVE CALLS INDICATE TO APPLICATION TO REMAP) */ - -FD_FN_CONST static inline ulong fd_groove_data_align ( void ) { return alignof( fd_groove_data_shmem_t ); } -FD_FN_CONST static inline ulong fd_groove_data_footprint( void ) { return sizeof ( fd_groove_data_shmem_t ); } - -void * fd_groove_data_new ( void * shmem ); -fd_groove_data_t * fd_groove_data_join ( void * ljoin, void * shdata, void * volume0, ulong volume_max, ulong cgroup_hint ); -void * fd_groove_data_leave ( fd_groove_data_t * join ); -void * fd_groove_data_delete( void * shdata ); - -/* fd_groove_data_{shdata,volume0,volume_max,cgroup_hint} return - {shdata,volume0,volume_max,cgroup_hint} used to join a - fd_groove_data. fd_groove_data_volume1 returns the end of the groove - region such that [volume0,volume1) is the groove data region. - Assumes data is a current local join. - fd_groove_data_{shdata,volume0,volume1}_const are const correct - versions. shdata,volume0,volume1 are in the caller's address space. */ - -FD_FN_PURE static inline void const * -fd_groove_data_shdata_const( fd_groove_data_t const * data ) { - return (void const *)((ulong)data->active_slot-FD_GROOVE_DATA_ALIGN); -} - -FD_FN_PURE static inline void const * -fd_groove_data_volume0_const( fd_groove_data_t const * data ) { - return fd_groove_volume_pool_shele_const( data->volume_pool ); -} - -FD_FN_PURE static inline void const * -fd_groove_data_volume1_const( fd_groove_data_t const * data ) { - return (void const *)( (fd_groove_volume_t *)fd_groove_volume_pool_shele_const( data->volume_pool ) - + fd_groove_volume_pool_ele_max( data->volume_pool ) ); -} - -FD_FN_PURE static inline ulong -fd_groove_data_volume_max( fd_groove_data_t const * data ) { - return fd_groove_volume_pool_ele_max( data->volume_pool ); -} - -FD_FN_PURE static inline ulong -fd_groove_data_cgroup_hint( fd_groove_data_t const * data ) { - return data->cgroup_hint; -} - -FD_FN_PURE static inline void * -fd_groove_data_shdata( fd_groove_data_t * data ) { - return (void *)((ulong)data->active_slot-FD_GROOVE_DATA_ALIGN); -} - -FD_FN_PURE static inline void * -fd_groove_data_volume0( fd_groove_data_t * data ) { - return fd_groove_volume_pool_shele( data->volume_pool ); -} - -FD_FN_PURE static inline void * -fd_groove_data_volume1( fd_groove_data_t * data ) { - return (void *)( (fd_groove_volume_t *)fd_groove_volume_pool_shele( data->volume_pool ) + - + fd_groove_volume_pool_ele_max( data->volume_pool ) ); -} - -/* fd_groove_data_hdr packs the given fields into a fd_groove_hdr_t. */ - -FD_FN_CONST static inline fd_groove_data_hdr_t -fd_groove_data_hdr( ulong type, /* assumed in [0,2^16) */ - ulong idx, /* assumed in [0,2^ 6) */ - ulong szc, /* assumed in [0,2^ 7) */ - ulong align, /* assumed in [0,2^10) */ - ulong sz, /* assumed in [0,2^25) */ - ulong info ) { /* arbitrary */ - fd_groove_data_hdr_t hdr; - hdr.bits = type | (idx<<16) | (szc<<22) | (align<<29) | (sz<<39); - hdr.info = info; - return hdr; -} - -/* fd_groove_data_hdr_* extract the given field from a fd_groove_hdr_t. */ - -FD_FN_CONST static inline ulong /* <2^16 */ fd_groove_data_hdr_type ( fd_groove_data_hdr_t h ) { return h.bits & 65535UL; } -FD_FN_CONST static inline ulong /* <2^ 6 */ fd_groove_data_hdr_idx ( fd_groove_data_hdr_t h ) { return (h.bits>>16) & 63UL; } -FD_FN_CONST static inline ulong /* <2^ 7 */ fd_groove_data_hdr_szc ( fd_groove_data_hdr_t h ) { return (h.bits>>22) & 127UL; } -FD_FN_CONST static inline ulong /* <2^10 */ fd_groove_data_hdr_align( fd_groove_data_hdr_t h ) { return (h.bits>>29) & 1023UL; } -FD_FN_CONST static inline ulong /* <2^25 */ fd_groove_data_hdr_sz ( fd_groove_data_hdr_t h ) { return h.bits>>39 ; } -FD_FN_CONST static inline ulong /* arb */ fd_groove_data_hdr_info ( fd_groove_data_hdr_t h ) { return h.info ; } - -/* fd_groove_data_{object,superblock}_hdr returns the header for a - groove object / object superblock. Assumes obj points to the first - object byte in the caller's address space. obj_szc is the object's - sizeclass and parent_idx is the object's index in its parent - superblock (and thus assumes the object is stored in a superblock). - fd_groove_data_{alloc,superblock}_hdr_const are const correct - versions. */ - -FD_FN_CONST static inline fd_groove_data_hdr_t * -fd_groove_data_object_hdr( void * obj ) { - return (fd_groove_data_hdr_t *)fd_ulong_align_dn( (ulong)obj - FD_GROOVE_DATA_HDR_FOOTPRINT, FD_GROOVE_BLOCK_ALIGN ); -} - -FD_FN_PURE static inline fd_groove_data_hdr_t * -fd_groove_data_superblock_hdr( void * obj, - ulong obj_szc, - ulong parent_idx ) { - return (fd_groove_data_hdr_t *) - ( fd_ulong_align_dn( (ulong)obj - FD_GROOVE_DATA_HDR_FOOTPRINT, FD_GROOVE_BLOCK_ALIGN ) - - parent_idx*(ulong)fd_groove_data_szc_cfg[ obj_szc ].obj_footprint - FD_GROOVE_BLOCK_FOOTPRINT ); -} - -FD_FN_CONST static inline fd_groove_data_hdr_t const * -fd_groove_data_object_hdr_const( void const * obj ) { - return (fd_groove_data_hdr_t const *)fd_ulong_align_dn( (ulong)obj - FD_GROOVE_DATA_HDR_FOOTPRINT, FD_GROOVE_BLOCK_ALIGN ); -} - -FD_FN_PURE static inline fd_groove_data_hdr_t const * -fd_groove_data_superblock_hdr_const( void const * obj, - ulong obj_szc, - ulong parent_idx ) { - return (fd_groove_data_hdr_t const *) - ( fd_ulong_align_dn( (ulong)obj - FD_GROOVE_DATA_HDR_FOOTPRINT, FD_GROOVE_BLOCK_ALIGN ) - - parent_idx*(ulong)fd_groove_data_szc_cfg[ obj_szc ].obj_footprint - FD_GROOVE_BLOCK_FOOTPRINT ); -} - -/* fd_groove_data_szc returns the index of the tightest fitting - sizeclass for footprint. The caller promises there is at least one - suitable sizeclass (i.e. footprint<=cfg[SZC_CNT-1].obj_footprint). - The return will be in [0,FD_GROOVE_DATA_SZC_CNT). */ - -static inline ulong -fd_groove_data_szc( ulong footprint ) { - ulong l = 0UL; - ulong h = FD_GROOVE_DATA_SZC_CNT - 1UL; - - /* Fixed count loop without early exit to make it easy for compiler to - unroll and nominally eliminate all branches for fast, highly - deterministic performance with no consumption of BTB resources. - FIXME: check the compiler is doing the right thing here. */ - - for( ulong r=0UL; r<5UL; r++ ) { /* Assumes SZC_CNT<=32 */ - - /* At this point sizeclasses in [0,l) are known too small and - sizeclasses [h,SZC_CNT) are known large enough. Sizeclasses in - [l,h) have not been tested. The size of the range will be - decreased to at least floor((h-l)/2) every iteration. If this - range is empty, m==l==h and c==1 as there m must be the tightest - size class such that the l/h updates are no-ops. */ - - ulong m = (l+h)>>1; /* No overflow for reasonable SZC_CNT */ - int c = (((ulong)fd_groove_data_szc_cfg[ m ].obj_footprint)>=footprint); - l = fd_ulong_if( c, l, m+1UL ); /* cmov */ - h = fd_ulong_if( c, m, h ); /* cmov */ - } - - return h; -} - -/* fd_groove_data_volume_{add,remove} are just thin wrappers around the - volume pool APIs for data's volume_pool. See the volume_pool API for - details. */ - -static inline int -fd_groove_data_volume_add( fd_groove_data_t * data, - void * volume, - ulong footprint, - void const * info, - ulong info_sz ) { - return fd_groove_volume_pool_add( data ? data->volume_pool : NULL, volume, footprint, info, info_sz ); -} - -static inline void * -fd_groove_data_volume_remove( fd_groove_data_t * data ) { - return fd_groove_volume_pool_remove( data ? data->volume_pool : NULL ); -} - -/* fd_groove_data_alloc creates a groove data object in the groove data - store with the given alignment, size and arbitrary user tag. Align - should be an integer power of 2 of at most - FD_GROOVE_DATA_ALLOC_ALIGN_MAX or 0 (0 indicates to use - FD_GROOVE_DATA_ALLOC_ALIGN_DEFAULT). Assumes data is current local - join. Because every allocation is independently tagged, a zero size - allocation will produce a unique non-NULL return. - - On success, returns a pointer in the caller's address to the created - object (will be aligned align with room for sz bytes). The lifetime - of the _object_ is until it is freed or the data store is destroyed. - The lifetime of the local _pointer_ is until the object is freed or - the join is left. If _opt_err is non-NULL, *_opt_err will be - FD_GROOVE_SUCCESS (zero). - - On failure, returns NULL. If _opt_err is non-NULL, *_opt_err will be - an FD_GROOVE_ERR code (negative). Reasons for failure include: - INVAL (logged) - bad input args (NULL data, align not power of 2, align too large, sz too large) - FULL (silent) - groove too full for allocation right now */ - -void * -fd_groove_data_alloc( fd_groove_data_t * data, - ulong align, - ulong sz, - ulong tag, - int * _opt_err ); - -/* fd_groove_data_free frees a groove data object in the groove data - store. Assumes obj points to the first byte in the object in the - caller's address space. On return, obj is no longer a groove data - obj. On success, returns FD_GROOVE_SUCCESS (0) and a FD_GROOVE_ERR - code (negative) on failure. Reasons for failure include: - - INVAL - NULL data (logged), NULL obj (silent), all sorts of - unexpected conditions that suggest buggy usage and/or data - corruption (logged), groove data state was not changed. - - CORRUPT - all sorts of unexpected conditions that suggest buggy - usage and/or data corruption (logged), groove data state might - have been changed before the unexpected condition was detected. */ - -int -fd_groove_data_private_free( fd_groove_data_t * data, - void * obj, - ulong exp_type ); - -static inline int -fd_groove_data_free( fd_groove_data_t * data, - void * obj ) { - return fd_groove_data_private_free( data, obj, FD_GROOVE_DATA_HDR_TYPE_ALLOC ); -} - -/* fd_groove_data_{align,sz,tag} return the values used when obj was - allocated. Assumes obj points in the caller's address space to a - current allocation in the groove data. */ - -FD_FN_PURE static inline ulong -fd_groove_data_alloc_align( void const * obj ) { - return fd_groove_data_hdr_align( *fd_groove_data_object_hdr_const( obj ) ); -} - -FD_FN_PURE static inline ulong -fd_groove_data_alloc_sz( void const * obj ) { - return fd_groove_data_hdr_sz( *fd_groove_data_object_hdr_const( obj ) ); -} - -FD_FN_PURE static inline ulong -fd_groove_data_alloc_tag( void const * obj ) { - return fd_groove_data_hdr_info( *fd_groove_data_object_hdr_const( obj ) ); -} - -/* fd_groove_data_{start,stop} return the actual range of addresses - [start,stop) in the caller's address space actually reserved for the - allocation obj. [obj,obj+sz) is completely covered by this region - (and obj will be aligned appropriately). - - fd_groove_data_{start,stop}_const are const correct versions. */ - -FD_FN_CONST static inline void * -fd_groove_data_alloc_start( void * obj ) { - return (void *)((ulong)fd_groove_data_object_hdr_const( obj ) + 16UL); -} - -FD_FN_PURE static inline void * -fd_groove_data_alloc_stop( void * obj ) { - fd_groove_data_hdr_t const * hdr = fd_groove_data_object_hdr_const( obj ); - return (void *)((ulong)hdr + (ulong)fd_groove_data_szc_cfg[ fd_groove_data_hdr_szc( *hdr ) ].obj_footprint); -} - -FD_FN_CONST static inline void const * -fd_groove_data_alloc_start_const( void const * obj ) { - return (void const *)((ulong)fd_groove_data_object_hdr_const( obj ) + 16UL); -} - -FD_FN_PURE static inline void const * -fd_groove_data_alloc_stop_const( void const * obj ) { - fd_groove_data_hdr_t const * hdr = fd_groove_data_object_hdr_const( obj ); - return (void const *)((ulong)hdr + (ulong)fd_groove_data_szc_cfg[ fd_groove_data_hdr_szc( *hdr ) ].obj_footprint); -} - -/* fd_groove_data_verify returns FD_GROOVE_SUCCESS if join appears to be - current local join to a valid groove data instance and - FD_GROOVE_ERR_CORRUPT otherwise (logs details). Assumes join is a - current local join and the groove data is idle. This only verifies - the groove data's state. Specifically, it verifies the join, the - data volume pool, the active superblocks and the inactive superblocks - look correct. It is does verify the entire contains of all volumes. - Use fd_groove_data_verify for that. */ - -int -fd_groove_data_verify( fd_groove_data_t const * data ); - -/* fd_groove_data_volume_verify returns FD_GROOVE_SUCCESS if the - groove volume mapped into the caller's address at _volume is appears - to be a valid groove volume and FD_GROOVE_ERR_CORRUPT otherwise (logs - details). Assumes join is a current local join and the groove data - is idle. It is fine to verify volumes in parallel (e.g. use hundreds - of cores to verify petabytes of groove data). */ - -int -fd_groove_data_volume_verify( fd_groove_data_t const * data, - fd_groove_volume_t const * _volume ); - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_groove_fd_groove_data_h */ diff --git a/src/groove/fd_groove_data_szc_cfg.c b/src/groove/fd_groove_data_szc_cfg.c deleted file mode 100644 index c7e2f9ddcf7..00000000000 --- a/src/groove/fd_groove_data_szc_cfg.c +++ /dev/null @@ -1,38 +0,0 @@ -#include "fd_groove_data.h" - -/* This table autogenerated */ - -fd_groove_data_szc_cfg_t const fd_groove_data_szc_cfg[32] = { - /* 0 */ { 512U, (uchar)38, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 1 */ { 1024U, (uchar)19, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 2 */ { 1536U, (uchar)12, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 3 */ { 2048U, (uchar) 9, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 4 */ { 2560U, (uchar) 7, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 5 */ { 3072U, (uchar) 6, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 6 */ { 3584U, (uchar) 5, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 7 */ { 4608U, (uchar) 4, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 8 */ { 6144U, (uchar) 3, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 9 */ { 9728U, (uchar) 2, (uchar)63, (uchar)11 }, // sb_footprint 19968U - /* 10 */ { 13312U, (uchar) 3, (uchar)31, (uchar)13 }, // sb_footprint 40448U - /* 11 */ { 19968U, (uchar) 2, (uchar)31, (uchar)13 }, // sb_footprint 40448U - /* 12 */ { 26624U, (uchar) 3, (uchar)15, (uchar)15 }, // sb_footprint 81408U - /* 13 */ { 40448U, (uchar) 2, (uchar)15, (uchar)15 }, // sb_footprint 81408U - /* 14 */ { 54272U, (uchar) 3, (uchar) 7, (uchar)18 }, // sb_footprint 163840U - /* 15 */ { 81408U, (uchar) 2, (uchar) 7, (uchar)18 }, // sb_footprint 163840U - /* 16 */ { 81920U, (uchar) 4, (uchar) 3, (uchar)20 }, // sb_footprint 328192U - /* 17 */ { 109056U, (uchar) 3, (uchar) 3, (uchar)20 }, // sb_footprint 328192U - /* 18 */ { 163840U, (uchar) 2, (uchar) 3, (uchar)20 }, // sb_footprint 328192U - /* 19 */ { 218624U, (uchar) 3, (uchar) 1, (uchar)22 }, // sb_footprint 657408U - /* 20 */ { 328192U, (uchar) 2, (uchar) 1, (uchar)22 }, // sb_footprint 657408U - /* 21 */ { 438272U, (uchar) 3, (uchar) 0, (uchar)24 }, // sb_footprint 1315328U - /* 22 */ { 657408U, (uchar) 2, (uchar) 0, (uchar)24 }, // sb_footprint 1315328U - /* 23 */ { 876544U, (uchar) 3, (uchar) 0, (uchar)26 }, // sb_footprint 2631168U - /* 24 */ { 1315328U, (uchar) 2, (uchar) 0, (uchar)26 }, // sb_footprint 2631168U - /* 25 */ { 1754112U, (uchar) 3, (uchar) 0, (uchar)28 }, // sb_footprint 5262848U - /* 26 */ { 2631168U, (uchar) 2, (uchar) 0, (uchar)28 }, // sb_footprint 5262848U - /* 27 */ { 3508224U, (uchar) 3, (uchar) 0, (uchar)30 }, // sb_footprint 10526208U - /* 28 */ { 5262848U, (uchar) 2, (uchar) 0, (uchar)30 }, // sb_footprint 10526208U - /* 29 */ { 7017472U, (uchar) 3, (uchar) 0, (uchar)31 }, // sb_footprint 21053440U - /* 30 */ { 10526208U, (uchar) 2, (uchar) 0, (uchar)31 }, // sb_footprint 21053440U - /* 31 */ { 21053440U, (uchar)51, (uchar) 0, (uchar)32 }, // sb_footprint 1073741312U -}; diff --git a/src/groove/fd_groove_meta.c b/src/groove/fd_groove_meta.c deleted file mode 100644 index 0742cccfb10..00000000000 --- a/src/groove/fd_groove_meta.c +++ /dev/null @@ -1,19 +0,0 @@ -#include "fd_groove_meta.h" - -#define MAP_NAME fd_groove_meta_map -#define MAP_ELE_T fd_groove_meta_t -#define MAP_KEY_T fd_groove_key_t -#define MAP_KEY_EQ fd_groove_key_eq -#define MAP_KEY_HASH fd_groove_key_hash -#define MAP_ELE_IS_FREE(ctx,ele) (!fd_groove_meta_bits_used( (ele)->bits )) -#define MAP_ELE_FREE(ctx,ele) do { (ele)->bits = fd_groove_meta_bits( 0,0,0, 0UL, 0UL ); } while(0) -#define MAP_ELE_MOVE(ctx,dst,src) do { \ - fd_groove_meta_t * _src = (src); \ - *(dst) = *_src; \ - _src->bits = fd_groove_meta_bits( 0,0,0, 0UL, 0UL ); \ - } while(0) -#define MAP_VERSION_T ushort -#define MAP_LOCK_MAX (8192) -#define MAP_MAGIC (0xfd67007e3e7a3a90UL) /* fd groove meta map version 0 */ -#define MAP_IMPL_STYLE 2 -#include "../util/tmpl/fd_map_slot_para.c" diff --git a/src/groove/fd_groove_meta.h b/src/groove/fd_groove_meta.h deleted file mode 100644 index 619a0d82a78..00000000000 --- a/src/groove/fd_groove_meta.h +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef HEADER_fd_src_groove_fd_groove_meta_h -#define HEADER_fd_src_groove_fd_groove_meta_h - -#include "fd_groove_base.h" /* includes ../util/fd_util.h */ - -/* fd_groove_meta_bits API ********************************************/ - -/* The groove key metadata contains a 64-bit wide bitfield used to hold - irregularly sized key metadata compactly. - - bits[ 0: 0] 1 -> used (map slot contains a key-meta pair) - bits[ 1: 1] 1 -> cold (val for key present in cold store) - bits[ 2: 2] 1 -> hot (val for key present in hot store) - bits[ 3:15] 13 -> - (available for additional use) - bits[16:39] 24 -> val_sz (num bytes for key's val) - bits[40:63] 24 -> val_max (max bytes for key's val, 0<=val_sz<=val_max) - - fd_groove_meta_bits pack the components used, cold, hot, val_sz and - val_max into this bitfield. used, cold and hot treat 0/non-zero as - 0/1. val_sz and val_max are assumed in [0,2^24). - - fd_grove_meta_bits_{used,cold,hot,val_sz,val_max} unpack this field - from the bitfield. used, cold and hot return a value in [0,1]. - val_sz and val_max will be in [0,2^24). */ - -FD_PROTOTYPES_BEGIN - -FD_FN_CONST static inline ulong -fd_groove_meta_bits( int used, - int cold, - int hot, - ulong val_sz, - ulong val_max ) { - return ((ulong)!!used) | (((ulong)!!cold)<<1) | (((ulong)!!hot)<<2) | (val_sz<<16) | (val_max<<40); -} - -FD_FN_CONST static inline int fd_groove_meta_bits_used ( ulong bits ) { return (int)( bits & 1UL); } -FD_FN_CONST static inline int fd_groove_meta_bits_cold ( ulong bits ) { return (int)((bits>> 1) & 1UL); } -FD_FN_CONST static inline int fd_groove_meta_bits_hot ( ulong bits ) { return (int)((bits>> 2) & 1UL); } -FD_FN_CONST static inline ulong fd_groove_meta_bits_val_sz ( ulong bits ) { return (bits>>16) & 16777215UL; } -FD_FN_CONST static inline ulong fd_groove_meta_bits_val_max( ulong bits ) { return bits>>40; } - -FD_PROTOTYPES_END - -/* fd_groove_meta API *************************************************/ - -/* FIXME: consider if memoizing is worth speed / footprint tradeoff */ - -struct fd_groove_meta { - fd_groove_key_t key; - ulong bits; /* groove metadata bit field */ - ulong val_off; /* if key's val is in the cold store, cold store bytes [val_off,val_off+val_sz) - hold the current val and bytes [val_off,val_off+val_max) are reserved for key's val. - Thus: 0 <= val_off <= val_off+val_sz <= val_off+val_max <= cold store addr space sz. - Further val's reserved bytes will all reside within a single cold store volume. */ -}; - -typedef struct fd_groove_meta fd_groove_meta_t; - -#define MAP_NAME fd_groove_meta_map -#define MAP_ELE_T fd_groove_meta_t -#define MAP_KEY_T fd_groove_key_t -#define MAP_KEY_EQ fd_groove_key_eq -#define MAP_KEY_HASH fd_groove_key_hash -#define MAP_ELE_IS_FREE(ctx,ele) (!fd_groove_meta_bits_used( (ele)->bits )) -#define MAP_ELE_FREE(ctx,ele) do (ele)->bits = fd_groove_meta_bits( 0,0,0, 0UL, 0UL ); while(0) -#define MAP_ELE_MOVE(ctx,dst,src) do { \ - fd_groove_meta_t * _src = (src); \ - *(dst) = *_src; \ - _src->bits = fd_groove_meta_bits( 0,0,0, 0UL, 0UL ); \ - } while(0) -#define MAP_VERSION_T ushort -#define MAP_LOCK_MAX (8192) -#define MAP_MAGIC (0xfd67007e3e7a3a90UL) /* fd groove meta map version 0 */ -#define MAP_IMPL_STYLE 1 -#include "../util/tmpl/fd_map_slot_para.c" - -#endif /* HEADER_fd_src_groove_fd_groove_meta_h */ diff --git a/src/groove/fd_groove_volume.c b/src/groove/fd_groove_volume.c deleted file mode 100644 index 4c9777412de..00000000000 --- a/src/groove/fd_groove_volume.c +++ /dev/null @@ -1,106 +0,0 @@ -#include "fd_groove_volume.h" - -#define POOL_NAME fd_groove_volume_pool -#define POOL_ELE_T fd_groove_volume_t -#define POOL_IDX_WIDTH (34) -#define POOL_MAGIC (0xfd67007e70190010UL) /* fd groove vol pool version 0 */ -#define POOL_IMPL_STYLE 2 -#include "../util/tmpl/fd_pool_para.c" - -int -fd_groove_volume_pool_add( fd_groove_volume_pool_t * pool, - void * shmem, - ulong footprint, - void const * info, - ulong info_sz ) { - - if( FD_UNLIKELY( !footprint ) ) return FD_GROOVE_SUCCESS; /* Nothing to do */ - - if( FD_UNLIKELY( !pool ) ) { - FD_LOG_WARNING(( "NULL pool" )); - return FD_GROOVE_ERR_INVAL; - } - - fd_groove_volume_t * _volume0 = (fd_groove_volume_t *)fd_groove_volume_pool_shele( pool ); - fd_groove_volume_t * _volume1 = _volume0 + fd_groove_volume_pool_ele_max( pool ); - - fd_groove_volume_t * _va = (fd_groove_volume_t *)shmem; - fd_groove_volume_t * _vb = (fd_groove_volume_t *)((ulong)shmem + footprint); - - if( FD_UNLIKELY( !( (_volume0<=_va) & (_va<_vb) & (_vb<=_volume1) & - fd_ulong_is_aligned( (ulong)_va-(ulong)_volume0, FD_GROOVE_VOLUME_FOOTPRINT ) & - fd_ulong_is_aligned( (ulong)_vb-(ulong)_volume0, FD_GROOVE_VOLUME_FOOTPRINT ) ) ) ) { - FD_LOG_WARNING(( "Invalid region" )); - return FD_GROOVE_ERR_INVAL; - } - - info_sz = fd_ulong_min( fd_ulong_if( !!info, info_sz, 0UL ), FD_GROOVE_VOLUME_INFO_MAX ); - - /* Format the volumes as empty and push into the free pool. We push - them in reverse order so the volumes will be preferentially used - from lowest to highest in future allocations. */ - - fd_groove_volume_t * _volume = _vb; - do { - _volume--; - - _volume->idx = (ulong)(_volume - _volume0); - //_volume->next initialized when released into the pool - _volume->info_sz = info_sz; - - memset( _volume->info, 0, FD_GROOVE_VOLUME_INFO_MAX ); - if( info_sz ) memcpy( _volume->info, info, info_sz ); - - //_volume->data initialized by caller - - FD_COMPILER_MFENCE(); - _volume->magic = ~FD_GROOVE_VOLUME_MAGIC; /* Mark groove volume as containing no data allocations */ - FD_COMPILER_MFENCE(); - - fd_groove_volume_pool_release( pool, _volume ); - - } while( _volume>_va ); - - return FD_GROOVE_SUCCESS; -} - -void * -fd_groove_volume_pool_remove( fd_groove_volume_pool_t * pool ) { - - if( FD_UNLIKELY( !pool ) ) { - FD_LOG_WARNING(( "NULL pool" )); - return NULL; - } - - fd_groove_volume_t * _volume = fd_groove_volume_pool_acquire( pool ); - - if( FD_LIKELY( _volume ) ) { - -# if FD_GROOVE_PARANOID - fd_groove_volume_t * _volume0 = fd_groove_volume_pool_shele( pool ); - fd_groove_volume_t * _volume1 = _volume0 + fd_groove_volume_pool_ele_max( pool ); - - ulong volume_off = (ulong)_volume - (ulong)_volume0; - - if( FD_UNLIKELY( !( (_volume0<=_volume) & (_volume<_volume1) & - fd_ulong_is_aligned( volume_off, FD_GROOVE_VOLUME_FOOTPRINT ) ) ) ) { - FD_LOG_WARNING(( "volume not at a valid groove data local address" )); - return NULL; - } - - if( FD_UNLIKELY( !( (_volume->magic ==~FD_GROOVE_VOLUME_MAGIC ) & - (_volume->idx*FD_GROOVE_VOLUME_FOOTPRINT==volume_off ) & - (_volume->info_sz <=FD_GROOVE_VOLUME_INFO_MAX) ) ) ) { - FD_LOG_WARNING(( "unexpected volume header" )); - return NULL; - } -# endif - - FD_COMPILER_MFENCE(); - _volume->magic = 0UL; /* mark as no longer a groove volume */ - FD_COMPILER_MFENCE(); - - } - - return (void *)_volume; -} diff --git a/src/groove/fd_groove_volume.h b/src/groove/fd_groove_volume.h deleted file mode 100644 index 28dc0a623d8..00000000000 --- a/src/groove/fd_groove_volume.h +++ /dev/null @@ -1,149 +0,0 @@ -#ifndef HEADER_fd_src_groove_fd_groove_volume_h -#define HEADER_fd_src_groove_fd_groove_volume_h - -#include "fd_groove_base.h" /* includes ../util/fd_util.h */ - -/* Groove data objects are stored in groove volumes. A volume has a - uniform size of FD_GROOVE_VOLUME_FOOTPRINT and a unique index. A - volume is mapped into the caller's address space starting at - (ulong)volume0+idx*FD_GROOVE_VOLUME_FOOTPRINT where volume0 is the - start of a region in the user's address space with enough room to - accommodate mapping current and future volumes. To support - persistent and IPC usage across different applications, concurrent - users can use different values for volume0. - - Volume indexing does not need to be contiguous or start at zero. - Further, there is no theoretical limit on the number of volumes. - Practically, the above implies that the address region footprint in - the caller's address space is - - (max_idx_in_use+1)*FD_GROOVE_VOLUME_FOOTPRINT. - - As the FD machine model targets 64-bit address spaces, this implies a - groove instance practically can hold at most < 2^64 B (16 EiB). - (Note that 50 billion data objects at a worst case 10 MiB per object - requires less than 0.5 EiB so this is not a limitation in practice.) - - To support commodity hardware and operating systems efficiently, - volume0 should have at least normal page alignment and - FD_GROOVE_VOLUME_FOOTPRINT should be a normal page multiple. For - highest performance on such systems, 1 GiB aligned volume0 with - gigantic page DRAM backed volumes are recommended but this is not - required. E.g. it is fine for volumes to be a memory mapped file - backed by a NVMe/SSD/RAID array. This implementation uses a - FD_GROOVE_VOLUME_FOOTPRINT of 1 GiB. - - Volumes can be backed by different technologies. Note that while - heterogeneous volumes are supported, this implementation does not - treat a volume backed by fast DRAM differently from a volume backed - by a tape drive. - - Empty groove volumes are stored on a lockfree stack to support adding - / removing volumes from a groove instance dynamically without - impacting on-going groove operations. E.g. to increase the groove's - capacity, create a volume at an unused index and lockfree push it - onto the groove's empty volume stack. To decrease capacity, pop the - stack and reclaim the returned volume. */ - -/* FIXME: consider allowing larger volume headers (or adding a - volume footer) to not waste ~15KiB space with padding */ - -#define FD_GROOVE_VOLUME_ALIGN (4096UL) -#define FD_GROOVE_VOLUME_FOOTPRINT (1UL<<30) -#define FD_GROOVE_VOLUME_MAGIC (0xfd67007e701c3300) /* fd groove volume version 0 */ -#define FD_GROOVE_VOLUME_INFO_MAX (FD_GROOVE_BLOCK_FOOTPRINT-32UL) -#define FD_GROOVE_VOLUME_DATA_MAX (FD_GROOVE_VOLUME_FOOTPRINT-FD_GROOVE_BLOCK_FOOTPRINT) - -struct __attribute__((aligned(FD_GROOVE_VOLUME_ALIGN))) fd_groove_volume { - - /* This point is aligned FD_GROOVE_VOLUME_ALIGN >= FD_GROOVE_BLOCK_ALIGN */ - - ulong magic; /* == FD_GROOVE_VOLUME_MAGIC if volume potentially contains groove data allocations, - ==~FD_GROOVE_VOLUME_MAGIC if volume contains no groove data allocations, - other values -> not a volume */ - ulong idx; /* Volume index (this is mapped into the user's address space at groove->volume0 + idx) */ - ulong next; /* Managed by the groove volume pool */ - ulong info_sz; /* in [0,INFO_MAX] */ - - /* This point is aligned 32 */ - - uchar info[ FD_GROOVE_VOLUME_INFO_MAX ]; /* info_sz bytes of user info, bytes [info_sz,INFO_MAX) are arbitrary */ - - /* This point is aligned FD_GROOVE_BLOCK_ALIGN */ - - uchar data[ FD_GROOVE_VOLUME_DATA_MAX ]; -}; - -typedef struct fd_groove_volume fd_groove_volume_t; - -/* Note: given a 2^30 volume footprint, a POOL_IDX_WIDTH of 34 - supports up ~2^34 volumes for up to a ~2^64 groove data storage. - (For the default pool index width, this could go as high as - fd_groove_pool_max_max ~ 2^43 but it would not be possible to memory - map such a set of volumes on 64-bit architectures.) */ - -#define POOL_NAME fd_groove_volume_pool -#define POOL_ELE_T fd_groove_volume_t -#define POOL_IDX_WIDTH (34) -#define POOL_MAGIC (0xfd67007e70190010UL) /* fd groove vol pool version 0 */ -#define POOL_IMPL_STYLE 1 -#include "../util/tmpl/fd_pool_para.c" - -FD_PROTOTYPES_BEGIN - -/* fd_groove_volume_pool_add adds the footprint sized memory region - whose first byte is at shmem in the caller's local address space to - the pool. Assumes pool is a current local join, the storage to add - is mapped into a memory region that starts and stops at a - FD_GROOVE_VOLUME_FOOTPRINT multiple offset in the pool's address - space, and this memory region does not conflict with any current - groove volumes. - - The volume info for these will be initialized to the info_sz bytes - pointed in the caller's local address space by info. info_sz will - treated as zero if info is NULL. info_sz>FD_GROOVE_VOLUME_INFO_MAX - will be treated as FD_GROOVE_VOLUME_INFO_MAX. If - info_szul[0]==limb[0] && ka->ul[1]==limb[1] && ka->ul[2]==limb[2] && ka->ul[3]==limb[3] && - kb->ul[0]==limb[4] && kb->ul[1]==limb[5] && kb->ul[2]==limb[6] && kb->ul[3]==limb[7] ); - - int eq = (ka->ul[0]==kb->ul[0]) && (ka->ul[1]==kb->ul[1]) && (ka->ul[2]==kb->ul[2]) && (ka->ul[3]==kb->ul[3]); - ulong ma = fd_groove_key_hash( ka, seed ); - ulong mb = fd_groove_key_hash( kb, seed ); - - FD_TEST( fd_groove_key_eq( ka, ka )== 1 ); FD_TEST( fd_groove_key_eq( ka, kb )==eq ); - FD_TEST( fd_groove_key_eq( kb, ka )==eq ); FD_TEST( fd_groove_key_eq( kb, kb )== 1 ); - if( eq ) FD_TEST( ma==mb ); - - /* zero padding copy */ - - ulong csz = seed % FD_GROOVE_KEY_FOOTPRINT; - - FD_TEST( fd_groove_key_init( kb, ka, csz )==kb ); - - eq = (ka->ul[0]==kb->ul[0]) && (ka->ul[1]==kb->ul[1]) && (ka->ul[2]==kb->ul[2]) && (ka->ul[3]==kb->ul[3]); - mb = fd_groove_key_hash( kb, seed ); - - for( ulong idx=csz; idxc[idx] ); - - FD_TEST( fd_groove_key_eq( ka, ka )== 1 ); FD_TEST( fd_groove_key_eq( ka, kb )==eq ); - FD_TEST( fd_groove_key_eq( kb, ka )==eq ); FD_TEST( fd_groove_key_eq( kb, kb )== 1 ); - if( eq ) FD_TEST( ma==mb ); - - /* normal copy and truncating copy */ - - csz += FD_GROOVE_KEY_FOOTPRINT; - - FD_TEST( fd_groove_key_init( kb, ka, csz )==kb ); - - eq = 1; - mb = fd_groove_key_hash( kb, seed ); - - FD_TEST( fd_groove_key_eq( ka, ka )== 1 ); FD_TEST( fd_groove_key_eq( ka, kb )==eq ); - FD_TEST( fd_groove_key_eq( kb, ka )==eq ); FD_TEST( fd_groove_key_eq( kb, kb )== 1 ); - if( eq ) FD_TEST( ma==mb ); - } - - fd_rng_delete( fd_rng_leave( rng ) ); - - FD_LOG_NOTICE(( "pass" )); - fd_halt(); - return 0; -} diff --git a/src/groove/test_groove_data.c b/src/groove/test_groove_data.c deleted file mode 100644 index 359e5767aef..00000000000 --- a/src/groove/test_groove_data.c +++ /dev/null @@ -1,498 +0,0 @@ -#include "fd_groove.h" - -FD_STATIC_ASSERT( FD_GROOVE_DATA_SZC_CNT ==32UL, unit_test ); -FD_STATIC_ASSERT( FD_GROOVE_DATA_SZC_CGROUP_MAX==64UL, unit_test ); - -FD_STATIC_ASSERT( FD_GROOVE_DATA_HDR_ALIGN ==16UL, unit_test ); -FD_STATIC_ASSERT( FD_GROOVE_DATA_HDR_FOOTPRINT==16UL, unit_test ); - -FD_STATIC_ASSERT( FD_GROOVE_DATA_HDR_TYPE_ALLOC ==0xfd67, unit_test ); -FD_STATIC_ASSERT( FD_GROOVE_DATA_HDR_TYPE_SUPERBLOCK==0x0298, unit_test ); - -FD_STATIC_ASSERT( FD_GROOVE_DATA_ALLOC_ALIGN_DEFAULT==16UL, unit_test ); -FD_STATIC_ASSERT( FD_GROOVE_DATA_ALLOC_ALIGN_MAX ==512UL, unit_test ); -FD_STATIC_ASSERT( FD_GROOVE_DATA_ALLOC_FOOTPRINT_MAX==(10UL<<20)+512UL, unit_test ); - -FD_STATIC_ASSERT( alignof(fd_groove_data_hdr_t)==FD_GROOVE_DATA_HDR_ALIGN, unit_test ); -FD_STATIC_ASSERT( sizeof (fd_groove_data_hdr_t)==FD_GROOVE_DATA_HDR_FOOTPRINT, unit_test ); - -FD_STATIC_ASSERT( FD_GROOVE_DATA_MAGIC==0xfd67007eda7a36c0, unit_test ); - -#define SHMEM_MAX (1UL<<20) - -static FD_TL uchar shmem[ SHMEM_MAX ]; -static FD_TL ulong shmem_cnt = 0UL; - -static void * -shmem_alloc( ulong a, - ulong s ) { - uchar * m = (uchar *)fd_ulong_align_up( (ulong)(shmem + shmem_cnt), a ); - shmem_cnt = (ulong)((m + s) - shmem); - FD_TEST( shmem_cnt <= SHMEM_MAX ); - return (void *)m; -} - -static ulong volume_avail_pmap = 0UL; - -static int -grow_data( fd_groove_data_t * data, - fd_rng_t * rng ) { - - fd_groove_volume_t * volume0 = (fd_groove_volume_t *)fd_groove_data_volume0( data ); - int volume_cnt = (int)fd_groove_data_volume_max( data ); /* Assumes<=64 */ - - /* Pick an available volume uniform IID random in a thread safe way */ - - int idx = -1; - - FD_TURNSTILE_BEGIN( 1 /* blocking */ ) { - - if( FD_LIKELY( volume_avail_pmap ) ) { - int sr = fd_rng_int_roll( rng, volume_cnt ); /* In [0,volume_cnt) */ - int sl = (volume_cnt-sr) & 63; /* In [0,volume_cnt), sl + sr = (volume_cnt % 64) */ - idx = fd_ulong_find_lsb( (volume_avail_pmap >> sr) | (volume_avail_pmap << sl) ) + sr; - idx -= fd_int_if( idx>=volume_cnt, volume_cnt, 0 ); - volume_avail_pmap ^= (1UL<> (64-sz_width), sz_max ); - ulong tag = fd_rng_ulong( rng ); - - /* Allocate it */ - - int err; - mem = (uchar *)fd_groove_data_alloc( data, align, sz, tag, &err ); - - if( err==FD_GROOVE_ERR_FULL ) { - FD_TEST( !mem ); - grow_data( data, rng ); - i--; /* do iteration over */ - FD_VOLATILE( test_slot[ idx ].lock ) = 0UL; - continue; - } - - align = fd_ulong_if( !!align, align, FD_GROOVE_DATA_ALLOC_ALIGN_DEFAULT ); - - FD_TEST( mem ); - FD_TEST( fd_ulong_is_aligned( (ulong)mem, align ) ); - FD_TEST( !err ); - - FD_TEST( fd_groove_data_alloc_align( mem )==align ); - FD_TEST( fd_groove_data_alloc_sz ( mem )==sz ); - FD_TEST( fd_groove_data_alloc_tag ( mem )==tag ); - - uchar * p = (uchar *)fd_groove_data_alloc_start( mem ); - uchar * stop = (uchar *)fd_groove_data_alloc_stop ( mem ); - - FD_TEST( (ulong)p <= (ulong)mem ); - FD_TEST( (ulong)mem <= (ulong)mem+sz ); - FD_TEST( (ulong)mem+sz <= (ulong)stop ); - - /* Fill it with a bit pattern unique to this allocation */ - - ulong pat = fd_ulong_hash( (tile_idx<<32) | (uint)tag ); - for( ; (p+7UL)page_sz; - page_cnt = info->page_cnt; - volume_cnt = (page_sz*page_cnt) / FD_GROOVE_VOLUME_FOOTPRINT; - - } else { - - FD_LOG_NOTICE(( "--name not specified, using anonymous shmem (--volume-cnt %lu --page-sz %s --near-cpu %lu)", - volume_cnt, _page_sz, near_cpu )); - - page_sz = fd_cstr_to_shmem_page_sz( _page_sz ); - if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "invalid page_sz" )); - page_cnt = (volume_cnt*FD_GROOVE_VOLUME_FOOTPRINT+page_sz-1UL) / page_sz; - volume = (fd_groove_volume_t *)fd_shmem_acquire_multi( page_sz, 1UL, &page_cnt, &near_cpu ); /* logs details */ - if( FD_UNLIKELY( !volume ) ) FD_LOG_ERR(( "fd_shmem_join failed" )); - - } - - volume_avail_pmap = fd_ulong_mask_lsb( (int)volume_cnt ); - - FD_LOG_NOTICE(( "Testing with %lu groove data volumes", volume_cnt )); - - FD_LOG_NOTICE(( "Testing groove data construction" )); - - ulong align = fd_groove_data_align(); FD_TEST( fd_ulong_is_pow2( align ) ); - ulong footprint = fd_groove_data_footprint(); FD_TEST( fd_ulong_is_aligned( footprint, align ) ); - - FD_TEST( !fd_groove_data_new( NULL ) ); /* NULL shmem */ - FD_TEST( !fd_groove_data_new( (void *)1UL ) ); /* misaligned shmem */ - - void * shdata = shmem_alloc( align, footprint ); - FD_TEST( fd_groove_data_new( shdata )==shdata ); - - fd_groove_data_t data[1]; - - ulong cgroup_hint = fd_rng_ulong( rng ); - - FD_TEST( !fd_groove_data_join( NULL, shdata, volume, volume_cnt, cgroup_hint ) ); /* NULL ljoin */ - FD_TEST( !fd_groove_data_join( (void *)1UL, shdata, volume, volume_cnt, cgroup_hint ) ); /* misaligned ljoin */ - FD_TEST( !fd_groove_data_join( data, NULL, volume, volume_cnt, cgroup_hint ) ); /* NULL shdata */ - FD_TEST( !fd_groove_data_join( data, (void *)1UL, volume, volume_cnt, cgroup_hint ) ); /* misaligned shdata */ - /* bad magic tested below */ - FD_TEST( !fd_groove_data_join( data, shdata, NULL, volume_cnt, cgroup_hint ) ); /* NULL volume */ - FD_TEST( !fd_groove_data_join( data, shdata, (void *)1UL, volume_cnt, cgroup_hint ) ); /* misaligned volume */ - FD_TEST( !fd_groove_data_join( data, shdata, volume, ULONG_MAX, cgroup_hint ) ); /* too large volume_cnt */ - /* cgroup_hint arbitrary */ - - FD_TEST( fd_groove_data_join( data, shdata, volume, volume_cnt, cgroup_hint )==data ); - - FD_TEST( !fd_groove_data_verify( data ) ); - for( ulong idx=0UL; idx>= 4; - int flags = (int)r; r >>= 32; - int blocking = !!(flags & FD_MAP_FLAG_BLOCKING); - int use_hint = !!(flags & FD_MAP_FLAG_USE_HINT); - int rdonly = !!(flags & FD_MAP_FLAG_RDONLY); - - fd_groove_meta_map_query_t query[1]; - - switch( op ) { - - case 0: { /* blocking read / bad insert (i.e. key already in map) */ - if( FD_UNLIKELY( !map_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, map_cnt ); - fd_groove_key_t * key = map_key + idx; - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_prepare( map, key, sentinel, query, flags ); - fd_groove_meta_t * ele = fd_groove_meta_map_query_ele( query ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - } else { - FD_TEST( (ulong)(ele-ele0)key, key ) ); - FD_TEST( fd_groove_meta_bits_used( ele->bits ) ); - FD_TEST( ele->val_off==(key->ul[0] ^ (ele->bits>>1)) ); - - fd_groove_meta_map_cancel( query ); - } - - break; - } - - case 1: { /* good insert (i.e. key not already in map) */ - fd_groove_key_t key[1]; fd_groove_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - ulong mod = 0UL; - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_prepare( map, key, sentinel, query, flags ); - fd_groove_meta_t * ele = fd_groove_meta_map_query_ele( query ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_FULL ); - } else { - FD_TEST( (ulong)(ele-ele0)bits ) ); - - ele->key = *key; - ele->val_off = key->ul[0] ^ mod; - ele->bits = 1UL | (mod<<1); - - fd_groove_meta_map_publish( query ); - - map_key[ map_cnt++ ] = *key; - local_key++; - } - - break; - } - - case 2: { /* bad remove (i.e. key not already in map) */ - fd_groove_key_t key[1]; fd_groove_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_remove( map, key, query, flags ); - - if( FD_LIKELY( err ) ) { - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_KEY ); - } - - break; - } - - case 3: { /* good remove (i.e. key already in map) */ - if( FD_UNLIKELY( !map_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, map_cnt ); - fd_groove_key_t * key = map_key + idx; - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_remove( map, key, query, flags ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - } else { - map_key[ idx ] = map_key[ --map_cnt ]; - } - - break; - } - - case 4: { /* bad modify (i.e. key not already in map) */ - fd_groove_key_t key[1]; fd_groove_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_prepare( map, key, sentinel, query, flags ); - fd_groove_meta_t * ele = fd_groove_meta_map_query_ele( query ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_FULL ); - } else { - FD_TEST( (ulong)(ele-ele0)bits ) ); - - fd_groove_meta_map_cancel( query ); - } - - break; - } - - case 5: { /* good modify (i.e. key already in map) */ - if( FD_UNLIKELY( !map_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, map_cnt ); - fd_groove_key_t * key = map_key + idx; - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_prepare( map, key, sentinel, query, flags ); - fd_groove_meta_t * ele = fd_groove_meta_map_query_ele( query ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - } else { - FD_TEST( (ulong)(ele-ele0)bits >> 1; - FD_TEST( fd_groove_key_eq( &ele->key, key ) ); - FD_TEST( ele->val_off==(key->ul[0] ^ mod) ); - FD_TEST( fd_groove_meta_bits_used( ele->bits ) ); - - mod++; - - ele->val_off = key->ul[0] ^ mod; - ele->bits = 1UL | (mod<<1); - - fd_groove_meta_map_publish( query ); - } - - break; - } - - case 6: { /* bad query (i.e. key not already in map) */ - fd_groove_key_t key[1]; fd_groove_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_query_try( map, key, sentinel, query, flags ); - fd_groove_meta_t const * ele = fd_groove_meta_map_query_ele_const( query ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - - FD_TEST( ele==sentinel ); - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_KEY ); - - break; - } - - case 7: { /* good query */ - if( FD_UNLIKELY( !map_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, map_cnt ); - fd_groove_key_t * key = map_key + idx; - ulong memo = fd_groove_meta_map_key_hash( key, seed ); - - if( use_hint ) { - fd_groove_meta_map_hint( map, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - } - - int err = fd_groove_meta_map_query_try( map, key, sentinel, query, flags ); - fd_groove_meta_t const * ele = fd_groove_meta_map_query_ele_const( query ); - FD_TEST( memo==fd_groove_meta_map_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - } else { - FD_TEST( (ulong)(ele-ele0)key; - ulong spec_mod = ele->bits >> 1; - ulong spec_val = ele->val_off; - - err = fd_groove_meta_map_query_test( query ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( tile_cnt>1UL ); - } else { - FD_TEST( fd_groove_key_eq( &spec_key, key ) ); - FD_TEST( spec_val==(key->ul[0] ^ spec_mod) ); - } - } - - break; - } - - case 8: { /* parallel iteration */ - ushort version[ lock_max ]; - - ulong range_start = fd_rng_ulong_roll( rng, lock_cnt ); - ulong range_cnt = fd_ulong_min( fd_rng_coin_tosses( rng ), lock_cnt ); - - int err = fd_groove_meta_map_lock_range( map, range_start, range_cnt, flags, version ); - - if( FD_UNLIKELY( err ) ) { - - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - - } else { - - ulong lock_idx = range_start; - for( ulong lock_rem=range_cnt; lock_rem; lock_rem-- ) { - ulong e0 = fd_groove_meta_map_lock_ele0( map, lock_idx ); - ulong e1 = fd_groove_meta_map_lock_ele1( map, lock_idx ); - - FD_TEST( e0< e1 ); - FD_TEST( e1<=ele_max ); - - for( ulong ele_idx=e0; ele_idxbits ) ) continue; - - ulong mod = ele->bits >> 1; - - FD_TEST( ele->val_off==(ele->key.ul[0] ^ mod) ); - - if( !rdonly ) { - mod++; - ele->bits = 1UL | (mod<<1); - ele->val_off = ele->key.ul[0] ^ mod; - } - } - - lock_idx = (lock_idx+1UL) & (lock_cnt-1UL); - } - - fd_groove_meta_map_unlock_range( map, range_start, range_cnt, version ); - - } - - break; - } - - case 9: { /* parallel memo iteration */ - ulong memo; - ulong iter_min; - if( FD_UNLIKELY( !map_cnt ) ) { /* pick a memo not likely in map */ - memo = fd_rng_ulong( rng ); - iter_min = 0UL; - } else { /* pick a memo in map at least once */ - ulong idx = fd_rng_ulong_roll( rng, map_cnt ); - fd_groove_key_t * key = map_key + idx; - memo = fd_groove_meta_map_key_hash( key, seed ); - iter_min = 1UL; - } - - fd_groove_meta_map_iter_t iter[1]; - int err = fd_groove_meta_map_iter_init( map, memo, flags, iter ); - - if( FD_UNLIKELY( err ) ) { - - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - - } else { - - ulong iter_cnt = 0UL; - while( !fd_groove_meta_map_iter_done( iter ) ) { - fd_groove_meta_t * ele = fd_groove_meta_map_iter_ele( iter ); - - FD_TEST( (ulong)(ele-ele0)bits >> 1; - - FD_TEST( fd_groove_meta_map_key_hash( &ele->key, seed )==memo ); - FD_TEST( fd_groove_meta_bits_used( ele->bits ) ); - FD_TEST( ele->val_off==(ele->key.ul[0] ^ mod) ); - - if( !rdonly ) { - mod++; - ele->bits = 1UL | (mod<<1); - ele->val_off = ele->key.ul[0] ^ mod; - } - - iter_cnt++; - - FD_TEST( fd_groove_meta_map_iter_next( iter )==iter ); - } - - FD_TEST( fd_groove_meta_map_iter_fini( iter )==iter ); - - FD_TEST( iter_cnt>=iter_min ); - } - - break; - } - - default: - break; - } - } - - /* Clean up for the next text battery */ - - for( ulong map_idx=0UL; map_idx>3) & 31UL); - - ulong bits = fd_groove_meta_bits( used<page_sz; - page_cnt = info->page_cnt; - volume_cnt = (page_sz*page_cnt) / FD_GROOVE_VOLUME_FOOTPRINT; - - } else { - - FD_LOG_NOTICE(( "--name not specified, using anonymous shmem (--volume-cnt %lu --page-sz %s --near-cpu %lu)", - volume_cnt, _page_sz, near_cpu )); - - page_sz = fd_cstr_to_shmem_page_sz( _page_sz ); - if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "invalid page_sz" )); - page_cnt = (volume_cnt*FD_GROOVE_VOLUME_FOOTPRINT + page_sz-1UL) / page_sz; - volume = (fd_groove_volume_t *)fd_shmem_acquire_multi( page_sz, 1UL, &page_cnt, &near_cpu ); /* logs details */ - if( FD_UNLIKELY( !volume ) ) FD_LOG_ERR(( "fd_shmem_join failed" )); - - } - - FD_LOG_NOTICE(( "Testing with %lu groove data volumes", volume_cnt )); - - FD_LOG_NOTICE(( "Creating test volume pool" )); - - void * shpool = shmem_alloc( fd_groove_volume_pool_align(), fd_groove_volume_pool_footprint() ); - FD_TEST( fd_groove_volume_pool_new( shpool )==shpool ); - - FD_LOG_NOTICE(( "Testing volume pool operations" )); - - fd_groove_volume_pool_t pool[1]; - FD_TEST( fd_groove_volume_pool_join( pool, shpool, volume, volume_cnt ) ); - -# define BUF_MAX (2UL*FD_GROOVE_VOLUME_INFO_MAX) - uchar buf[ BUF_MAX ]; - - ulong vfp = FD_GROOVE_VOLUME_FOOTPRINT; - FD_TEST( !fd_groove_volume_pool_add( 0UL, NULL, 0UL, NULL, 0UL ) ); - FD_TEST( fd_groove_volume_pool_add( NULL, volume, vfp, NULL, 0UL )==FD_GROOVE_ERR_INVAL ); - FD_TEST( fd_groove_volume_pool_add( pool, NULL, vfp, NULL, 0UL )==FD_GROOVE_ERR_INVAL ); - FD_TEST( fd_groove_volume_pool_add( pool, volume, -vfp, NULL, 0UL )==FD_GROOVE_ERR_INVAL ); - FD_TEST( fd_groove_volume_pool_add( pool, (uchar *)volume + 1UL, vfp, NULL, 0UL )==FD_GROOVE_ERR_INVAL ); - FD_TEST( fd_groove_volume_pool_add( pool, volume, 1UL, NULL, 0UL )==FD_GROOVE_ERR_INVAL ); - - FD_TEST( !fd_groove_volume_pool_remove( NULL ) ); - - for( ulong rem=100000UL; rem; rem-- ) { - ulong i0 = fd_rng_ulong_roll( rng, volume_cnt ); - ulong i1 = fd_rng_ulong_roll( rng, volume_cnt ); - fd_swap_if( i1FD_GROOVE_VOLUME_INFO_MAX ) info_sz = FD_GROOVE_VOLUME_INFO_MAX; - - for( ulong i=i0; i Date: Fri, 23 Oct 2026 21:44:17 +0000 Subject: [PATCH 3/5] Remove vinyl --- src/vinyl/Local.mk | 13 - src/vinyl/bstream/Local.mk | 4 - src/vinyl/bstream/fd_vinyl_bstream.c | 262 ------ src/vinyl/bstream/fd_vinyl_bstream.h | 471 ---------- src/vinyl/bstream/test_vinyl_bstream.c | 242 ------ src/vinyl/cq/Local.mk | 4 - src/vinyl/cq/fd_vinyl_cq.c | 108 --- src/vinyl/cq/fd_vinyl_cq.h | 261 ------ src/vinyl/cq/test_vinyl_cq.c | 207 ----- src/vinyl/data/Local.mk | 7 - src/vinyl/data/fd_vinyl_data.c | 594 ------------- src/vinyl/data/fd_vinyl_data.h | 479 ----------- src/vinyl/data/fd_vinyl_data_szc_cfg.c | 100 --- src/vinyl/data/gen_szc_cfg.m | 150 ---- src/vinyl/data/test_vinyl_data.c | 279 ------ src/vinyl/fd_vinyl.c | 171 ---- src/vinyl/fd_vinyl.h | 282 ------ src/vinyl/fd_vinyl_base.c | 16 - src/vinyl/fd_vinyl_base.h | 305 ------- src/vinyl/fd_vinyl_case_acquire.c | 397 --------- src/vinyl/fd_vinyl_case_erase.c | 108 --- src/vinyl/fd_vinyl_case_fetch.c | 116 --- src/vinyl/fd_vinyl_case_flush.c | 68 -- src/vinyl/fd_vinyl_case_move.c | 326 ------- src/vinyl/fd_vinyl_case_release.c | 352 -------- src/vinyl/fd_vinyl_case_test.c | 39 - src/vinyl/fd_vinyl_case_try.c | 180 ---- src/vinyl/fd_vinyl_cmd.c | 147 ---- src/vinyl/fd_vinyl_compact.c | 377 -------- src/vinyl/fd_vinyl_ctl.c | 712 --------------- src/vinyl/fd_vinyl_ctl_help | 91 -- src/vinyl/fd_vinyl_exec.c | 702 --------------- src/vinyl/fd_vinyl_recover.c | 731 ---------------- src/vinyl/fd_vinyl_recover_serial.c | 351 -------- src/vinyl/io/Local.mk | 8 - src/vinyl/io/fd_vinyl_io.c | 229 ----- src/vinyl/io/fd_vinyl_io.h | 557 ------------ src/vinyl/io/fd_vinyl_io_bd.c | 665 -------------- src/vinyl/io/fd_vinyl_io_mm.c | 652 -------------- src/vinyl/io/test_vinyl_io_bd.c | 175 ---- src/vinyl/io/test_vinyl_io_common.c | 231 ----- src/vinyl/io/test_vinyl_io_mm.c | 141 --- src/vinyl/line/Local.mk | 4 - src/vinyl/line/fd_vinyl_line.c | 94 -- src/vinyl/line/fd_vinyl_line.h | 140 --- src/vinyl/line/test_vinyl_line.c | 92 -- src/vinyl/meta/Local.mk | 4 - src/vinyl/meta/fd_vinyl_meta.c | 161 ---- src/vinyl/meta/fd_vinyl_meta.h | 239 ----- src/vinyl/meta/test_vinyl_meta.c | 866 ------------------- src/vinyl/rq/Local.mk | 4 - src/vinyl/rq/fd_vinyl_rq.c | 108 --- src/vinyl/rq/fd_vinyl_rq.h | 540 ------------ src/vinyl/rq/test_vinyl_rq.c | 207 ----- src/vinyl/test_vinyl_base.c | 98 --- src/vinyl/test_vinyl_req.c | 1101 ------------------------ 56 files changed, 14968 deletions(-) delete mode 100644 src/vinyl/Local.mk delete mode 100644 src/vinyl/bstream/Local.mk delete mode 100644 src/vinyl/bstream/fd_vinyl_bstream.c delete mode 100644 src/vinyl/bstream/fd_vinyl_bstream.h delete mode 100644 src/vinyl/bstream/test_vinyl_bstream.c delete mode 100644 src/vinyl/cq/Local.mk delete mode 100644 src/vinyl/cq/fd_vinyl_cq.c delete mode 100644 src/vinyl/cq/fd_vinyl_cq.h delete mode 100644 src/vinyl/cq/test_vinyl_cq.c delete mode 100644 src/vinyl/data/Local.mk delete mode 100644 src/vinyl/data/fd_vinyl_data.c delete mode 100644 src/vinyl/data/fd_vinyl_data.h delete mode 100644 src/vinyl/data/fd_vinyl_data_szc_cfg.c delete mode 100644 src/vinyl/data/gen_szc_cfg.m delete mode 100644 src/vinyl/data/test_vinyl_data.c delete mode 100644 src/vinyl/fd_vinyl.c delete mode 100644 src/vinyl/fd_vinyl.h delete mode 100644 src/vinyl/fd_vinyl_base.c delete mode 100644 src/vinyl/fd_vinyl_base.h delete mode 100644 src/vinyl/fd_vinyl_case_acquire.c delete mode 100644 src/vinyl/fd_vinyl_case_erase.c delete mode 100644 src/vinyl/fd_vinyl_case_fetch.c delete mode 100644 src/vinyl/fd_vinyl_case_flush.c delete mode 100644 src/vinyl/fd_vinyl_case_move.c delete mode 100644 src/vinyl/fd_vinyl_case_release.c delete mode 100644 src/vinyl/fd_vinyl_case_test.c delete mode 100644 src/vinyl/fd_vinyl_case_try.c delete mode 100644 src/vinyl/fd_vinyl_cmd.c delete mode 100644 src/vinyl/fd_vinyl_compact.c delete mode 100644 src/vinyl/fd_vinyl_ctl.c delete mode 100644 src/vinyl/fd_vinyl_ctl_help delete mode 100644 src/vinyl/fd_vinyl_exec.c delete mode 100644 src/vinyl/fd_vinyl_recover.c delete mode 100644 src/vinyl/fd_vinyl_recover_serial.c delete mode 100644 src/vinyl/io/Local.mk delete mode 100644 src/vinyl/io/fd_vinyl_io.c delete mode 100644 src/vinyl/io/fd_vinyl_io.h delete mode 100644 src/vinyl/io/fd_vinyl_io_bd.c delete mode 100644 src/vinyl/io/fd_vinyl_io_mm.c delete mode 100644 src/vinyl/io/test_vinyl_io_bd.c delete mode 100644 src/vinyl/io/test_vinyl_io_common.c delete mode 100644 src/vinyl/io/test_vinyl_io_mm.c delete mode 100644 src/vinyl/line/Local.mk delete mode 100644 src/vinyl/line/fd_vinyl_line.c delete mode 100644 src/vinyl/line/fd_vinyl_line.h delete mode 100644 src/vinyl/line/test_vinyl_line.c delete mode 100644 src/vinyl/meta/Local.mk delete mode 100644 src/vinyl/meta/fd_vinyl_meta.c delete mode 100644 src/vinyl/meta/fd_vinyl_meta.h delete mode 100644 src/vinyl/meta/test_vinyl_meta.c delete mode 100644 src/vinyl/rq/Local.mk delete mode 100644 src/vinyl/rq/fd_vinyl_rq.c delete mode 100644 src/vinyl/rq/fd_vinyl_rq.h delete mode 100644 src/vinyl/rq/test_vinyl_rq.c delete mode 100644 src/vinyl/test_vinyl_base.c delete mode 100644 src/vinyl/test_vinyl_req.c diff --git a/src/vinyl/Local.mk b/src/vinyl/Local.mk deleted file mode 100644 index f3f3519f76d..00000000000 --- a/src/vinyl/Local.mk +++ /dev/null @@ -1,13 +0,0 @@ -ifdef FD_HAS_LZ4 -$(call make-lib,fd_vinyl) -$(call add-hdrs,fd_vinyl_base.h fd_vinyl.h) -$(call add-objs,fd_vinyl_base fd_vinyl_recover fd_vinyl_compact fd_vinyl_cmd fd_vinyl fd_vinyl_exec,fd_vinyl) -ifdef FD_HAS_HOSTED -$(call make-bin,fd_vinyl_ctl,fd_vinyl_ctl,fd_vinyl fd_tango fd_util_test fd_util) -endif -$(call make-unit-test,test_vinyl_base,test_vinyl_base,fd_vinyl fd_tango fd_util_test fd_util) -ifdef FD_HAS_HOSTED -$(call make-unit-test,test_vinyl_req,test_vinyl_req,fd_vinyl fd_tango fd_util_test fd_util) -endif -$(call run-unit-test,test_vinyl_base) -endif diff --git a/src/vinyl/bstream/Local.mk b/src/vinyl/bstream/Local.mk deleted file mode 100644 index 634e228a7c7..00000000000 --- a/src/vinyl/bstream/Local.mk +++ /dev/null @@ -1,4 +0,0 @@ -$(call add-hdrs,fd_vinyl_bstream.h) -$(call add-objs,fd_vinyl_bstream,fd_vinyl) -$(call make-unit-test,test_vinyl_bstream,test_vinyl_bstream,fd_vinyl fd_tango fd_util_test fd_util) -$(call run-unit-test,test_vinyl_bstream) diff --git a/src/vinyl/bstream/fd_vinyl_bstream.c b/src/vinyl/bstream/fd_vinyl_bstream.c deleted file mode 100644 index e26ff6df270..00000000000 --- a/src/vinyl/bstream/fd_vinyl_bstream.c +++ /dev/null @@ -1,262 +0,0 @@ -#include "fd_vinyl_bstream.h" - -void -fd_vinyl_bstream_pair_hash( ulong seed, - fd_vinyl_bstream_block_t * hdr ) { - - ulong ctl = hdr->ctl; - ulong val_esz = fd_vinyl_bstream_ctl_sz( ctl ); - - ulong pair_sz = fd_vinyl_bstream_pair_sz( val_esz ); - - fd_vinyl_bstream_block_t * ftr = (fd_vinyl_bstream_block_t *)((uchar *)hdr + pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ); - - ulong off = sizeof(fd_vinyl_bstream_phdr_t) + val_esz; - ulong zsz = pair_sz - off; /* covers zpad and footer so at least FD_VINYL_BSTREAM_FTR_SZ */ - - memset( (uchar *)hdr + off, 0, zsz ); - - ulong hash_trail = fd_vinyl_bstream_hash( seed, hdr+1, pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ ); - ulong hash_blocks = fd_vinyl_bstream_hash( hash_trail, hdr, FD_VINYL_BSTREAM_BLOCK_SZ ); - - ftr->ftr.hash_trail = hash_trail; - ftr->ftr.hash_blocks = hash_blocks; -} - -char const * -fd_vinyl_bstream_pair_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * hdr, - ulong buf_sz ) { - (void)seq; - - if( FD_UNLIKELY( !hdr ) ) return "NULL buf"; - if( FD_UNLIKELY( buf_szctl; - int pair_type = fd_vinyl_bstream_ctl_type( pair_ctl ); - ulong pair_val_esz = fd_vinyl_bstream_ctl_sz ( pair_ctl ); - ulong pair_val_sz = (ulong)hdr->phdr.info.val_sz; - - ulong pair_sz = fd_vinyl_bstream_pair_sz( pair_val_esz ); - - if( FD_UNLIKELY( pair_type != FD_VINYL_BSTREAM_CTL_TYPE_PAIR ) ) return "unexpected type"; - if( FD_UNLIKELY( pair_val_sz > FD_VINYL_VAL_MAX ) ) return "unexpected val size"; - if( FD_UNLIKELY( buf_sz < pair_sz ) ) return "truncated pair"; - - fd_vinyl_bstream_block_t * ftr = (fd_vinyl_bstream_block_t *)((ulong)hdr + pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ); - - ulong hash_trail = ftr->ftr.hash_trail; - ulong hash_blocks = ftr->ftr.hash_blocks; - - ftr->ftr.hash_trail = 0UL; - ftr->ftr.hash_blocks = 0UL; - - /* FIXME: test zero padding? */ - - if( FD_UNLIKELY( fd_vinyl_bstream_hash( seed, hdr+1, pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ ) != hash_trail ) ) - return "unexpected trailing hash"; - - if( FD_UNLIKELY( fd_vinyl_bstream_hash( hash_trail, hdr, FD_VINYL_BSTREAM_BLOCK_SZ ) != hash_blocks ) ) - return "unexpected pair hash"; - - return NULL; -} - -char const * -fd_vinyl_bstream_pair_test_fast( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t const * hdr, - fd_vinyl_bstream_block_t * ftr ) { - (void)seq; - - if( FD_UNLIKELY( !hdr ) ) return "NULL hdr"; - if( FD_UNLIKELY( !ftr ) ) return "NULL ftr"; - - ulong pair_ctl = hdr->ctl; - int pair_type = fd_vinyl_bstream_ctl_type( pair_ctl ); - ulong pair_val_esz = fd_vinyl_bstream_ctl_sz ( pair_ctl ); - ulong pair_val_sz = (ulong)hdr->phdr.info.val_sz; - - ulong pair_sz = fd_vinyl_bstream_pair_sz( pair_val_esz ); - - if( FD_UNLIKELY( pair_type != FD_VINYL_BSTREAM_CTL_TYPE_PAIR ) ) return "unexpected type"; - if( FD_UNLIKELY( pair_val_sz > FD_VINYL_VAL_MAX ) ) return "unexpected pair val size"; - - ulong hash_trail = ftr->ftr.hash_trail; - ulong hash_blocks = ftr->ftr.hash_blocks; - - ftr->ftr.hash_trail = 0UL; - ftr->ftr.hash_blocks = 0UL; - - if( FD_UNLIKELY( pair_sz <= 2UL*FD_VINYL_BSTREAM_BLOCK_SZ ) && - FD_UNLIKELY( fd_vinyl_bstream_hash( seed, ftr, pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ ) != hash_trail ) ) - return "unexpected trailing hash"; - - if( FD_UNLIKELY( fd_vinyl_bstream_hash( hash_trail, hdr, FD_VINYL_BSTREAM_BLOCK_SZ ) != hash_blocks ) ) - return "unexpected pair hash"; - - return NULL; -} - -char const * -fd_vinyl_bstream_dead_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block ) { - - if( FD_UNLIKELY( !block ) ) return "NULL block"; - - ulong dead_info_sz = block->dead.info_sz; - - int bad_ctl = (block->dead.ctl != fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_DEAD, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - FD_VINYL_BSTREAM_BLOCK_SZ ) ); - int bad_match = !fd_vinyl_seq_eq( block->dead.seq, seq ); - int bad_seq = (!fd_ulong_is_aligned( block->dead.seq, FD_VINYL_BSTREAM_BLOCK_SZ )); - int bad_pair_type = (fd_vinyl_bstream_ctl_type( block->dead.phdr.ctl ) != FD_VINYL_BSTREAM_CTL_TYPE_PAIR); - int bad_pair_val_sz = ((ulong)block->dead.phdr.info.val_sz > FD_VINYL_VAL_MAX); - int bad_info_sz = (dead_info_sz > FD_VINYL_BSTREAM_DEAD_INFO_MAX); - - if( FD_UNLIKELY( bad_ctl | bad_match | bad_seq | bad_pair_type | bad_pair_val_sz | bad_info_sz ) ) - return bad_ctl ? "unexpected dead ctl" : - bad_match ? "mismatched dead seq" : - bad_seq ? "misaligned dead seq" : - bad_pair_type ? "unexpected dead pair type" : - bad_pair_val_sz ? "unexpected dead pair val size" : - "unexpected dead info size"; - - for( ulong zpad_idx=dead_info_sz; zpad_idxdead.info[ zpad_idx ] ) ) return "data in zero padding"; - - if( FD_UNLIKELY( fd_vinyl_bstream_block_test( seed, block ) ) ) return "corrupt block"; - - return NULL; -} - -char const * -fd_vinyl_bstream_move_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block, - fd_vinyl_bstream_block_t * dst ) { - - if( FD_UNLIKELY( !block ) ) return "NULL block"; - if( FD_UNLIKELY( !dst ) ) return "NULL dst"; - - ulong move_info_sz = block->move.info_sz; - - int bad_ctl = (block->move.ctl != fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_MOVE, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - FD_VINYL_BSTREAM_BLOCK_SZ ) ); - int bad_match = !fd_vinyl_seq_eq( block->move.seq, seq ); - int bad_seq = (!fd_ulong_is_aligned( block->move.seq, FD_VINYL_BSTREAM_BLOCK_SZ )); - int bad_src_type = (fd_vinyl_bstream_ctl_type( block->move.src.ctl ) != FD_VINYL_BSTREAM_CTL_TYPE_PAIR); - int bad_src_val_sz = ((ulong)block->move.src.info.val_sz > FD_VINYL_VAL_MAX); - int bad_move = fd_vinyl_key_eq( &block->move.src.key, &block->move.dst ); - int bad_info_sz = (move_info_sz > FD_VINYL_BSTREAM_MOVE_INFO_MAX); - int bad_dst_type = (fd_vinyl_bstream_ctl_type( dst->phdr.ctl ) != FD_VINYL_BSTREAM_CTL_TYPE_PAIR); - int bad_dst_key = (!fd_vinyl_key_eq( &block->move.dst, &dst->phdr.key )); - int bad_dst_info = (!!memcmp( &block->move.src.info, &dst->phdr.info, FD_VINYL_INFO_SZ )); - - if( FD_UNLIKELY( bad_ctl | bad_match | bad_seq | bad_src_type | bad_src_val_sz | bad_move | bad_info_sz | - bad_dst_type | bad_dst_key | bad_dst_info ) ) - return bad_ctl ? "unexpected move ctl" : - bad_match ? "mismatched move seq" : - bad_seq ? "misaligned move seq" : - bad_src_type ? "unexpected move src type" : - bad_src_val_sz ? "unexpected move src val size" : - bad_move ? "dst key matches src key" : - bad_info_sz ? "unexpected move info size" : - bad_dst_type ? "mismatched move dst type" : - bad_dst_key ? "mismatched move dst key" : - "mismatched move dst info"; - -# if 0 /* Note: with BLOCK_SZ==128, MOVE_INFO_MAX=0 and there's no move.info field due to language limitations */ - for( ulong zpad_idx=move_info_sz; zpad_idxmove.info[ zpad_idx ] ) ) return "data in zero padding"; -# endif - - if( FD_UNLIKELY( fd_vinyl_bstream_block_test( seed, block ) ) ) return "corrupt block"; - - return NULL; -} - -char const * -fd_vinyl_bstream_part_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block ) { - - if( FD_UNLIKELY( !block ) ) return "NULL block"; - - ulong part_seq = block->part.seq; - ulong part_seq0 = block->part.seq0; - ulong part_dead_cnt = block->part.dead_cnt; - ulong part_move_cnt = block->part.move_cnt; - ulong part_info_sz = block->part.info_sz; - - ulong part_block_cnt = (part_seq - part_seq0) / FD_VINYL_BSTREAM_BLOCK_SZ; - - int bad_ctl = (block->part.ctl != fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PART, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - FD_VINYL_BSTREAM_BLOCK_SZ ) ); - int bad_match = !fd_vinyl_seq_eq( block->part.seq, seq ); - int bad_seq = (!fd_ulong_is_aligned( part_seq, FD_VINYL_BSTREAM_BLOCK_SZ )); - int bad_seq0 = (!fd_ulong_is_aligned( part_seq0, FD_VINYL_BSTREAM_BLOCK_SZ )); - int bad_order = fd_vinyl_seq_gt( part_seq0, part_seq ); - int bad_dead_cnt = (part_dead_cnt > part_block_cnt); - int bad_move_cnt = ((2UL*part_move_cnt) > part_block_cnt); - int bad_block_cnt = ((part_dead_cnt+2UL*part_move_cnt) > part_block_cnt); - int bad_info_sz = (part_info_sz > FD_VINYL_BSTREAM_PART_INFO_MAX); - - if( FD_UNLIKELY( bad_ctl | bad_match | bad_seq | bad_seq0 | bad_order | - bad_dead_cnt | bad_move_cnt | bad_block_cnt | bad_info_sz ) ) - return bad_ctl ? "unexpected partition ctl" : - bad_match ? "mismatched partition seq" : - bad_seq ? "misaligned partition seq" : - bad_seq0 ? "misaligned partition seq previous" : - bad_order ? "unordered partition seq/seq previous" : - bad_dead_cnt ? "unexpected partition dead count" : - bad_move_cnt ? "unexpected partition move count" : - bad_block_cnt ? "unexpected partition block count" : - "unexpected partition info size"; - - for( ulong zpad_idx=part_info_sz; zpad_idxpart.info[ zpad_idx ] ) ) return "data in zero padding"; - - if( FD_UNLIKELY( fd_vinyl_bstream_block_test( seed, block ) ) ) return "corrupt block"; - - return NULL; -} - -char const * -fd_vinyl_bstream_zpad_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block ) { - (void)seed; (void)seq; - - if( FD_UNLIKELY( !block ) ) return "NULL block"; - - ulong const * buf = (ulong const *)block->zpad; - ulong cnt = FD_VINYL_BSTREAM_BLOCK_SZ / sizeof(ulong); - - for( ulong idx=0UL; idx 0L; } -FD_FN_CONST static inline int fd_vinyl_seq_le( ulong a, ulong b ) { return ((long)(a-b))<=0L; } -FD_FN_CONST static inline int fd_vinyl_seq_ge( ulong a, ulong b ) { return ((long)(a-b))>=0L; } -FD_FN_CONST static inline int fd_vinyl_seq_eq( ulong a, ulong b ) { return ((long)(a-b))==0L; } -FD_FN_CONST static inline int fd_vinyl_seq_ne( ulong a, ulong b ) { return ((long)(a-b))!=0L; } - -/* fd_vinyl_bstream_ctl return the bstream control field that encodes a - FD_VINYL_BSTREAM_CTL_TYPE_* type, FD_VINYL_BSTREAM_CTL_STYLE_* style, - and a byte size in [0,2^36). fd_vinyl_bstream_ctl_* return the - eponymous field from the given bstream ctl. */ - -FD_FN_CONST static inline ulong -fd_vinyl_bstream_ctl( int type, - int style, - ulong val_sz ) { - return ((ulong)type) | (((ulong)style)<<16) | (val_sz<<28); -} - -FD_FN_CONST static inline int fd_vinyl_bstream_ctl_type ( ulong ctl ) { return (int)( ctl & 65535UL); } -FD_FN_CONST static inline int fd_vinyl_bstream_ctl_style( ulong ctl ) { return (int)((ctl>>16) & 4095UL); } -FD_FN_CONST static inline ulong fd_vinyl_bstream_ctl_sz ( ulong ctl ) { return ctl>>28; } - -/* fd_vinyl_bstream_pair_sz returns the byte footprint of a pair with a - val_esz encoded value byte size. Assumes val_esz in [0,2^48) which - implies the worst case maximum encoded value size allowed is in - [0,2^48). Returns a positive FD_VINYL_BSTREAM_BLOCK_SZ multiple. */ - -FD_FN_CONST static inline ulong -fd_vinyl_bstream_pair_sz( ulong val_esz ) { - return fd_ulong_align_up( sizeof(fd_vinyl_bstream_phdr_t) + val_esz + FD_VINYL_BSTREAM_FTR_SZ, FD_VINYL_BSTREAM_BLOCK_SZ ); -} - -/* fd_vinyl_bstream_hash returns a 64-bit hash of the 64-bit seed and - the sz byte buffer buf. Assumes buf is points to well aligned range - of blocks stable for the duration of the call and sz is a multiple of - FD_VINYL_BSTREAM_BLOCK_SZ. sz 0 returns seed. Retains no interest - in buf. */ - -FD_FN_PURE static inline ulong -fd_vinyl_bstream_hash( ulong seed, - fd_vinyl_bstream_block_t const * buf, - ulong buf_sz ) { - return FD_LIKELY( buf_sz ) ? fd_hash( seed, buf, buf_sz ) : seed; -} - -/* fd_vinyl_bstream_block_hash initializes the hash fields of a bstream - object that fits into a single block (i.e. a small val pair, sync - block, a dead block, a move block or a part block). block's hash - fields and any zero padding should be zero on entry. block's hash - fields will be set correctly on exit. Assumes block is valid. */ - -static inline void -fd_vinyl_bstream_block_hash( ulong seed, - fd_vinyl_bstream_block_t * block ) { - - ulong hash_trail = seed; - ulong hash_blocks = fd_vinyl_bstream_hash( hash_trail, block, FD_VINYL_BSTREAM_BLOCK_SZ ); - - block->ftr.hash_trail = hash_trail; - block->ftr.hash_blocks = hash_blocks; - -} - -/* fd_vinyl_bstream_block_test returns FD_VINYL_SUCCESS if a bstream - object that fits into a single bstream block (i.e. a small val pair, - a sync block, a dead block, a move block or a part block) has valid - data integrity hashes and FD_VINYL_ERR_CORRUPT if not. seed is the - bstream's data integrity seed. Assumes block points to the valid - stable location. block's hash fields will be clobbered on return - (will be zero on success). */ - -static inline int -fd_vinyl_bstream_block_test( ulong seed, - fd_vinyl_bstream_block_t * block ) { - - ulong hash_trail = block->ftr.hash_trail; - ulong hash_blocks = block->ftr.hash_blocks; - - block->ftr.hash_trail = 0UL; - block->ftr.hash_blocks = 0UL; - - if( FD_UNLIKELY( hash_trail != seed ) || - FD_UNLIKELY( fd_vinyl_bstream_hash( seed, block, FD_VINYL_BSTREAM_BLOCK_SZ ) != hash_blocks ) ) - return FD_VINYL_ERR_CORRUPT; - - return FD_VINYL_SUCCESS; -} - -/* fd_vinyl_bstream_pair_hash clears the footer region and any zero - padding region of pair and then populates the hash fields - appropriately. seed is the bstream data integrity seed. Assumes - pair points to any appropriately sized and aligned memory region and - the phdr and val fields are correctly populated (in particular, - val_esz is set correctly). */ - -void -fd_vinyl_bstream_pair_hash( ulong seed, - fd_vinyl_bstream_block_t * phdr ); - -/* fd_vinyl_bstream_*_test returns NULL if bstream object at seq is - well formed and an infinite lifetime human-readable cstr describing - the issue detected if not. The footer has been clobbered on return - (will be zero on success). seed is the bstream's data integrity - seed. Assumes pair points to a valid pair_sz footprint location. - - For fd_vinyl_bstream_pair_test, the pair is assumed fit within the - the buf_sz buffer buf, not exactly fill the buffer. - - fd_vinyl_bstream_pair_test_fast is the same thing but omits testing - any interior blocks for use in fast iteration. hdr / ftr point - locations holding the first/last block in the pair. These should - point to the same location if the pair fits into a single block. */ - -char const * -fd_vinyl_bstream_pair_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * buf, - ulong buf_sz ); - -char const * -fd_vinyl_bstream_pair_test_fast( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t const * hdr, - fd_vinyl_bstream_block_t * ftr ); - -char const * -fd_vinyl_bstream_dead_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block ); - -char const * -fd_vinyl_bstream_move_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block, - fd_vinyl_bstream_block_t * dst ); - -char const * -fd_vinyl_bstream_part_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block ); - -char const * -fd_vinyl_bstream_zpad_test( ulong seed, - ulong seq, - fd_vinyl_bstream_block_t * block ); - -/* fd_vinyl_bstream_ctl_style_cstr returns an infinite lifetime human - readable cstr for the given style. Return value is always non-NULL - (if style is not a valid style, the string will be "unk"). - - fd_cstr_to_vinyl_bstream_ctl_style returns the style corresponding to - the given cstr. Returns -1 if cstr does not correspond to a - supported style. */ - -char const * -fd_vinyl_bstream_ctl_style_cstr( int style ); - -int -fd_cstr_to_vinyl_bstream_ctl_style( char const * cstr ); - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_bstream_fd_vinyl_bstream_h */ diff --git a/src/vinyl/bstream/test_vinyl_bstream.c b/src/vinyl/bstream/test_vinyl_bstream.c deleted file mode 100644 index f1a75fd4b05..00000000000 --- a/src/vinyl/bstream/test_vinyl_bstream.c +++ /dev/null @@ -1,242 +0,0 @@ -#include "../fd_vinyl.h" -#include - -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_BLOCK_SZ==128UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_BLOCK_LG_SZ==7, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_TYPE_PAIR==(int)0x9a17, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_TYPE_SYNC==(int)0x512c, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_TYPE_DEAD==(int)0xdead, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_TYPE_MOVE==(int)0x30c3, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_TYPE_PART==(int)0xd121, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_TYPE_ZPAD==0, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_STYLE_RAW==(int)0x7a3, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_CTL_STYLE_LZ4==(int)0x124, unit_test ); - -FD_STATIC_ASSERT( alignof(fd_vinyl_bstream_phdr_t)==8UL, unit_test ); -FD_STATIC_ASSERT( sizeof( fd_vinyl_bstream_phdr_t)==8UL+sizeof(fd_vinyl_key_t)+sizeof(fd_vinyl_info_t), unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_FTR_SZ==16UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_LZ4_VAL_THRESH==56UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_SYNC_INFO_MAX==80UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_DEAD_INFO_MAX==32UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_MOVE_INFO_MAX== 0UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_BSTREAM_PART_INFO_MAX==64UL, unit_test ); - -FD_STATIC_ASSERT( alignof(fd_vinyl_bstream_block_t)==128UL, unit_test ); -FD_STATIC_ASSERT( sizeof (fd_vinyl_bstream_block_t)==128UL, unit_test ); - -FD_STATIC_ASSERT( offsetof( fd_vinyl_bstream_block_t, ftr .hash_trail )==112UL, unit_test ); -FD_STATIC_ASSERT( offsetof( fd_vinyl_bstream_block_t, sync.hash_trail )==112UL, unit_test ); -FD_STATIC_ASSERT( offsetof( fd_vinyl_bstream_block_t, dead.hash_trail )==112UL, unit_test ); -FD_STATIC_ASSERT( offsetof( fd_vinyl_bstream_block_t, move.hash_trail )==112UL, unit_test ); -FD_STATIC_ASSERT( offsetof( fd_vinyl_bstream_block_t, part.hash_trail )==112UL, unit_test ); - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - fd_rng_t rng[1]; fd_rng_join( fd_rng_new( rng, 0U, 0UL ) ); - - for( ulong rem=1000000UL; rem; rem-- ) { - - ulong r = fd_rng_ulong( rng ); - ulong s = fd_rng_ulong( rng ); - - /* Test sequence comparison */ - - long d = (long)(r-s); - FD_TEST( fd_vinyl_seq_lt( r, s )==(d< 0L) ); FD_TEST( fd_vinyl_seq_lt( r, r )==0 ); FD_TEST( fd_vinyl_seq_lt( s, s )==0 ); - FD_TEST( fd_vinyl_seq_gt( r, s )==(d> 0L) ); FD_TEST( fd_vinyl_seq_gt( r, r )==0 ); FD_TEST( fd_vinyl_seq_gt( s, s )==0 ); - FD_TEST( fd_vinyl_seq_le( r, s )==(d<=0L) ); FD_TEST( fd_vinyl_seq_le( r, r )==1 ); FD_TEST( fd_vinyl_seq_le( s, s )==1 ); - FD_TEST( fd_vinyl_seq_ge( r, s )==(d>=0L) ); FD_TEST( fd_vinyl_seq_ge( r, r )==1 ); FD_TEST( fd_vinyl_seq_ge( s, s )==1 ); - FD_TEST( fd_vinyl_seq_eq( r, s )==(d==0L) ); FD_TEST( fd_vinyl_seq_eq( r, r )==1 ); FD_TEST( fd_vinyl_seq_eq( s, s )==1 ); - FD_TEST( fd_vinyl_seq_ne( r, s )==(d!=0L) ); FD_TEST( fd_vinyl_seq_ne( r, r )==0 ); FD_TEST( fd_vinyl_seq_ne( s, s )==0 ); - - /* Test ctl encoding decoding */ - - int type = fd_vinyl_bstream_ctl_type ( r ); FD_TEST( (0<=type ) & (type <65536) ); - int style = fd_vinyl_bstream_ctl_style( r ); FD_TEST( (0<=style) & (style< 4096) ); - ulong sz = fd_vinyl_bstream_ctl_sz ( r ); FD_TEST( sz<(1UL<<36) ); - - FD_TEST( fd_vinyl_bstream_ctl( type, style, sz )==r ); - - /* Test pair_sz */ - - ulong pair_sz = fd_vinyl_bstream_pair_sz( (r>>16) ); - FD_TEST( pair_sz==fd_ulong_align_up( sizeof(fd_vinyl_bstream_phdr_t) + (r>>16) + FD_VINYL_BSTREAM_FTR_SZ, - FD_VINYL_BSTREAM_BLOCK_SZ ) ); - - r |= 1UL; /* guarantee non-zero r */ - - /* Single block hash tests */ - - fd_vinyl_bstream_block_t hdr[1]; - ulong * buf = (ulong *)hdr->data; - ulong cnt = FD_VINYL_BSTREAM_BLOCK_SZ / sizeof(ulong); - for( ulong idx=0UL; idxftr.hash_trail = 0UL; - hdr->ftr.hash_blocks = 0UL; - ulong hash_blocks = fd_vinyl_bstream_hash( s, hdr, FD_VINYL_BSTREAM_BLOCK_SZ ); - - fd_vinyl_bstream_block_hash( s, hdr ); - FD_TEST( hdr->ftr.hash_trail ==s ); - FD_TEST( hdr->ftr.hash_blocks==hash_blocks ); - - FD_TEST( !fd_vinyl_bstream_block_test( s, hdr ) ); - - hdr->ftr.hash_trail = r ^ 1UL; - hdr->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_block_test( s, hdr )==FD_VINYL_ERR_CORRUPT ); - - hdr->ftr.hash_trail = r; - hdr->ftr.hash_blocks = hash_blocks ^ 1UL; - FD_TEST( fd_vinyl_bstream_block_test( s, hdr )==FD_VINYL_ERR_CORRUPT ); - - /* Pair tests */ - -# define BUF_SZ (3UL*FD_VINYL_BSTREAM_BLOCK_SZ) -# define VAL_MAX (BUF_SZ - sizeof(fd_vinyl_bstream_phdr_t) - FD_VINYL_BSTREAM_FTR_SZ) - - union { - fd_vinyl_bstream_block_t block[ BUF_SZ / FD_VINYL_BSTREAM_BLOCK_SZ ]; - fd_vinyl_bstream_phdr_t phdr; - uchar buf[ BUF_SZ ]; - } cache; - - ulong val_sz = r % (VAL_MAX+1UL); - /**/ pair_sz = fd_vinyl_bstream_pair_sz( val_sz ); - - ulong phdr_ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ); - - cache.phdr.ctl = phdr_ctl; - memset( &cache.phdr.key, (int)((r>> 8) & 255UL), FD_VINYL_KEY_FOOTPRINT ); - memset( &cache.phdr.info, (int)((r>>16) & 255UL), FD_VINYL_INFO_SZ ); - memset( cache.buf + sizeof(fd_vinyl_bstream_phdr_t), (int)((r>>24) & 255UL), val_sz ); - cache.phdr.info.val_sz = (uint)val_sz; - - fd_vinyl_bstream_pair_hash( s, cache.block ); - - fd_vinyl_bstream_block_t * end = (fd_vinyl_bstream_block_t *)(cache.buf + pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ); - ulong hash_trail = end->ftr.hash_trail; - /**/ hash_blocks = end->ftr.hash_blocks; - - /* Slow tests */ - - /* NULL buf */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test( s, 0UL, NULL, pair_sz ) ); - - /* Too small buf */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test( s, 0UL, cache.block, FD_VINYL_BSTREAM_BLOCK_SZ-1UL ) ); - - /* Bad type (and bad hash but type will hit first) */ - cache.phdr.ctl = fd_vinyl_bstream_ctl( 0, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ); - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test( s, 0UL, cache.block, pair_sz ) ); - cache.phdr.ctl = phdr_ctl; - - /* Bad val_sz (and bad hash but val_sz will hit first) */ - cache.phdr.info.val_sz = (uint)(FD_VINYL_VAL_MAX+1UL); - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test( s, 0UL, cache.block, pair_sz ) ); - cache.phdr.info.val_sz = (uint)val_sz; - - /* Truncated */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test( s, 0UL, cache.block, pair_sz-1UL ) ); - - /* Bad trailing hash */ - end->ftr.hash_trail = hash_trail ^ 1UL; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test( s, 0UL, cache.block, pair_sz ) ); - - /* Bad end-to-end hash */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks ^ 1UL; - FD_TEST( fd_vinyl_bstream_pair_test( s, 0UL, cache.block, pair_sz ) ); - - /* Good */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( !fd_vinyl_bstream_pair_test( s, 0UL, cache.block, pair_sz ) ); - - /* Fast tests */ - - /* NULL hdr */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test_fast( s, 0UL, NULL, end ) ); - - /* NULL ftr */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test_fast( s, 0UL, cache.block, NULL ) ); - - /* Bad type (and bad hash but type will hit first) */ - cache.phdr.ctl = fd_vinyl_bstream_ctl( 0, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ); - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test_fast( s, 0UL, cache.block, end ) ); - cache.phdr.ctl = phdr_ctl; - - /* Bad val_sz (and bad hash but val_sz will hit first) */ - cache.phdr.info.val_sz = (uint)(FD_VINYL_VAL_MAX+1UL); - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test_fast( s, 0UL, cache.block, end ) ); - cache.phdr.info.val_sz = (uint)val_sz; - - /* Bad trailing hash (fast will detect as leading blocks not matched to trailing blocks) */ - end->ftr.hash_trail = hash_trail ^ 1UL; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( fd_vinyl_bstream_pair_test_fast( s, 0UL, cache.block, end ) ); - - /* Bad end-to-end hash */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks ^ 1UL; - FD_TEST( fd_vinyl_bstream_pair_test_fast( s, 0UL, cache.block, end ) ); - - /* Good */ - end->ftr.hash_trail = hash_trail; - end->ftr.hash_blocks = hash_blocks; - FD_TEST( !fd_vinyl_bstream_pair_test_fast( s, 0UL, cache.block, end ) ); - - /* FIXME: add coverage for dead / move / part tests */ - - /* Zero pad tests */ - - FD_TEST( fd_vinyl_bstream_zpad_test( s, 0UL, hdr ) ); - - memset( hdr, 0, FD_VINYL_BSTREAM_BLOCK_SZ ); - - FD_TEST( !fd_vinyl_bstream_zpad_test( s, 0UL, hdr ) ); - - } - - FD_LOG_NOTICE(( "style_cstr( -1 ): %s", fd_vinyl_bstream_ctl_style_cstr( -1 ) )); - FD_LOG_NOTICE(( "style_cstr( RAW ): %s", fd_vinyl_bstream_ctl_style_cstr( FD_VINYL_BSTREAM_CTL_STYLE_RAW ) )); - FD_LOG_NOTICE(( "style_cstr( LZ4 ): %s", fd_vinyl_bstream_ctl_style_cstr( FD_VINYL_BSTREAM_CTL_STYLE_LZ4 ) )); - - FD_TEST( fd_cstr_to_vinyl_bstream_ctl_style( NULL )==-1 ); - FD_TEST( fd_cstr_to_vinyl_bstream_ctl_style( "foo" )==-1 ); - FD_TEST( fd_cstr_to_vinyl_bstream_ctl_style( "raw" )==FD_VINYL_BSTREAM_CTL_STYLE_RAW ); - FD_TEST( fd_cstr_to_vinyl_bstream_ctl_style( "lz4" )==FD_VINYL_BSTREAM_CTL_STYLE_LZ4 ); - - fd_rng_delete( fd_rng_leave( rng ) ); - - FD_LOG_NOTICE(( "pass" )); - fd_halt(); - return 0; -} diff --git a/src/vinyl/cq/Local.mk b/src/vinyl/cq/Local.mk deleted file mode 100644 index 82ac81a0b84..00000000000 --- a/src/vinyl/cq/Local.mk +++ /dev/null @@ -1,4 +0,0 @@ -$(call add-hdrs,fd_vinyl_cq.h) -$(call add-objs,fd_vinyl_cq,fd_vinyl) -$(call make-unit-test,test_vinyl_cq,test_vinyl_cq,fd_vinyl fd_tango fd_util_test fd_util) -$(call run-unit-test,test_vinyl_cq) diff --git a/src/vinyl/cq/fd_vinyl_cq.c b/src/vinyl/cq/fd_vinyl_cq.c deleted file mode 100644 index f05ced6f8db..00000000000 --- a/src/vinyl/cq/fd_vinyl_cq.c +++ /dev/null @@ -1,108 +0,0 @@ -#include "fd_vinyl_cq.h" - -ulong -fd_vinyl_cq_align( void ) { - return alignof(fd_vinyl_cq_t); -} - -ulong -fd_vinyl_cq_footprint( ulong comp_cnt ) { - if( FD_UNLIKELY( !((4UL<=comp_cnt) & (comp_cnt<(1UL<<63)/sizeof(fd_vinyl_comp_t)) & fd_ulong_is_pow2( comp_cnt )) ) ) return 0UL; - return fd_ulong_align_up( sizeof(fd_vinyl_cq_t) + comp_cnt*sizeof(fd_vinyl_comp_t), alignof(fd_vinyl_cq_t) ); /* no overflow */ -} - -void * -fd_vinyl_cq_new( void * shmem, - ulong comp_cnt ) { - fd_vinyl_cq_t * cq = (fd_vinyl_cq_t *)shmem; - - if( FD_UNLIKELY( !cq ) ) { - FD_LOG_WARNING(( "NULL shmem")); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)cq, fd_vinyl_cq_align() ) ) ) { - FD_LOG_WARNING(( "bad align")); - return NULL; - } - - ulong footprint = fd_vinyl_cq_footprint( comp_cnt ); - if( FD_UNLIKELY( !footprint) ) { - FD_LOG_WARNING(( "bad comp_cnt")); - return NULL; - } - - memset( cq, 0, footprint ); - - cq->comp_cnt = comp_cnt; - cq->seq = 0UL; - - fd_vinyl_comp_t * comp = fd_vinyl_cq_comp( cq ); - - for( ulong seq=0UL; seqmagic = FD_VINYL_CQ_MAGIC; - FD_COMPILER_MFENCE(); - - return cq; -} - -fd_vinyl_cq_t * -fd_vinyl_cq_join ( void * shcq ) { - fd_vinyl_cq_t * cq = (fd_vinyl_cq_t *)shcq; - - if( FD_UNLIKELY( !cq ) ) { - FD_LOG_WARNING(( "NULL shcq")); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)cq, fd_vinyl_cq_align() ) ) ) { - FD_LOG_WARNING(( "bad align")); - return NULL; - } - - if( FD_UNLIKELY( cq->magic!=FD_VINYL_CQ_MAGIC ) ) { - FD_LOG_WARNING(( "bad magic")); - return NULL; - } - - return (fd_vinyl_cq_t *)shcq; -} - -void * -fd_vinyl_cq_leave( fd_vinyl_cq_t * cq ) { - - if( FD_UNLIKELY( !cq ) ) { - FD_LOG_WARNING(( "NULL cq")); - return NULL; - } - - return cq; -} - -void * -fd_vinyl_cq_delete( void * shcq ) { - fd_vinyl_cq_t * cq = (fd_vinyl_cq_t *)shcq; - - if( FD_UNLIKELY( !cq ) ) { - FD_LOG_WARNING(( "NULL shcq")); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)cq, fd_vinyl_cq_align() ) ) ) { - FD_LOG_WARNING(( "bad align")); - return NULL; - } - - if( FD_UNLIKELY( cq->magic!=FD_VINYL_CQ_MAGIC ) ) { - FD_LOG_WARNING(( "bad magic")); - return NULL; - } - - FD_COMPILER_MFENCE(); - cq->magic = 0UL; - FD_COMPILER_MFENCE(); - - return cq; -} diff --git a/src/vinyl/cq/fd_vinyl_cq.h b/src/vinyl/cq/fd_vinyl_cq.h deleted file mode 100644 index e38bce20964..00000000000 --- a/src/vinyl/cq/fd_vinyl_cq.h +++ /dev/null @@ -1,261 +0,0 @@ -#ifndef HEADER_fd_src_vinyl_cq_fd_vinyl_cq_h -#define HEADER_fd_src_vinyl_cq_fd_vinyl_cq_h - -/* A fd_vinyl_comp_t provides details about vinyl request completions. */ - -#include "../fd_vinyl_base.h" - -/* FD_VINYL_COMP_{ALIGN,FOOTPRINT} give the byte alignment and footprint - of a fd_vinyl_comp_t. ALIGN is a reasonable power-of-2. FOOTPRINT - is a multiple of ALIGN. */ - -#define FD_VINYL_COMP_ALIGN (32UL) -#define FD_VINYL_COMP_FOOTPRINT (32UL) - -/* FD_VINYL_COMP_QUOTA_MAX gives the maximum client acquire quota that - can be returned by a completion. */ - -#define FD_VINYL_COMP_QUOTA_MAX (65535UL) - -/* FIXME: consider eching back the val_gaddr and err_gaddr array back - too? Maybe helpful in the case of read pipelining (e.g. request made - on one thread, completion received on a different thread). But the - req_id can probably encode this for a case like this without needing - to bloat the completion footprint. */ - -struct __attribute__((aligned(FD_VINYL_COMP_ALIGN))) fd_vinyl_comp { - ulong seq; /* Completion sequence number */ - ulong req_id; /* Echoed from corresponding request */ - ulong link_id; /* Echoed from corresponding request */ - short err; /* FD_VINYL_SUCCESS (0) if the request was processed (see request err array for individual failure details) - FD_VINYL_ERR_* (negative) otherwise (no items in the request were processed) */ - ushort batch_cnt; /* Num items requested */ - ushort fail_cnt; /* If a successful completion, num items that failed processing, in [0,batch_cnt]. - If a failed completion, 0 (no items were processed). */ - ushort quota_rem; /* Client quota remaining when request completed processing. - 0<=quota_rem<=client quota_max<=FD_VINYL_COMP_QUOTA_MAX */ -}; - -typedef struct fd_vinyl_comp fd_vinyl_comp_t; - -/* A fd_vinyl_cq_t is an interprocess sharable persistent SPMC - queue used to communicate fd_vinyl_comp_t completions from a vinyl - tile to clients. It is implemented as a hybrid direct mapped cache / - queue with similar lockfree properties to a tango mcache. - Specifically, when publishing a completion into a cq, a producer: - - - Assigns the completion the cq's next sequence number and determines - the corresponding cache line. - - - Sets the cache line seq to seq-1. Because the cq's cache has at - least 4 lines (2 is enough for this particular case), this is a seq - that will never be held in that line. This atomically indicates to - a concurrent consumer that seq-comp_cnt is no longer available in - cache and that seq is in the process of being published into the - line. - - - Set the rest of the completion fields. - - - Sets the cache line seq to seq. This atomically indicates that - completion seq is ready. - - - Advances the cq's seq cursor. From a consumer's point of view, - this cursor has the property that that [0,seq) have been published - and (seq,ULONG_MAX] have not been published. This cursor should - only be used by consumers to synchronize their local cursors at - initialization. - - When reading completions from a cq, a consumer: - - - Determines the cache line for consumer's seq. - - - Reads the cache line seq. - - - Reads the rest of the completion fields. - - - Reads the cache line seq again. - - If the first and second reads do not match, the vinyl tile is in the - process of updating that cache line (most likely the consumer is - caught up and producer is about to produce seq ... the consumer - should try again soon). - - Otherwise, if the read sequence numbers match the consumer's seq, the - consumer received completion seq and should advance their local seq. - - If they are behind the consumer's seq, the producer has not written - seq yet and the consumer should try again later. - - If they are ahead of the consumer's seq, the producer has overrun the - consumer (the amount ahead gives a ballpark how far the consumer fell - behind) and the consumer should recover / reinit. As such, flow - control is managed by the application (e.g. if the application - ensures that there are at most comp_cnt completion generating vinyl - requests pending at any given time on this cq, no overruns are - possible). - - Note that because fd_vinyl_comp_t are AVX-2 friendly, it is possible - to SIMD accelerate producers and consumers (also similar to - fd_tango). */ - -/* FIXME: consider making comp_cnt a compile time constant? */ - -#define FD_VINYL_CQ_MAGIC (0xfd3a7352dc03a6c0UL) /* fd warm snd cq magc version 0 */ - -struct __attribute__((aligned(128))) fd_vinyl_cq_private { - - ulong magic; /* ==FD_VINYL_CQ_MAGIC */ - ulong comp_cnt; /* Number of completions that can be in flight on this cq at any given time, power of 2 of least 4 */ - uchar _[ 112 ]; /* Put seq on a separate cache line pair */ - ulong seq; /* Completion sequence number to publish next */ - /* padding to 128 alignment */ - /* fd_vinyl_comp_t comp[ comp_cnt ] here, seq number at idx = seq & (comp_cnt-1UL) when available */ - /* padding to 128 alignment */ - -}; - -typedef struct fd_vinyl_cq_private fd_vinyl_cq_t; - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_cq_{align,footprint,new,join,leave_delete} have the usual - interprocess shared persistent memory object semantics. comp_cnt is - a power-of-2 of at least 4 that gives the number completions that can - be in flight on this cq. */ - -FD_FN_CONST ulong fd_vinyl_cq_align ( void ); -FD_FN_CONST ulong fd_vinyl_cq_footprint( ulong comp_cnt ); -void * fd_vinyl_cq_new ( void * shmem, ulong comp_cnt ); -fd_vinyl_cq_t * fd_vinyl_cq_join ( void * shcq ); -void * fd_vinyl_cq_leave ( fd_vinyl_cq_t * cq ); -void * fd_vinyl_cq_delete ( void * shcq ); - -/* fd_vinyl_cq_comp returns the location in the caller's address space - of the cq's completion array. fd_vinyl_cq_comp_const is a const - correct version. fd_vinyl_cq_comp_cnt is the size of this array. - The lifetime of the returned array is the lifetime of the local join. - These assume cq is a current local join. - - fd_vinyl_cq_comp_idx gives the array index that will cache completion - seq in a cq completion array with comp_cnt elements. */ - -FD_FN_CONST static inline fd_vinyl_comp_t * -fd_vinyl_cq_comp( fd_vinyl_cq_t * cq ) { - return (fd_vinyl_comp_t *)(cq+1); -} - -FD_FN_CONST static inline fd_vinyl_comp_t const * -fd_vinyl_cq_comp_const( fd_vinyl_cq_t const * cq ) { - return (fd_vinyl_comp_t const *)(cq+1); -} - -FD_FN_PURE static inline ulong fd_vinyl_cq_comp_cnt( fd_vinyl_cq_t const * cq ) { return cq->comp_cnt; } - -FD_FN_CONST static inline ulong fd_vinyl_cq_comp_idx( ulong seq, ulong comp_cnt ) { return seq & (comp_cnt-1UL); } - -/* fd_vinyl_cq_seq returns the position of the producer's sequence - number cursor. Specifically, at some point during the call, - completions [0,seq) were published, completions (seq,ULONG_MAX] were - not been published, and completion seq was either published, being - published or not published. This is used for initial synchronization - between producer and consumers. This is a compiler fence. */ - -static inline ulong -fd_vinyl_cq_seq( fd_vinyl_cq_t const * cq ) { - FD_COMPILER_MFENCE(); - ulong seq = cq->seq; - FD_COMPILER_MFENCE(); - return seq; -} - -/* fd_vinyl_cq_send sends a completion. If comp is non-NULL, the - completion will be written out-of-band to the location comp (assumes - comp->seq is not 1 on entry and will set to 1 once the send it done). - Otherwise, if cq is non-NULL, the completion will be enquened into - the given cq (assumes cq is a current local join). If both cq and - comp are NULL, this is a no-op. This is a compiler fence. */ - -/* FIXME: consider SIMD accelerating */ - -static inline void -fd_vinyl_cq_send( fd_vinyl_cq_t * cq, - fd_vinyl_comp_t * comp, - ulong req_id, - ulong link_id, - int err, /* In [-2^15,2^15) */ - ulong batch_cnt, /* In [0,2^16) */ - ulong fail_cnt, /* In [0,2^16) */ - ulong quota_rem ) { /* In [0,2^16) */ - - ulong stack_seq[1]; - - ulong seq; - ulong * _seq; - - if( FD_UNLIKELY( comp ) ) { /* Send directly */ - - seq = 1UL; /* For direct sends, comp->seq should have already been set to something != 1 */ - _seq = stack_seq; - - } else if( FD_LIKELY( cq ) ) { /* Send via cq */ - - seq = cq->seq; - _seq = &cq->seq; - comp = fd_vinyl_cq_comp( cq ) + fd_vinyl_cq_comp_idx( seq, cq->comp_cnt ); - - FD_COMPILER_MFENCE(); - comp->seq = seq - 1UL; /* Mark completion seq being written */ - - } else { /* No place to send completion */ - - FD_COMPILER_MFENCE(); /* Consistent semantics in all cases */ - return; - - } - - FD_COMPILER_MFENCE(); - comp->req_id = req_id; - comp->link_id = link_id; - comp->err = (short)err; - comp->batch_cnt = (ushort)batch_cnt; - comp->fail_cnt = (ushort)fail_cnt; - comp->quota_rem = (ushort)quota_rem; - FD_COMPILER_MFENCE(); - comp->seq = seq; /* Mark completion seq as written */ - FD_COMPILER_MFENCE(); - *_seq = seq + 1UL; /* Record that completions [0,seq) are published */ - FD_COMPILER_MFENCE(); - -} - -/* fd_vinyl_cq_recv receives completion seq from the given cq. Returns - 0 on success, positive if completion seq has not been published yet - and negative if the consumer has been overrun by the producer. On - return, *dst contains the desired completion on success and is - clobbered otherwise. Assumes cq is a current local join and dst is - valid. This is a compiler fence. */ - -/* FIXME: consider SIMD accelerating */ - -static inline long -fd_vinyl_cq_recv( fd_vinyl_cq_t const * cq, - ulong seq, - fd_vinyl_comp_t * dst ) { - fd_vinyl_comp_t const * src = fd_vinyl_cq_comp_const( cq ) + fd_vinyl_cq_comp_idx( seq, cq->comp_cnt ); - - FD_COMPILER_MFENCE(); - ulong seq0 = src->seq; - FD_COMPILER_MFENCE(); - *dst = *src; - FD_COMPILER_MFENCE(); - ulong seq1 = src->seq; - FD_COMPILER_MFENCE(); - - long diff0 = (long)(seq-seq0); - long diff1 = (long)(seq-seq1); - return fd_long_if( !diff0, diff1, diff0 ); -} - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_cq_fd_vinyl_cq_h */ diff --git a/src/vinyl/cq/test_vinyl_cq.c b/src/vinyl/cq/test_vinyl_cq.c deleted file mode 100644 index 20b7f0e54e8..00000000000 --- a/src/vinyl/cq/test_vinyl_cq.c +++ /dev/null @@ -1,207 +0,0 @@ -#include "../fd_vinyl.h" - -FD_STATIC_ASSERT( FD_VINYL_COMP_ALIGN ==32UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_COMP_FOOTPRINT==32UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_COMP_ALIGN ==alignof(fd_vinyl_comp_t), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_COMP_FOOTPRINT==sizeof (fd_vinyl_comp_t), unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_COMP_QUOTA_MAX==65535UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_CQ_MAGIC==0xfd3a7352dc03a6c0UL, unit_test ); - -static FD_FOR_ALL_BEGIN( test_tile, 1L ) { - fd_vinyl_cq_t * cq = (fd_vinyl_cq_t *)arg[0]; - long t_stop = (long) arg[1]; - - static fd_vinyl_comp_t oob_comp[2]; - - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, (uint)(tpool_t0+1UL), 0UL ) ); - - if( !tpool_t0 ) { /* Comp sender */ - - memset( oob_comp, 0, 2*sizeof(fd_vinyl_comp_t) ); /* seq should be !=1 before sender sends to it and receiver expects it */ - - FD_LOG_NOTICE(( "Starting transmitter" )); - - for(;;) { - - if( FD_UNLIKELY( fd_log_wallclock() > t_stop ) ) break; - - ulong val = fd_rng_ulong( rng ); - - val += (ulong)!val; /* Use req_id==0 as termination for receivers in this test */ - - ulong req_id = val++; - ulong link_id = val++; - int err = (int) (val++); - ulong batch_cnt = (ulong)(val++); - ulong fail_cnt = (ulong)(val++); - ulong quota_rem = (ulong)(val++); - - fd_vinyl_cq_send( cq, NULL, req_id, link_id, err, batch_cnt, fail_cnt, quota_rem ); - - } - - /* Send terminate */ - - fd_vinyl_cq_send( cq, NULL, 0UL, 0UL, 0, 0UL, 0UL, 0UL ); - - /* Do some oob and no-op sending */ - - fd_vinyl_cq_send( NULL, NULL, 1UL, 2UL, 3, 4UL, 5UL, 6UL ); /* no-op */ - fd_vinyl_cq_send( NULL, &oob_comp[0], 2UL, 3UL, 4, 5UL, 6UL, 7UL ); /* to oob_comp 0 (no cq provided) */ - fd_vinyl_cq_send( cq, &oob_comp[1], 3UL, 4UL, 5, 6UL, 7UL, 8UL ); /* to oob_comp 1 (cq override) */ - - } else { - - FD_LOG_NOTICE(( "Starting receiver %lu", tpool_t0-1UL )); - - ulong seq = fd_vinyl_cq_seq( cq ); - - for(;;) { - - fd_vinyl_comp_t comp[1]; - long diff = fd_vinyl_cq_recv( cq, seq, comp ); - - if( FD_LIKELY( diff ) ) { - if( FD_UNLIKELY( diff<0L ) ) { - FD_LOG_WARNING(( "Receiver %lu overrun; resynchronizing", tpool_t0-1UL )); - seq = fd_vinyl_cq_seq( cq ); - } - FD_SPIN_PAUSE(); - continue; - } - - ulong val = comp->req_id; - if( FD_UNLIKELY( !val ) ) break; - - FD_TEST( comp->req_id == val ); val++; - FD_TEST( comp->link_id == val ); val++; - FD_TEST( comp->err == (short) val ); val++; - FD_TEST( comp->batch_cnt == (ushort)val ); val++; - FD_TEST( comp->fail_cnt == (ushort)val ); val++; - FD_TEST( comp->quota_rem == (ushort)val ); val++; - - seq++; - } - - ulong volatile * _seq; - - /* Wait to receive oob_comp 0 */ - - _seq = &oob_comp[0].seq; while( !_seq[0] ) FD_SPIN_PAUSE(); - - FD_TEST( oob_comp[0].req_id == 2UL ); - FD_TEST( oob_comp[0].link_id == 3UL ); - FD_TEST( oob_comp[0].err == (schar) 4UL ); - FD_TEST( oob_comp[0].batch_cnt == (ushort)5UL ); - FD_TEST( oob_comp[0].fail_cnt == (ushort)6UL ); - FD_TEST( oob_comp[0].quota_rem == (ushort)7UL ); - - /* Wait to receive oob_comp 1 */ - - _seq = &oob_comp[1].seq; while( !_seq[0] ) FD_SPIN_PAUSE(); - - FD_TEST( oob_comp[1].req_id == 3UL ); - FD_TEST( oob_comp[1].link_id == 4UL ); - FD_TEST( oob_comp[1].err == (schar) 5UL ); - FD_TEST( oob_comp[1].batch_cnt == (ushort)6UL ); - FD_TEST( oob_comp[1].fail_cnt == (ushort)7UL ); - FD_TEST( oob_comp[1].quota_rem == (ushort)8UL ); - - } - - fd_rng_delete( fd_rng_leave( rng ) ); - -} FD_FOR_ALL_END - -#define SHMEM_ALIGN (128) -#define SHMEM_FOOTPRINT (1UL<<20) - -static uchar shmem[ SHMEM_FOOTPRINT ] __attribute__((aligned(SHMEM_ALIGN))); - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - ulong comp_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--comp-cnt", NULL, 8192UL ); - - FD_LOG_NOTICE(( "Testing (--comp-cnt %lu)", comp_cnt )); - - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, 0U, 0UL ) ); - - ulong thread_cnt = fd_tile_cnt(); - - FD_LOG_NOTICE(( "Creating tpool from all %lu tiles", thread_cnt )); - - static uchar _tpool[ FD_TPOOL_FOOTPRINT( FD_TILE_MAX ) ] __attribute__((aligned(FD_TPOOL_ALIGN))); - - fd_tpool_t * tpool = fd_tpool_init( _tpool, thread_cnt, 0UL ); /* logs details */ - if( FD_UNLIKELY( !tpool ) ) FD_LOG_ERR(( "fd_tpool_init failed" )); - - for( ulong thread_idx=1UL; thread_idx SHMEM_ALIGN) | (footprint > SHMEM_FOOTPRINT) ) ) - FD_LOG_ERR(( "Update SHMEM_ALIGN and/or SHMEM_FOOTPRINT for this comp_cnt" )); - - FD_TEST( !fd_vinyl_cq_new( NULL, comp_cnt ) ); /* NULL shmem */ - FD_TEST( !fd_vinyl_cq_new( (void *)1UL, comp_cnt ) ); /* misaligned shmem */ - FD_TEST( !fd_vinyl_cq_new( shmem, 0UL ) ); /* bad comp_cnt */ - void * shcq = fd_vinyl_cq_new( shmem, comp_cnt ); FD_TEST( !!shcq ); - - FD_TEST( !fd_vinyl_cq_join( NULL ) ); /* NULL shmem */ - FD_TEST( !fd_vinyl_cq_join( (void *)1UL ) ); /* misaligned shmem */ - fd_vinyl_cq_t * cq = fd_vinyl_cq_join( shcq ); FD_TEST( !!cq ); - - fd_vinyl_comp_t * comp = fd_vinyl_cq_comp( cq ); FD_TEST( !!comp ); - FD_TEST( fd_vinyl_cq_comp_const( cq )==comp ); - - FD_TEST( fd_vinyl_cq_comp_cnt( cq )==comp_cnt ); - - for( ulong rem=10000UL; rem; rem-- ) { - ulong seq = fd_rng_ulong( rng ); - FD_TEST( fd_vinyl_cq_comp_idx( seq, comp_cnt )==(seq & (comp_cnt-1UL)) ); - } - - FD_TEST( !fd_vinyl_cq_seq( cq ) ); /* Initial sequence number at zero */ - - long t_stop = fd_log_wallclock() + (long)1e9; - - FD_FOR_ALL( test_tile, tpool,0UL,thread_cnt, 0L,(long)thread_cnt, cq, t_stop ); - - FD_TEST( !fd_vinyl_cq_leave( NULL ) ); /* NULL cq */ - FD_TEST( fd_vinyl_cq_leave( cq )==shcq ); - - FD_TEST( !fd_vinyl_cq_delete( NULL ) ); /* NULL shmem */ - FD_TEST( !fd_vinyl_cq_delete( (void *)1UL ) ); /* misaligned shmem */ - FD_TEST( fd_vinyl_cq_delete( shcq )==shmem ); - - FD_TEST( !fd_vinyl_cq_join ( shcq ) ); /* bad magic */ - FD_TEST( !fd_vinyl_cq_delete( shcq ) ); /* bad magic */ - - FD_LOG_NOTICE(( "Cleaning up" )); - - /* Note: fini automatically pops all worker threads */ - - fd_tpool_fini( tpool ); /* logs details */ - - fd_rng_delete( fd_rng_leave( rng ) ); - - FD_LOG_NOTICE(( "pass" )); - fd_halt(); - return 0; -} diff --git a/src/vinyl/data/Local.mk b/src/vinyl/data/Local.mk deleted file mode 100644 index 9cee44aae71..00000000000 --- a/src/vinyl/data/Local.mk +++ /dev/null @@ -1,7 +0,0 @@ -$(call add-hdrs,fd_vinyl_data.h) -$(call add-objs,fd_vinyl_data fd_vinyl_data_szc_cfg,fd_vinyl) - -ifdef FD_HAS_HOSTED -$(call make-unit-test,test_vinyl_data,test_vinyl_data,fd_vinyl fd_tango fd_util_test fd_util) -$(call run-unit-test,test_vinyl_data) -endif diff --git a/src/vinyl/data/fd_vinyl_data.c b/src/vinyl/data/fd_vinyl_data.c deleted file mode 100644 index cd0484421ea..00000000000 --- a/src/vinyl/data/fd_vinyl_data.c +++ /dev/null @@ -1,594 +0,0 @@ -#include "fd_vinyl_data.h" - -FD_FN_PURE static inline ulong /* FIXME: FD_FN_CONST? (i.e. szc_cfg is const) */ -fd_vinyl_data_szc_all_blocks( ulong szc ) { - /* sigh ... deal with wide shift UB (obj_cnt in [1,64]) */ - return ((1UL << ((int)fd_vinyl_data_szc_cfg[ szc ].obj_cnt - 1)) << 1) - 1UL; -} - -FD_FN_CONST static inline ulong -fd_vinyl_data_obj_off( void const * laddr0, - fd_vinyl_data_obj_t const * obj ) { - return fd_ulong_if( !!obj, (ulong)obj - (ulong)laddr0, 0UL ); -} - -FD_FN_CONST static inline fd_vinyl_data_obj_t * -fd_vinyl_data_obj_ptr( void const * laddr0, - ulong off ) { - return (fd_vinyl_data_obj_t *)fd_ulong_if( !!off, (ulong)laddr0 + off, 0UL ); -} - -FD_FN_PURE static int -fd_vinyl_data_superblock_test( fd_vinyl_data_t const * data, - fd_vinyl_data_obj_t const * superblock, - ulong szc ) { - - /* Test that superblock seems to be point to a valid location for a - superblock that holds szc objects. */ - - ulong sb0 = (ulong)superblock; - - if( FD_UNLIKELY( !fd_ulong_is_aligned( sb0, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) return FD_VINYL_ERR_CORRUPT; /* misaligned */ - - ulong parent_szc = fd_vinyl_data_szc_cfg[ szc ].parent_szc; - - ulong vol_idx = (sb0 - (ulong)data->vol) / FD_VINYL_DATA_VOL_FOOTPRINT; - if( FD_UNLIKELY( vol_idx >= data->vol_cnt ) ) return FD_VINYL_ERR_CORRUPT; /* not in a volume */ - - ulong obj_cnt = (ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt; - ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc ); - - ulong sb1 = sb0 + sizeof(fd_vinyl_data_obj_t) + obj_cnt*obj_footprint; - - int not_volume_sb = parent_szcvol[ vol_idx ]; - ulong vol1 = vol0 + FD_VINYL_DATA_VOL_FOOTPRINT; - - if( FD_LIKELY( not_volume_sb ) ) vol0 += sizeof(fd_vinyl_data_obj_t); /* only volume sb can be at head of volume */ - else { - if( FD_UNLIKELY( sb0!=vol0 ) ) return FD_VINYL_ERR_CORRUPT; /* vol sb not a vol */ - if( FD_UNLIKELY( superblock->idx!=vol_idx ) ) return FD_VINYL_ERR_CORRUPT; /* mismatched idx */ - } - - if( FD_UNLIKELY( !((vol0<=sb0) & (sb1<=vol1) ) ) ) return FD_VINYL_ERR_CORRUPT; /* out of bounds */ - - if( FD_LIKELY( not_volume_sb ) && FD_UNLIKELY( superblock->idx >= fd_vinyl_data_szc_cfg[ parent_szc ].obj_cnt ) ) - return FD_VINYL_ERR_CORRUPT; - - /* At this point, superblock is at an appropriate position. Test that - its type and szc correspond to a superblock for objects of this - size class and that free_blocks doesn't have any stray bits set in - it. - - Note that we can't next_off fully here because we don't know if SB - is on the inactive stack. Even without testing here, it is fully - tested because we test the result of all inactive stack pops (but - it might be nice to catch issues with next_off corruption earlier). */ - - ulong all_blocks = fd_vinyl_data_szc_all_blocks( szc ); - - if( FD_UNLIKELY( !( (superblock->type ==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK) & - ((ulong)superblock->child_szc==szc ) & - ((ulong)superblock->szc ==parent_szc ) & - (!(superblock->free_blocks & ~all_blocks) ) ) ) ) return FD_VINYL_ERR_CORRUPT; - - return FD_VINYL_SUCCESS; -} - -/**********************************************************************/ - -ulong -fd_vinyl_data_align( void ) { - return alignof(fd_vinyl_data_t); -} - -ulong -fd_vinyl_data_footprint( void ) { - return sizeof(fd_vinyl_data_t); -} - -fd_vinyl_data_t * -fd_vinyl_data_init( void * lmem, - void * shmem, - ulong shmem_sz, - void * laddr0 ) { - - if( FD_UNLIKELY( !lmem ) ) { - FD_LOG_WARNING(( "NULL lmem" )); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)lmem, fd_vinyl_data_align() ) ) ) { - FD_LOG_WARNING(( "misaligned lmem" )); - return NULL; - } - - ulong _laddr0 = (ulong)laddr0; - if( FD_UNLIKELY( !fd_ulong_is_aligned( _laddr0, FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) { - FD_LOG_WARNING(( "misaligned laddr0" )); - return NULL; - } - - /* Note that the shmem has a larger guard region at the end such that - a "try" for a block at the end of the shmem will have large enough - guard region that it is safe to read a val up to FD_VINYL_VAL_MAX - in size (as the val_sz returned as part of the speculation might be - bogus and the user of a try should recognize this as part of their - speculation). */ - - ulong _shmem0 = (ulong)shmem; - ulong _shmem1 = _shmem0 + shmem_sz - fd_vinyl_bstream_pair_sz( FD_VINYL_VAL_MAX ) + FD_VINYL_BSTREAM_BLOCK_SZ; - ulong _vol0 = fd_ulong_align_up( _shmem0, FD_VINYL_BSTREAM_BLOCK_SZ ); - ulong vol_cnt = (_shmem1-_vol0) / FD_VINYL_DATA_VOL_FOOTPRINT; - ulong _vol1 = _vol0 + vol_cnt*FD_VINYL_DATA_VOL_FOOTPRINT; - - if( FD_UNLIKELY( !((_laddr0<_shmem0) & (_shmem0<=_vol0) & (_vol0<_vol1) & (_vol1<=_shmem1)) ) ) { - FD_LOG_WARNING(( "bad shmem region" )); - return NULL; - } - - fd_vinyl_data_t * data = (fd_vinyl_data_t *)lmem; - - memset( data, 0, fd_vinyl_data_footprint() ); - - data->shmem = shmem; - data->shmem_sz = shmem_sz; - data->laddr0 = laddr0; - data->vol = (fd_vinyl_data_vol_t *)_vol0; - data->vol_cnt = vol_cnt; - - return data; -} - -void * -fd_vinyl_data_fini( fd_vinyl_data_t * data ) { - - if( FD_UNLIKELY( !data ) ) { - FD_LOG_WARNING(( "NULL data" )); - return NULL; - } - - return data; -} - -/* Note: the algorithms below is identical to fd_alloc. But since it - is running single threaded and non-persistent, there's less atomic - operation and/or address translation shenanigans going on. See - fd_alloc for more in depth discussions. */ - -fd_vinyl_data_obj_t * -fd_vinyl_data_alloc( fd_vinyl_data_t * data, - ulong szc ) { - - FD_DCHECK_CRIT( data, "NULL data" ); - FD_DCHECK_CRIT( szcladdr0; - fd_vinyl_data_vol_t * vol = data->vol; - fd_vinyl_data_obj_t ** _active = &data->superblock[ szc ].active; - fd_vinyl_data_obj_t ** _inactive_top = &data->superblock[ szc ].inactive_top; - - /* Acquire a superblock with space for a szc object. We first look - if there is an active superblock for this szc. If not, we look at - the inactive stack for this sizeclass. If not, we allocate - superblock suitable for holding objects of this sizeclass. This - will be either from this allocator or new data volume. (We don't - need to use global addresses for superblock->next_off but doing so - allows tools to non-invasively inspect the data cache real time.) */ - - fd_vinyl_data_obj_t * superblock = *_active; - - if( FD_LIKELY( superblock ) ) { - - FD_DCHECK_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" ); - - *_active = NULL; - - } else { - - superblock = *_inactive_top; - - if( FD_LIKELY( superblock ) ) { - - FD_DCHECK_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" ); - - *_inactive_top = fd_vinyl_data_obj_ptr( laddr0, superblock->next_off ); - - } else { - - ulong parent_szc = (ulong)fd_vinyl_data_szc_cfg[ szc ].parent_szc; - if( FD_LIKELY( parent_szctype init by obj_alloc to ALLOC, reset below */ - /* superblock->szc init by obj_alloc */ - /* superblock->idx init by obj_alloc */ - /* superblock->child_szc init below */ - /* superblock->free_blocks init below */ - /* superblock->next_off init when pushed onto inactive stack */ - - } else { - - ulong vol_idx = data->vol_idx_free; - if( FD_UNLIKELY( vol_idx >= data->vol_cnt ) ) return NULL; - data->vol_idx_free = vol[ vol_idx ].obj->idx; - - superblock = vol[ vol_idx ].obj; - - /* superblock->type init below */ - superblock->szc = (ushort)FD_VINYL_DATA_SZC_CNT; - superblock->idx = vol_idx; - /* superblock->child_szc init below */ - /* superblock->free_blocks init below */ - /* superblock->next_off init when pushed onto inactive stack */ - - } - - superblock->type = FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK; - superblock->child_szc = (ushort)szc; - superblock->free_blocks = fd_vinyl_data_szc_all_blocks( szc ); - /*superblock->next_off init when pushed onto inactive stack */ - - } - } - - /* At this point, superblock has at least 1 free szc object, is not - in circulation and szc has no active superblock. Allocate the - first free object in it. */ - - ulong free_blocks = superblock->free_blocks; - - FD_DCHECK_CRIT( free_blocks, "corruption detected" ); - - ulong idx = (ulong)fd_ulong_find_lsb( free_blocks ); - - free_blocks = fd_ulong_pop_lsb( free_blocks ); - - superblock->free_blocks = free_blocks; - - /* If this superblock still has free blocks in it, return it to - circulation for future allocation as szc's active superblock, - pushing any displaced superblock onto szc's inactive superblock - stack. Other strategies are possible, see fd_alloc for discussion - of tradeoffs. */ - -# if 0 - - if( FD_LIKELY( free_blocks ) ) { - - fd_vinyl_data_obj_t * displaced_superblock = *_active; - *_active = superblock; - - if( FD_UNLIKELY( displaced_superblock ) ) { - - FD_DCHECK_ALERT( !fd_vinyl_data_superblock_test( data, displaced_superblock, szc ), "corruption detected" ); - - displaced_superblock->next_off = fd_vinyl_data_obj_off( laddr0, *_inactive_top ); - *_inactive_top = displaced_superblock; - - } - - } - -# else - - /* For a non-concurrent implementation, we know szc has no active - superblock active at this point (because their's no concurrent - alloc or free that could have set it behind our back). We don't - have to worry about displacing a superblock, simplifying the - above. */ - - fd_vinyl_data_obj_t * tmp[1]; - *(free_blocks ? _active : tmp) = superblock; /* branchless conditional store */ - -# endif - - /* Initialize the allocated object metadata and return. */ - - fd_vinyl_data_obj_t * obj = (fd_vinyl_data_obj_t *)( (ulong)superblock + sizeof(fd_vinyl_data_obj_t) - + idx*fd_vinyl_data_szc_obj_footprint( szc ) ); - - obj->type = FD_VINYL_DATA_OBJ_TYPE_ALLOC; - obj->szc = (ushort)szc; - obj->idx = idx; -//obj->child_szc = ... d/c (not a superblock) -//obj->free_blocks = ... d/c (not a superblock) -//obj->next_off = ... d/c (not a superblock) - - return obj; -} - -void -fd_vinyl_data_free( fd_vinyl_data_t * data, - fd_vinyl_data_obj_t * obj ) { - - FD_DCHECK_CRIT( data, "NULL data" ); - - if( FD_UNLIKELY( !obj ) ) return; - - FD_DCHECK_CRIT( fd_ulong_is_aligned( (ulong)obj, FD_VINYL_BSTREAM_BLOCK_SZ ), "obj misaligned" ); - FD_DCHECK_CRIT( ((ulong)data->vol<=(ulong)obj) & ((ulong)obj<(ulong)(data->vol+data->vol_cnt)), "obj not in data cache" ); - FD_DCHECK_CRIT( (obj->type==FD_VINYL_DATA_OBJ_TYPE_ALLOC) | (obj->type==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK), "obj not freeable" ); - - /* At this point, obj appears to be a freeable obj in the data. - Determine how obj was allocated. If obj is a vol, push obj onto - the vol free stack. */ - - ulong szc = (ulong)obj->szc; - ulong idx = obj->idx; - - FD_DCHECK_CRIT( szc<=FD_VINYL_DATA_SZC_CNT, "corruption detected" ); /* valid szc */ - - if( FD_UNLIKELY( szc>=FD_VINYL_DATA_SZC_CNT ) ) { - FD_DCHECK_CRIT( idx < data->vol_cnt, "corruption detected" ); /* valid idx for vol */ - - obj->type = FD_VINYL_DATA_OBJ_TYPE_FREEVOL; /* Mark as on the free stack */ - obj->idx = data->vol_idx_free; - data->vol_idx_free = idx; - - return; - } - - FD_DCHECK_CRIT( idx<(ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt, "corruption detected" ); /* valid idx for szc */ - - /* At this point, obj appears to be contained in a superblock at - position idx. Mark the object as free in the superblock. */ - - fd_vinyl_data_obj_t * superblock = (fd_vinyl_data_obj_t *) - ((ulong)obj - sizeof(fd_vinyl_data_obj_t) - idx*fd_vinyl_data_szc_obj_footprint( szc )); - - FD_DCHECK_ALERT( !fd_vinyl_data_superblock_test( data, superblock, szc ), "corruption detected" ); - - ulong free_blocks = superblock->free_blocks; - ulong block = 1UL << idx; - - FD_DCHECK_CRIT( !(free_blocks & block), "obj already free" ); - - obj->type = 0UL; /* Mark this as no longer an object (not strictly necessary but useful for things like double free detection) */ - - free_blocks |= block; - - superblock->free_blocks = free_blocks; - - /* If this superblock was not in circulation for szc allocations (i.e. - had no free objects in it before the free we just did), we return - it to circulation as szc's active superblock, pushing any displaced - superblock onto the szc's inactive superblock stack. - - Otherwise, if this free made the superblock totally empty, we check - if the szc'c inactive superblock top is also totally empty. If so, - we pop the inactive stack and free that. - - This keeps a small bounded supply empty superblocks around for fast - future allocations in this szc while allowing memory to reclaimed - for different szc objs. Note that we can't just free superblock if - it is totally empty fast O(1) because we don't know where it is in - circulation (and, even if we did, this is a bad idea). Other - strategies are possible, see fd_alloc for discussion of tradeoffs. */ - - if( FD_UNLIKELY( free_blocks==block ) ) { - - fd_vinyl_data_obj_t * displaced_superblock = data->superblock[ szc ].active; - data->superblock[ szc ].active = superblock; - - if( displaced_superblock ) { - - FD_DCHECK_ALERT( !fd_vinyl_data_superblock_test( data, displaced_superblock, szc ), "corruption detected" ); - - displaced_superblock->next_off = fd_vinyl_data_obj_off( data->laddr0, data->superblock[ szc ].inactive_top ); - data->superblock[ szc ].inactive_top = displaced_superblock; - - } - - } else { - - ulong all_blocks = fd_vinyl_data_szc_all_blocks( szc ); - - if( FD_UNLIKELY( free_blocks==all_blocks ) ) { - - fd_vinyl_data_obj_t * candidate_superblock = data->superblock[ szc ].inactive_top; - - if( FD_UNLIKELY( candidate_superblock ) ) { - - FD_DCHECK_ALERT( !fd_vinyl_data_superblock_test( data, candidate_superblock, szc ), "corruption detected" ); - - if( FD_UNLIKELY( candidate_superblock->free_blocks==all_blocks ) ) { - - data->superblock[ szc ].inactive_top = fd_vinyl_data_obj_ptr( data->laddr0, candidate_superblock->next_off ); - - fd_vinyl_data_free( data, candidate_superblock ); - } - - } - - } - - } - -} - -static FD_FOR_ALL_BEGIN( fd_vinyl_data_reset_task, 1L ) { - fd_vinyl_data_t * data = (fd_vinyl_data_t *)arg[0]; - int level = (int) arg[1]; - - void * shmem = data->shmem; - ulong shmem_sz = data->shmem_sz; - fd_vinyl_data_vol_t * vol = data->vol; - ulong vol_cnt = data->vol_cnt; - - ulong vol0 = (ulong)block_i0; - ulong vol1 = (ulong)block_i1; - - /* At this point, we have been assigned the non-empty set of volumes - [vol0,vol1) to reset. If this is a hard reset, we zero out the - volume we have been assigned. If we are responsible for zeroing - the leading/trailing volume, we also handle any leading/trailing - zero padding in the shmem. */ - - if( level ) { - void * mem = (void *)(vol + vol0); - ulong mem_sz = (vol1-vol0)*FD_VINYL_DATA_VOL_FOOTPRINT; - - if( vol0==0UL ) mem_sz += (ulong)vol - (ulong)shmem, mem = shmem; - if( vol1==vol_cnt ) mem_sz += shmem_sz - vol_cnt*FD_VINYL_DATA_VOL_FOOTPRINT; - - memset( mem, 0, mem_sz ); /* mem_sz guaranteed non-zero */ - } - - /* Mark the volumes as free and join them in a linked list */ - - for( ulong vol_idx=vol0; vol_idxtype = FD_VINYL_DATA_OBJ_TYPE_FREEVOL; - vol[ vol_idx ].obj->idx = vol_idx + 1UL; - } - -} FD_FOR_ALL_END - -void -fd_vinyl_data_reset( fd_tpool_t * tpool, ulong t0, ulong t1, int level, - fd_vinyl_data_t * data ) { - - FD_FOR_ALL( fd_vinyl_data_reset_task, tpool,t0,t1, 0L,(long)data->vol_cnt, data, level ); - - data->vol_idx_free = 0UL; - - for( ulong szc=0UL; szcsuperblock[ szc ].active = NULL; - data->superblock[ szc ].inactive_top = NULL; - } - -} - -/* FIXME: consider adding a compact function? */ - -#define TEST( c ) \ - do { if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "corruption detected (%s)", #c )); return FD_VINYL_ERR_CORRUPT; } } while(0) - -/* fd_vinyl_data_verify_obj returns FD_VINYL_SUCCESS (0) if a type - SUPERBLOCK or type ALLOC obj and all its children appear to be valid - and FD_VINYL_ERR_CORRUPT (negative) if memory corruption was detected - (logs details). */ - -static int -fd_vinyl_data_verify_superblock( fd_vinyl_data_t const * data, - fd_vinyl_data_obj_t const * superblock ) { - - TEST( superblock ); - TEST( fd_ulong_is_aligned( (ulong)superblock, alignof(fd_vinyl_data_obj_t) ) ); - ulong szc = (ulong)superblock->child_szc; - - TEST( !fd_vinyl_data_superblock_test( data, superblock, szc ) ); - - ulong free_blocks = superblock->free_blocks; - - ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc ); - - for( ulong rem=fd_vinyl_data_szc_all_blocks( szc ) & ~free_blocks; rem; rem=fd_ulong_pop_lsb( rem ) ) { - ulong obj_idx = (ulong)fd_ulong_find_lsb( rem ); - - fd_vinyl_data_obj_t const * obj = (fd_vinyl_data_obj_t const *) - ((ulong)superblock + sizeof(fd_vinyl_data_obj_t) + obj_idx*obj_footprint); - - ulong type = obj->type; - if( type==FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK ) TEST( !fd_vinyl_data_verify_superblock( data, obj ) ); - else TEST( type==FD_VINYL_DATA_OBJ_TYPE_ALLOC ); - } - - return FD_VINYL_SUCCESS; -} - -int -fd_vinyl_data_verify( fd_vinyl_data_t const * data ) { - - /* Verify data looks like a fd_vinyl_data_t */ - - TEST( data ); - TEST( fd_ulong_is_aligned( (ulong)data, alignof(fd_vinyl_data_t) ) ); - - ulong vol_cnt = data->vol_cnt; - - ulong laddr0 = (ulong)data->laddr0; - ulong shmem0 = (ulong)data->shmem; - ulong shmem1 = (ulong)data->shmem + data->shmem_sz; - ulong vol0 = (ulong)&data->vol[0 ]; - ulong vol1 = (ulong)&data->vol[vol_cnt]; - - TEST( fd_ulong_is_aligned( laddr0, FD_VINYL_BSTREAM_BLOCK_SZ ) ); - TEST( fd_ulong_is_aligned( vol0, FD_VINYL_BSTREAM_BLOCK_SZ ) ); - TEST( fd_ulong_is_aligned( vol1, FD_VINYL_BSTREAM_BLOCK_SZ ) ); - - TEST( (laddr0vol; - ulong vol_idx = data->vol_idx_free; - for(;;) { - if( vol_idx>=vol_cnt ) break; - fd_vinyl_data_obj_t const * obj = vol[ vol_idx ].obj; - TEST( vol_free_cnttype==FD_VINYL_DATA_OBJ_TYPE_FREEVOL ); /* volume is marked as free */ - vol_free_cnt++; - vol_idx = obj->idx; - } - - /* Verify volumes */ - - ulong vol_free_rem = vol_free_cnt; - ulong vol_used_rem = vol_cnt - vol_free_cnt; - - for( vol_idx=0UL; vol_idxtype; - - if( type==FD_VINYL_DATA_OBJ_TYPE_FREEVOL ) { /* Free volume */ - TEST( vol_free_rem ); - /* obj->szc ... d/c for a free vol */ - /* obj->idx ... validated above */ - /* obj->child_szc ... d/c for a free vol */ - /* obj->free_blocks ... d/c for a free vol */ - /* obj->next_off ... d/c for a free vol */ - vol_free_rem--; - continue; - } - - TEST( vol_used_rem ); - TEST( !fd_vinyl_data_verify_superblock( data, obj ) ); - vol_used_rem--; - } - - TEST( !vol_free_rem ); - TEST( !vol_used_rem ); - - /* Verify active superblocks and inactive stacks */ - - for( ulong szc=0UL; szcsuperblock[ szc ].active; - if( active ) { - TEST( !fd_vinyl_data_superblock_test( data, active, szc ) ); - TEST( active->free_blocks ); - } - - ulong obj_footprint = fd_vinyl_data_szc_obj_footprint( szc ); - ulong obj_cnt = (ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt; - ulong superblock_footprint = sizeof(fd_vinyl_data_obj_t) + obj_cnt*obj_footprint; - ulong rem = (vol1 - vol0 + superblock_footprint - 1UL) / superblock_footprint; - - fd_vinyl_data_obj_t * superblock = data->superblock[ szc ].inactive_top; - while( superblock ) { - TEST( rem ); rem--; /* avoid cycles */ - TEST( superblock!=active ); - TEST( !fd_vinyl_data_superblock_test( data, superblock, szc ) ); - TEST( superblock->free_blocks ); - superblock = fd_vinyl_data_obj_ptr( (void *)laddr0, superblock->next_off ); - } - } - - return FD_VINYL_SUCCESS; -} diff --git a/src/vinyl/data/fd_vinyl_data.h b/src/vinyl/data/fd_vinyl_data.h deleted file mode 100644 index ef95c6c6659..00000000000 --- a/src/vinyl/data/fd_vinyl_data.h +++ /dev/null @@ -1,479 +0,0 @@ -#ifndef HEADER_fd_src_vinyl_data_fd_vinyl_data_h -#define HEADER_fd_src_vinyl_data_fd_vinyl_data_h - -/* fd_vinyl_data provides a data structure suitable for in-memory - caching extremely of large amounts of variable sized pairs in memory. - The memory layouts are such that cached pairs can be zero copy - lockfree operated on by multiple threads in other address spaces and - async direct I/O hardware concurrently. - - Note that, though pairs are cached in a shared memory region, this is - not a persistent or concurrent datastructure. Specifically, only the - vinyl tile can allocate or free objects from it and then can do only - sequentially. - - Notes: - - - The shared memory region is divided into a fixed number of fixed - sized volumes (similar to fd_groove). - - Volumes flexibly store data objects. - - Volumes not in use are stored on a free volume stack. - - The size of a data object is determined by the object's size class. - - A data object can either be an allocation (i.e. a cached pair val - that fits in that object's sizeclass) or a superblock (a collection - of smaller data objects from the same sizeclass). - - The algorithms that manage the allocations are virtually identical to - fd_groove and fd_alloc. But they have been simplified, customized - and optimized for this use case (e.g. minimal need for address - translation, no need for atomic operations, no need for concurrency - group optimizations, no need to layout cache for concurrent access, - much more fine grained size classes for minimal data store overheads, - etc). This also does extensive (and compile time configurable) - memory data integrity continuously to help catch memory corruption - (either due to hardware failures, buggy usage or malicious usage). - - I/O alignment requirements quantize the data cache footprint of a - val_sz pair to BLOCK_SZ+align_up(pair_sz(val_sz),BLOCK_SZ). This - unavoidable quantization dominates allocation footprint efficiency - for the smallest values (e.g. a val_sz 1 pair will occupy 2 blocks - for the I/O alignment requirements and metadata). This is negligible - for large pairs (e.g. ~0.0024% for a VAL_MAX ~ 10 MiB val). - - Rounding an allocation to the smallest compatible size class adds no - additional overhead for smallest sizes (every possible quantization - of a val_sz less then ~4-8 KiB has a dedicated sizeclass). For - object sizes << VAL_MAX, the worst case overhead is better than ~2%. - For the object sizes ~ VAL_MAX, if the volume size is ~O(1) VAL_MAX, - volume divisibility starts to impact this overhead. It still less - than ~20% for the current config (and such sizes should be rare in - the practical usage). This effect can eliminated by using volume - sizes much larger than VAL_MAX (at the expense of creating a deeper - sizeclass nesting). - - The packing of objects into nested superblocks also incurs a small - amount of additional overhead. The smallest footprint object (2 - blocks or 256B) will be in a leaf superblock with 64 objects. The - leaf superblock overhead (128B) amortized over these 64 objects is - thus 2 bytes per object. This leaf superblock will be nested in a - larger superblock with 2 leaf superblocks. With 128B additional - overhead amortized over 128 objects, this yields 1 more byte overhead - per object. And so forth. For this sizeclass, the overall - superblock overhead converges to <~4 bytes per object absolute or - <~1.5% relative. This is a rough relative upper bound for all - sizeclasses. Specifically, for leaf superblocks with less than 64 - objects, there is more absolute superblock overhead per object but - the object itself is large enough to compensate. And for objects in - large superblocks, the objects more than large enough to compensate. - - The allocator will also implicitly adaptively preallocate space for - frequently used sizeclasses to speed up allocations. For - asymptotically large data caches relative to the worst case object - sizes, the amount of preallocation is very small. - - TL;DR Allocator footprint overhead is the unavoidable BLOCK_SZ - quantization plus a couple percent typically. */ - -#include "../io/fd_vinyl_io.h" -#include "../../util/tpool/fd_tpool.h" - -#define FD_VINYL_DATA_VOL_FOOTPRINT (41944192UL) /* autogenerated */ -#define FD_VINYL_DATA_SZC_CNT (83UL) /* autogenerated */ -#define FD_VINYL_DATA_SZC_ITER_MAX (7UL) /* autogenerated */ - -/* fd_vinyl_data_szc **************************************************/ - -struct __attribute__((aligned(8))) fd_vinyl_data_szc_cfg { - uint val_max; /* max pair val byte size that can be stored in an object in this size class. - The object is aligned in memory with FD_VINYL_BSTREAM_BLOCK_SZ alignment and with a footprint of: - FD_VINYL_BSTREAM_BLOCK_SZ + sizeof(fd_vinyl_bstream_phdr_t) + val_max + FD_VINYL_BSTREAM_FTR_SZ - The footprint is a FD_VINYL_BSTREAM_BLOCK_SZ multiple. */ - ushort obj_cnt; /* ==num objects in the containing superblock, in [2,64] */ - ushort parent_szc; /* size class of the superblock that contains objects of this size class, - FD_VINYL_DATA_SZC_CNT indicates superblocks for objects of this size class fill an entire volume, - (the superblock footprint is FD_VINYL_BSTREAM_BLOCK_SZ + obj_cnt*obj_footprint) */ -}; - -typedef struct fd_vinyl_data_szc_cfg fd_vinyl_data_szc_cfg_t; - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_data_szc_cfg describes the sizeclasses used by the data - cache. Indexed [0,FD_VINYL_DATA_SZC_CNT). */ - -extern fd_vinyl_data_szc_cfg_t const fd_vinyl_data_szc_cfg[ FD_VINYL_DATA_SZC_CNT ]; - -/* fd_vinyl_data_szc_obj_footprint returns the in-memory footprint for - an object with the given size class. - - fd_vinyl_data_szc_val_max returns the largest pair val that can be - cached in an object with the given size class. Assumes szc is in - [O,FD_VINYL_DATA_SZC_CNT). Return will be in - [0,cfg(SZC_CNT-1).val_max]. - - fd_vinyl_data_szc returns the tightest fitting size class that can - cache a pair val with a maximum size of val_max. Assumes val_max is - in [0,szc.cfg(SZC_CNT-1.val_max]. Return will be in - [0,FD_VINYL_DATA_SZC_CNT). The returned size class is typically able - to hold a val_max ~1-2% larger than the given val_max. - - Note that the size classes are configured such that, given val_sz in - [0,FD_VINYL_VAL_MAX]: - - fd_vinyl_data_szc_val_max( fd_vinyl_data_szc( val_sz ) ) <= FD_VINYL_VAL_MAX. - - where equality is achieved when val_sz==FD_VINYL_VAL_MAX. - - FIXME: should these be FD_FN_CONST (szc_cfg is const)? */ - -FD_FN_PURE static inline ulong -fd_vinyl_data_szc_obj_footprint( ulong szc ) { - return FD_VINYL_BSTREAM_BLOCK_SZ + - sizeof(fd_vinyl_bstream_phdr_t) + (ulong)fd_vinyl_data_szc_cfg[ szc ].val_max + FD_VINYL_BSTREAM_FTR_SZ; -} - -FD_FN_PURE static inline ulong -fd_vinyl_data_szc_val_max( ulong szc ) { - return (ulong)fd_vinyl_data_szc_cfg[ szc ].val_max; -} - -FD_FN_PURE static inline ulong -fd_vinyl_data_szc( ulong val_max ) { - - ulong l = 0UL; - ulong h = FD_VINYL_DATA_SZC_CNT-1UL; - - for( ulong rem=FD_VINYL_DATA_SZC_ITER_MAX; rem; rem-- ) { - - /* At this point, szc in [0,l) aren't suitable, szc in [h,CNT) are - suitable and szc in [l,h) are untested. See fd_alloc for more - detail on using fixed count loop. */ - - ulong m = (l+h) >> 1; - int c = (((ulong)fd_vinyl_data_szc_cfg[ m ].val_max) >= val_max); - l = fd_ulong_if( c, l, m+1UL ); - h = fd_ulong_if( c, m, h ); - - } - - return l; -} - -FD_PROTOTYPES_END - -/* fd_vinyl_data_obj **************************************************/ - -#define FD_VINYL_DATA_OBJ_TYPE_FREEVOL (0xf7eef7eef7eef7eeUL) /* free, object is a free volume */ -#define FD_VINYL_DATA_OBJ_TYPE_ALLOC (0xa11ca11ca11ca11cUL) /* allc, object is an allocation */ -#define FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK (0x59e759e759e759e7UL) /* sper, object is a superblock */ - -#define FD_VINYL_DATA_OBJ_GUARD_SZ (FD_VINYL_BSTREAM_BLOCK_SZ - sizeof(fd_vinyl_io_rd_t) - 8UL*sizeof(ulong)) - -struct fd_vinyl_data_obj; -typedef struct fd_vinyl_data_obj fd_vinyl_data_obj_t; - -struct __attribute__((aligned(FD_VINYL_BSTREAM_BLOCK_SZ))) fd_vinyl_data_obj { - - /* type gives the object type. A FD_VINYL_DATA_OBJ_TYPE_*. - - For type SUPERBLOCK objects, child_szc gives the size class of the - objects contained in this superblock. In - [0,FD_VINYL_DATA_SZC_CNT). Ignored for other types of objects. - - For type SUPERBLOCK and type ALLOC objects, szc gives the size - class of the object. In [0,FD_VINYL_DATA_SZC_CNT]. Values less - then SZC_CNT indicate an object contained in a superblock. Equal - to SZC_CNT indicates an object that fills an entire volume. For - other types of objects, ignored. - - For objects contained in a parent superblock, idx gives the index - of the object in its parent, in [0,szc.obj_cnt). For objects that - fill an entire volume, idx gives the data volume index, in - [0,vol_cnt). - - For type SUPERBLOCK objects, free_blocks gives a bit field - identifying which blocks are free. Ignored for other types of - objects. - - For inactive type SUPERBLOCK objects, next_off gives the byte - offset from laddr0 of the next inactive superblock and 0UL if no - more inactive superblocks. next_off is ignored in other - circumstances (but see note about pending I/O ops). Note that this - implies laddr0 must less than the local address of vol (such that - data gaddr==0 never points to an object). Note also that, if - laddr0 is 0, next_off will be just a pointer in the local address - space. - - Note that I/O acceleration may require memory alignment and I/O - device alignment to match. So we need to put all the object - allocator data its own block. This can leave a lot of extra space. - We put this space up front in the block to that it can act as a - guard region for whatever precedes it (applications could even use - this guard region to stash extra info but this is not recommended - because of false sharing conflicts in might induce between - different threads using adjacent in memory objects). Likewise, - because we have all this space from block quantization, we don't - try to be hyperefficient with the packing (like we do for, say, - fd_alloc). */ - -# if 0 /* Note: with BLOCK_SZ==128, GUARD_SZ=0 so there's no guard field due to language limitations */ - uchar guard[ FD_VINYL_DATA_OBJ_GUARD_SZ ]; -# endif - - /* rd on its own cache line */ - - fd_vinyl_io_rd_t rd[1]; /* rd: ctx is element idx */ - - /* allocator metadata on its own cache line */ - - ulong unused[1]; /* unused space */ - schar * rd_err; /* rd: client req_err (or dummy location if no client req_err) */ - short rd_active; /* rd: is a read in progress on this obj */ - ushort _unused; /* unused space */ - ushort szc; /* data: allocation size class */ - ushort child_szc; /* data: (superblock) contains allocations of this sizeclass */ - ulong line_idx; /* vinyl line_idx that is responsible for this object, in [0,line_cnt), ignored if not type alloc */ - ulong type; /* data: allocation type (alloc or superblock) */ - ulong idx; /* data: (alloc or superblock) index of this allocation in its parent superblock, (vol) vol idx */ - ulong free_blocks; /* data: (superblock) bit field free allocations */ - ulong next_off; /* data: (inactive superblock) pointer to next inactive superblock or 0 last */ - - /* This point is FD_VINYL_BSTREAM_BLOCK_SZ aligned */ - - /* Space for sizeof(fd_vinyl_bstream_phdr_t) + fd_vinyl_data_szc_cfg[szc].val_max + FD_VINYL_BSTREAM_FTR_SZ - - Note that is is a FD_VINYL_BSTREAM_BLOCK_SZ multiple so that the - entire region starting from phdr can submitted zero copy for - streaming to hardware async direct I/O friendly. */ - - /* This point is FD_VINYL_BSTREAM_BLOCK_SZ aligned */ - - /* There is an implied FD_VINYL_DATA_OBJ_GUARD_SZ region here as per - note above. It is not considered part of _this_ data_obj_t though. */ - -}; - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_data_obj_* returns a pointer to the eponymous field in - the given data object. Assumes obj is valid. Returns value for - fd_vinyl_data_obj_phdr will be FD_VINYL_BSTREAM_BLOCK_SZ aligned. - fd_vinyl_data_* mirror the above but they take the value region as - input. */ - -FD_FN_CONST static inline fd_vinyl_bstream_phdr_t * -fd_vinyl_data_obj_phdr( fd_vinyl_data_obj_t const * obj ) { - return (fd_vinyl_bstream_phdr_t *)((ulong)obj + sizeof(fd_vinyl_data_obj_t)); -} - -FD_FN_CONST static inline fd_vinyl_key_t * -fd_vinyl_data_obj_key( fd_vinyl_data_obj_t const * obj ) { - return (fd_vinyl_key_t *)((ulong)obj + sizeof(fd_vinyl_data_obj_t) + sizeof(ulong)); -} - -FD_FN_CONST static inline fd_vinyl_info_t * -fd_vinyl_data_obj_info( fd_vinyl_data_obj_t const * obj ) { - return (fd_vinyl_info_t *)((ulong)obj + sizeof(fd_vinyl_data_obj_t) + sizeof(ulong) + sizeof(fd_vinyl_key_t)); -} - -FD_FN_CONST static inline void * -fd_vinyl_data_obj_val( fd_vinyl_data_obj_t const * obj ) { - return (void *)((ulong)obj + sizeof(fd_vinyl_data_obj_t) + sizeof(fd_vinyl_bstream_phdr_t)); -} - -FD_FN_PURE static inline ulong -fd_vinyl_data_obj_val_sz( fd_vinyl_data_obj_t const * obj ) { - return (ulong)fd_vinyl_data_obj_info( obj )->val_sz; -} - -FD_FN_PURE static inline ulong -fd_vinyl_data_obj_val_max( fd_vinyl_data_obj_t const * obj ) { - return fd_vinyl_data_szc_val_max( (ulong)obj->szc ); -} - -FD_FN_CONST static inline fd_vinyl_data_obj_t * -fd_vinyl_data_obj( void const * val ) { - return (fd_vinyl_data_obj_t *)((ulong)val - sizeof(fd_vinyl_bstream_phdr_t) - sizeof(fd_vinyl_data_obj_t)); -} - -FD_FN_CONST static inline fd_vinyl_bstream_phdr_t * -fd_vinyl_data_phdr( void const * val ) { - return (fd_vinyl_bstream_phdr_t *)((ulong)val - sizeof(fd_vinyl_bstream_phdr_t)); -} - -FD_FN_CONST static inline fd_vinyl_key_t * -fd_vinyl_data_key( void const * val ) { - return (fd_vinyl_key_t *)((ulong)val - sizeof(fd_vinyl_info_t) - sizeof(fd_vinyl_key_t)); -} - -FD_FN_CONST static inline fd_vinyl_info_t * -fd_vinyl_data_info( void const * val ) { - return (fd_vinyl_info_t *)((ulong)val - sizeof(fd_vinyl_info_t)); -} - -FD_FN_PURE static inline ulong -fd_vinyl_data_val_sz( void const * val ) { - return (ulong)fd_vinyl_data_info( val )->val_sz; -} - -FD_FN_PURE static inline ulong -fd_vinyl_data_val_max( void const * val ) { - return fd_vinyl_data_szc_val_max( (ulong)fd_vinyl_data_obj( val )->szc ); -} - -FD_PROTOTYPES_END - -/* fd_vinyl_data_vol **************************************************/ - -struct fd_vinyl_data_vol { - fd_vinyl_data_obj_t obj[1]; - uchar data[ FD_VINYL_DATA_VOL_FOOTPRINT - sizeof(fd_vinyl_data_obj_t) ]; -}; - -typedef struct fd_vinyl_data_vol fd_vinyl_data_vol_t; - -/* fd_vinyl_data ******************************************************/ - -#define FD_VINYL_DATA_ALIGN (128UL) -#define FD_VINYL_DATA_FOOTPRINT sizeof(fd_vinyl_data_t) - -struct __attribute((aligned(FD_VINYL_DATA_ALIGN))) fd_vinyl_data { - void * shmem; /* Raw shared memory region */ - ulong shmem_sz; /* Raw shared memory region size */ - void * laddr0; /* Location where gaddr 0 points in the local address space - (FD_VINYL_BSTREAM_BLOCK_SZ aligned) */ - fd_vinyl_data_vol_t * vol; /* Vols, indexed [0,vol_cnt), in raw shared memory region */ - ulong vol_cnt; /* Num vols, in [0,FD_VINYL_DATA_VOL_MAX) */ - ulong vol_idx_free; /* Idx of first free volume if in [0,vol_cnt), no free volumes o.w. */ - struct { - fd_vinyl_data_obj_t * active; /* active superblock for this size class */ - fd_vinyl_data_obj_t * inactive_top; /* top of the inactive superblock stack for this size class */ - } superblock[ FD_VINYL_DATA_SZC_CNT ]; -}; - -typedef struct fd_vinyl_data fd_vinyl_data_t; - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_data_{align,footprint} return the alignment and footprint - needed for a local memory region to hold the state of a data cache. - align will be a power of 2 and footprint will be a multiple of align. - Matches FD_VINYL_DATA_{ALIGN,FOOTPRINT}. */ - -FD_FN_CONST ulong fd_vinyl_data_align ( void ); -FD_FN_CONST ulong fd_vinyl_data_footprint( void ); - -/* fd_vinyl_data_init formats a suitable local memory region lmem and an - arbitrary shared memory region shmem with byte size shmem_sz as a - vinyl data cache. laddr0 gives the location in the caller's local - address space that corresponds to data object global address 0. It - should be FD_VINYL_BSTREAM_BLOCK_SZ aligned and before shmem. E.g. - laddr0==NULL could be used when the data objects aren't shared with - threads in different processes while laddr0==wksp could be used for - data objects that are shared and backed by a wksp. - - IMPORTANT SAFETY TIP! This does _not_ do the initial formatting of - the shmem region into free data volumes (e.g. the caller can use the - data shmem region as a scratch during thread parallel resume and then - format it appropriately). The caller is responsible for calling - fd_vinyl_data_reset before using data as an object store. - - Returns a handle to the data cache on success (data cache owns the - memory regions) and NULL on failure (bad lmem, bad shmem, too small - size ... logs details, no ownership changes). */ - -fd_vinyl_data_t * -fd_vinyl_data_init( void * lmem, - void * shmem, - ulong shmem_sz, - void * laddr0 ); - -/* fd_vinyl_data_fini stops using lmem and shmem as a data cache. - Returns lmem on success and NULL on failure (logs details). */ - -void * -fd_vinyl_data_fini( fd_vinyl_data_t * data ); - -/* fd_vinyl_data_{laddr0,shmem,shmem_sz} return the address translation - and shared memory region used by the data cache. */ - -FD_FN_PURE static inline void * fd_vinyl_data_laddr0 ( fd_vinyl_data_t const * data ) { return (void *)data->laddr0; } -FD_FN_PURE static inline void * fd_vinyl_data_shmem ( fd_vinyl_data_t const * data ) { return (void *)data->shmem; } -FD_FN_PURE static inline ulong fd_vinyl_data_shmem_sz( fd_vinyl_data_t const * data ) { return data->shmem_sz; } - -/* fd_vinyl_data_is_valid_obj returns 1 if laddr appears to point to - a valid data object and 0 if not. vol points to data volume 0 in the - local address space and vol_cnt is the number of data volumes. */ - -FD_FN_PURE static inline int -fd_vinyl_data_is_valid_obj( void const * laddr, - fd_vinyl_data_vol_t const * vol, - ulong vol_cnt ) { - - ulong vol_idx = ((ulong)laddr - (ulong)vol) / FD_VINYL_DATA_VOL_FOOTPRINT; - - if( FD_UNLIKELY( !( ((ulong)vol<=(ulong)laddr) & - (vol_idxszc; - - if( FD_UNLIKELY( !((obj->type==FD_VINYL_DATA_OBJ_TYPE_ALLOC) & (szcidx<(ulong)fd_vinyl_data_szc_cfg[ szc ].obj_cnt) & (end<=(ulong)&vol[vol_idx+1UL])) ) ) return 0; - - return 1; -} - -/* fd_vinyl_data_alloc acquires an object of sizeclass szc from the data - cache. Returns a pointer to the object on success and NULL if there - is no space available in the data. Will FD_LOG_CRIT if anything - wonky is detected (bad, memory corruption, etc). */ - -fd_vinyl_data_obj_t * -fd_vinyl_data_alloc( fd_vinyl_data_t * data, - ulong szc ); - -/* fd_vinyl_data_free releases obj to the data cache. This cannot fail - from the caller's perspective. Will FD_LOG_CRIT if anything wonky is - detected (bad args, memory corruption, etc). */ - -void -fd_vinyl_data_free( fd_vinyl_data_t * data, - fd_vinyl_data_obj_t * obj ); - -/* fd_vinyl_data_reset uses the caller and tpool threads (t0,t1) to free - all objects from the data cache. level zero/non-zero indicates to do - soft/hard reset. In a hard reset, the shmem region is zero'd before - formatting it into a set of free data volumes. This cannot fail from - the caller's perspective. Assumes tpool threads (t0,t1) are - available for dispatch. Retains no interest in tpool and tpool - threads (t0,t1) will be available for dispatch on return. */ - -void -fd_vinyl_data_reset( fd_tpool_t * tpool, ulong t0, ulong t1, int level, - fd_vinyl_data_t * data ); - -/* fd_vinyl_data_verify returns FD_VINYL_SUCCESS (0) if the given data - appears to be a valid vinyl data and FD_VINYL_ERR_CORRUPT (negative) - otherwise (logs details). This only verifies the vinyl data's state - and superblock hierarchy are intact. It does not test any of the - allocations for correctness (but could given access to the bstream, - line and/or meta). */ - -int -fd_vinyl_data_verify( fd_vinyl_data_t const * data ); - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_data_fd_vinyl_data_h */ diff --git a/src/vinyl/data/fd_vinyl_data_szc_cfg.c b/src/vinyl/data/fd_vinyl_data_szc_cfg.c deleted file mode 100644 index d97e35f9092..00000000000 --- a/src/vinyl/data/fd_vinyl_data_szc_cfg.c +++ /dev/null @@ -1,100 +0,0 @@ -#include "fd_vinyl_data.h" - -/* This table autogenerated - io_block 128 - ctl_sz 8 - key_sz 32 - info_sz 16 - ftr_sz 16 - obj_cnt_min 4 - obj_cnt_max 16 - val_min 0 - val_max 10485816 - root_sb_footprint 41944192 - gamma 1.066667e+00 */ - -fd_vinyl_data_szc_cfg_t const fd_vinyl_data_szc_cfg[ FD_VINYL_DATA_SZC_CNT ] = { - /* 0 */ { 56U, (ushort) 9, (ushort) 11 }, // obj_footprint 256 parent_footprint 2432 slack 0 - /* 1 */ { 184U, (ushort) 6, (ushort) 11 }, // obj_footprint 384 parent_footprint 2432 slack 0 - /* 2 */ { 312U, (ushort) 4, (ushort) 11 }, // obj_footprint 512 parent_footprint 2432 slack 256 - /* 3 */ { 440U, (ushort)15, (ushort) 22 }, // obj_footprint 640 parent_footprint 10112 slack 384 - /* 4 */ { 568U, (ushort)13, (ushort) 22 }, // obj_footprint 768 parent_footprint 10112 slack 0 - /* 5 */ { 696U, (ushort)11, (ushort) 22 }, // obj_footprint 896 parent_footprint 10112 slack 128 - /* 6 */ { 824U, (ushort) 9, (ushort) 22 }, // obj_footprint 1024 parent_footprint 10112 slack 768 - /* 7 */ { 952U, (ushort) 8, (ushort) 22 }, // obj_footprint 1152 parent_footprint 10112 slack 768 - /* 8 */ { 1208U, (ushort) 7, (ushort) 22 }, // obj_footprint 1408 parent_footprint 10112 slack 128 - /* 9 */ { 1464U, (ushort) 6, (ushort) 22 }, // obj_footprint 1664 parent_footprint 10112 slack 0 - /* 10 */ { 1720U, (ushort) 5, (ushort) 22 }, // obj_footprint 1920 parent_footprint 10112 slack 384 - /* 11 */ { 2232U, (ushort)16, (ushort) 34 }, // obj_footprint 2432 parent_footprint 40832 slack 1792 - /* 12 */ { 2488U, (ushort)15, (ushort) 34 }, // obj_footprint 2688 parent_footprint 40832 slack 384 - /* 13 */ { 2872U, (ushort)13, (ushort) 34 }, // obj_footprint 3072 parent_footprint 40832 slack 768 - /* 14 */ { 3128U, (ushort)12, (ushort) 34 }, // obj_footprint 3328 parent_footprint 40832 slack 768 - /* 15 */ { 3384U, (ushort)11, (ushort) 34 }, // obj_footprint 3584 parent_footprint 40832 slack 1280 - /* 16 */ { 3768U, (ushort)10, (ushort) 34 }, // obj_footprint 3968 parent_footprint 40832 slack 1024 - /* 17 */ { 4280U, (ushort) 9, (ushort) 34 }, // obj_footprint 4480 parent_footprint 40832 slack 384 - /* 18 */ { 4792U, (ushort) 8, (ushort) 34 }, // obj_footprint 4992 parent_footprint 40832 slack 768 - /* 19 */ { 5560U, (ushort) 7, (ushort) 34 }, // obj_footprint 5760 parent_footprint 40832 slack 384 - /* 20 */ { 6584U, (ushort) 6, (ushort) 34 }, // obj_footprint 6784 parent_footprint 40832 slack 0 - /* 21 */ { 7864U, (ushort) 5, (ushort) 34 }, // obj_footprint 8064 parent_footprint 40832 slack 384 - /* 22 */ { 9912U, (ushort)16, (ushort) 46 }, // obj_footprint 10112 parent_footprint 163712 slack 1792 - /* 23 */ { 10680U, (ushort)15, (ushort) 46 }, // obj_footprint 10880 parent_footprint 163712 slack 384 - /* 24 */ { 11448U, (ushort)14, (ushort) 46 }, // obj_footprint 11648 parent_footprint 163712 slack 512 - /* 25 */ { 12344U, (ushort)13, (ushort) 46 }, // obj_footprint 12544 parent_footprint 163712 slack 512 - /* 26 */ { 13368U, (ushort)12, (ushort) 46 }, // obj_footprint 13568 parent_footprint 163712 slack 768 - /* 27 */ { 14648U, (ushort)11, (ushort) 46 }, // obj_footprint 14848 parent_footprint 163712 slack 256 - /* 28 */ { 16056U, (ushort)10, (ushort) 46 }, // obj_footprint 16256 parent_footprint 163712 slack 1024 - /* 29 */ { 17976U, (ushort) 9, (ushort) 46 }, // obj_footprint 18176 parent_footprint 163712 slack 0 - /* 30 */ { 20152U, (ushort) 8, (ushort) 46 }, // obj_footprint 20352 parent_footprint 163712 slack 768 - /* 31 */ { 23096U, (ushort) 7, (ushort) 46 }, // obj_footprint 23296 parent_footprint 163712 slack 512 - /* 32 */ { 27064U, (ushort) 6, (ushort) 46 }, // obj_footprint 27264 parent_footprint 163712 slack 0 - /* 33 */ { 32440U, (ushort) 5, (ushort) 46 }, // obj_footprint 32640 parent_footprint 163712 slack 384 - /* 34 */ { 40632U, (ushort)16, (ushort) 58 }, // obj_footprint 40832 parent_footprint 655360 slack 1920 - /* 35 */ { 43448U, (ushort)15, (ushort) 58 }, // obj_footprint 43648 parent_footprint 655360 slack 512 - /* 36 */ { 46520U, (ushort)14, (ushort) 58 }, // obj_footprint 46720 parent_footprint 655360 slack 1152 - /* 37 */ { 50104U, (ushort)13, (ushort) 58 }, // obj_footprint 50304 parent_footprint 655360 slack 1280 - /* 38 */ { 54328U, (ushort)12, (ushort) 58 }, // obj_footprint 54528 parent_footprint 655360 slack 896 - /* 39 */ { 59320U, (ushort)11, (ushort) 58 }, // obj_footprint 59520 parent_footprint 655360 slack 512 - /* 40 */ { 65208U, (ushort)10, (ushort) 58 }, // obj_footprint 65408 parent_footprint 655360 slack 1152 - /* 41 */ { 72504U, (ushort) 9, (ushort) 58 }, // obj_footprint 72704 parent_footprint 655360 slack 896 - /* 42 */ { 81592U, (ushort) 8, (ushort) 58 }, // obj_footprint 81792 parent_footprint 655360 slack 896 - /* 43 */ { 93368U, (ushort) 7, (ushort) 58 }, // obj_footprint 93568 parent_footprint 655360 slack 256 - /* 44 */ { 108984U, (ushort) 6, (ushort) 58 }, // obj_footprint 109184 parent_footprint 655360 slack 128 - /* 45 */ { 130744U, (ushort) 5, (ushort) 58 }, // obj_footprint 130944 parent_footprint 655360 slack 512 - /* 46 */ { 163512U, (ushort)16, (ushort) 70 }, // obj_footprint 163712 parent_footprint 2621440 slack 1920 - /* 47 */ { 174520U, (ushort)15, (ushort) 70 }, // obj_footprint 174720 parent_footprint 2621440 slack 512 - /* 48 */ { 186936U, (ushort)14, (ushort) 70 }, // obj_footprint 187136 parent_footprint 2621440 slack 1408 - /* 49 */ { 201400U, (ushort)13, (ushort) 70 }, // obj_footprint 201600 parent_footprint 2621440 slack 512 - /* 50 */ { 218168U, (ushort)12, (ushort) 70 }, // obj_footprint 218368 parent_footprint 2621440 slack 896 - /* 51 */ { 238008U, (ushort)11, (ushort) 70 }, // obj_footprint 238208 parent_footprint 2621440 slack 1024 - /* 52 */ { 261816U, (ushort)10, (ushort) 70 }, // obj_footprint 262016 parent_footprint 2621440 slack 1152 - /* 53 */ { 291000U, (ushort) 9, (ushort) 70 }, // obj_footprint 291200 parent_footprint 2621440 slack 512 - /* 54 */ { 327352U, (ushort) 8, (ushort) 70 }, // obj_footprint 327552 parent_footprint 2621440 slack 896 - /* 55 */ { 374200U, (ushort) 7, (ushort) 70 }, // obj_footprint 374400 parent_footprint 2621440 slack 512 - /* 56 */ { 436664U, (ushort) 6, (ushort) 70 }, // obj_footprint 436864 parent_footprint 2621440 slack 128 - /* 57 */ { 523960U, (ushort) 5, (ushort) 70 }, // obj_footprint 524160 parent_footprint 2621440 slack 512 - /* 58 */ { 655160U, (ushort)16, (ushort) 82 }, // obj_footprint 655360 parent_footprint 10486016 slack 128 - /* 59 */ { 698808U, (ushort)15, (ushort) 82 }, // obj_footprint 699008 parent_footprint 10486016 slack 768 - /* 60 */ { 748728U, (ushort)14, (ushort) 82 }, // obj_footprint 748928 parent_footprint 10486016 slack 896 - /* 61 */ { 806328U, (ushort)13, (ushort) 82 }, // obj_footprint 806528 parent_footprint 10486016 slack 1024 - /* 62 */ { 873528U, (ushort)12, (ushort) 82 }, // obj_footprint 873728 parent_footprint 10486016 slack 1152 - /* 63 */ { 953016U, (ushort)11, (ushort) 82 }, // obj_footprint 953216 parent_footprint 10486016 slack 512 - /* 64 */ { 1048376U, (ushort)10, (ushort) 82 }, // obj_footprint 1048576 parent_footprint 10486016 slack 128 - /* 65 */ { 1164856U, (ushort) 9, (ushort) 82 }, // obj_footprint 1165056 parent_footprint 10486016 slack 384 - /* 66 */ { 1310520U, (ushort) 8, (ushort) 82 }, // obj_footprint 1310720 parent_footprint 10486016 slack 128 - /* 67 */ { 1497784U, (ushort) 7, (ushort) 82 }, // obj_footprint 1497984 parent_footprint 10486016 slack 0 - /* 68 */ { 1747384U, (ushort) 6, (ushort) 82 }, // obj_footprint 1747584 parent_footprint 10486016 slack 384 - /* 69 */ { 2096952U, (ushort) 5, (ushort) 82 }, // obj_footprint 2097152 parent_footprint 10486016 slack 128 - /* 70 */ { 2621240U, (ushort)16, (ushort) 83 }, // obj_footprint 2621440 parent_footprint 41944192 slack 1024 - /* 71 */ { 2795960U, (ushort)15, (ushort) 83 }, // obj_footprint 2796160 parent_footprint 41944192 slack 1664 - /* 72 */ { 2995768U, (ushort)14, (ushort) 83 }, // obj_footprint 2995968 parent_footprint 41944192 slack 512 - /* 73 */ { 3226168U, (ushort)13, (ushort) 83 }, // obj_footprint 3226368 parent_footprint 41944192 slack 1280 - /* 74 */ { 3495096U, (ushort)12, (ushort) 83 }, // obj_footprint 3495296 parent_footprint 41944192 slack 512 - /* 75 */ { 3812792U, (ushort)11, (ushort) 83 }, // obj_footprint 3812992 parent_footprint 41944192 slack 1152 - /* 76 */ { 4194104U, (ushort)10, (ushort) 83 }, // obj_footprint 4194304 parent_footprint 41944192 slack 1024 - /* 77 */ { 4660152U, (ushort) 9, (ushort) 83 }, // obj_footprint 4660352 parent_footprint 41944192 slack 896 - /* 78 */ { 5242808U, (ushort) 8, (ushort) 83 }, // obj_footprint 5243008 parent_footprint 41944192 slack 0 - /* 79 */ { 5991736U, (ushort) 7, (ushort) 83 }, // obj_footprint 5991936 parent_footprint 41944192 slack 512 - /* 80 */ { 6990392U, (ushort) 6, (ushort) 83 }, // obj_footprint 6990592 parent_footprint 41944192 slack 512 - /* 81 */ { 8388536U, (ushort) 5, (ushort) 83 }, // obj_footprint 8388736 parent_footprint 41944192 slack 384 - /* 82 */ { 10485816U, (ushort) 4, (ushort) 83 }, // obj_footprint 10486016 parent_footprint 41944192 slack 0 -}; diff --git a/src/vinyl/data/gen_szc_cfg.m b/src/vinyl/data/gen_szc_cfg.m deleted file mode 100644 index 972c861d00e..00000000000 --- a/src/vinyl/data/gen_szc_cfg.m +++ /dev/null @@ -1,150 +0,0 @@ -close all -clear all - -io_block = 128; - -ctl_sz = 8; -key_sz = 32; -info_sz = 16; -ftr_sz = 16; - -pair_framing = ctl_sz + key_sz + info_sz + ftr_sz; - -val_min = 0; -val_max = 10*2^20 + io_block - pair_framing; - -obj_cnt_min = 4; -obj_cnt_max = 16; -obj_footprint_min = io_block*ceil( ( io_block + pair_framing + val_min ) / io_block ); -obj_footprint_max = io_block*ceil( ( io_block + pair_framing + val_max ) / io_block ); -root_sb_footprint = io_block + obj_cnt_min*obj_footprint_max; -gamma = obj_cnt_max / (obj_cnt_max-1); - -szc_cnt = 0; - -sb_footprint = root_sb_footprint; -obj_footprint_target = obj_footprint_max; -while obj_footprint_target>=obj_footprint_min, - - ofp = io_block*round( obj_footprint_target / io_block ); - while 1, - ocnt = floor( (sb_footprint - io_block) / ofp ); - if ocnt<=obj_cnt_max, break; endif - - % More than obj_cnt_max objects fit into the current superblock. - % Pick a new superblock size to be the smallest object footprint in - % an already computed sizeclass sufficient to hold a superblock with - % obj_cnt_min ofp sized objects. If there are none (i.e. the needed - % superblock size is larger than the object footprints for all the - % already computed sizeclasses), use the already computed sizeclass - % with the largest object footprint (the resulting superblock for - % ofp sized objects will have fewer than obj_cnt_min objects but - % that's unavoidable). - - idx = find( obj_footprint >= io_block + ofp*obj_cnt_min, 1, 'last' ); - if isempty(idx), idx = 1; endif - sb_footprint = obj_footprint( idx ); - endwhile - - if ~((2<=ocnt) & (ocnt<=64)), error( 'hmmm' ); endif - - szc_cnt++; - obj_footprint (szc_cnt) = io_block*floor( (sb_footprint - io_block) / (io_block*ocnt) ); - obj_val_max (szc_cnt) = obj_footprint(szc_cnt) - io_block - pair_framing; - obj_cnt (szc_cnt) = ocnt; - parent_footprint(szc_cnt) = sb_footprint; - - obj_footprint_target /= gamma; -endwhile - -% Sort by object footprint, discarding redundant and/or runt sizeclasses - -[~,idx] = unique( obj_footprint, 'first' ); -obj_footprint = obj_footprint (idx); -obj_val_max = obj_val_max (idx); -obj_cnt = obj_cnt (idx); -parent_footprint = parent_footprint(idx); - -keep = diff(obj_footprint) > 0.25*(gamma-1)*obj_footprint(1:(end-1)); -keep(end+1) = 1; - -obj_footprint = obj_footprint (keep); -obj_val_max = obj_val_max (keep); -obj_cnt = obj_cnt (keep); -parent_footprint = parent_footprint(keep); - -szc_cnt = length(obj_footprint); - -% Determine sizeclass nesting - -for szc=1:szc_cnt, - pfp = io_block + obj_cnt(szc)*obj_footprint(szc); - pszc = find( pfp<=obj_footprint, 1, 'first' ); - if isempty(pszc), - parent_footprint(szc) = root_sb_footprint; - parent_szc(szc) = szc_cnt+1; - else - parent_footprint(szc) = obj_footprint(pszc); - parent_szc(szc) = pszc; - endif -endfor - -szc_thresh = szc_cnt; - -printf( '#define FD_VINYL_VAL_MAX (%uUL) /* autogenerated */\n', val_max ); -printf( '\n' ); -printf( '#define FD_VINYL_DATA_VOL_FOOTPRINT (%uUL) /* autogenerated */\n', max( parent_footprint ) ); -printf( '#define FD_VINYL_DATA_SZC_CNT (%uUL) /* autogenerated */\n', szc_cnt ); -printf( '#define FD_VINYL_DATA_SZC_ITER_MAX (%uUL) /* autogenerated */\n', ceil( log2( szc_cnt ) ) ); -printf( '\n' ); -printf( '######################################################################################################\n' ); -printf( '#include \"fd_vinyl_data.h\"\n' ); -printf( '\n' ); -printf( '/* This table autogenerated\n' ); -printf( ' io_block %i\n', io_block ); -printf( ' ctl_sz %i\n', ctl_sz ); -printf( ' key_sz %i\n', key_sz ); -printf( ' info_sz %i\n', info_sz ); -printf( ' ftr_sz %i\n', ftr_sz ); -printf( ' obj_cnt_min %i\n', obj_cnt_min ); -printf( ' obj_cnt_max %i\n', obj_cnt_max ); -printf( ' val_min %i\n', val_min ); -printf( ' val_max %i\n', val_max ); -printf( ' root_sb_footprint %i\n', root_sb_footprint ); -printf( ' gamma %.6e */\n', gamma ); -printf( '\n' ); -printf( 'fd_vinyl_data_szc_cfg_t const fd_vinyl_data_szc_cfg[ FD_VINYL_DATA_SZC_CNT ] = {\n' ); -for szc=1:szc_cnt, - ocnt = obj_cnt (szc); - ofp = obj_footprint (szc); - pszc = parent_szc (szc); - pfp = parent_footprint(szc); - printf( ' /* %3u */ { %10uU, (ushort)%2u, (ushort)%3u }, // obj_footprint %10u parent_footprint %10u slack %5u\n', ... - szc-1, obj_val_max(szc), ocnt, pszc-1, ofp, pfp, pfp - (io_block+ocnt*ofp) ); -endfor -printf( '};\n' ); - -obj_val_min = shift(obj_val_max,1) + 1; obj_val_min(1) = 1; -obj_val_avg = 0.5*(obj_val_min+obj_val_max); - -denom = io_block./obj_cnt + obj_footprint; % Ideally would have SB nesting accounted too (but its in the noise) -eff_min = obj_val_min./denom; -eff_avg = obj_val_avg./denom; -eff_max = obj_val_max./denom; - -szc_idx = 0:(szc_cnt-1); - -subplot(2,1,1); -semilogy( szc_idx, [ obj_val_min; obj_val_avg; obj_val_max ] ); -axis tight; -grid; -xlabel( 'sizeclass index' ); -ylabel( 'object size (min/avg/max)' ); - -subplot(2,1,2); -semilogy( szc_idx, 1 - [eff_min; eff_avg; eff_max] ); -axis tight; -grid; -xlabel( 'sizeclass index' ); -ylabel( 'footprint overhead frac (min/avg/max)' ); - diff --git a/src/vinyl/data/test_vinyl_data.c b/src/vinyl/data/test_vinyl_data.c deleted file mode 100644 index 2187bc7f09d..00000000000 --- a/src/vinyl/data/test_vinyl_data.c +++ /dev/null @@ -1,279 +0,0 @@ -#include "../fd_vinyl.h" - -FD_STATIC_ASSERT( FD_VINYL_DATA_SZC_CNT==83UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_DATA_OBJ_TYPE_FREEVOL ==0xf7eef7eef7eef7eeUL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_DATA_OBJ_TYPE_ALLOC ==0xa11ca11ca11ca11cUL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_DATA_OBJ_TYPE_SUPERBLOCK==0x59e759e759e759e7UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_DATA_OBJ_GUARD_SZ==0UL, unit_test ); - -FD_STATIC_ASSERT( alignof(fd_vinyl_data_obj_t)==FD_VINYL_BSTREAM_BLOCK_SZ, unit_test ); -FD_STATIC_ASSERT( sizeof (fd_vinyl_data_obj_t)==FD_VINYL_BSTREAM_BLOCK_SZ, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_DATA_VOL_FOOTPRINT==41944192UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_DATA_ALIGN ==alignof(fd_vinyl_data_t), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_DATA_FOOTPRINT==sizeof (fd_vinyl_data_t), unit_test ); - -FD_STATIC_ASSERT( alignof(fd_vinyl_data_vol_t)==FD_VINYL_BSTREAM_BLOCK_SZ, unit_test ); -FD_STATIC_ASSERT( sizeof (fd_vinyl_data_vol_t)==FD_VINYL_DATA_VOL_FOOTPRINT, unit_test ); - -#define ALLOC_MAX (65536UL) - -fd_vinyl_data_obj_t * alloc[ ALLOC_MAX ]; - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - char const * name = fd_env_strip_cmdline_cstr ( &argc, &argv, "--name", NULL, NULL ); - ulong page_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--page-cnt", NULL, 1UL ); - char const * _page_sz = fd_env_strip_cmdline_cstr ( &argc, &argv, "--page-sz", NULL, "gigantic" ); - ulong near_cpu = fd_env_strip_cmdline_ulong( &argc, &argv, "--near-cpu", NULL, fd_log_cpu_id() ); - ulong iter_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--iter-cnt", NULL, 2000000 ); - int level = fd_env_strip_cmdline_int ( &argc, &argv, "--level", NULL, 0 ); - - fd_rng_t rng[1]; fd_rng_join( fd_rng_new( rng, 0U, 0UL ) ); - - fd_tpool_t * tpool = NULL; - - ulong thread_cnt = fd_tile_cnt(); - - if( thread_cnt>1UL ) { - FD_LOG_NOTICE(( "Creating tpool from all %lu tiles", thread_cnt )); - - static uchar _tpool[ FD_TPOOL_FOOTPRINT( FD_TILE_MAX ) ] __attribute__((aligned(FD_TPOOL_ALIGN))); - - tpool = fd_tpool_init( _tpool, thread_cnt, 0UL ); /* logs details */ - if( FD_UNLIKELY( !tpool ) ) FD_LOG_ERR(( "fd_tpool_init failed" )); - - for( ulong thread_idx=1UL; thread_idx= FD_VINYL_VAL_MAX ); - FD_TEST( fd_vinyl_data_szc( 0UL )==0UL ); - - FD_TEST( fd_vinyl_data_szc_val_max( fd_vinyl_data_szc( FD_VINYL_VAL_MAX ) )==FD_VINYL_VAL_MAX ); - - ulong val_max = 0UL; - for( ulong szc=0UL; szc val_max_prev ); /* Sorted by val_max */ - FD_TEST( (2UL<=obj_cnt) & (obj_cnt<=64UL) ); /* Valid obj_cnt */ - FD_TEST( (szcval_max_prev; val_test-- ) FD_TEST( fd_vinyl_data_szc( val_test )==szc ); - - /* fd_vinyl_data_t specific tests */ - - /* object footprint correct for object in memory layout */ - ulong _obj_footprint = sizeof(fd_vinyl_data_obj_t) + sizeof(fd_vinyl_bstream_phdr_t) + val_max + FD_VINYL_BSTREAM_FTR_SZ; - FD_TEST( obj_footprint==_obj_footprint ); - FD_TEST( fd_ulong_is_aligned( obj_footprint, FD_VINYL_BSTREAM_BLOCK_SZ ) ); - FD_TEST( fd_ulong_is_aligned( sizeof(fd_vinyl_data_obj_t), FD_VINYL_BSTREAM_BLOCK_SZ ) ); - } - - FD_LOG_NOTICE(( "Testing fd_vinyl_data obj accessors" )); - - for( ulong rem=1000000UL; rem; rem-- ) { - -# define VAL_MAX (FD_VINYL_BSTREAM_BLOCK_SZ - sizeof(fd_vinyl_bstream_phdr_t) - FD_VINYL_BSTREAM_FTR_SZ) - struct { - fd_vinyl_data_obj_t obj; - fd_vinyl_bstream_phdr_t phdr; - uchar val[ VAL_MAX ]; - ulong hash_trail; - ulong hash_blocks; - } cache; - - ulong r = fd_rng_ulong( rng ); - ulong szc = r % FD_VINYL_DATA_SZC_CNT; - ulong val_sz = (ulong)(uint)r; - - cache.obj.szc = (ushort)szc; - cache.phdr.info.val_sz = (uint)val_sz; - - FD_TEST( fd_vinyl_data_obj_phdr ( &cache.obj ) == &cache.phdr ); - FD_TEST( fd_vinyl_data_obj_key ( &cache.obj ) == &cache.phdr.key ); - FD_TEST( fd_vinyl_data_obj_info ( &cache.obj ) == &cache.phdr.info ); - FD_TEST( fd_vinyl_data_obj_val ( &cache.obj ) == (void *)cache.val ); - FD_TEST( fd_vinyl_data_obj_val_sz ( &cache.obj ) == val_sz ); - FD_TEST( fd_vinyl_data_obj_val_max( &cache.obj ) == fd_vinyl_data_szc_val_max( szc ) ); - - FD_TEST( fd_vinyl_data_obj ( cache.val ) == &cache.obj ); - FD_TEST( fd_vinyl_data_phdr ( cache.val ) == &cache.phdr ); - FD_TEST( fd_vinyl_data_key ( cache.val ) == &cache.phdr.key ); - FD_TEST( fd_vinyl_data_info ( cache.val ) == &cache.phdr.info ); - FD_TEST( fd_vinyl_data_val_sz ( cache.val ) == val_sz ); - FD_TEST( fd_vinyl_data_val_max ( cache.val ) == fd_vinyl_data_szc_val_max( szc ) ); - - } - - FD_LOG_NOTICE(( "Acquiring local and shared memory" )); - - fd_vinyl_data_t lmem[1]; - - void * shmem; - ulong page_sz; - if( name ) { - - FD_LOG_NOTICE(( "Joining to --name %s", name )); - fd_shmem_join_info_t info[1]; - shmem = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, 0, NULL, NULL, info ); /* logs details */ - if( FD_UNLIKELY( !shmem ) ) FD_LOG_ERR(( "fd_shmem_join failed" )); - page_cnt = info->page_cnt; - page_sz = info->page_sz; - - } else { - - FD_LOG_NOTICE(( "--name not specified, using anonymous shmem (--page-cnt %lu --page-sz %s --near-cpu %lu)", - page_cnt, _page_sz, near_cpu )); - page_sz = fd_cstr_to_shmem_page_sz( _page_sz ); - if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "invalid page_sz" )); - shmem = fd_shmem_acquire_multi( page_sz, 1UL, &page_cnt, &near_cpu ); /* logs details */ - if( FD_UNLIKELY( !shmem ) ) FD_LOG_ERR(( "fd_shmem_acquire_multi failed" )); - - } - - ulong shmem_sz = page_sz*page_cnt; - - FD_LOG_NOTICE(( "Testing with %lu size data cache", shmem_sz )); - - FD_LOG_NOTICE(( "Testing fd_vinyl_data_t construction" )); - - ulong align = fd_vinyl_data_align (); FD_TEST( fd_ulong_is_pow2( align ) ); - ulong footprint = fd_vinyl_data_footprint(); FD_TEST( fd_ulong_is_aligned( footprint, align ) ); - - ulong sz_too_small = FD_VINYL_DATA_VOL_FOOTPRINT - 1UL; - - FD_TEST( !fd_vinyl_data_init( NULL, shmem, shmem_sz, NULL ) ); /* NULL lmem */ - FD_TEST( !fd_vinyl_data_init( (void *)1UL, shmem, shmem_sz, NULL ) ); /* misaligned lmem */ - FD_TEST( !fd_vinyl_data_init( lmem, NULL, shmem_sz, NULL ) ); /* NULL shmem */ - FD_TEST( !fd_vinyl_data_init( lmem, (void *)ULONG_MAX, shmem_sz, NULL ) ); /* shmem wraps */ - FD_TEST( !fd_vinyl_data_init( lmem, shmem, sz_too_small, NULL ) ); /* sz too small */ - FD_TEST( !fd_vinyl_data_init( lmem, shmem, shmem_sz, shmem ) ); /* shmem at or before laddr0 */ - FD_TEST( !fd_vinyl_data_init( lmem, shmem, shmem_sz, (void *)1UL ) ); /* misaligned laddr0 */ - - fd_vinyl_data_t * data = fd_vinyl_data_init( lmem, shmem, shmem_sz, NULL ); - FD_TEST( data ); - - FD_LOG_NOTICE(( "Testing fd_vinyl_data_t reset (--level %i)", level )); - - fd_vinyl_data_reset( tpool,0UL,thread_cnt, level, data ); - - FD_TEST( !fd_vinyl_data_verify( data ) ); - - FD_LOG_NOTICE(( "Testing fd_vinyl data_t accessors" )); - - FD_TEST( fd_vinyl_data_laddr0 ( data )==NULL ); - FD_TEST( fd_vinyl_data_shmem ( data )==shmem ); - FD_TEST( fd_vinyl_data_shmem_sz( data )==shmem_sz ); - - FD_LOG_NOTICE(( "Testing alloc/free (--iter-cnt %lu)", iter_cnt )); - - ulong verify_rem = 0UL; - for( ulong iter_idx=0UL; iter_idxvol, data->vol_cnt ) ); - - obj->unused[0] = pat; - - uchar * p = (uchar *)(obj+1); - for( ulong rem=fd_vinyl_bstream_pair_sz( fd_vinyl_data_szc_val_max( szc ) ) >> 3; rem; p+=8UL, rem-- ) *(ulong *)p = pat; - - alloc[ alloc_idx ] = obj; - - } else { - uchar const * p = (uchar const *)(obj+1); - ulong pat = obj->unused[0]; - - for( ulong rem=fd_vinyl_bstream_pair_sz( fd_vinyl_data_szc_val_max( (ulong)obj->szc ) ) >> 3; rem; p+=8UL, rem-- ) - FD_TEST( *((ulong const *)p)==pat ); - - FD_TEST( fd_vinyl_data_is_valid_obj( obj, data->vol, data->vol_cnt ) ); - - fd_vinyl_data_free( data, obj ); - - FD_TEST( !fd_vinyl_data_is_valid_obj( obj, data->vol, data->vol_cnt ) ); - - alloc[ alloc_idx ] = NULL; - } - } - - FD_TEST( !fd_vinyl_data_verify( data ) ); - - for( ulong alloc_idx=0UL; alloc_idxunused[0]; - - for( ulong rem=fd_vinyl_bstream_pair_sz( fd_vinyl_data_szc_val_max( (ulong)obj->szc ) ) >> 3; rem; p+=8UL, rem-- ) - FD_TEST( *((ulong const *)p)==pat ); - - fd_vinyl_data_free( data, obj ); - - alloc[ alloc_idx ] = NULL; - } - } - - FD_TEST( !fd_vinyl_data_verify( data ) ); - - FD_LOG_NOTICE(( "Testing fd_vinyl_data_t destruction" )); - - FD_TEST( !fd_vinyl_data_fini( NULL ) ); - - FD_TEST( fd_vinyl_data_fini( lmem )==lmem ); - - FD_LOG_NOTICE(( "Cleaning up" )); - - if( name ) fd_shmem_leave ( shmem, NULL, NULL ); /* logs details */ - else fd_shmem_release( shmem, page_sz, page_cnt ); /* logs details */ - - if( tpool ) fd_tpool_fini( tpool ); /* logs details, note: fini automatically pops all worker threads */ - - fd_rng_delete( fd_rng_leave( rng ) ); - - FD_LOG_NOTICE(( "pass" )); - fd_halt(); - return 0; -} diff --git a/src/vinyl/fd_vinyl.c b/src/vinyl/fd_vinyl.c deleted file mode 100644 index bb5e0f10cd0..00000000000 --- a/src/vinyl/fd_vinyl.c +++ /dev/null @@ -1,171 +0,0 @@ -#include "fd_vinyl.h" - -ulong -fd_vinyl_align( void ) { - return alignof(fd_vinyl_t); -} - -ulong -fd_vinyl_footprint( void ) { - return sizeof(fd_vinyl_t); -} - -fd_vinyl_t * -fd_vinyl_init( fd_tpool_t * tpool, ulong t0, ulong t1, int level, - void * _vinyl, - void * _cnc, ulong cnc_footprint, - void * _meta, ulong meta_footprint, - void * _line, ulong line_footprint, - void * _ele, ulong ele_footprint, - void * _obj, ulong obj_footprint, - fd_vinyl_io_t * io, - ulong seed, - void * obj_laddr0, - ulong async_min, - ulong async_max, - ulong part_thresh, - ulong gc_thresh, - int gc_eager, - int style ) { - if( t1<=t0 ) t0 = 0UL, t1 = 1UL; - - FD_LOG_NOTICE(( "Testing vinyl configuration" )); - -# define TEST( c ) do { if( FD_UNLIKELY( !(c) ) ) { FD_LOG_WARNING(( "FAIL: %s", #c )); return NULL; } } while(0) - - TEST( _vinyl ); TEST( fd_ulong_is_aligned( (ulong)_vinyl, fd_vinyl_align() ) ); - TEST( _cnc ); TEST( fd_ulong_is_aligned( (ulong)_cnc, fd_cnc_align() ) ); - TEST( _meta ); TEST( fd_ulong_is_aligned( (ulong)_meta, fd_vinyl_meta_align() ) ); - TEST( _line ); TEST( fd_ulong_is_aligned( (ulong)_line, alignof(fd_vinyl_line_t) ) ); - TEST( _ele ); TEST( fd_ulong_is_aligned( (ulong)_ele, alignof(fd_vinyl_meta_ele_t) ) ); - TEST( _obj ); TEST( fd_ulong_is_aligned( (ulong)_obj, alignof(fd_vinyl_data_obj_t) ) ); - - TEST( cnc_footprint >= fd_cnc_footprint( FD_VINYL_CNC_APP_SZ ) ); - - ulong ele_max = fd_ulong_pow2_dn( ele_footprint / sizeof( fd_vinyl_meta_ele_t ) ); - ulong lock_cnt = fd_vinyl_meta_lock_cnt_est( ele_max ); - ulong probe_max = ele_max; - - TEST( ele_max>=4UL ); - TEST( meta_footprint >= fd_vinyl_meta_footprint( ele_max, lock_cnt, probe_max ) ); - - ulong pair_max = ele_max - 1UL; - ulong line_cnt = fd_ulong_min( line_footprint / sizeof( fd_vinyl_line_t ), pair_max ); - - TEST( (3UL<=line_cnt) & (line_cnt<=FD_VINYL_LINE_MAX) ); - - TEST( io ); - - /* seed is arb */ - - TEST( (0ULcnc = fd_cnc_join( fd_cnc_new( _cnc, FD_VINYL_CNC_APP_SZ, FD_VINYL_CNC_TYPE, fd_log_wallclock() ) ); TEST( vinyl->cnc ); - vinyl->line = (fd_vinyl_line_t *)_line; - vinyl->io = io; - - vinyl->line_cnt = line_cnt; - vinyl->pair_max = pair_max; - vinyl->async_min = async_min; - vinyl->async_max = async_max; - - vinyl->part_thresh = part_thresh; - vinyl->gc_thresh = gc_thresh; - vinyl->gc_eager = gc_eager; - vinyl->style = style; - vinyl->line_idx_lru = 0U; - vinyl->pair_cnt = 0UL; - vinyl->garbage_sz = 0UL; - - TEST( fd_vinyl_meta_join( vinyl->meta, fd_vinyl_meta_new( _meta, ele_max, lock_cnt, probe_max, seed ), _ele )==vinyl->meta ); - - TEST( fd_vinyl_data_init( vinyl->data, _obj, obj_footprint, obj_laddr0 )==vinyl->data ); - - vinyl->cnc_footprint = cnc_footprint; - vinyl->meta_footprint = meta_footprint; - vinyl->line_footprint = line_footprint; - vinyl->ele_footprint = ele_footprint; - vinyl->obj_footprint = obj_footprint; - - FD_LOG_NOTICE(( "Recovering bstream past (level %i)", level )); - - TEST( fd_vinyl_seq_eq( fd_vinyl_recover( tpool,t0,t1, level, vinyl ), fd_vinyl_io_seq_present( io ) ) ); - -# undef TEST - - FD_LOG_NOTICE(( "Initializing complete" )); - - return vinyl; -} - -void * -fd_vinyl_fini( fd_vinyl_t * vinyl ) { - - if( FD_UNLIKELY( !vinyl ) ) { - FD_LOG_WARNING(( "NULL vinyl" )); - return NULL; - } - - /* Note: does not sync. App should decide if sync is appropriate - before calling fini. */ - - fd_vinyl_data_fini( vinyl->data ); - - void * _meta = fd_vinyl_meta_shmap( vinyl->meta ); - fd_vinyl_meta_leave( vinyl->meta ); - fd_vinyl_meta_delete( _meta ); - - fd_cnc_delete( fd_cnc_leave( vinyl->cnc ) ); - - return vinyl; -} - -char * -fd_vinyl_cnc_signal_cstr( ulong signal, - char * buf ) { - if( FD_LIKELY( buf ) ) { - switch( signal ) { - case FD_VINYL_CNC_SIGNAL_RUN: strcpy( buf, "run" ); break; - case FD_VINYL_CNC_SIGNAL_BOOT: strcpy( buf, "boot" ); break; - case FD_VINYL_CNC_SIGNAL_FAIL: strcpy( buf, "fail" ); break; - case FD_VINYL_CNC_SIGNAL_HALT: strcpy( buf, "halt" ); break; - case FD_VINYL_CNC_SIGNAL_SYNC: strcpy( buf, "sync" ); break; - case FD_VINYL_CNC_SIGNAL_GET: strcpy( buf, "get" ); break; - case FD_VINYL_CNC_SIGNAL_SET: strcpy( buf, "set" ); break; - case FD_VINYL_CNC_SIGNAL_CLIENT_JOIN: strcpy( buf, "client_join" ); break; - case FD_VINYL_CNC_SIGNAL_CLIENT_LEAVE: strcpy( buf, "client_leave" ); break; - default: fd_cstr_printf( buf, FD_VINYL_CNC_SIGNAL_CSTR_BUF_MAX, NULL, "%lu", signal ); break; - } - } - return buf; -} diff --git a/src/vinyl/fd_vinyl.h b/src/vinyl/fd_vinyl.h deleted file mode 100644 index 96999c3962e..00000000000 --- a/src/vinyl/fd_vinyl.h +++ /dev/null @@ -1,282 +0,0 @@ -#ifndef HEADER_fd_src_vinyl_fd_vinyl_h -#define HEADER_fd_src_vinyl_fd_vinyl_h - -//#include "fd_vinyl_base.h" /* includes ../tango/fd_tango.h, ../util/tmpl/fd_map.h */ -//#include "bstream/fd_vinyl_bstream.h" /* includes fd_vinyl_base.h */ -//#include "io/fd_vinyl_io.h" /* includes bstream/fd_vinyl_bstream.h */ -//#include "meta/fd_vinyl_meta.h" /* includes bstream/fd_vinyl_bstream.h */ -//#include "data/fd_vinyl_data.h" /* includes io/fd_vinyl_io.h */ -#include "line/fd_vinyl_line.h" /* includes meta/fd_vinyl_meta.h data/fd_vinyl_data.h */ -#include "rq/fd_vinyl_rq.h" /* includes fd_vinyl_base.h */ -#include "cq/fd_vinyl_cq.h" /* includes fd_vinyl_base.h */ - -#define FD_VINYL_CNC_TYPE (0xFDC12C2CUL) /* FD VIN CNC */ - -#define FD_VINYL_CNC_SIGNAL_RUN FD_CNC_SIGNAL_RUN -#define FD_VINYL_CNC_SIGNAL_BOOT FD_CNC_SIGNAL_BOOT -#define FD_VINYL_CNC_SIGNAL_FAIL FD_CNC_SIGNAL_FAIL -#define FD_VINYL_CNC_SIGNAL_HALT FD_CNC_SIGNAL_HALT -#define FD_VINYL_CNC_SIGNAL_SYNC (4UL) /* Signal that recovery should resume from the current bstream past */ -#define FD_VINYL_CNC_SIGNAL_GET (5UL) /* Get the value of the given option */ -#define FD_VINYL_CNC_SIGNAL_SET (6UL) /* Set the value of the given option */ -#define FD_VINYL_CNC_SIGNAL_CLIENT_JOIN (7UL) /* Have a client join the vinyl tile */ -#define FD_VINYL_CNC_SIGNAL_CLIENT_LEAVE (8UL) /* Have a client leave the vinyl tile */ - -#define FD_VINYL_OPT_PART_THRESH (0) -#define FD_VINYL_OPT_GC_THRESH (1) -#define FD_VINYL_OPT_GC_EAGER (2) -#define FD_VINYL_OPT_STYLE (3) - -union fd_vinyl_cmd { - struct { - int err; - ulong link_id; - ulong burst_max; - ulong quota_max; - char rq [ FD_WKSP_CSTR_MAX ]; - char cq [ FD_WKSP_CSTR_MAX ]; - char wksp[ FD_SHMEM_NAME_MAX ]; - } join; - struct { - int err; - ulong link_id; - } leave; - struct { - int err; - int opt; - ulong val; - } get; - struct { - int err; - int opt; - ulong val; - } set; -}; - -typedef union fd_vinyl_cmd fd_vinyl_cmd_t; - -/* FIXME: ADD ADDITIONAL DIAGNOSTICS (LIKE DISK USAGE, PAIR_CNT, ETC) */ - -#define FD_VINYL_DIAG_DROP_LINK (0) -#define FD_VINYL_DIAG_DROP_COMP (1) -#define FD_VINYL_DIAG_CNT (2UL) - -#define FD_VINYL_CNC_APP_SZ (sizeof(fd_vinyl_cmd_t) + sizeof(ulong)*FD_VINYL_DIAG_CNT) - -struct __attribute__((aligned(128))) fd_vinyl_private { - - /* Underlying shared objects */ - - fd_cnc_t * cnc; - fd_vinyl_line_t * line; /* Indexed [0,line_cnt) */ - fd_vinyl_io_t * io; - - /* Config */ - - ulong line_cnt; /* Max pairs data that can be cached, in [3,min(pair_max,FD_VINYL_LINE_MAX)] */ - ulong pair_max; /* Max pairs that can be tracked globally, in [3,meta->ele_max) */ - ulong async_min; /* Min run loop iterations per async handling, positive */ - ulong async_max; /* Max run loop iterations per async handling, >=async_min */ - - /* State */ - - ulong part_thresh; /* Insert partition blocks roughly every part_thresh bytes for parallel recovery */ - ulong gc_thresh; /* Only compact if past_sz > gc_thresh */ - int gc_eager; /* Compact until <~ 2^-gc_eager fraction is garbage, in [-1,63], -1 disables */ - int style; /* Preferred bstream io encoding (i.e. is background compressed enabled) */ - uint line_idx_lru; /* Index of the least recently used cache line, in [0,line_cnt] */ - ulong pair_cnt; /* Num pair meta cached, in [0,pair_max] */ - ulong garbage_sz; /* Num bytes garbage in the bstream's past */ - - fd_vinyl_meta_t meta[1]; /* probe_max==ele_max, ele_max is a power 2 of at least 4, seed is meta_seed */ - fd_vinyl_data_t data[1]; /* data_laddr0 is local address of data cache gaddr0, obj contains data volumes */ - - ulong cnc_footprint; - ulong meta_footprint; - ulong line_footprint; - ulong ele_footprint; - ulong obj_footprint; - - /* Padding to 128 byte aligned */ - -}; - -typedef struct fd_vinyl_private fd_vinyl_t; - -FD_PROTOTYPES_BEGIN - -FD_FN_CONST ulong fd_vinyl_align ( void ); -FD_FN_CONST ulong fd_vinyl_footprint( void ); - -/* fd_vinyl_init uses the the caller (typically tpool thread t0) and - tpool threads (t0,t1) to init the vinyl structure (this structure can - be extremely large ... hundreds of gigabytes to terabytes in memory - for petabytes or more in persistent storage ... so it is worthwhile - to parallelize the initialization). The bstream's past will be used - to recover the vinyl instance to the bstream's seq_present. The - recovery level is given by level (see fd_vinyl_recover below). - Assumes tpool threads (t0,t1) are available for dispatch. These - threads will be available for dispatch on return. Retain no interest - in tpool. If tpool is NULL and/or the set [t0,t1) is empty/invalid, - uses a serial algorithm for initialization. */ - -fd_vinyl_t * -fd_vinyl_init( fd_tpool_t * tpool, ulong t0, ulong t1, int level, - void * lmem, /* memory region to hold the vinyl's state */ - void * shcnc, ulong cnc_footprint, /* memory region to use for the tile cnc */ - void * shmeta, ulong meta_footprint, /* memory region to use for the cached pair metadata state */ - void * shline, ulong line_footprint, /* memory region to use for the cached pair state */ - void * shele, ulong ele_footprint, /* memory region to use for the cached pair metadata */ - void * shobj, ulong obj_footprint, /* memory region to use for the cached pairs */ - fd_vinyl_io_t * io, /* interface to the underlying bstream */ - ulong seed, - void * obj_laddr0, - ulong async_min, - ulong async_max, - ulong part_thresh, - ulong gc_thresh, - int gc_eager, - int style ); - -void * -fd_vinyl_fini( fd_vinyl_t * vinyl ); - -/* Accessors */ - -/* fd_vinyl_* return the eponymous field. Where applicable - fd_vinyl_*_const are const correct versions. These assume vinyl is - valid. fd_vinyl_pair_cnt returns the num live pairs in the vinyl - instance (e.g. all pairs that exist at bstream's seq_present and all - pairs in the process of being created). Will be in [0,pair_max]. - Assumes vinyl is valid. */ - -FD_FN_PURE static inline void * -fd_vinyl_shcnc( fd_vinyl_t const * vinyl ) { - return (void *)fd_cnc_shmem_const( vinyl->cnc ); -} - -FD_FN_PURE static inline void * -fd_vinyl_shmeta( fd_vinyl_t const * vinyl ) { - return (void *)fd_vinyl_meta_shmap_const( vinyl->meta ); -} - -FD_FN_PURE static inline void * -fd_vinyl_shline( fd_vinyl_t const * vinyl ) { - return (void *)vinyl->line; -} - -FD_FN_PURE static inline void * -fd_vinyl_shele( fd_vinyl_t const * vinyl ) { - return (void *)fd_vinyl_meta_shele_const( vinyl->meta ); -} - -FD_FN_PURE static inline void * -fd_vinyl_shobj( fd_vinyl_t const * vinyl ) { - return fd_vinyl_data_shmem( vinyl->data ); -} - -FD_FN_PURE static inline ulong fd_vinyl_cnc_footprint ( fd_vinyl_t const * vinyl ) { return vinyl->cnc_footprint; } -FD_FN_PURE static inline ulong fd_vinyl_meta_footprint_( fd_vinyl_t const * vinyl ) { return vinyl->meta_footprint; } /* FIXME: sigh */ -FD_FN_PURE static inline ulong fd_vinyl_line_footprint ( fd_vinyl_t const * vinyl ) { return vinyl->line_footprint; } -FD_FN_PURE static inline ulong fd_vinyl_ele_footprint ( fd_vinyl_t const * vinyl ) { return vinyl->ele_footprint; } -FD_FN_PURE static inline ulong fd_vinyl_obj_footprint ( fd_vinyl_t const * vinyl ) { return vinyl->obj_footprint; } - -FD_FN_PURE static inline fd_vinyl_io_t * -fd_vinyl_io( fd_vinyl_t const * vinyl ) { - return (fd_vinyl_io_t *)vinyl->io; -} - -FD_FN_PURE static inline ulong -fd_vinyl_seed( fd_vinyl_t const * vinyl ) { - return fd_vinyl_meta_seed( vinyl->meta ); -} - -FD_FN_PURE static inline void * -fd_vinyl_obj_laddr0( fd_vinyl_t const * vinyl ) { - return fd_vinyl_data_laddr0( vinyl->data ); -} - -FD_FN_PURE static inline ulong fd_vinyl_async_min ( fd_vinyl_t const * vinyl ) { return vinyl->async_min; } -FD_FN_PURE static inline ulong fd_vinyl_async_max ( fd_vinyl_t const * vinyl ) { return vinyl->async_max; } -FD_FN_PURE static inline ulong fd_vinyl_part_thresh( fd_vinyl_t const * vinyl ) { return vinyl->part_thresh; } -FD_FN_PURE static inline ulong fd_vinyl_gc_thresh ( fd_vinyl_t const * vinyl ) { return vinyl->gc_thresh; } -FD_FN_PURE static inline int fd_vinyl_gc_eager ( fd_vinyl_t const * vinyl ) { return vinyl->gc_eager; } -FD_FN_PURE static inline int fd_vinyl_style ( fd_vinyl_t const * vinyl ) { return vinyl->style; } - -/* fd_vinyl_compact does up to compact_max rounds of compaction to the - bstream's past. This cannot fail from the caller's perspective (will - FD_LOG_CRIT if any corruption is detected). */ -/* FIXME: PRIVATE */ - -void -fd_vinyl_compact( fd_vinyl_t * vinyl, - ulong compact_max ); - -/* fd_vinyl_recover uses the caller (typically tpool thread t0) and - tpool threads (t0,t1) to reset the vinyl meta cache, reset the vinyl - data cache, reset vinyl cache line eviction priorities and repopulate - the vinyl meta data cache from the current state of the bstream's - past to the bstream's seq_present. level zero/non-zero indicates to - do a soft/hard reset. In a soft reset, the data cache region is - minimally cleared. In a hard reset, it is fully cleared. A hard - reset is recommended for most usage but a soft reset can allow faster - startup for rapid iteration during development. - - Returns the bstream sequence number of how far recovery got (if this - is not seq_present, the recovery was partial and it is theoretically - moves in the recovery were not processed atomically). Logs details - of any issues encountered. - - Assumes the tpool threads (t0,t1) are available for dispatch. - Retains no interest in tpool and threads (t0,t1) will be available - for dispatch on return. */ -/* FIXME: PRIVATE */ - -ulong -fd_vinyl_recover( fd_tpool_t * tpool, ulong t0, ulong t1, int level, - fd_vinyl_t * vinyl ); - -/* fd_vinyl_exec runs a vinyl tile on the caller. */ - -void -fd_vinyl_exec( fd_vinyl_t * vinyl ); - -int -fd_vinyl_halt( fd_cnc_t * cnc ); - -int -fd_vinyl_sync( fd_cnc_t * cnc ); - -int -fd_vinyl_get( fd_cnc_t * cnc, - int opt, - ulong * opt_val ); - -int -fd_vinyl_set( fd_cnc_t * cnc, - int opt, - ulong val, - ulong * opt_val ); - -int -fd_vinyl_client_join( fd_cnc_t * cnc, - fd_vinyl_rq_t * rq, - fd_vinyl_cq_t * cq, - fd_wksp_t * wksp, - ulong link_id, - ulong burst_max, - ulong quota_max ); - -int -fd_vinyl_client_leave( fd_cnc_t * cnc, - ulong link_id ); - -#define FD_VINYL_CNC_SIGNAL_CSTR_BUF_MAX (21UL) - -char * -fd_vinyl_cnc_signal_cstr( ulong signal, - char * buf ); - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_fd_vinyl_h */ diff --git a/src/vinyl/fd_vinyl_base.c b/src/vinyl/fd_vinyl_base.c deleted file mode 100644 index c793e694043..00000000000 --- a/src/vinyl/fd_vinyl_base.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "fd_vinyl_base.h" - -char const * -fd_vinyl_strerror( int err ) { - switch( err ) { - case FD_VINYL_SUCCESS: return "success"; - case FD_VINYL_ERR_INVAL: return "inval"; - case FD_VINYL_ERR_AGAIN: return "again"; - case FD_VINYL_ERR_CORRUPT: return "corrupt"; - case FD_VINYL_ERR_EMPTY: return "empty"; - case FD_VINYL_ERR_FULL: return "full"; - case FD_VINYL_ERR_KEY: return "key"; - default: break; - } - return "unknown"; -} diff --git a/src/vinyl/fd_vinyl_base.h b/src/vinyl/fd_vinyl_base.h deleted file mode 100644 index ea04b20b0af..00000000000 --- a/src/vinyl/fd_vinyl_base.h +++ /dev/null @@ -1,305 +0,0 @@ -#ifndef HEADER_fd_src_vinyl_fd_vinyl_base_h -#define HEADER_fd_src_vinyl_fd_vinyl_base_h - -/* Vinyl implements a persistent interprocess shared key-val store. - Unlike other fd key-val stores (including groove and funk_rec), - vinyl's key value store is backed by non-volatile storage. Features - include: - - - Massive batch requests processed async with zero-copy cut-through - from the underlying I/O layer to an arbitrary number of clients. - - - Runtime plugin I/O backends to support various styles of non-volatile - storage. This includes kernel-bypassing the VM and/or the file - system (e.g. direct I/O to raw block devices, I/O uring, memory - mapped I/O, posix I/O volumes, etc). - - - Massive amounts of transparent in memory LRU data caching with fine - grained user control of cache eviction priorities. - - - Acquire/release semantics with explicit completions for explicit - control of pointer lifetimes. - - - Optional lockfree O(1) metadata queries. E.g. hot-or-not query in - sig-verify with I/O cost aware downstream packing. - - - Always in memory application specific metadata. E.g. expirations, - balances, etc are always available fast O(1). - - - Application controllable persistence guarantees (including massively - parallel recovery / fast startup, recovery from power failures, and - recovery to earlier known good states). - - - Support for practically unlimited number of keys. - - - Non-blocking speculative reads - - - Fast ignoring existing values (e.g. fast overwriting). - - - Extensive user and internal consistency checking - - - Transparent background meta and data integrity checking - - - Transparent background data compression (with dynamic load adaption - and user configuration for low jitter) . - - - Live runtime statistics. - - - ... - - The most basic configuration is shown below: - - (optional) - (read only) +------------+ - ..........| vinyl_meta |<------------+ - . +------------+ | - . | - . +------+ | - . +------>| cnc |<------+ | - . | +------+ | | - v v v v vinyl_bstream - ========== +----------+ ============== vinyl_io +--------------------+ - = client =-->| vinyl_rq |-->= vinyl_tile = <--------> | non-volatile store | - ========== +----------+ ============== +--------------------+ - ^ ^ | ^ - | | +----------+ | | - | +-----| vinyl_cq |-----+ | - | +----------+ | - | | - | +------------+ | - +-------->| vinyl_data |<------------+ - +------------+ - - - To interact with a vinyl instance, a client joins the vinyl - command-and-control (cnc), vinyl data cache (vinyl_data) and (if - useful to the client) vinyl metadata cache (vinyl_meta). - - - The client uses the cnc to connect to the vinyl instance and - monitor the status of a vinyl instance. When connecting, the - client specifies the vinyl request queue (vinyl_rq) the client will - use to send batch requests to the vinyl instance and the vinyl - completion queue (vinyl_cq) the client will use to receive batch - request completion notifications. - - - Most client-vinyl interactions have acquire/release semantics. - - - When a client wants to read, modify, etc a batch of key-val pairs, - the client issues a batch request to acquire the keys. The vinyl - tile will order all incoming requests by clients for performance - and do the necessary I/O operations asynchronously in the - background. When finished, the vinyl instance will notify the - client where the values and metadata are for the keys are located - in the vinyl data cache on completion. This location will be - stable until the client releases the keys. As such, the client can - operate on the values and metadata in-place zero-copy as - appropriate for the type of request. - - - When the client is done operating on some or all of these keys, - the client issues a second request to release them. The vinyl - instance will update the data, meta and non-volatile storage - appropriately and notify the client when done. - - - The atomicity and ordering guarantees for requests from multiple - are described in detail in fd_vinyl_rq.h. - - - Advanced optimizations support out-of-band completions, ignoring - completions, and cut-through batch processing without waiting for a - completion. Likewise, a rq can be shared by multiple vinyl - instances (e.g. sharding the store over multiple vinyl instances). - Further, a cq can be shared by multiple clients (and not - necessarily the same clients issuing requests) to support advanced - thread modelings (e.g. read pipelining). - - - ... */ - -#include "../tango/fd_tango.h" /* For util and tango/cnc functionality */ - -/* fd_vinyl error code API ********************************************/ - -/* These mirror fd_map error codes exactly */ - -#include "../util/tmpl/fd_map.h" - -#define FD_VINYL_SUCCESS FD_MAP_SUCCESS -#define FD_VINYL_ERR_INVAL FD_MAP_ERR_INVAL -#define FD_VINYL_ERR_AGAIN FD_MAP_ERR_AGAIN -#define FD_VINYL_ERR_CORRUPT FD_MAP_ERR_CORRUPT -#define FD_VINYL_ERR_EMPTY FD_MAP_ERR_EMPTY -#define FD_VINYL_ERR_FULL FD_MAP_ERR_FULL -#define FD_VINYL_ERR_KEY FD_MAP_ERR_KEY - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_strerror converts an FD_VINYL_SUCCESS / FD_VINYL_ERR_* code - into a human readable cstr. The lifetime of the returned pointer is - infinite. The returned pointer is always to a non-NULL cstr. */ - -FD_FN_CONST char const * -fd_vinyl_strerror( int err ); - -FD_PROTOTYPES_END - -/* fd_vinyl_key_t API *************************************************/ - -/* A fd_vinyl_key_t specifies the key type for vinyl key-val pair. - Compact binary keys are encouraged but a cstr can be used so long as - it has strlen(cstr)c; - ulong csz = fd_ulong_min( src_sz, FD_VINYL_KEY_FOOTPRINT ); /* typically compile time */ - ulong zsz = FD_VINYL_KEY_FOOTPRINT - csz; /* " */ - if( zsz ) memset( dst, 0, FD_VINYL_KEY_FOOTPRINT ); /* " */ - if( csz ) memcpy( dst, src, csz ); /* " */ - return k; -} - -/* fd_vinyl_key_ulong populates the fd_vinyl_key_t compatible memory - region mem with the given ulongs. Assumes mem points to a stable - valid region in the caller's address space for the duration of the - call. Retains no interest in mem. Returns mem as a fd_vinyl_key_t. */ - -static inline fd_vinyl_key_t * -fd_vinyl_key_init_ulong( void * mem, - ulong k0, - ulong k1, - ulong k2, - ulong k3 ) { - fd_vinyl_key_t * k = (fd_vinyl_key_t *)mem; - k->ul[0] = k0; k->ul[1] = k1; k->ul[2] = k2; k->ul[3] = k3; - return k; -} - -/* fd_vinyl_key_eq tests keys ka and kb for equality. Assumes ka and - kb point in the caller's address space to stable valid keys for the - duration of the call. Retains no interest in ka or kb. Returns 1 if - the keys are equal and 0 otherwise. */ - -FD_FN_PURE static inline int -fd_vinyl_key_eq( fd_vinyl_key_t const * ka, - fd_vinyl_key_t const * kb ) { - ulong const * a = ka->ul; - ulong const * b = kb->ul; - return !((a[0]^b[0]) | (a[1]^b[1]) | (a[2]^b[2]) | (a[3]^b[3])); /* tons of ILP and vectorizable */ -} - -/* fd_vinyl_key_memo hashes the arbitrary 64-bit integer seed and the - key pointed to by k to a uniform quasi-random 64-bit integer. This - hash function is meant to be high quality but not necessarily - cryptographically secure. Assumes k points in the caller's address - space to a valid key for the duration of the call. Retains no - interest in k. Returns the hash (arbitrary). */ - -#if FD_HAS_INT128 - -/* If the target supports uint128, fd_vinyl_key_memo is seeded - xxHash3 with 64-bit output size. (open source BSD licensed) */ - -static inline ulong -fd_xxh3_mul128_fold64_( ulong lhs, ulong rhs ) { - uint128 product = (uint128)lhs * (uint128)rhs; - return (ulong)product ^ (ulong)( product>>64 ); -} - -static inline ulong -fd_xxh3_mix16b_( ulong i0, ulong i1, - ulong s0, ulong s1, - ulong seed ) { - return fd_xxh3_mul128_fold64_( i0 ^ (s0 + seed), i1 ^ (s1 - seed) ); -} - -FD_FN_PURE static inline ulong -fd_vinyl_key_memo( ulong seed, - fd_vinyl_key_t const * k ) { - ulong k0 = k->ul[0]; - ulong k1 = k->ul[1]; - ulong k2 = k->ul[2]; - ulong k3 = k->ul[3]; - ulong acc = 32 * 0x9E3779B185EBCA87ULL; - acc += fd_xxh3_mix16b_( k0, k1, 0xbe4ba423396cfeb8UL, 0x1cad21f72c81017cUL, seed ); - acc += fd_xxh3_mix16b_( k2, k3, 0xdb979083e96dd4deUL, 0x1f67b3b7a4a44072UL, seed ); - acc = acc ^ (acc >> 37); - acc *= 0x165667919E3779F9ULL; - acc = acc ^ (acc >> 32); - return acc; -} - -#else /* Portable variant */ - -FD_FN_PURE static inline ulong -fd_vinyl_key_memo( ulong seed, - fd_vinyl_key_t const * k ) { - ulong const * a = k->ul; - return fd_ulong_hash( a[0] ^ seed ) ^ fd_ulong_hash( a[1] ^ ( seed ^ 0x5555555555555555UL) ) ^ - fd_ulong_hash( a[2] ^ ( seed ^ 0xaaaaaaaaaaaaaaaaUL) ) ^ fd_ulong_hash( a[3] ^ ( seed ^ 0x5a5a5a5a5a5a5a5aUL) ); - /* tons of ILP and vectorizable */ -} - -#endif - -FD_PROTOTYPES_END - -/* fd_vinyl_info_t ABI ************************************************/ - -/* A fd_vinyl_info_t gives the pair decoded value size and any - application specific info (e.g. ctime, mtime, balance, expire, etc). - This structure is directly used in the bstream, vinyl meta cache, and - vinyl val data cache to reduce data marshalling, memory copies, - random access, etc between clients, tiles, caches and I/O layers and - to allow extra integrity checking. The application can use whatever - structure for the fd_vinyl_info_t so long as it is 8 byte aligned, - FD_VINYL_INFO_SZ in size and the first 4 bytes are the decoded pair - val size as a uint, in [0,FD_VINYL_VAL_MAX]. */ - -#define FD_VINYL_INFO_SZ (16UL) - -union fd_vinyl_info { - uint val_sz; - uchar uc[ FD_VINYL_INFO_SZ ]; - uint ui[ FD_VINYL_INFO_SZ / sizeof(uint ) ]; - ulong ul[ FD_VINYL_INFO_SZ / sizeof(ulong) ]; -}; - -typedef union fd_vinyl_info fd_vinyl_info_t; - -/* FD_VINYL_VAL_MAX is the maximum encoded and decoded byte size of a - pair val. The value below is: - - 10MiB + block_sz - sizeof(ctl) - sizeof(key) - sizeof(info) - sizeof(ftr) - 128 8 32 16 16 - - where the adjustments make it possible create a vinyl_data sizeclass - whose val_max is exactly FD_VINYL_VAL_MAX. */ - -#define FD_VINYL_VAL_MAX (10485816UL) /* autogenerated */ - -#endif /* HEADER_fd_src_vinyl_fd_vinyl_base_h */ diff --git a/src/vinyl/fd_vinyl_case_acquire.c b/src/vinyl/fd_vinyl_case_acquire.c deleted file mode 100644 index 801cf87d79e..00000000000 --- a/src/vinyl/fd_vinyl_case_acquire.c +++ /dev/null @@ -1,397 +0,0 @@ - case FD_VINYL_REQ_TYPE_ACQUIRE: { - ulong req_flags = (ulong)req->flags; - fd_vinyl_key_t const * req_key = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t, batch_cnt ); - ulong * req_val_gaddr = MAP_REQ_GADDR( req->val_gaddr_gaddr, ulong, batch_cnt ); - schar * req_err = MAP_REQ_GADDR( req->err_gaddr, schar, batch_cnt ); - - int req_flag_modify = fd_vinyl_req_flag_modify( req_flags ); - int req_flag_ignore = fd_vinyl_req_flag_ignore( req_flags ); - int req_flag_create = fd_vinyl_req_flag_create( req_flags ); - int req_flag_excl = fd_vinyl_req_flag_excl ( req_flags ); - int req_evict_prio = fd_vinyl_req_evict_prio ( req_flags ); - - int bad_gaddr = (!!batch_cnt) & ((!req_key) | (!req_val_gaddr) | (!req_err)); - int bad_quota = quota_remFD_VINYL_VAL_MAX ) ) DONE( FD_VINYL_ERR_INVAL ); - } - - /* Query vinyl meta for key */ - - fd_vinyl_key_t const * key = req_key + batch_idx; - - ulong memo = fd_vinyl_key_memo( meta_seed, key ); - - ulong _ele_idx; /* avoid pointer escape */ - int err = fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &_ele_idx ); - ulong ele_idx = _ele_idx; /* In [0,ele_max) */ - - if( FD_LIKELY( !err ) ) { /* pair key meta cached */ - - /* At this point, pair key either exists at bstream seq_present - or is in the process of being created. If pair key is being - created, fail with AGAIN (it must be acquired for modify). */ - - ulong pair_ctl = ele0[ ele_idx ].phdr.ctl; - - FD_DCHECK_CRIT( (fd_vinyl_bstream_ctl_type( pair_ctl )==FD_VINYL_BSTREAM_CTL_TYPE_PAIR) | (pair_ctl==ULONG_MAX), - "corruption detected" ); - - if( FD_UNLIKELY( pair_ctl==ULONG_MAX ) ) DONE( FD_VINYL_ERR_AGAIN ); - - /* At this point, pair key exists at bstream seq_present. */ - - ulong val_sz = (ulong)ele0[ ele_idx ].phdr.info.val_sz; - ulong line_idx = ele0[ ele_idx ].line_idx; - - FD_DCHECK_CRIT( val_sz<=FD_VINYL_VAL_MAX, "corruption detected" ); - FD_DCHECK_CRIT( (line_idxline_idx==line_idx, "corruption detected" ); - - ulong line_ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( line_ctl ); - long ref = fd_vinyl_line_ctl_ref( line_ctl ); - - if( FD_LIKELY( !req_flag_modify ) ) { - - /* At this point, we are acquiring a cached pair for read. - If the line is acquired for modify, fail with AGAIN. If - there are too many acquires for read on this pair, CRIT - (could consider AGAIN here). Otherwise, we update the - ref count (don't change the ver), point the client at the - line caching pair key to finish the acquire. Note that - we don't validate the pair header if we detect that an - earlier acquire in this batch started fetching the pair - because the read might still be in progress (see note - below for more details). */ - - if( FD_UNLIKELY( ref<0L ) ) DONE( FD_VINYL_ERR_AGAIN ); - if( FD_UNLIKELY( ref>=FD_VINYL_LINE_REF_MAX ) ) FD_LOG_CRIT(( "too many acquires for read on this pair" )); - - if( FD_LIKELY( !obj->rd_active ) ) { - fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj ); - - FD_DCHECK_CRIT( fd_vinyl_data_obj_val_max( obj ) >= val_sz, "corruption detected" ); - FD_DCHECK_CRIT( phdr->ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ), "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_key_eq( &phdr->key, key ), "corruption detected" ); - FD_DCHECK_CRIT( !memcmp( &phdr->info, &ele0[ ele_idx ].phdr.info, sizeof(fd_vinyl_info_t) ), "corruption detected" ); - } - - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver, ref+1L ); /* don't bump ver */ - - req_val_gaddr[ batch_idx ] = (ulong)fd_vinyl_data_obj_val( obj ) - data_laddr0; - - DONE( FD_VINYL_SUCCESS ); - - } - - /* At this point, we are acquiring a cached pair for modify. - If we are not allowed to acquire an existing pair for - modify (INVAL) or if the line line_idx is already acquired - for anything (AGAIN), fail. */ - - if( FD_UNLIKELY( ref ) ) DONE( FD_VINYL_ERR_AGAIN ); - if( FD_UNLIKELY( req_flag_excl ) ) DONE( FD_VINYL_ERR_INVAL ); - - fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj ); - - FD_DCHECK_CRIT( !obj->rd_active, "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_data_obj_val_max( obj ) >= val_sz, "corruption detected" ); - FD_DCHECK_CRIT( phdr->ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ), "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_key_eq( &phdr->key, key ), "corruption detected" ); - FD_DCHECK_CRIT( !memcmp( &phdr->info, &ele0[ ele_idx ].phdr.info, sizeof(fd_vinyl_info_t) ), "corruption detected" ); - - /* If the ignore flag is set, set the cached value size to 0. */ - - if( req_flag_ignore ) { - phdr->info.val_sz = 0U; - val_sz = 0UL; - } - - /* If the current location for the pair key's data isn't - sufficient to hold the worst case val_sz that the client - might modify the pair's value into, adjust the space - available for the pair to the user's val_max. Because we - might be ignoring the existing value, this could be smaller - than the current object. (We could chose to not trim in - this case because it will get trimmed again on release. - But doing so makes a more consistent guarantee to the - client and makes testing easier.) */ - - ulong csz = sizeof(fd_vinyl_bstream_phdr_t) + val_sz; - - ulong szc_new = fd_vinyl_data_szc( fd_ulong_max( val_sz, req_val_max ) ); - ulong szc_old = (ulong)obj->szc; - - if( FD_UNLIKELY( szc_new != szc_old ) ) { - - fd_vinyl_data_obj_t * obj_new = fd_vinyl_data_alloc( data, szc_new ); - if( FD_UNLIKELY( !obj_new ) ) FD_LOG_CRIT(( "increase data cache size" )); - - fd_vinyl_bstream_phdr_t * phdr_new = fd_vinyl_data_obj_phdr( obj_new ); - - memcpy( phdr_new, phdr, csz ); - - fd_vinyl_data_free( data, obj ); - - phdr = phdr_new; - obj = obj_new; - - line[ line_idx ].obj = obj; obj->line_idx = line_idx; obj->rd_active = (short)0; - } - - /* Zero out any remaining space in the pair. */ - - ulong zsz = fd_vinyl_bstream_pair_sz( fd_vinyl_data_szc_val_max( szc_new ) ) - csz; - memset( ((uchar *)phdr) + csz, 0, zsz ); - - /* Finish up acquiring for modify */ - - //line[ line_idx ].obj = ... already init; - //line[ line_idx ].ele_idx = ... already init; - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, -1L ); /* bump ver */ - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - //phdr->ctl = ... already init - //phdr->key = ... already init - //phdr->info = ... already init - - //ele0[ ele_idx ] = ... already init - - req_val_gaddr[ batch_idx ] = (ulong)fd_vinyl_data_obj_val( obj ) - data_laddr0; - - DONE( FD_VINYL_SUCCESS ); - - } /* pair key data cached */ - - /* At this point, pair key is not cached. If we are not allowed - to acquire this pair, fail. Otherwise, evict the least - recently used evictable line (this should always be possible - if quotas are configured correctly) to make room to cache this - pair. Connect this line to meta element ele_idx, set the - line's reference count appropriately, bump the line's version - and move the line to the desired location in the eviction - sequence. We don't modify any shared fields in meta element - ele_idx so we can do the modification fast. - - We do this upfront to free data cache for the alloc if the - LRU line is in use and to handle the same pair appearing - multiple times in an acquire. - - That is, if req_key appears multiple times in an acquire to - modify, the trailing redundant acquires will see the object - as cached with ref==-1 and fail with AGAIN. If the key - appears multiple times in an acquire for read, the trailing - redundant acquires will see the object as cached with ref>0 - and rd_active==1, conclude that the first redundant acquire - is in the process of reading the pair into cache, skip any - racy metadata checks, increase the ref count and succeed. - - IMPORTANT SAFETY TIP! Note that this implies that client - doing an acquire-for-read with redundant keys and with - speculative processing will see req_err transition to success - for the trailing redundant items for a key before the leading - item of that key transitions to success (and thus before the - object is fully read / verified and/or decoded). It is up to - the client doing speculative cut through processing to avoid - redundant keys or react accordingly. */ - - if( FD_UNLIKELY( req_flag_modify & req_flag_excl ) ) DONE( FD_VINYL_ERR_INVAL ); - - line_idx = fd_vinyl_line_evict_lru( &vinyl->line_idx_lru, line, line_cnt, ele0, ele_max, data ); - - ulong line_ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( line_ctl ); - - line[ line_idx ].ele_idx = ele_idx; ele0[ ele_idx ].line_idx = line_idx; - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, req_flag_modify ? -1L : 1L ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - /* Allocate an appropriately sized object to hold this pair, - connect it to this line and report the location to the client. */ - - ulong val_max = fd_ulong_if( !req_flag_modify, val_sz, - fd_ulong_if( !req_flag_ignore, fd_ulong_max( val_sz, req_val_max ), - req_val_max ) ); - - ulong szc = fd_vinyl_data_szc( val_max ); - - fd_vinyl_data_obj_t * obj = fd_vinyl_data_alloc( data, szc ); - if( FD_UNLIKELY( !obj ) ) FD_LOG_CRIT(( "increase data cache size" )); - - line[ line_idx ].obj = obj; obj->line_idx = line_idx; - - void * val = fd_vinyl_data_obj_val( obj ); - - req_val_gaddr[ batch_idx ] = (ulong)val - data_laddr0; - - /* If we need to do I/O, start reading encoded pair data and - defer the data integrity and decoding to later (and then in - whatever order the I/O layer sees fit). */ - - if( FD_LIKELY( !(req_flag_modify & req_flag_ignore) ) ) { - obj->rd_active = (short)1; - - int style = fd_vinyl_bstream_ctl_style( pair_ctl ); - ulong val_esz = fd_vinyl_bstream_ctl_sz ( pair_ctl ); - - FD_DCHECK_CRIT( val_esz<=FD_VINYL_VAL_MAX, "corruption detected" ); - FD_DCHECK_CRIT( (style!=FD_VINYL_BSTREAM_CTL_STYLE_RAW) | (val_sz==val_esz), "corruption detected" ); - - fd_vinyl_data_obj_t * cobj; - - if( FD_LIKELY( style==FD_VINYL_BSTREAM_CTL_STYLE_RAW ) ) cobj = obj; - else { - cobj = fd_vinyl_data_alloc( data, fd_vinyl_data_szc( val_esz ) ); - if( FD_UNLIKELY( !cobj ) ) FD_LOG_CRIT(( "increase data cache size" )); - } - - cobj->rd->ctx = (ulong)obj; - cobj->rd->seq = ele0[ ele_idx ].seq; - cobj->rd->dst = fd_vinyl_data_obj_phdr( cobj ); - cobj->rd->sz = fd_vinyl_bstream_pair_sz( val_esz ); - - cobj->rd_err = req_err + batch_idx; - - fd_vinyl_io_read( io, cobj->rd ); - read_cnt++; - - quota_rem--; - goto next_acquire; - } - - /* At this point, we are acquiring to modify but we don't need - the existing value. We populate the cached pair header - appropriately for the modify and zero the rest to complete - this request immediately. */ - - obj->rd_active = (short)0; - - fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj ); - - phdr->ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ); - phdr->key = *key; - phdr->info = ele0[ ele_idx ].phdr.info; - - phdr->info.val_sz = 0U; - - memset( val, 0, fd_vinyl_data_szc_obj_footprint( szc ) - sizeof(fd_vinyl_data_obj_t) - sizeof(fd_vinyl_bstream_phdr_t) ); - - DONE( FD_VINYL_SUCCESS ); - - } /* pair key meta cached */ - - /* At this point, pair key does not exist at bstream seq_present - and is not in the process of being created. If we aren't - allowed to create pair key, fail. Otherwise, evict the least - recently used evictable line (this should always be possible if - quotas are configured correctly) to make room to cache this - pair, set the line's reference count appropriately, bump the - version and move the line to the desired location in the - eviction sequence. We do this upfront to free data cache for - the alloc if the LRU line is in use. */ - - if( FD_UNLIKELY( !(req_flag_modify & req_flag_create) ) ) DONE( FD_VINYL_ERR_KEY ); - - ulong line_idx = fd_vinyl_line_evict_lru( &vinyl->line_idx_lru, line, line_cnt, ele0, ele_max, data ); - - ulong line_ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( line_ctl ); - - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, -1L ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - /* Allocate an appropriately sized object to hold this pair and - connect it to this line. */ - - ulong szc = fd_vinyl_data_szc( req_val_max ); - - fd_vinyl_data_obj_t * obj = fd_vinyl_data_alloc( data, szc ); - if( FD_UNLIKELY( !obj ) ) FD_LOG_CRIT(( "increase data cache size" )); - - line[ line_idx ].obj = obj; obj->line_idx = line_idx; obj->rd_active = (short)0; - - /* Allocate a meta element to hold metadata for this pair and - connect it to this line. Since we are inserting at meta - element ele_idx, we don't need to lock anything so long as we - mark the element as in use very last. */ - - ulong pair_cnt = vinyl->pair_cnt; - if( FD_UNLIKELY( pair_cnt>=pair_max ) ) FD_LOG_CRIT(( "increase meta cache size" )); - vinyl->pair_cnt = pair_cnt + 1UL; - - ele0[ ele_idx ].memo = memo; - //ele0[ ele_idx ].phdr.ctl init below - ele0[ ele_idx ].phdr.key = *key; - memset( &ele0[ ele_idx ].phdr.info, 0, sizeof(fd_vinyl_info_t) ); /* sets val_sz to 0 */ - ele0[ ele_idx ].line_idx = line_idx; - ele0[ ele_idx ].seq = 0UL; /* Will be init on release */ - - FD_COMPILER_MFENCE(); - ele0[ ele_idx ].phdr.ctl = ULONG_MAX; /* Mark as being created */ - FD_COMPILER_MFENCE(); - - line[ line_idx ].ele_idx = ele_idx; - - /* Initialize the data region for a new pair */ - - *fd_vinyl_data_obj_phdr( obj ) = ele0[ ele_idx ].phdr; - - uchar * val = (uchar *)fd_vinyl_data_obj_val( obj ); - - memset( val, 0, fd_vinyl_data_szc_obj_footprint( szc ) - sizeof(fd_vinyl_data_obj_t) - sizeof(fd_vinyl_bstream_phdr_t) ); - - req_val_gaddr[ batch_idx ] = (ulong)val - data_laddr0; - - DONE( FD_VINYL_SUCCESS ); - - next_acquire: /* silly language restriction */; - -# undef DONE - - } /* for batch_idx */ - - FD_DCHECK_CRIT( (!read_cnt) | (!(req_flag_modify & req_flag_ignore)), "corruption detected" ); - - comp_err = FD_VINYL_SUCCESS; - break; - } diff --git a/src/vinyl/fd_vinyl_case_erase.c b/src/vinyl/fd_vinyl_case_erase.c deleted file mode 100644 index e97ac68da81..00000000000 --- a/src/vinyl/fd_vinyl_case_erase.c +++ /dev/null @@ -1,108 +0,0 @@ - case FD_VINYL_REQ_TYPE_ERASE: { - - fd_vinyl_key_t const * req_key = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t, batch_cnt ); - schar * req_err = MAP_REQ_GADDR( req->err_gaddr, schar, batch_cnt ); - - if( FD_UNLIKELY( (!!batch_cnt) & ((!req_key) | (!req_err)) ) ) { - comp_err = FD_VINYL_ERR_INVAL; - break; - } - - for( ulong batch_idx=0UL; batch_idxline_idx==line_idx, "corruption detected" ); - FD_DCHECK_CRIT ( !obj->rd_active, "corruption detected" ); - - ulong ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( ctl ); - long ref = fd_vinyl_line_ctl_ref( ctl ); - - if( FD_UNLIKELY( ref ) ) { - FD_COMPILER_MFENCE(); - req_err[ batch_idx ] = (schar)FD_VINYL_ERR_AGAIN; - FD_COMPILER_MFENCE(); - fail_cnt++; - continue; - } - - line[ line_idx ].obj = NULL; - line[ line_idx ].ele_idx = ULONG_MAX; //ele0[ ele_idx ].line_idx = ULONG_MAX; /* Technically not necessary given below */ - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, 0L ); /* bump version */ - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, FD_VINYL_LINE_EVICT_PRIO_LRU ); - - fd_vinyl_data_free( data, obj ); - - } else { - - FD_DCHECK_CRIT( line_idx==ULONG_MAX, "corruption detected" ); - - } - - /* At this point, pair key exists and is not cached. Append a - dead block and remove it from the meta. This generates two - pieces of bstream garbage: the old pair and the dead block - itself (the dead block is only needed for recovery and then - only while the old pair is in the bstream's past). */ - - /* FIXME: COMPACT SEQUENTIAL DEADS IN THE BSTREAM TO BE MORE - SPACE EFFICIENT? */ - - ulong val_esz = fd_vinyl_bstream_ctl_sz( ele0[ ele_idx ].phdr.ctl ); - - accum_garbage_cnt += 2UL; - accum_garbage_sz += fd_vinyl_bstream_pair_sz( val_esz ) + FD_VINYL_BSTREAM_BLOCK_SZ; - - fd_vinyl_io_append_dead( io, &ele0[ ele_idx ].phdr, NULL, 0UL ); - append_cnt++; - accum_dead_cnt++; - - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, line, line_cnt, ele_idx ); - - ulong pair_cnt = vinyl->pair_cnt; - FD_DCHECK_CRIT( pair_cnt, "corruption detected" ); - vinyl->pair_cnt = pair_cnt - 1UL; - - FD_COMPILER_MFENCE(); - req_err[ batch_idx ] = (schar)FD_VINYL_SUCCESS; - FD_COMPILER_MFENCE(); - } - - comp_err = FD_VINYL_SUCCESS; - break; - } diff --git a/src/vinyl/fd_vinyl_case_fetch.c b/src/vinyl/fd_vinyl_case_fetch.c deleted file mode 100644 index ff5c4c953d4..00000000000 --- a/src/vinyl/fd_vinyl_case_fetch.c +++ /dev/null @@ -1,116 +0,0 @@ - case FD_VINYL_REQ_TYPE_FETCH: { - fd_vinyl_key_t const * req_key = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t, batch_cnt ); - - if( FD_UNLIKELY( (!!batch_cnt) & (!req_key) ) ) break; - - for( ulong batch_idx=0UL; batch_idxline_idx_lru, line, line_cnt, line_idx, FD_VINYL_LINE_EVICT_PRIO_MRU ); - - continue; - } - - /* At this point, pair key existsat seq_present but is not cached. - Evict the least recently used evictable line to make room to - cache this pair. Connect this line to meta element ele_idx, - set the line's reference count to zero, bump the line's version - and set the eviction priority to MRU. We don't modify any - shared fields in meta element ele_idx so we can do the - modification fast. - - We do this upfront to free data cache for the alloc if the LRU - line is in use and to handle the same pair appearing multiple - times in an acquire. - - The mechanics for fetch requests with redundant keys are - similar to acquire-for-read requests. In this case, trailing - redundant fetches will see the pair as cached (due to the first - redundant fetch ... this one), set the eviction priority to MRU - (again) and then continue. */ - - ulong pair_ctl = ele0[ ele_idx ].phdr.ctl; - ulong val_sz = (ulong)ele0[ ele_idx ].phdr.info.val_sz; - - FD_DCHECK_CRIT( fd_vinyl_bstream_ctl_type( pair_ctl )==FD_VINYL_BSTREAM_CTL_TYPE_PAIR, "corruption detected" ); - FD_DCHECK_CRIT( val_sz<=FD_VINYL_VAL_MAX, "corruption detected" ); - - line_idx = fd_vinyl_line_evict_lru( &vinyl->line_idx_lru, line, line_cnt, ele0, ele_max, data ); - - ulong line_ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( line_ctl ); - - line[ line_idx ].ele_idx = ele_idx; ele0[ ele_idx ].line_idx = line_idx; - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, 0L ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, FD_VINYL_LINE_EVICT_PRIO_MRU ); - - /* Allocate an appropriately sized object to hold this pair, - connect it to this line and start reading the encoded pair data - into obj. */ - - ulong szc = fd_vinyl_data_szc( val_sz ); - - fd_vinyl_data_obj_t * obj = fd_vinyl_data_alloc( data, szc ); - if( FD_UNLIKELY( !obj ) ) FD_LOG_CRIT(( "increase data cache size" )); - - line[ line_idx ].obj = obj; obj->line_idx = line_idx; - - /* Start reading encoded pair data and defer the validation and - decoding to later (and then in whatever order the I/O layer - sees fit). */ - - obj->rd_active = (short)1; - - int style = fd_vinyl_bstream_ctl_style( pair_ctl ); - ulong val_esz = fd_vinyl_bstream_ctl_sz ( pair_ctl ); - - FD_DCHECK_CRIT( val_esz<=FD_VINYL_VAL_MAX, "corruption detected" ); - FD_DCHECK_CRIT( (style!=FD_VINYL_BSTREAM_CTL_STYLE_RAW) | (val_sz==val_esz), "corruption detected" ); - - fd_vinyl_data_obj_t * cobj; - - if( FD_LIKELY( style==FD_VINYL_BSTREAM_CTL_STYLE_RAW ) ) cobj = obj; - else { - cobj = fd_vinyl_data_alloc( data, fd_vinyl_data_szc( val_esz ) ); - if( FD_UNLIKELY( !cobj ) ) FD_LOG_CRIT(( "increase data cache size" )); - } - - cobj->rd->ctx = (ulong)obj; - cobj->rd->seq = ele0[ ele_idx ].seq; - cobj->rd->dst = fd_vinyl_data_obj_phdr( cobj ); - cobj->rd->sz = fd_vinyl_bstream_pair_sz( val_esz ); - - cobj->rd_err = (schar *)cobj->unused; - - fd_vinyl_io_read( io, cobj->rd ); - read_cnt++; - } - - break; - } diff --git a/src/vinyl/fd_vinyl_case_flush.c b/src/vinyl/fd_vinyl_case_flush.c deleted file mode 100644 index ec87537a054..00000000000 --- a/src/vinyl/fd_vinyl_case_flush.c +++ /dev/null @@ -1,68 +0,0 @@ - case FD_VINYL_REQ_TYPE_FLUSH: { - - fd_vinyl_key_t const * req_key = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t, batch_cnt ); - - if( FD_UNLIKELY( (!!batch_cnt) & (!req_key) ) ) break; /* flushes don't generate completions */ - - for( ulong batch_idx=0UL; batch_idxline_idx==line_idx, "corruption detected" ); - FD_DCHECK_CRIT ( !obj->rd_active, "corruption detected" ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, FD_VINYL_LINE_EVICT_PRIO_LRU ); - - ulong ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( ctl ); - long ref = fd_vinyl_line_ctl_ref( ctl ); - - if( FD_UNLIKELY( ref ) ) continue; - - /* At this point, pair key is cached, not acquired and the line - is at LRU position. Flush the cached data. We don't modify - any shared fields of meta element ele_idx so we can do this - fast. */ - - line[ line_idx ].obj = NULL; - line[ line_idx ].ele_idx = ULONG_MAX; - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, 0UL ); - /* evict prio updated above */ - - ele0[ ele_idx ].line_idx = ULONG_MAX; - - fd_vinyl_data_free( data, obj ); - - } - - break; - } diff --git a/src/vinyl/fd_vinyl_case_move.c b/src/vinyl/fd_vinyl_case_move.c deleted file mode 100644 index 94b33686802..00000000000 --- a/src/vinyl/fd_vinyl_case_move.c +++ /dev/null @@ -1,326 +0,0 @@ - case FD_VINYL_REQ_TYPE_MOVE: { - - fd_vinyl_key_t const * req_key_src = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t, batch_cnt ); - fd_vinyl_key_t const * req_key_dst = MAP_REQ_GADDR( req->val_gaddr_gaddr, fd_vinyl_key_t, batch_cnt ); - schar * req_err = MAP_REQ_GADDR( req->err_gaddr, schar, batch_cnt ); - - if( FD_UNLIKELY( (!!batch_cnt) & ((!req_key_src) | (!req_key_dst) | (!req_err)) ) ) { - comp_err = FD_VINYL_ERR_INVAL; - break; - } - - for( ulong batch_idx=0UL; batch_idxline_idx==line_idx_src, "corruption detected" ); - - phdr_src = fd_vinyl_data_obj_phdr( obj_src ); - - line[ line_idx_src ].ctl = fd_vinyl_line_ctl( ver_src+1UL, 0L ); - - } else { - - FD_DCHECK_CRIT( line_idx_src==ULONG_MAX, "corruption detected" ); - - /* Read the encoded pair from the bstream */ - - ulong ctl = ele0[ ele_idx_src ].phdr.ctl; - - int type = fd_vinyl_bstream_ctl_type ( ctl ); - int style = fd_vinyl_bstream_ctl_style( ctl ); - ulong val_esz = fd_vinyl_bstream_ctl_sz ( ctl ); - - FD_DCHECK_CRIT( type==FD_VINYL_BSTREAM_CTL_TYPE_PAIR, "corruption detected" ); - FD_DCHECK_CRIT( (style==FD_VINYL_BSTREAM_CTL_STYLE_RAW) | (style==FD_VINYL_BSTREAM_CTL_STYLE_LZ4), "corruption detected" ); - FD_DCHECK_CRIT( val_esz<=FD_VINYL_VAL_MAX, "corruption detected" ); - - fd_vinyl_data_obj_t * cobj = fd_vinyl_data_alloc( data, fd_vinyl_data_szc( val_esz ) ); - if( FD_UNLIKELY( !cobj ) ) FD_LOG_CRIT(( "increase data cache size" )); - - fd_vinyl_bstream_phdr_t * cphdr = fd_vinyl_data_obj_phdr( cobj ); - ulong cpair_sz = fd_vinyl_bstream_pair_sz( val_esz ); - - fd_vinyl_io_read_imm( io, seq_src, cphdr, cpair_sz ); - /* not an async read (so no read_cnt increment) */ - - /* Verify data integrity */ - - FD_DCHECK_ALERT( !fd_vinyl_bstream_pair_test( io_seed, seq_src, (fd_vinyl_bstream_block_t *)cphdr, cpair_sz ), - "corruption detected" ); - - /* Decode the pair */ - - if( FD_LIKELY( style==FD_VINYL_BSTREAM_CTL_STYLE_RAW ) ) { - - FD_DCHECK_CRIT( val_esz==val_sz, "corruption detected" ); - - obj_src = cobj; - phdr_src = cphdr; - - } else { - - obj_src = fd_vinyl_data_alloc( data, fd_vinyl_data_szc( val_sz ) ); - if( FD_UNLIKELY( !obj_src ) ) FD_LOG_CRIT(( "increase data cache size" )); - - char const * cval = (char const *)fd_vinyl_data_obj_val( cobj ); - char * val = (char *) fd_vinyl_data_obj_val( obj_src ); - if( FD_UNLIKELY( (ulong)LZ4_decompress_safe( cval, val, (int)val_esz, (int)val_sz )!=val_sz ) ) - FD_LOG_CRIT(( "LZ4_decompress_safe failed" )); - - phdr_src = fd_vinyl_data_obj_phdr( obj_src ); - - phdr_src->ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ); - phdr_src->key = cphdr->key; - phdr_src->info = cphdr->info; - - fd_vinyl_data_free( data, cobj ); - - } - - line_idx_src = fd_vinyl_line_evict_lru( &vinyl->line_idx_lru, line, line_cnt, ele0, ele_max, data ); - - ulong line_ctl_src = line[ line_idx_src ].ctl; - - ulong ver_src = fd_vinyl_line_ctl_ver( line_ctl_src ); - - line[ line_idx_src ].obj = obj_src; obj_src->line_idx = line_idx_src; obj_src->rd_active = (short)0; - line[ line_idx_src ].ele_idx = ele_idx_src; ele0[ ele_idx_src ].line_idx = line_idx_src; - line[ line_idx_src ].ctl = fd_vinyl_line_ctl( ver_src+1UL, 0L ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx_src, FD_VINYL_LINE_EVICT_PRIO_LRU ); - - if( line_idx_src==line_idx_dst ) line_idx_dst = ULONG_MAX; /* Handle evict_lru evicting the dst */ - - } - - /* At this point, pair key_src is cached but not acquired and pair - key_dst is not acquired. We are clear to move. If pair - key_dst exists, we are replacing pair key_dst with pair - key_src. In this case, we remove pair key_dst from cache and - remove pair key_dst from the meta. This remove might move the - location of pair key_src's meta element. So we reload if - necessary. */ - - FD_DCHECK_CRIT( fd_vinyl_bstream_ctl_type( phdr_src->ctl )==fd_vinyl_bstream_ctl_type( ele0[ ele_idx_src ].phdr.ctl ), - "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_key_eq( &phdr_src->key, &ele0[ ele_idx_src ].phdr.key ), "corruption detected" ); - FD_DCHECK_CRIT( !memcmp( &phdr_src->info, &ele0[ ele_idx_src ].phdr.info, sizeof(fd_vinyl_info_t) ), "corruption detected" ); - - accum_garbage_cnt += 2UL; /* old src and new move block */ - accum_garbage_sz += fd_vinyl_bstream_pair_sz( fd_vinyl_bstream_ctl_sz( ele0[ ele_idx_src ].phdr.ctl ) ) + - FD_VINYL_BSTREAM_BLOCK_SZ; - - if( FD_UNLIKELY( !err_dst ) ) { - - accum_garbage_cnt++; /* old dst */ - accum_garbage_sz += fd_vinyl_bstream_pair_sz( fd_vinyl_bstream_ctl_sz( ele0[ ele_idx_dst ].phdr.ctl ) ); - - if( FD_UNLIKELY( line_idx_dst < line_cnt ) ) { - - FD_DCHECK_CRIT( line[ line_idx_dst ].ele_idx==ele_idx_dst, "corruption detected" ); - - fd_vinyl_data_obj_t * obj_dst = line[ line_idx_dst ].obj; - - FD_DCHECK_ALERT( fd_vinyl_data_is_valid_obj( obj_dst, vol, vol_cnt ), "corruption detected" ); - FD_DCHECK_CRIT ( obj_dst->line_idx==line_idx_dst, "corruption detected" ); - - ulong line_ctl_dst = line[ line_idx_dst ].ctl; - - ulong ver_dst = fd_vinyl_line_ctl_ver( line_ctl_dst ); - - fd_vinyl_data_free( data, obj_dst ); - - line[ line_idx_dst ].obj = NULL; - line[ line_idx_dst ].ele_idx = ULONG_MAX; // ele0[ ele_idx_dst ].line_idx = ULONG_MAX; /* Technically not necessary given below */ - line[ line_idx_dst ].ctl = fd_vinyl_line_ctl( ver_dst+1UL, 0L ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx_dst, FD_VINYL_LINE_EVICT_PRIO_LRU ); - } - - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, line, line_cnt, ele_idx_dst ); /* See note below about atomicity for concurrent meta readers */ - - ulong pair_cnt = vinyl->pair_cnt; - FD_DCHECK_CRIT( pair_cnt, "corruption detected" ); - vinyl->pair_cnt = pair_cnt - 1UL; - - err_src = fd_vinyl_meta_query_fast( ele0, ele_max, key_src, memo_src, &_ele_idx_src ); - ele_idx_src = _ele_idx_src; /* In [0,ele_max) */ - FD_DCHECK_CRIT( !err_src, "corruption detected" ); - /* Note: could test other fields post move too */ - - } - - /* At this point, pair key_src is cached but not acquired and pair - key_dst is not cached and not in the meta (the move block that - will official erase if it already exists will be written - below). Update the cached phdr to reflect the move. Remove - the meta entry for pair key_src and insert a meta entry for - pair key_dst. - - Note: this means from the point of view of concurrent meta - queries, there will be a brief time interval when pair key_src - and pair key_dst are both reported as not existing. - - As an alternative with more overhead we could instead insert - the meta element for key_dst, remove the meta element for - key_src and requery meta for key_dst (as the remove could move - it). In this case, there will be a gap where both key_src and - key_dst are both reported as available (and they will point to - the same cache entry during this interval). - - With even more complexity and overhead, we could eliminate the - gap and overhead and make this atomic from the point of view of - concurrent meta readers. (Would have compute a lock set that - cover the target key_dst insert location and the key_src probe - sequence assuming key_dst has been inserted, lock the locks, do - the insert, do the remove without any locking behavior, free - the lock set and then requery where key_dst ended up.) Also - note that, if we are replacing pair key_dst, at this point, - pair key_dst is already reported to concurrent meta readers as - not existing. Would need to extend this to the above. - - But it isn't clear that concurrent meta readers care at all. - So we go with the fast simple method below (it still is atomic - from the point of view of clients and the bstream). */ - - ulong pair_sz = fd_vinyl_bstream_pair_sz( val_sz ); - ulong seq_move = fd_vinyl_io_hint( io, FD_VINYL_BSTREAM_BLOCK_SZ + pair_sz ); - ulong seq_dst = seq_move + FD_VINYL_BSTREAM_BLOCK_SZ; - - //phdr_src->ctl = ... already init - phdr_src->key = *key_dst; - //phdr_src->info = ... already init - - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, line, line_cnt, ele_idx_src ); - - err_dst = fd_vinyl_meta_query_fast( ele0, ele_max, key_dst, memo_dst, &_ele_idx_dst ); - ele_idx_dst = _ele_idx_dst; /* In [0,ele_max) */ - - FD_DCHECK_CRIT( err_dst==FD_VINYL_ERR_KEY, "corruption detected" ); - - ele0[ ele_idx_dst ].memo = memo_dst; - //ele0[ ele_idx_dst ].phdr.ctl = ... init below for concurrent safe insert - ele0[ ele_idx_dst ].phdr.key = phdr_src->key; - ele0[ ele_idx_dst ].phdr.info = phdr_src->info; - ele0[ ele_idx_dst ].line_idx = line_idx_src; - ele0[ ele_idx_dst ].seq = seq_dst; - - FD_COMPILER_MFENCE(); - ele0[ ele_idx_dst ].phdr.ctl = phdr_src->ctl; - FD_COMPILER_MFENCE(); - - line[ line_idx_src ].ele_idx = ele_idx_dst; - - fd_vinyl_io_append_move( io, phdr_src, key_dst, NULL, 0UL ); - append_cnt++; - accum_move_cnt++; - - fd_vinyl_bstream_pair_hash( io_seed, (fd_vinyl_bstream_block_t *)phdr_src ); - - ulong seq = fd_vinyl_io_append( io, phdr_src, pair_sz ); - append_cnt++; - FD_DCHECK_CRIT( fd_vinyl_seq_eq( seq, seq_dst ), "unexpected append location" ); - - DONE( FD_VINYL_SUCCESS ); - - next_move: /* silly language restriction */; - -# undef DONE - - } - - comp_err = FD_VINYL_SUCCESS; - break; - } diff --git a/src/vinyl/fd_vinyl_case_release.c b/src/vinyl/fd_vinyl_case_release.c deleted file mode 100644 index 6d1d187b3f3..00000000000 --- a/src/vinyl/fd_vinyl_case_release.c +++ /dev/null @@ -1,352 +0,0 @@ - case FD_VINYL_REQ_TYPE_RELEASE: { - - ulong req_flags = (ulong)req->flags; - fd_vinyl_key_t const * req_key = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t const, batch_cnt ); - ulong * req_val_gaddr = MAP_REQ_GADDR( req->val_gaddr_gaddr, ulong, batch_cnt ); - schar * req_err = MAP_REQ_GADDR( req->err_gaddr, schar, batch_cnt ); - - int req_flag_modify = fd_vinyl_req_flag_modify( req_flags ); - int req_flag_ignore = fd_vinyl_req_flag_ignore( req_flags ); - int req_flag_erase = fd_vinyl_req_flag_erase ( req_flags ); - int req_flag_by_key = fd_vinyl_req_flag_by_key( req_flags ); - int req_evict_prio = fd_vinyl_req_evict_prio ( req_flags ); - - if( FD_UNLIKELY( (!!batch_cnt) & ( ((!req_key ) & req_flag_by_key ) | - ((!req_val_gaddr) & (!req_flag_by_key)) | - ( !req_err ) ) ) ) { - comp_err = FD_VINYL_ERR_INVAL; - break; - } - - for( ulong batch_idx=0UL; batch_idxrd_active ) ) DONE( FD_VINYL_ERR_INVAL ); - - line_idx = obj->line_idx; - if( FD_UNLIKELY( line_idx>=line_cnt ) || FD_UNLIKELY( obj!=line[ line_idx ].obj ) ) DONE( FD_VINYL_ERR_INVAL ); - - ele_idx = line[ line_idx ].ele_idx; - if( FD_UNLIKELY( ele_idx>=ele_max ) || FD_UNLIKELY( ele0[ ele_idx ].line_idx!=line_idx ) ) DONE( FD_VINYL_ERR_INVAL ); - /* FIXME: MAKE SURE ELE0[ ELE_IDX ] IS IN USE FOR DATA INTEGRITY! */ - - ulong ctl = line[ line_idx ].ctl; - - ver = fd_vinyl_line_ctl_ver( ctl ); - ref = fd_vinyl_line_ctl_ref( ctl ); - - if( FD_UNLIKELY( !ref ) ) DONE( FD_VINYL_ERR_INVAL ); /* Pair key exists and is cached ... but not acquired */ - - } else { /* Release by key */ - - fd_vinyl_key_t const * key = req_key + batch_idx; - - ulong memo = fd_vinyl_key_memo( meta_seed, key ); /* This can be slow which is why releasing by val_gaddr is preferred */ - - ulong _ele_idx; /* avoid pointer escape */ - int err = fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &_ele_idx ); - ele_idx = _ele_idx; /* in [0,ele_max) */ - - if( FD_UNLIKELY( err ) ) DONE( FD_VINYL_ERR_INVAL ); /* Pair key does not exist ... can't have been acquired */ - - line_idx = ele0[ ele_idx ].line_idx; - - if( FD_UNLIKELY( line_idx>=line_cnt ) ) { /* Pair key exists but is not cached ... can't have been acquired */ - FD_DCHECK_CRIT( line_idx==ULONG_MAX, "corruption detected" ); - DONE( FD_VINYL_ERR_INVAL ); - } - - FD_DCHECK_CRIT( ele_idx==line[ line_idx ].ele_idx, "corruption detected" ); - - obj = line[ line_idx ].obj; - - FD_DCHECK_ALERT( fd_vinyl_data_is_valid_obj( obj, vol, vol_cnt ), "corruption detected" ); - FD_DCHECK_CRIT ( obj->line_idx==line_idx, "corruption detected" ); - FD_DCHECK_CRIT ( !obj->rd_active, "corruption detected" ); - - ulong ctl = line[ line_idx ].ctl; - - ver = fd_vinyl_line_ctl_ver( ctl ); - ref = fd_vinyl_line_ctl_ref( ctl ); - - if( FD_UNLIKELY( !ref ) ) DONE( FD_VINYL_ERR_INVAL ); /* Pair key exists and is cached ... but not acquired */ - - } - - /* At this point, we are releasing an acquire of the object obj, - cached at line line_idx with metadata at ele_idx. */ - - fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj ); - - if( FD_LIKELY( ref>0L ) ) { - - /* At this point, we are releasing an acquire for read. If - the client indicated they modified pair key, we don't have - data integrity anymore and we CRIT. Otherwise, we update - line eviction priority and ref count to do the release. */ - - if( FD_UNLIKELY( req_flag_modify ) ) FD_LOG_CRIT(( "client modified read only acquire" )); - - FD_DCHECK_CRIT( phdr->ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - (ulong)ele0[ ele_idx ].phdr.info.val_sz ), "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_key_eq( &phdr->key, &ele0[ ele_idx ].phdr.key ), "corruption detected" ); - FD_DCHECK_CRIT( !memcmp( &phdr->info, &ele0[ ele_idx ].phdr.info, sizeof(fd_vinyl_info_t) ), "corruption detected" ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver, ref-1L ); /* don't bump ver */ - - DONE( FD_VINYL_SUCCESS ); - } - - /* At this point, we are releasing an acquire for modify */ - - ulong phdr_ctl = phdr->ctl; - - int modifying_existing = (phdr_ctl!=ULONG_MAX); - - if( FD_LIKELY( req_flag_modify & (!req_flag_erase) ) ) { - - /* At this point, we are either finishing up modifying an - existing pair (modifying_existing 1) or finishing up creating - a new pair (modifying_existing 0). Cache the object in the - smallest size class that supports it. Note that the client - could have modified info so we only validate ctl and key - (FIXME: consider validating memo too?). */ - - FD_DCHECK_CRIT( (!modifying_existing) | - (phdr_ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - (ulong)ele0[ ele_idx ].phdr.info.val_sz )), "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_key_eq( &phdr->key, &ele0[ ele_idx ].phdr.key ), "corruption detected" ); - - ulong val_sz_after = (ulong)phdr->info.val_sz; - - if( FD_UNLIKELY( val_sz_after > fd_vinyl_data_obj_val_max( obj ) ) ) FD_LOG_CRIT(( "client overran memory" )); - - ulong szc_before = (ulong)obj->szc; - ulong szc_after = fd_vinyl_data_szc( val_sz_after ); - - if( FD_UNLIKELY( szc_before!=szc_after ) ) { - - FD_DCHECK_CRIT( szc_afterctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz_after ); - /*phdr->key already init */ - /*phdr->info already init */ - - int style_after; - ulong val_esz_after; - ulong seq_after = fd_vinyl_io_append_pair_inplace( io, vinyl->style, phdr, &style_after, &val_esz_after ); - append_cnt++; - - /* Update the line and meta to match. Note that setting meta - element ele_idx phdr.ctl to something other than ULONG_MAX - marks a pair that was being created as no longer being - created. For a pair that already existed, we also need to - update phdr.ctl to reflect that we might be storing this in - the stream in a different format than it was stored in - bstream before. Since we are changing shared fields of meta - element ele_idx, we need to use prepare / publish semantics. */ - - line[ line_idx ].obj = obj; obj->line_idx = line_idx; obj->rd_active = (short)0; - //line[ line_idx ].ele_idx ... already init - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1L, 0L ); /* bump ver */ - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - fd_vinyl_meta_prepare_fast( lock, lock_shift, ele_idx ); - - //ele0[ ele_idx ].memo = already init - ele0[ ele_idx ].phdr.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, style_after, val_esz_after ); - //ele0[ ele_idx ].phdr.key = already init - ele0[ ele_idx ].phdr.info = phdr->info; - ele0[ ele_idx ].seq = seq_after; - //ele0[ ele_idx ].line_idx = already init - - fd_vinyl_meta_publish_fast( lock, lock_shift, ele_idx ); - - DONE( FD_VINYL_SUCCESS ); - - } - - /* At this point, we are either canceling a modification (modify - 0, erase d/c) or the modification is to erase the pair (modify - 1, erase 1). If we are canceling the modification of an - existing pair and the client indicated the cached pair info and - cached pair val are still valid, (i.e. release-cancel of an - acquire-for-modify of an existing pair), we revert the line - state and adjust the line evict priority. (This code path can - be omitted if we don't trust the clients to report correctly. - We do test at least the client is correctly reporting the info - is not modified.) Note that we might have put this in a larged - sized obj when we acquired it for modify. So we also move the - object to the tightest location. */ - - if( FD_LIKELY( modifying_existing & (!req_flag_modify) & (!req_flag_ignore) ) ) { - - /* FIXME: consider allowing the client to always clobber the - pair info and just restore info from the meta cache? */ - - if( FD_UNLIKELY( !( (phdr->ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - (ulong)ele0[ ele_idx ].phdr.info.val_sz ) ) & - (fd_vinyl_key_eq( &phdr->key, &ele0[ ele_idx ].phdr.key ) ) & - (!memcmp( &phdr->info, &ele0[ ele_idx ].phdr.info, sizeof(fd_vinyl_info_t) )) ) ) ) - FD_LOG_CRIT(( "client clobbered pair info" )); - - ulong val_sz_before = (ulong)phdr->info.val_sz; - - ulong szc_after = (ulong)obj->szc; - ulong szc_before = fd_vinyl_data_szc( val_sz_before ); - - if( FD_UNLIKELY( szc_before!=szc_after ) ) { - - FD_DCHECK_CRIT( szc_beforeline_idx = line_idx; obj_before->rd_active = (short)0; - - } - - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver-1UL, 0L ); /* revert ver */ - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - DONE( FD_VINYL_SUCCESS ); - - } - - /* At this point, we are canceling a modification of an existing - pair that no longer has valid cached pair info or cached pair - val, erasing an existing pair, canceling the creation of a new - pair or erasing a pair in the process of being created (which - we treat the same as cancelling the creation). - - Since there was nothing cached originally (canceling / erasing - a pair being created), the cached data is no longer valid - (cancel with ignore of an existing pair) or the the cached data - is no longer needed (erase of an existing pair), we free the - data obj, mark the line as empty, move the line to LRU - position. */ - - /* FIXME: INTEGRITY CHECKS ON PHDR HERE? (TRICKY AS WE'D HAVE TO - MAP OUT EXACTLY WHICH FIELDS CAN BE TRUSTED AT THIS POINT AND - IT ISN'T OBVIOUS IT MATTERS) */ - - fd_vinyl_data_free( data, obj ); - - line[ line_idx ].obj = NULL; - line[ line_idx ].ele_idx = ULONG_MAX; ele0[ ele_idx ].line_idx = ULONG_MAX; - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, 0L ); /* bump ver */ - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, FD_VINYL_LINE_EVICT_PRIO_LRU ); - - /* If we are erasing an existing pair, append a dead block to - the bstream. This generates two pieces of bstream garbage (the - old pair and the dead block itself). Likewise, if we are - erasing an existing pair or cancelling / erasing a pair - creation, remove the element from the meta. Note that - req_flag_modify==1 implies req_flag_erase==1 but not vice versa - at this point. */ - - if( FD_LIKELY( req_flag_modify & modifying_existing ) ) { - - ulong val_esz_before = fd_vinyl_bstream_ctl_sz( ele0[ ele_idx ].phdr.ctl ); - - accum_garbage_cnt += 2UL; - accum_garbage_sz += fd_vinyl_bstream_pair_sz( val_esz_before ) + FD_VINYL_BSTREAM_BLOCK_SZ; - - fd_vinyl_io_append_dead( io, &ele0[ ele_idx ].phdr, NULL, 0UL ); - append_cnt++; - accum_dead_cnt++; - - } - - if( FD_LIKELY( req_flag_modify | (!modifying_existing) ) ) { - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, line, line_cnt, ele_idx ); - - ulong pair_cnt = vinyl->pair_cnt; - FD_DCHECK_CRIT( (0ULpair_cnt = pair_cnt - 1UL; - } - - DONE( FD_VINYL_SUCCESS ); - - next_release: /* silly language restriction */; - -# undef DONE - - } /* for batch_idx */ - - comp_err = FD_VINYL_SUCCESS; - break; - } diff --git a/src/vinyl/fd_vinyl_case_test.c b/src/vinyl/fd_vinyl_case_test.c deleted file mode 100644 index 3e03319aeea..00000000000 --- a/src/vinyl/fd_vinyl_case_test.c +++ /dev/null @@ -1,39 +0,0 @@ - case FD_VINYL_REQ_TYPE_TEST: { - - ulong const * req_val_gaddr = MAP_REQ_GADDR( req->val_gaddr_gaddr, ulong, 2UL*batch_cnt ); - schar * req_err = MAP_REQ_GADDR( req->err_gaddr, schar, batch_cnt ); - - if( FD_UNLIKELY( (!!batch_cnt) & ((!req_val_gaddr) | (!req_err)) ) ) { - comp_err = FD_VINYL_ERR_INVAL; - break; - } - - ulong const * req_try = req_val_gaddr + batch_cnt; - - for( ulong batch_idx=0UL; batch_idx> 32; - ulong line_idx = try & FD_VINYL_LINE_MAX; - - int err = FD_UNLIKELY( line_idx>=line_cnt ) ? FD_VINYL_ERR_INVAL - : FD_UNLIKELY( fd_vinyl_line_ctl_ver( line[ line_idx ].ctl )!=ver ) ? FD_VINYL_ERR_CORRUPT - : FD_VINYL_SUCCESS; - - FD_COMPILER_MFENCE(); - req_err[ batch_idx ] = (schar)err; - FD_COMPILER_MFENCE(); - - fail_cnt += (ulong)!!err; - - } - - comp_err = FD_VINYL_SUCCESS; - break; - } diff --git a/src/vinyl/fd_vinyl_case_try.c b/src/vinyl/fd_vinyl_case_try.c deleted file mode 100644 index db610799b4b..00000000000 --- a/src/vinyl/fd_vinyl_case_try.c +++ /dev/null @@ -1,180 +0,0 @@ - case FD_VINYL_REQ_TYPE_TRY: { - - FD_STATIC_ASSERT( FD_VINYL_LINE_VER_MAX==((1UL<<32)-1UL), update_impl_for_ver_max ); - - ulong req_flags = (ulong)req->flags; - fd_vinyl_key_t const * req_key = MAP_REQ_GADDR( req->key_gaddr, fd_vinyl_key_t, batch_cnt ); - ulong * req_val_gaddr = MAP_REQ_GADDR( req->val_gaddr_gaddr, ulong, 2UL*batch_cnt ); - schar * req_err = MAP_REQ_GADDR( req->err_gaddr, schar, batch_cnt ); - - int req_evict_prio = fd_vinyl_req_evict_prio( req_flags ); - - if( FD_UNLIKELY( (!!batch_cnt) & ((!req_key) | (!req_val_gaddr) | (!req_err)) ) ) { - comp_err = FD_VINYL_ERR_INVAL; - break; - } - - ulong * req_try = req_val_gaddr + batch_cnt; - - for( ulong batch_idx=0UL; batch_idxline_idx==line_idx, "corruption detected" ); - - ulong line_ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( line_ctl ); - long ref = fd_vinyl_line_ctl_ref( line_ctl ); - - if( FD_UNLIKELY( ref<0L ) ) DONE( FD_VINYL_ERR_AGAIN, ULONG_MAX ); - - if( FD_LIKELY( !obj->rd_active ) ) { - fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj ); - - FD_DCHECK_CRIT( fd_vinyl_data_obj_val_max( obj ) >= val_sz, "corruption detected" ); - FD_DCHECK_CRIT( phdr->ctl==fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ), "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_key_eq( &phdr->key, key ), "corruption detected" ); - FD_DCHECK_CRIT( !memcmp( &phdr->info, &ele0[ ele_idx ].phdr.info, sizeof(fd_vinyl_info_t) ), "corruption detected" ); - } - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - req_val_gaddr[ batch_idx ] = (ulong)fd_vinyl_data_obj_val( obj ) - data_laddr0; - - DONE( FD_VINYL_SUCCESS, (ver<<32) | line_idx ); - - } - - /* At this point, pair key exists but is not cached. Evict the - least recently used evictable line to make room to cache this - pair. Connect this line to meta element ele_idx, set the - line's reference count to zero, bump the line's version and set - the eviction priority as desired. We don't modify any shared - fields in meta element ele_idx so we can do the modification - fast. - - We do this upfront to free data cache for the alloc if the LRU - line is in use and to handle the same pair appearing multiple - times in an acquire. - - The mechanics for try requests with redundant keys are the same - as acquire-for-read requests. */ - - line_idx = fd_vinyl_line_evict_lru( &vinyl->line_idx_lru, line, line_cnt, ele0, ele_max, data ); - - ulong line_ctl = line[ line_idx ].ctl; - - ulong ver = fd_vinyl_line_ctl_ver( line_ctl ); - - line[ line_idx ].ele_idx = ele_idx; ele0[ ele_idx ].line_idx = line_idx; - line[ line_idx ].ctl = fd_vinyl_line_ctl( ver+1UL, 0L ); - - fd_vinyl_line_evict_prio( &vinyl->line_idx_lru, line, line_cnt, line_idx, req_evict_prio ); - - /* Allocate an appropriately sized object to hold this pair, - connect it to this line and report the location to the client. */ - - ulong szc = fd_vinyl_data_szc( val_sz ); - - fd_vinyl_data_obj_t * obj = fd_vinyl_data_alloc( data, szc ); - if( FD_UNLIKELY( !obj ) ) FD_LOG_CRIT(( "increase data cache size" )); - - line[ line_idx ].obj = obj; obj->line_idx = line_idx; - - void * val = fd_vinyl_data_obj_val( obj ); - - req_val_gaddr[ batch_idx ] = (ulong)val - data_laddr0; - req_try [ batch_idx ] = ((ver+1UL)<<32) | line_idx; - - /* Start reading encoded pair data and defer validation and - decoding to later (and then in whatever order the I/O layer - sees fit). */ - - obj->rd_active = (short)1; - - int style = fd_vinyl_bstream_ctl_style( pair_ctl ); - ulong val_esz = fd_vinyl_bstream_ctl_sz ( pair_ctl ); - - FD_DCHECK_CRIT( val_esz<=FD_VINYL_VAL_MAX, "corruption detected" ); - FD_DCHECK_CRIT( (style!=FD_VINYL_BSTREAM_CTL_STYLE_RAW) | (val_sz==val_esz), "corruption detected" ); - - fd_vinyl_data_obj_t * cobj; - - if( FD_LIKELY( style==FD_VINYL_BSTREAM_CTL_STYLE_RAW ) ) cobj = obj; - else { - cobj = fd_vinyl_data_alloc( data, fd_vinyl_data_szc( val_esz ) ); - if( FD_UNLIKELY( !cobj ) ) FD_LOG_CRIT(( "increase data cache size" )); - } - - cobj->rd->ctx = (ulong)obj; - cobj->rd->seq = ele0[ ele_idx ].seq; - cobj->rd->dst = fd_vinyl_data_obj_phdr( cobj ); - cobj->rd->sz = fd_vinyl_bstream_pair_sz( val_esz ); - - cobj->rd_err = req_err + batch_idx; - - fd_vinyl_io_read( io, cobj->rd ); - read_cnt++; - - next_try: /* silly language restriction */; - -# undef DONE - - } /* for batch_idx */ - - comp_err = FD_VINYL_SUCCESS; - break; - } diff --git a/src/vinyl/fd_vinyl_cmd.c b/src/vinyl/fd_vinyl_cmd.c deleted file mode 100644 index 5b72fc5dbf7..00000000000 --- a/src/vinyl/fd_vinyl_cmd.c +++ /dev/null @@ -1,147 +0,0 @@ -#include "fd_vinyl.h" - -/* FIXME: HAVE ERR_TIMEOUT? */ - -int -fd_vinyl_halt( fd_cnc_t * cnc ) { - - int err = fd_cnc_open( cnc ); /* logs details */ - if( FD_UNLIKELY( err ) ) return err==FD_CNC_ERR_AGAIN ? FD_VINYL_ERR_AGAIN : FD_VINYL_ERR_CORRUPT; - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_HALT ); - - err = FD_LIKELY( fd_cnc_wait( cnc, FD_VINYL_CNC_SIGNAL_HALT, (long)5e9, NULL )==FD_VINYL_CNC_SIGNAL_BOOT ) - ? FD_VINYL_SUCCESS : FD_VINYL_ERR_CORRUPT; - - fd_cnc_close( cnc ); - - return err; -} - -int -fd_vinyl_sync( fd_cnc_t * cnc ) { - - int err = fd_cnc_open( cnc ); /* logs details */ - if( FD_UNLIKELY( err ) ) return err==FD_CNC_ERR_AGAIN ? FD_VINYL_ERR_AGAIN : FD_VINYL_ERR_CORRUPT; - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_SYNC ); - - err = FD_LIKELY( fd_cnc_wait( cnc, FD_VINYL_CNC_SIGNAL_SYNC, (long)5e9, NULL )==FD_VINYL_CNC_SIGNAL_RUN ) - ? FD_VINYL_SUCCESS : FD_VINYL_ERR_CORRUPT; - - fd_cnc_close( cnc ); - - return err; -} - -int -fd_vinyl_get( fd_cnc_t * cnc, - int opt, - ulong * opt_val ) { - - int err = fd_cnc_open( cnc ); /* logs details */ - if( FD_UNLIKELY( err ) ) return err==FD_CNC_ERR_AGAIN ? FD_VINYL_ERR_AGAIN : FD_VINYL_ERR_CORRUPT; - - fd_vinyl_cmd_t * cmd = (fd_vinyl_cmd_t *)fd_cnc_app_laddr( cnc ); - - cmd->get.opt = opt; - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_GET ); - - err = FD_LIKELY( fd_cnc_wait( cnc, FD_VINYL_CNC_SIGNAL_GET, (long)5e9, NULL )==FD_VINYL_CNC_SIGNAL_RUN ) - ? cmd->get.err : FD_VINYL_ERR_CORRUPT; - - fd_cnc_close( cnc ); - - if( opt_val ) *opt_val = cmd->get.val; - return err; -} - -int -fd_vinyl_set( fd_cnc_t * cnc, - int opt, - ulong val, - ulong * opt_val ) { - - int err = fd_cnc_open( cnc ); /* logs details */ - if( FD_UNLIKELY( err ) ) return err==FD_CNC_ERR_AGAIN ? FD_VINYL_ERR_AGAIN : FD_VINYL_ERR_CORRUPT; - - fd_vinyl_cmd_t * cmd = (fd_vinyl_cmd_t *)fd_cnc_app_laddr( cnc ); - - cmd->set.opt = opt; - cmd->set.val = val; - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_SET ); - - err = FD_LIKELY( fd_cnc_wait( cnc, FD_VINYL_CNC_SIGNAL_SET, (long)5e9, NULL )==FD_VINYL_CNC_SIGNAL_RUN ) - ? cmd->set.err : FD_VINYL_ERR_CORRUPT; - - fd_cnc_close( cnc ); - - if( opt_val ) *opt_val = cmd->set.val; - return err; -} - -int -fd_vinyl_client_join( fd_cnc_t * cnc, - fd_vinyl_rq_t * rq, - fd_vinyl_cq_t * cq, - fd_wksp_t * wksp, - ulong link_id, - ulong burst_max, - ulong quota_max ) { - - int err = fd_cnc_open( cnc ); /* logs details */ - if( FD_UNLIKELY( err ) ) return err==FD_CNC_ERR_AGAIN ? FD_VINYL_ERR_AGAIN : FD_VINYL_ERR_CORRUPT; - - /* At this point, we have a command session on cnc and a superficially - valid request. Issue the command and wait for the response. */ - - fd_vinyl_cmd_t * cmd = (fd_vinyl_cmd_t *)fd_cnc_app_laddr( cnc ); - - cmd->join.link_id = link_id; - cmd->join.burst_max = burst_max; - cmd->join.quota_max = quota_max; - - if( FD_UNLIKELY( !fd_wksp_cstr_laddr( rq, cmd->join.rq ) ) || - FD_UNLIKELY( !fd_wksp_cstr_laddr( cq, cmd->join.cq ) ) || - FD_UNLIKELY( !wksp ) ) { - fd_cnc_close( cnc ); - return FD_VINYL_ERR_INVAL; - } - - strcpy( cmd->join.wksp, fd_wksp_name( wksp ) ); - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_CLIENT_JOIN ); - - err = FD_LIKELY( fd_cnc_wait( cnc, FD_VINYL_CNC_SIGNAL_CLIENT_JOIN, (long)5e9, NULL )==FD_VINYL_CNC_SIGNAL_RUN ) - ? cmd->join.err : FD_VINYL_ERR_CORRUPT; - - fd_cnc_close( cnc ); - - return err; -} - -int -fd_vinyl_client_leave( fd_cnc_t * cnc, - ulong link_id ) { - - int err = fd_cnc_open( cnc ); /* logs details */ - if( FD_UNLIKELY( err ) ) return err==FD_CNC_ERR_AGAIN ? FD_VINYL_ERR_AGAIN : FD_VINYL_ERR_CORRUPT; - - /* At this point, we have a command session on cnc and a superficially - valid request. Issue the command and wait for the response. */ - - fd_vinyl_cmd_t * cmd = (fd_vinyl_cmd_t *)fd_cnc_app_laddr( cnc ); - - cmd->leave.link_id = link_id; - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_CLIENT_LEAVE ); - - err = FD_LIKELY( fd_cnc_wait( cnc, FD_VINYL_CNC_SIGNAL_CLIENT_LEAVE, (long)5e9, NULL )==FD_VINYL_CNC_SIGNAL_RUN ) - ? cmd->leave.err : FD_VINYL_ERR_CORRUPT; - - fd_cnc_close( cnc ); - - return err; -} diff --git a/src/vinyl/fd_vinyl_compact.c b/src/vinyl/fd_vinyl_compact.c deleted file mode 100644 index e2b1068c174..00000000000 --- a/src/vinyl/fd_vinyl_compact.c +++ /dev/null @@ -1,377 +0,0 @@ -#include -#include "fd_vinyl.h" - -void -fd_vinyl_compact( fd_vinyl_t * vinyl, - ulong compact_max ) { - - fd_vinyl_io_t * io = vinyl->io; - ulong gc_thresh = vinyl->gc_thresh; - int gc_eager = vinyl->gc_eager; - int style = vinyl->style; - - ulong io_seed = fd_vinyl_io_seed ( io ); (void)io_seed; - ulong seq_past = fd_vinyl_io_seq_past ( io ); - ulong seq_present = fd_vinyl_io_seq_present( io ); - - if( FD_UNLIKELY( (!compact_max) | ((seq_present-seq_past)<=gc_thresh) | (gc_eager<0) ) ) return; - - fd_vinyl_meta_t * meta = vinyl->meta; - fd_vinyl_line_t * line = vinyl->line; - ulong line_cnt = vinyl->line_cnt; - ulong garbage_sz = vinyl->garbage_sz; - - fd_vinyl_meta_ele_t * ele0 = meta->ele; - ulong ele_max = meta->ele_max; - ulong meta_seed = meta->seed; - - fd_vinyl_data_t * data = vinyl->data; - - fd_vinyl_data_vol_t * vol = data->vol; (void)vol; - ulong vol_cnt = data->vol_cnt; (void)vol_cnt; - - ulong seq = seq_past; - - for( ulong rem=compact_max; rem; rem-- ) { - - /* At this point, we've compacted [seq_past,seq) (cyclic), with - items still needed in this range at [seq_present,seq_future) - (cyclic). We still have [seq,seq_present) (cyclic), containing - garbage_sz bytes to compact. - - If the new past region is small enough or there is a relatively - small amount of garbage in this region, we consider the bstream's - past fully compacted. */ - - ulong past_sz_new = fd_vinyl_io_seq_future( io ) - seq; - if( FD_UNLIKELY( (past_sz_new <= gc_thresh ) | - (garbage_sz <= (past_sz_new >> gc_eager)) | - (fd_vinyl_seq_ge( seq, seq_present ) ) ) ) { - FD_DCHECK_CRIT( fd_vinyl_seq_le( seq, seq_present ), "corruption detected" ); - if( FD_UNLIKELY( fd_vinyl_seq_eq( seq, seq_present ) ) ) FD_DCHECK_CRIT( !garbage_sz, "corruption detected" ); - break; - } - - /* At this point, there is enough garbage to do some more - compaction. Load the leading block of the object at seq and - determine if this object is needed to recover the bstream's state - at seq_present. - - That is, we determine if the object at bstream_past_new is the - version of a pair that exists at bstream seq_present. If so, we - append a copy to the bstream's present. - - When compacting is complete, we forget the region containing the - copy at seq. This then effectively moves the copy from seq to - seq_future without any risk of losing data while allowing - compaction to be done with large amounts of async I/O overlapped - with compaction processing (metadata lookups, hash validation, - etc). - - This move will not move the pair past any conflicting operations - later in the bstream's past (almost definitionally so as the pair - is the most recent version). Thus set of pairs recovered at - seq_future will be identical to the set of pairs recovered at - seq_present. */ - - fd_vinyl_bstream_block_t block[1]; - - fd_vinyl_io_read_imm( io, seq, block, FD_VINYL_BSTREAM_BLOCK_SZ ); - - ulong ctl = block->ctl; - - int type = fd_vinyl_bstream_ctl_type( ctl ); - - switch( type ) { - - case FD_VINYL_BSTREAM_CTL_TYPE_PAIR: { - - /* At this point, we've read a pair's leading block into block. - Validate the pair was completely written. It's okay if we are - in a move (move block processing the previous iteration already - confirmed this pair is the proper). */ - - int pair_style = fd_vinyl_bstream_ctl_style( ctl ); - ulong pair_val_esz = fd_vinyl_bstream_ctl_sz ( ctl ); - fd_vinyl_key_t const * pair_key = &block->phdr.key; - ulong pair_val_sz = (ulong)block->phdr.info.val_sz; - - ulong pair_sz = fd_vinyl_bstream_pair_sz( pair_val_esz ); - - int truncated = (pair_sz > (seq_present - seq)); /* Wrapping safe */ - int bad_esz = (pair_val_esz > FD_VINYL_VAL_MAX); - int bad_sz = (pair_val_sz > FD_VINYL_VAL_MAX); - - FD_DCHECK_CRIT( !(truncated | bad_esz | bad_sz), truncated ? "truncated pair" : - bad_esz ? "unexpected pair value encoded size" : - "pair value size too large" ); - -# if FD_PARANOID - fd_vinyl_bstream_block_t _ftr[1]; - fd_vinyl_bstream_block_t * ftr = _ftr; - - if( FD_UNLIKELY( pair_sz <= FD_VINYL_BSTREAM_BLOCK_SZ ) ) ftr = block; - else fd_vinyl_io_read_imm( io, seq + pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ, ftr, FD_VINYL_BSTREAM_BLOCK_SZ ); - - FD_DCHECK_ALERT( !fd_vinyl_bstream_pair_test_fast( io_seed, seq, block, ftr ), "corruption detected" ); -# endif - - /* At this point, we appear to have a valid pair. Query the - vinyl's meta to determine if this is the version of the pair at - bstream seq_present. Since this implementation is doing single - threaded recovery, we can use the single threaded optimized - meta APIs. */ - - ulong pair_memo = fd_vinyl_key_memo( meta_seed, pair_key ); - - ulong _ele_idx; /* avoid pointer escape */ - int err = fd_vinyl_meta_query_fast( ele0, ele_max, pair_key, pair_memo, &_ele_idx ); - ulong ele_idx = _ele_idx; - - if( FD_LIKELY( !err ) ) { - - /* At this point, a version of pair key is mapped */ - - if( FD_LIKELY( fd_vinyl_meta_ele_in_bstream( &ele0[ ele_idx ] ) ) ) { - - /* At this point, a version of pair key exists at bstream - seq_present (i.e. is not in the process of being created by - a client). */ - - ulong pair_seq = ele0[ ele_idx ].seq; - - if( FD_LIKELY( fd_vinyl_seq_eq( pair_seq, seq ) ) ) { - - /* At this point, the version of pair key at seq is the - version of pair key that exists at bstream seq_present. - Validate the metadata. */ - - FD_DCHECK_CRIT( !memcmp( &ele0[ ele_idx ].phdr, &block->phdr, sizeof(fd_vinyl_bstream_phdr_t) ), "corruption detected" ); - - /* If the pair is cached and not acquired for modify, append - the cached copy in the target style. Otherwise, append a - (possibly recoded) copy from the bstream. */ - - int pair_style_new; - ulong pair_val_esz_new; - ulong pair_seq_new; - - int do_copy = 1; - - ulong line_idx = ele0[ ele_idx ].line_idx; - - if( FD_LIKELY( line_idx!=ULONG_MAX ) ) { /* Pair is in cache */ - - FD_DCHECK_CRIT( line_idxline_idx==line_idx, "corruption detected" ); - FD_DCHECK_CRIT ( !obj->rd_active, "corruption detected" ); - - ulong line_ctl = line[ line_idx ].ctl; - - if( FD_LIKELY( fd_vinyl_line_ctl_ref( line_ctl )>=0L ) ) { /* Pair cached and not acquired for modify */ - - fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj ); - - FD_DCHECK_ALERT( !memcmp( phdr, &block->phdr, sizeof(fd_vinyl_bstream_phdr_t) ), "corruption detected" ); - - pair_seq_new = fd_vinyl_io_append_pair_inplace( io, style, phdr, &pair_style_new, &pair_val_esz_new ); - - do_copy = 0; - - } - - } - - if( do_copy ) { /* Pair is either not in cache or acquired for modify, append from the bstream */ - - if( FD_LIKELY( (pair_style!=FD_VINYL_BSTREAM_CTL_STYLE_RAW) | - (style ==FD_VINYL_BSTREAM_CTL_STYLE_RAW) | - (pair_sz ==FD_VINYL_BSTREAM_BLOCK_SZ ) ) ) { - - /* At this point, the pair is already stored in an - encoded format, the preferred format for storing - encoded pairs is raw and/or encoding the pair will - not make it any smaller in the bstream. Copy the - pair as is from seq to seq_future. The reason we - don't re-encode the pair in the second case is that - this pair has likely not been touched since it last - got to the bstream's seq_past. It would be waste to - compute and bstream storage to uncompress it as we - copy it. */ - - pair_style_new = pair_style; - pair_val_esz_new = fd_vinyl_bstream_ctl_sz( ele0[ ele_idx ].phdr.ctl ); - pair_seq_new = fd_vinyl_io_copy( io, pair_seq, pair_sz ); - - } else { - - /* At this point, the pair is stored in a raw encoded - format, the preferred format is an encoded format and - there is a possibility that encoding it will make it - smaller. Encode the pair as we copy it from seq to - seq_future. - - To do this, we allocate enough scratch from the io - append spad to cover the worst case encoded pair and - the raw pair (this sets the lower bound on how large - the io append spad must be). Then we read the raw - pair into the trailing part of the scratch and encode - from that into the leading part of the scratch. - - We play some games with the spad_used so that the - append_pair_inplace will not invalidate the read and - so that we use scratch as efficiently as possible - when there is lots of stuff to compress. */ - - ulong cpair_max = fd_vinyl_bstream_pair_sz( (ulong)LZ4_COMPRESSBOUND( (int)pair_val_sz ) ); - ulong scratch_max = cpair_max + pair_sz; - - fd_vinyl_bstream_phdr_t * cphdr = (fd_vinyl_bstream_phdr_t *) - fd_vinyl_io_alloc( io, scratch_max, FD_VINYL_IO_FLAG_BLOCKING ); - - fd_vinyl_bstream_phdr_t * phdr = (fd_vinyl_bstream_phdr_t *)((ulong)cphdr + cpair_max); - - fd_vinyl_io_read_imm( io, seq, phdr, pair_sz ); - - fd_vinyl_io_trim( io, scratch_max ); - - pair_seq_new = fd_vinyl_io_append_pair_inplace( io, style, phdr, &pair_style_new, &pair_val_esz_new ); - - /* At this point, we either are appending the encoded - pair from the leading part of the scratch and - spad_used is correct or we are appending the pair - from the trailing part and spad_used does not include - it. Adjust the spad used for the later case. In - this second case, we end up with a temporary hole in - the scratch when we decided not to copy into an - encoded form. This just scratch is used less - efficiently in the unlikely case in order to use it - more efficiently in the likely case (the correct - tradeoff). */ - - if( FD_UNLIKELY( pair_style_new==FD_VINYL_BSTREAM_CTL_STYLE_RAW ) ) io->spad_used += scratch_max; - - } - } - - /* Note: we don't need to prepare here because we aren't - modifying shared fields. */ - - ele0[ ele_idx ].phdr.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, pair_style_new, pair_val_esz_new ); - ele0[ ele_idx ].seq = pair_seq_new; - - } else { - - /* The version of the pair at bstream seq was replaced. The - most recent version of this pair is at pair_seq. */ - - FD_DCHECK_CRIT( fd_vinyl_seq_gt( pair_seq, seq ), "corruption detected" ); - - garbage_sz -= pair_sz; - - } - - } else { - - /* The pair at bstream seq does not exist in the bstream at - bstream seq_present. It is in the vinyl meta because it is - being created. We wouldn't be in the process of creating - it unless this pair (or a subsequent version of it) was - erased or moved before seq_present. So this pair is - garbage. */ - - garbage_sz -= pair_sz; - - } - - } else { - - /* The pair at bstream seq does not exist in the bstream at - bstream seq_present. This pair (or a subsequent version of - it) was erased or moved before seq_present. So this pair - is garbage. */ - - garbage_sz -= pair_sz; - - } - - seq += pair_sz; - break; - - } - - case FD_VINYL_BSTREAM_CTL_TYPE_DEAD: - case FD_VINYL_BSTREAM_CTL_TYPE_MOVE: - case FD_VINYL_BSTREAM_CTL_TYPE_PART: { - - /* DEAD blocks can always be compacted out because the version of - the pair they reference is not in the current view of the - bstream (because that version was unmapped when the DEAD was - written), that version was located at an earlier location than - the DEAD (because blocks are appended sequentially) and thus - that version has already been compacted out (because a previous - iteration of this would have encountered it before getting this - DEAD block, would have detecting that version was no longer - needed and compacted it at that time instead of moving it to a - higher sequence number). - - MOVE blocks can always be compacted out for the same reasons as - the above with the twist that, compacting the move block makes - the pair following look like a create from the point of view of - a recovery starting at the pair. This is immaterial though - because doesn't change the recovered view if recovery starts - on the block after the move. - - PART blocks can always be compacted because they are just - informational (to help partition the bstream past in parallel - recovery) and this partition ends bstream blocks that have - already been compacted out. - - We validate the block because we already have the data anyway. */ - - FD_DCHECK_ALERT( !fd_vinyl_bstream_block_test( io_seed, block ), "corruption detected" ); - - garbage_sz -= FD_VINYL_BSTREAM_BLOCK_SZ; - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - - } - - case FD_VINYL_BSTREAM_CTL_TYPE_ZPAD: { - - /* ZPAD blocks can always be compacted out because they are no-ops - from the point of view of bstream processing (the underlying - I/O layer can insert these so that, for example, a multi-block - pair is never split across two different physical volumes). - Note that zpad blocks aren't included in garbage_sz because we - don't control when they get created (and thus can't easily - update garbage_sz to account for them when they are created). */ - - FD_DCHECK_ALERT( !fd_vinyl_bstream_zpad_test( io_seed, seq, block ), "corruption detected" ); - - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - - } - - default: FD_LOG_CRIT(( "%016lx: unknown type (%x)", seq, (uint)type )); - - } - - } - - /* At this point, we've made copies of all info in [seq_past,seq) - (cyclic) to [seq_present,seq_future) (cyclic) needed to recover the - bstream's state at seq_present. We commit the new, forget the old - and update the garbage size to finish this compaction. */ - - fd_vinyl_io_commit( io, FD_VINYL_IO_FLAG_BLOCKING ); - fd_vinyl_io_forget( io, seq ); - - vinyl->garbage_sz = garbage_sz; -} diff --git a/src/vinyl/fd_vinyl_ctl.c b/src/vinyl/fd_vinyl_ctl.c deleted file mode 100644 index 3a5ade25bf8..00000000000 --- a/src/vinyl/fd_vinyl_ctl.c +++ /dev/null @@ -1,712 +0,0 @@ -/* For O_DIRECT and O_NOATIME */ -#define _GNU_SOURCE - -#include "fd_vinyl.h" -#include "../util/pod/fd_pod.h" - -#include -#include -#include -#include -#include - -FD_IMPORT_CSTR( fd_vinyl_ctl_help, "src/vinyl/fd_vinyl_ctl_help" ); - -static int -fd_vinyl_main( int argc, - char ** argv ) { - - ulong seed_default = fd_cstr_hash_append( (ulong)fd_log_wallclock(), fd_log_host() ); - - char const * _pod = fd_env_strip_cmdline_cstr ( &argc, &argv, "--pod", NULL, NULL ); - char const * _cfg = fd_env_strip_cmdline_cstr ( &argc, &argv, "--cfg", NULL, NULL ); - ulong seed = fd_env_strip_cmdline_ulong( &argc, &argv, "--seed", NULL, seed_default ); - char const * type = fd_env_strip_cmdline_cstr ( &argc, &argv, "--type", NULL, "mm" ); - char const * path = fd_env_strip_cmdline_cstr ( &argc, &argv, "--path", NULL, NULL ); - int dsync = fd_env_strip_cmdline_int ( &argc, &argv, "--dsync", NULL, 0 ); - int direct = fd_env_strip_cmdline_int ( &argc, &argv, "--direct", NULL, 0 ); - int noatime = fd_env_strip_cmdline_int ( &argc, &argv, "--noatime", NULL, 0 ); - char const * _page_sz = fd_env_strip_cmdline_cstr ( &argc, &argv, "--page-sz", NULL, "gigantic" ); - ulong page_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--page-cnt", NULL, 1UL ); - ulong near_cpu = fd_env_strip_cmdline_ulong( &argc, &argv, "--near-cpu", NULL, fd_log_cpu_id() ); - int reset = fd_env_strip_cmdline_int ( &argc, &argv, "--reset", NULL, 0 ); - char const * info = fd_env_strip_cmdline_cstr ( &argc, &argv, "--info", NULL, NULL ); - ulong io_seed = fd_env_strip_cmdline_ulong( &argc, &argv, "--io-seed", NULL, 0UL ); - - int open_flags = O_RDWR | (dsync ? O_DSYNC : 0 ) | (direct ? O_DIRECT : 0) | (noatime ? O_NOATIME : 0); - ulong page_sz = fd_cstr_to_shmem_page_sz( _page_sz ); - ulong info_sz = info ? (strlen( info )+1UL) : 0UL; - - if( FD_UNLIKELY( !_pod ) ) FD_LOG_ERR(( "--pod not specified" )); - if( FD_UNLIKELY( !page_sz ) ) FD_LOG_ERR(( "bad --page-sz" )); - - FD_LOG_NOTICE(( "Attaching to --pod %s", _pod )); - - uchar const * pod = fd_wksp_pod_attach( _pod ); /* logs details, guaranteed to succeed */ - uchar const * cfg; - if( FD_UNLIKELY( !_cfg ) ) { - FD_LOG_NOTICE(( "--cfg not specified (using pod root for config)" )); - cfg = pod; - } else { - FD_LOG_NOTICE(( "Finding config --cfg %s", _cfg )); - cfg = fd_pod_query_subpod( pod, _cfg ); - if( FD_UNLIKELY( !cfg ) ) FD_LOG_ERR(( "config not found" )); - } - - FD_LOG_NOTICE(( "Extracting pod configuration" )); - - /* See below for explanation of defaults */ - ulong spad_max = fd_pod_query_ulong( cfg, "spad_max", fd_vinyl_io_spad_est() ); - ulong async_min = fd_pod_query_ulong( cfg, "async_min", 2UL ); - ulong async_max = fd_pod_query_ulong( cfg, "async_max", 2UL*async_min ); - ulong part_thresh = fd_pod_query_ulong( cfg, "part_thresh", 1UL<<30 ); - ulong gc_thresh = fd_pod_query_ulong( cfg, "gc_thresh", 8UL<<30 ); - int gc_eager = fd_pod_query_int ( cfg, "gc_eager", 2 ); - int style = fd_pod_query_int ( cfg, "style", FD_VINYL_BSTREAM_CTL_STYLE_LZ4 ); - int level = fd_pod_query_int ( cfg, "level", 1 ); - - FD_LOG_NOTICE(( "Processing command line configuration overrides" )); - - char const * _style = fd_env_strip_cmdline_cstr( &argc, &argv, "--style", NULL, NULL ); - if( _style ) style = fd_cstr_to_vinyl_bstream_ctl_style( _style ); - - spad_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--spad-max", NULL, spad_max ); - async_min = fd_env_strip_cmdline_ulong( &argc, &argv, "--async-min", NULL, async_min ); - async_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--async-max", NULL, async_max ); - part_thresh = fd_env_strip_cmdline_ulong( &argc, &argv, "--part-thresh", NULL, part_thresh ); - gc_thresh = fd_env_strip_cmdline_ulong( &argc, &argv, "--gc-thresh", NULL, gc_thresh ); - gc_eager = fd_env_strip_cmdline_int ( &argc, &argv, "--gc-eager", NULL, gc_eager ); - level = fd_env_strip_cmdline_int ( &argc, &argv, "--level", NULL, level ); - - FD_LOG_NOTICE(( "Mapping vinyl memory regions" )); - - void * _vinyl = fd_wksp_pod_map( cfg, "vinyl" ); ulong vinyl_footprint = fd_pod_query_ulong( cfg, "vinyl_footprint", 0UL ); - void * _cnc = fd_wksp_pod_map( cfg, "cnc" ); ulong cnc_footprint = fd_pod_query_ulong( cfg, "cnc_footprint", 0UL ); - void * _meta = fd_wksp_pod_map( cfg, "meta" ); ulong meta_footprint = fd_pod_query_ulong( cfg, "meta_footprint", 0UL ); - void * _line = fd_wksp_pod_map( cfg, "line" ); ulong line_footprint = fd_pod_query_ulong( cfg, "line_footprint", 0UL ); - void * _io = fd_wksp_pod_map( cfg, "io" ); ulong io_footprint = fd_pod_query_ulong( cfg, "io_footprint", 0UL ); - void * _ele = fd_wksp_pod_map( cfg, "ele" ); ulong ele_footprint = fd_pod_query_ulong( cfg, "ele_footprint", 0UL ); - void * _obj = fd_wksp_pod_map( cfg, "obj" ); ulong obj_footprint = fd_pod_query_ulong( cfg, "obj_footprint", 0UL ); - -# define TEST( c, msg ) do { \ - if( FD_UNLIKELY( !(c) ) ) FD_LOG_ERR(( "FAIL: %s (%s)", #c, (msg) )); \ - } while(0) - - fd_wksp_t * wksp = fd_wksp_containing( _obj ); - TEST( wksp, "fd_wksp_containing failed" ); - - TEST( fd_ulong_is_aligned( (ulong)_vinyl, fd_vinyl_io_mm_align() ), "bad alloc" ); - TEST( vinyl_footprint >= fd_vinyl_footprint(), "bad alloc" ); - - int is_mmio = !strcmp( type, "mm" ); - - FD_LOG_NOTICE(( "io config" - "\n\t--type \"%s\"" - "\n\t--spad-max %lu bytes" - "\n\t--path \"%s\"" - "\n\t--dsync %i" - "\n\t--direct %i" - "\n\t--noatime %i" - "\n\t--page-sz \"%s\"%s" - "\n\t--page-cnt %lu pages%s" - "\n\t--near-cpu %lu%s" - "\n\t--reset %i" - "\n\t--info \"%s\" (info_sz %lu bytes%s)" - "\n\t--io-seed 0x%016lx%s", - type, spad_max, path ? path : "(null)", dsync, direct, noatime, - _page_sz, is_mmio && !path ? "" : " (ignored)", - page_cnt, is_mmio && !path ? "" : " (ignored)", - near_cpu, is_mmio && !path ? "" : " (ignored)", - reset, info ? info : "(null)", info_sz, reset ? "" : ", ignored", io_seed, reset ? "" : " (ignored)" )); - - FD_LOG_NOTICE(( "Joining bstream" )); - - int bstream_type; - int fd = -1; - void * mmio; - ulong mmio_sz; - - fd_vinyl_io_t * io; - - if( FD_LIKELY( is_mmio ) ) { - - if( FD_LIKELY( path ) ) { - - fd = open( path, open_flags, (mode_t)0 ); - - if( FD_LIKELY( fd!=-1 ) ) { /* --path seems to be file (e.g. testing or basic I/O with weak persistence) */ - - TEST( !direct, "--direct 1 not supported with --type mm and file --path" ); - /* FIXME: is dsync valid for mmio? (unclear) noatime? (probably) */ - - FD_LOG_NOTICE(( "Using file at --path as a memory mapped bstream" )); - - bstream_type = 0; - - int err = fd_io_mmio_init( fd, FD_IO_MMIO_MODE_READ_WRITE, &mmio, &mmio_sz ); - if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "fd_io_mmio_init failed (%i-%s)", err, fd_io_strerror( err ) )); - - } else { /* --path doesn't seem to be a file, use shmem (e.g. testing or ultra HPC with weak persistence) */ - - FD_LOG_NOTICE(( "Using shmem region at --path as a memory mapped bstream (ignoring --dsync, --direct and --noatime)" )); - - bstream_type = 1; - - fd_shmem_join_info_t info[1]; - mmio = fd_shmem_join( path, FD_SHMEM_JOIN_MODE_READ_WRITE, 0, NULL, NULL, info ); - TEST( mmio, "fd_shmem_join failed" ); - mmio_sz = info->page_sz * info->page_cnt; - - } - - } else { /* No --path, use an anonymous region (e.g. testing or ultra HPC with no persistence) */ - - FD_LOG_NOTICE(( "Using an anonymous shmem region as a memory mapped bstream " - "(ignoring --dsync, --direct and --noatime, setting --reset to 1)" )); - - bstream_type = 2; - reset = 1; - - mmio = fd_shmem_acquire( page_sz, page_cnt, near_cpu ); - TEST( mmio, "fd_shmem_acquire failed" ); - mmio_sz = page_sz*page_cnt; - - } - - TEST( fd_ulong_is_aligned( (ulong)_io, fd_vinyl_io_mm_align() ), "bad alloc" ); - TEST( io_footprint >= fd_vinyl_io_mm_footprint( spad_max ), "bad alloc" ); - - io = fd_vinyl_io_mm_init( _io, spad_max, mmio, mmio_sz, reset, info, info_sz, io_seed ); - TEST( io, "fd_vinyl_io mm_init failed" ); - - } else if( !strcmp( type, "bd" ) ) { - - if( FD_VINYL_BSTREAM_BLOCK_SZ<512UL ) TEST( !direct, "--direct 1 not supported with --type bd and BLOCK_SZ<512" ); - - TEST( path, "--path not specified for --type bd" ); - - FD_LOG_NOTICE(( "Using --path as a block device bstream" )); - - bstream_type = 3; - - fd = open( path, open_flags, 0 ); - if( FD_UNLIKELY( fd==-1 ) ) FD_LOG_ERR(( "open failed (%i-%s)", errno, fd_io_strerror( errno ) )); - - TEST( fd_ulong_is_aligned( (ulong)_io, fd_vinyl_io_bd_align() ), "bad wksp alloc" ); - TEST( io_footprint >= fd_vinyl_io_bd_footprint( spad_max ), "bad wksp alloc" ); - - io = fd_vinyl_io_bd_init( _io, spad_max, fd, reset, info, info_sz, io_seed ); - TEST( io, "fd_vinyl_io bd_init failed" ); - - } else { - - FD_LOG_ERR(( "Unsupported io type" )); - - } - - FD_LOG_NOTICE(( "Creating vinyl" )); - - fd_tpool_t * tpool = NULL; - - ulong thread_cnt = fd_tile_cnt(); - - if( thread_cnt>1UL ) { - FD_LOG_NOTICE(( "Creating temporary tpool from all %lu tiles for thread parallel init", thread_cnt )); - - static uchar _tpool[ FD_TPOOL_FOOTPRINT( FD_TILE_MAX ) ] __attribute__((aligned(FD_TPOOL_ALIGN))); - - tpool = fd_tpool_init( _tpool, thread_cnt, 0UL ); /* logs details */ - if( FD_UNLIKELY( !tpool ) ) FD_LOG_ERR(( "fd_tpool_init failed" )); - - for( ulong thread_idx=1UL; thread_idx~ 8 GiB used */ - int gc_eager = 2; /* target <~25% garbage items */ - int style = FD_VINYL_BSTREAM_CTL_STYLE_LZ4; /* enable data compression */ - int level = 1; /* do a hard reset by default */ - ulong obj_footprint_avg = 2UL*FD_VINYL_BSTREAM_BLOCK_SZ + 8UL + 1UL; /* see note above */ - - int err = 0; - int cnt = 0; - - while( argc ) { - char const * cmd = argv[0]; - SHIFT(1); - - if( !strcmp( cmd, "help" ) ) { - - fflush( stdout ); fflush( stderr ); - fputs( fd_vinyl_ctl_help, stdout ); - fflush( stdout ); fflush( stderr ); - - FD_LOG_NOTICE(( "%i: %s: success", cnt, cmd )); - - } else if( !strcmp( cmd, "set" ) ) { - - if( FD_UNLIKELY( argc<2 ) ) FD_LOG_ERR(( "%i: %s: too few arguments\n\tDo %s help for help", cnt, cmd, bin )); - - char const * key = argv[0]; - char const * val = argv[1]; - - /**/ if( !strcmp( key, "wksp_tag" ) ) wksp_tag = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "pod_max" ) ) pod_max = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "cfg_path" ) ) cfg_path = val; - else if( !strcmp( key, "cnc_app_sz" ) ) cnc_app_sz = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "spad_max" ) ) spad_max = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "async_min" ) ) async_min = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "async_max" ) ) async_max = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "part_thresh" ) ) part_thresh = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "gc_thresh" ) ) gc_thresh = fd_cstr_to_ulong ( val ); - else if( !strcmp( key, "gc_eager" ) ) gc_eager = fd_cstr_to_int ( val ); - else if( !strcmp( key, "style" ) ) style = fd_cstr_to_vinyl_bstream_ctl_style( val ); - else if( !strcmp( key, "level" ) ) level = fd_cstr_to_int ( val ); - else if( !strcmp( key, "obj_footprint_avg" ) ) obj_footprint_avg = fd_cstr_to_ulong ( val ); - else FD_LOG_ERR(( "%i: %s %s %s: unknown key", cnt, cmd, key, val)); - - FD_LOG_NOTICE(( "%i: %s %s %s: success", cnt, cmd, key, val )); - SHIFT(2); - - } else if( !strcmp( cmd, "alloc-memory" ) ) { - - if( FD_UNLIKELY( argc<5 ) ) FD_LOG_ERR(( "%i: %s: too few arguments\n\tDo %s help for help", cnt, cmd, bin )); - - char const * mem = argv[0]; - ulong page_cnt = fd_cstr_to_ulong ( argv[1] ); - ulong page_sz = fd_cstr_to_shmem_page_sz( argv[2] ); - char const * seq = argv[3]; - ulong mode = fd_cstr_to_ulong_octal ( argv[4] ); - - if( FD_UNLIKELY( !page_cnt ) ) - FD_LOG_ERR(( "%i: %s %s %lu %s %s 0%03lo: bad page count\n\t" - "Do %s help for help", cnt, cmd, mem, page_cnt, argv[2], seq, mode, bin )); - - if( FD_UNLIKELY( !page_sz ) ) - FD_LOG_ERR(( "%i: %s %s %lu %s %s 0%03lo: bad page size\n\t" - "Do %s help for help", cnt, cmd, mem, page_cnt, argv[2], seq, mode, bin )); - - /* Partition the pages over the seq */ - - ulong sub_page_cnt[ 512UL ]; - ulong sub_cpu_idx [ 512UL ]; - ulong sub_cnt = fd_cstr_to_ulong_seq( seq, sub_cpu_idx, 512UL ); - - if( FD_UNLIKELY( !sub_cnt ) ) - FD_LOG_ERR(( "%i: %s %s %lu %s %s 0%03lo: empty or invalid cpu sequence\n\t" - "Do %s help for help", cnt, cmd, mem, page_cnt, argv[2], seq, mode, bin )); - - if( FD_UNLIKELY( sub_cnt>512UL ) ) - FD_LOG_ERR(( "%i: %s %s %lu %s %s 0%03lo: sequence too long, increase limit in fd_vinyl_ctl.c\n\t" - "Do %s help for help", cnt, cmd, mem, page_cnt, argv[2], seq, mode, bin )); - - /* TODO: consider striping instead of blocking */ - - ulong sub_page_min = page_cnt / sub_cnt; - ulong sub_page_rem = page_cnt % sub_cnt; - for( ulong sub_idx=0UL; sub_idx(1UL<<32)) ) ) - FD_LOG_ERR(( "%i: %s %s %lu 0%03lo: bad number of gigabytes\n\t" - "Do %s help for help", cnt, cmd, path, GiB_cnt, mode, bin )); - - ulong sz = GiB_cnt << 30; - - int fd = open( path, O_RDWR | O_CREAT | O_EXCL, (mode_t)mode ); - if( FD_UNLIKELY( fd==-1 ) ) - FD_LOG_ERR(( "%i: %s %s %lu 0%03lo: open failed (%i-%s)\n\tDo %s help for help", - cnt, cmd, path, GiB_cnt, mode, errno, fd_io_strerror( errno ), bin )); - - int err = fd_io_truncate( fd, sz ); - if( FD_UNLIKELY( err ) ) - FD_LOG_ERR(( "%i: %s %s %lu 0%03lo: fd_io_truncate failed (%i-%s)\n\tDo %s help for help", - cnt, cmd, path, GiB_cnt, mode, err, fd_io_strerror( err ), bin )); - - if( FD_UNLIKELY( close( fd ) ) ) - FD_LOG_WARNING(( "%i: %s %s %lu 0%03lo: close failed (%i-%s); attempting to continue", - cnt, cmd, path, GiB_cnt, mode, errno, fd_io_strerror( errno ) )); - - FD_LOG_NOTICE(( "%i: %s %s %lu 0%03lo: success", cnt, cmd, path, GiB_cnt, mode )); - SHIFT(3); - - } else if( !strcmp( cmd, "free-storage" ) ) { - - if( FD_UNLIKELY( argc<1 ) ) - FD_LOG_ERR(( "%i: %s: too few arguments\n\tDo %s help for help", cnt, cmd, bin )); - - char const * store = argv[0]; - - if( FD_UNLIKELY( unlink( store ) ) ) - FD_LOG_ERR(( "%i: %s %s: unlink failed (%i-%s)\n\tDo %s help for help", - cnt, cmd, store, errno, fd_io_strerror( errno ), bin )); - - FD_LOG_NOTICE(( "%i: %s %s: success", cnt, cmd, store )); - SHIFT(1); - - } else if( !strcmp( cmd, "new" ) ) { - - if( FD_UNLIKELY( argc<3 ) ) - FD_LOG_ERR(( "%i: %s: too few arguments\n\tDo %s help for help", cnt, cmd, bin )); - - char const * mem = argv[0]; - ulong pair_max = fd_cstr_to_ulong( argv[1] ); - ulong GiB_max = fd_cstr_to_ulong( argv[2] ); - -# define TEST( c, msg ) do { \ - if( FD_UNLIKELY( !(c) ) ) \ - FD_LOG_ERR(( "%i: %s %s %lu %lu: FAIL %s (%s)\n\tDo %s help for help", \ - cnt, cmd, mem, pair_max, GiB_max, #c, (msg), bin )); \ - } while(0) - - ulong ele_max = fd_ulong_pow2_up( pair_max + 1UL ); - ulong lock_cnt = fd_vinyl_meta_lock_cnt_est( ele_max ); - ulong probe_max = ele_max; - - TEST( (0UL=3UL, "increase maximum GiB allowed and/or decrease pair_max / spad_max / pod_max / cnc_app_sz" ); - - ulong line_footprint = sizeof(fd_vinyl_line_t)*line_max; - - mem_req += line_footprint; - - ulong obj_footprint = fd_ulong_align_dn( mem_max - mem_req, alignof(fd_vinyl_data_obj_t) ); - - mem_req += obj_footprint; - - TEST( mem_req<=mem_max, "internal error" ); - - /* Attach to the memory that will contain this vinyl instance */ - - fd_wksp_t * wksp = fd_wksp_attach( mem ); - TEST( wksp, "fd_wksp_attach failed" ); - - /* Allocate all the needed regions. Note that, even though the - vinyl io tile state is neither shared nor persistent, we - allocate it here so the vinyl tile itself doesn't have to - allocate it (it is dynamically sized and rather large). Since - we want the vinyl tile to be able to pick the type of io - interface and bstream store at startup without creating a new - vinyl instance, we allocated an upper bound for all supported - io types above (they are all roughly the same size anyway). - - Alternatively, we could have the vinyl tile do this allocation - at tile startup. But this would create some additional - complexity: the vinyl tile would need an allocator (and then - one potentially has allocations left over from previous runs - that did not terminate cleanly). - - Similar considerations apply for the data cache state, vinyl - tile state, lines and data objects. - - Note also that, though meta is shared and persistent, - persistence should only be used for post mortem debugging (the - meta cache is recreated from scratch on vinyl tile startup). */ - - void * _pod = fd_wksp_alloc_laddr( wksp, pod_align, pod_footprint, wksp_tag ); - void * _vinyl = fd_wksp_alloc_laddr( wksp, vinyl_align, vinyl_footprint, wksp_tag ); - void * _cnc = fd_wksp_alloc_laddr( wksp, cnc_align, cnc_footprint, wksp_tag ); - void * _meta = fd_wksp_alloc_laddr( wksp, meta_align, meta_footprint, wksp_tag ); - void * _io = fd_wksp_alloc_laddr( wksp, io_align, io_footprint, wksp_tag ); - void * _line = fd_wksp_alloc_laddr( wksp, line_align, line_footprint, wksp_tag ); /* This is kinda big */ - void * _ele = fd_wksp_alloc_laddr( wksp, ele_align, ele_footprint, wksp_tag ); /* This is really big */ - void * _obj = fd_wksp_alloc_laddr( wksp, obj_align, obj_footprint, wksp_tag ); - - /* Note: the bigger obj gets, the better the performance (until it - is large enough pairs always fit in cache but that would dwarf - ele). In typical use cases, this is probably smaller to - comparable to ele (resulting in much cheaper hardware at - comparable speeds for typical usage patterns but less robust - performance for extreme usage patterns). */ - - TEST( (!!_pod) & (!!_vinyl) & (!!_cnc) & (!!_io) & (!!_line) & (!!_ele) & (!!_obj), - "fd_wksp_alloc_laddr failed (free unneeded allocs or increase wksp size or partitions)" ); - - /* Format and the join the pod and create the cfg subpod as - necessary. */ - - uchar * pod = fd_pod_join( fd_pod_new( _pod, pod_max ) ); - TEST( pod, "internal error" ); - - uchar * cfg; - if( !cfg_path ) cfg = pod; - else { - ulong off = fd_pod_alloc_subpod( pod, cfg_path, 1024UL ); - TEST( off, "use shorter cfg_path or increase pod_max?" ); - cfg = pod + off; - } - - /* Populate the pod */ - - char tmp[ FD_WKSP_CSTR_MAX ]; - - TEST( fd_pod_insert_cstr( cfg, "vinyl", fd_wksp_cstr_laddr( _vinyl, tmp ) ), "increase pod_max?" ); - TEST( fd_pod_insert_cstr( cfg, "cnc", fd_wksp_cstr_laddr( _cnc, tmp ) ), "increase pod_max?" ); - TEST( fd_pod_insert_cstr( cfg, "meta", fd_wksp_cstr_laddr( _meta, tmp ) ), "increase pod_max?" ); - TEST( fd_pod_insert_cstr( cfg, "io", fd_wksp_cstr_laddr( _io, tmp ) ), "increase pod_max?" ); - TEST( fd_pod_insert_cstr( cfg, "line", fd_wksp_cstr_laddr( _line, tmp ) ), "increase pod_max?" ); - TEST( fd_pod_insert_cstr( cfg, "ele", fd_wksp_cstr_laddr( _ele, tmp ) ), "increase pod_max?" ); - TEST( fd_pod_insert_cstr( cfg, "obj", fd_wksp_cstr_laddr( _obj, tmp ) ), "increase pod_max?" ); - - TEST( fd_pod_insert_ulong( cfg, "vinyl_footprint", vinyl_footprint ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "cnc_footprint", cnc_footprint ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "meta_footprint", meta_footprint ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "io_footprint", io_footprint ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "line_footprint", line_footprint ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "ele_footprint", ele_footprint ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "obj_footprint", obj_footprint ), "increase pod_max?" ); - - TEST( fd_pod_insert_ulong( cfg, "spad_max", spad_max ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "pair_max", pair_max ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "line_max", line_max ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "async_min", async_min ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "async_max", async_max ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "part_thresh", part_thresh ), "increase pod_max?" ); - TEST( fd_pod_insert_ulong( cfg, "gc_thresh", gc_thresh ), "increase pod_max?" ); - TEST( fd_pod_insert_int ( cfg, "gc_eager", gc_eager ), "increase pod_max?" ); - TEST( fd_pod_insert_int ( cfg, "style", style ), "increase pod_max?" ); - TEST( fd_pod_insert_int ( cfg, "level", level ), "increase pod_max?" ); - - /* Tell the operator where the pod is */ - /* FIXME: consider putting the config pod in a normal page named - shmem region or a flat file instead? Probably easier to pass - between applications than a wksp gaddr. */ - - printf( "%s\n", fd_wksp_cstr_laddr( _pod, tmp ) ); - - /* Clean up */ - - if( cfg!=pod ) TEST( fd_pod_compact( cfg, 1 ), "internal error" ); - - TEST( fd_pod_leave( pod )==_pod, "internal error" ); - - TEST( !fd_wksp_detach( wksp ), "internal error" ); - -# undef TEST - - FD_LOG_NOTICE(( "%i: %s %s %lu %lu: success", cnt, cmd, mem, pair_max, GiB_max )); - SHIFT(3); - - } else if( !strcmp( cmd, "delete" ) ) { - - if( FD_UNLIKELY( argc<1 ) ) - FD_LOG_ERR(( "%i: %s: too few arguments\n\tDo %s help for help", cnt, cmd, bin )); - - char const * cstr = argv[0]; - -# define TEST( c, msg ) do { \ - if( FD_UNLIKELY( !(c) ) ) \ - FD_LOG_ERR(( "%i: %s %s: FAIL %s (%s)\n\tDo %s help for help", \ - cnt, cmd, cstr, #c, (msg), bin )); \ - } while(0) - - uchar const * pod = fd_pod_join( fd_wksp_map( cstr ) ); /* logs details */ - TEST( pod, "unable to join pod" ); - - uchar const * cfg; - if( !cfg_path ) cfg = pod; - else { - cfg = fd_pod_query_subpod( pod, cfg_path ); - TEST( cfg, "cfg not found at cfg_path" ); - } - - fd_wksp_cstr_free( fd_pod_query_cstr( cfg, "obj", NULL ) ); - fd_wksp_cstr_free( fd_pod_query_cstr( cfg, "ele", NULL ) ); - fd_wksp_cstr_free( fd_pod_query_cstr( cfg, "line", NULL ) ); - fd_wksp_cstr_free( fd_pod_query_cstr( cfg, "io", NULL ) ); - fd_wksp_cstr_free( fd_pod_query_cstr( cfg, "meta", NULL ) ); - fd_wksp_cstr_free( fd_pod_query_cstr( cfg, "cnc", NULL ) ); - fd_wksp_cstr_free( fd_pod_query_cstr( cfg, "vinyl", NULL ) ); - - fd_wksp_unmap( fd_pod_leave( pod ) ); - - fd_wksp_cstr_free( cstr ); - - FD_LOG_NOTICE(( "%i: %s %s: success", cnt, cmd, cstr )); - SHIFT(1); - - } else if( !strcmp( cmd, "exec" ) ) { - - err = fd_vinyl_main( argc, argv ); - break; - - } else { - - FD_LOG_ERR(( "%i: %s: unknown command\n\t" - "Do %s help for help", cnt, cmd, bin )); - - } - cnt++; - } - - if( FD_UNLIKELY( cnt<1 ) ) FD_LOG_NOTICE(( "processed %i commands\n\tDo %s help for help", cnt, bin )); - else FD_LOG_NOTICE(( "processed %i commands", cnt )); - -# undef SHIFT - - fd_halt(); - return err; -} diff --git a/src/vinyl/fd_vinyl_ctl_help b/src/vinyl/fd_vinyl_ctl_help deleted file mode 100644 index e8a821cfd13..00000000000 --- a/src/vinyl/fd_vinyl_ctl_help +++ /dev/null @@ -1,91 +0,0 @@ - -Usage: fd_vinyl_ctl [cmd] [cmd args] [cmd] [cmd args] ... - -Commands are: - -help -- Prints this message. - -set [key] [val] -- Set the [key] to [val]. keys include: - - key | val type | default | notes - ------------------+----------+--------------------+----------------------------------------------------------- - wksp_tag | ulong | 0xfdc12113c597a600 | allocation tag used for vinyl wksp allocs - pod_max | ulong | 4KiB | byte size for the vinyl pod - cfg_path | cstr | NULL | path to the vinyl cfg in the vinyl pod (NULL use pod root) - cnc_app_sz | ulong | vinyl tile minimum | - - spad_max | ulong | 32MiB | io append scratch pad byte size - exec_max | ulong | 2UL | vinyl tile max request executed per run loop iteration - async_min | ulong | 2UL | min run loop iterations per async handling - async_max | ulong | 4UL | max run loop iterations per async handling - part_thresh | ulong | 1GiB | target partition size - gc_thresh | ulong | 8GiB | min bstream past size to consider compaction - gc_eager | int | 2 | target less than <~2^-gc_eager garbage items - style | cstr | "lz4" | preferred bstream pair encoding ("raw" and "lz4" are supported) - level | int | 1 | preferred reset level at startup (0 soft, 1 hard) - obj_footprint_avg | ulong | 1KiB + 9B | marginal byte cost of object in the data cache - -alloc-memory wksp page_cnt page_sz cpu_idx_seq mode - -- Create a workspace named wksp from page_cnt page_sz pages distributed - over numa nodes near cpu_idx_seq. The region will have the unix - permissions specified by mode (assumed octal). See fd_shmem_ctl help - for more details how the pages will be distributed over numa nodes. - -- The purpose of this is to create a workspace which can be used to hold - the memory regions needed by a vinyl tile. Since this is a normal - workspace though, it can be used for other application allocations. - (Or vice versa.) - -- This is identical to fd_wksp_ctl new and provided here as a - convenience. - -free-memory wksp - -- Delete a workspace named wksp. If multiple shmem regions exist with - same name, try to use the shmem region backed by the largest page size - -- This is identical to fd_wksp_ctl delete and provided here as a - convenience and for symmetry with alloc-memory. - -alloc-storage path sz_in_GiB mode - -- Create a file at path with the given sz filled with zeros. The file - will have the permissions specified by mode (assumed octal). Fails - if the file already exists. - -- The purpose of this is create a file which can be used to hold - the bstream needed by a vinyl tile. There is no requirement to use - this though. - -- This is provided as a convenience. The user is free to create bstream - storage via whatever means suits them. This includes using - pre-existing block devices directly. - -free-storage path - -- Destroys the file at path. Fails if the file could not be destroyed. - -- This is provided here as a convenience and for symmetry with - alloc-storage. - -new wksp pair_max sz_in_GiB - -- Allocate memory resources needed for a vinyl tile in the workspace - wksp. The vinyl tile will be able to track up to pair_max key-val - pairs total. The target amount of wksp memory to devote to the vinyl - tile should be sz_in_GiB. - -- One of these memory resources will be a pod. The locations of these - memory resources will be stashed in in this pod as wksp cstr gaddrs - (along with other vinyl tile configurations). - -- The wksp cstr of this pod will be printed to stdout. - -delete pod - -- Frees all memory resources allocated for a vinyl tile. The location - of the resources are given by pod. This includes freeing the pod - itself. The tile should not be running when this is done. - diff --git a/src/vinyl/fd_vinyl_exec.c b/src/vinyl/fd_vinyl_exec.c deleted file mode 100644 index a543388af71..00000000000 --- a/src/vinyl/fd_vinyl_exec.c +++ /dev/null @@ -1,702 +0,0 @@ -#include "fd_vinyl.h" -#include "../util/pod/fd_pod.h" -#include -#include -#include -#include - -struct fd_vinyl_client { - fd_vinyl_rq_t * rq; /* Channel for requests from this client (could be shared by multiple vinyl instances) */ - fd_vinyl_cq_t * cq; /* Channel for completions from this client to this vinyl instance - (could be shared by multiple receivers of completions from this vinyl instance). */ - ulong burst_max; /* Max requests receive from this client at a time */ - ulong seq; /* Sequence number of the next request to receive in the rq */ - ulong link_id; /* Identifies requests from this client to this vinyl instance in the rq */ - ulong laddr0; /* A valid non-zero gaddr from this client maps to the vinyl instance's laddr laddr0 + gaddr ... */ - ulong laddr1; /* ... and thus is in (laddr0,laddr1). A zero gaddr maps to laddr NULL. */ - ulong quota_rem; /* Num of remaining acquisitions this client is allowed on this vinyl instance */ - ulong quota_max; /* Max quota */ -}; - -typedef struct fd_vinyl_client fd_vinyl_client_t; - -/* MAP_REQ_GADDR maps a request global address req_gaddr to an array of - cnt T's into the local address space as a T * pointer. If the result - is not properly aligned or the entire range does not completely fall - within the shared region with the client, returns NULL. Likewise, - gaadr 0 maps to NULL. Assumes sizeof(T)*(n) does not overflow (which - is true where as n is at most batch_cnt which is at most 2^32 and - sizeof(T) is at most 40. */ - -#define MAP_REQ_GADDR( gaddr, T, n ) ((T *)fd_vinyl_laddr( (gaddr), alignof(T), sizeof(T)*(n), client_laddr0, client_laddr1 )) - -FD_FN_CONST static inline void * -fd_vinyl_laddr( ulong req_gaddr, - ulong align, - ulong footprint, - ulong client_laddr0, - ulong client_laddr1 ) { - ulong req_laddr0 = client_laddr0 + req_gaddr; - ulong req_laddr1 = req_laddr0 + footprint; - return (void *)fd_ulong_if( (!!req_gaddr) & fd_ulong_is_aligned( req_laddr0, align ) & - (client_laddr0<=req_laddr0) & (req_laddr0<=req_laddr1) & (req_laddr1<=client_laddr1), - req_laddr0, 0UL ); -} - -/* FIXME: STASH THESE IN THE VINYL TOO? */ -#define FD_VINYL_CLIENT_MAX (1024UL) -#define FD_VINYL_REQ_MAX (1024UL) - -void -fd_vinyl_exec( fd_vinyl_t * vinyl ) { - - /* Unpack shared objects */ - - fd_cnc_t * cnc = vinyl->cnc; - fd_vinyl_io_t * io = vinyl->io; - fd_vinyl_line_t * line = vinyl->line; - fd_vinyl_meta_t * meta = vinyl->meta; - fd_vinyl_data_t * data = vinyl->data; - - /* Unpack config */ - - ulong line_cnt = vinyl->line_cnt; - ulong pair_max = vinyl->pair_max; - ulong async_min = vinyl->async_min; - ulong async_max = vinyl->async_max; - - /* Unpack cnc */ - - if( FD_UNLIKELY( fd_cnc_signal_query( cnc )!=FD_VINYL_CNC_SIGNAL_BOOT ) ) { - FD_LOG_WARNING(( "cnc not booting (restarting after an unclean termination?); forcing to boot and attempting to continue" )); - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_BOOT ); - } - - fd_vinyl_cmd_t * cmd = (fd_vinyl_cmd_t *)fd_cnc_app_laddr( cnc ); - ulong * diag = (ulong *)(cmd+1); - - /* Unpack io */ - - ulong io_seed = fd_vinyl_io_seed( io ); - - /* Unpack meta */ - - fd_vinyl_meta_ele_t * ele0 = meta->ele; - ulong ele_max = meta->ele_max; - ulong meta_seed = meta->seed; - ulong * lock = meta->lock; - int lock_shift = meta->lock_shift; - - /* Unpack data */ - - ulong data_laddr0 = (ulong)data->laddr0; - fd_vinyl_data_vol_t const * vol = data->vol; - ulong vol_cnt = data->vol_cnt; - - /* Connected clients */ - - fd_vinyl_client_t _client[ FD_VINYL_CLIENT_MAX ]; - ulong client_cnt = 0UL; /* In [0,client_max) */ - ulong client_idx = 0UL; /* If client_cnt>0, next client to poll for requests, d/c otherwise */ - - ulong quota_free = line_cnt - 1UL; - - /* Received requests */ - - fd_vinyl_req_t _req[ FD_VINYL_REQ_MAX ]; - ulong req_head = 0UL; /* Requests [0,req_head) have been processed */ - ulong req_tail = 0UL; /* Requests [req_head,req_tail) are pending */ - /* Requests [req_tail,ULONG_MAX) have not been received */ - ulong burst_free = FD_VINYL_REQ_MAX; - ulong exec_max = 0UL; - - /* accum_dead_cnt is the number of dead blocks that have been - written since the last partition block. - - accum_move_cnt is the number of move blocks that have been - written since this last partition block. - - accum_garbage_cnt / sz is the number of items / bytes garbage in - the bstream that have accumulated since the last time we compacted - the bstream. We use this to estimate the number of rounds of - compaction to do in async handling. - - accum_drop_link is the number of requests that were silently - dropped because the request link_id did not match the client's - link_id. - - accum_drop_comp is the number of requests that were silently - dropped because an out-of-band completion was requested to be sent - to an unmappable client address. - - accumt_req_full is the number of times we detected the pending - request queue being completely full. */ - - ulong accum_dead_cnt = 0UL; - ulong accum_move_cnt = 0UL; - ulong accum_garbage_cnt = 0UL; - ulong accum_garbage_sz = 0UL; - ulong accum_drop_link = 0UL; - ulong accum_drop_comp = 0UL; - - ulong seq_part = fd_vinyl_io_seq_present( io ); - - /* Run */ - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_RUN ); - - ulong async_rem = 1UL; - - for(;;) { - - /* Process background tasks this iteration if necessary */ - - if( FD_UNLIKELY( !(--async_rem) ) ) { - long now = fd_log_wallclock(); - async_rem = async_min + (fd_ulong_hash( (ulong)now ) % (async_max-async_min+1UL)); /* FIXME: FASTER ALGO */ - - fd_cnc_heartbeat( cnc, now ); - - /* If we've written enough to justify appending a parallel - recovery partition, append one. */ - - ulong seq_future = fd_vinyl_io_seq_future( io ); - if( FD_UNLIKELY( (seq_future - seq_part) > vinyl->part_thresh ) ) { - - ulong seq = fd_vinyl_io_append_part( io, seq_part, accum_dead_cnt, accum_move_cnt, NULL, 0UL ); - FD_DCHECK_CRIT( fd_vinyl_seq_eq( seq, seq_future ), "corruption detected" ); - seq_part = seq + FD_VINYL_BSTREAM_BLOCK_SZ; - - accum_dead_cnt = 0UL; - accum_move_cnt = 0UL; - - accum_garbage_cnt++; - accum_garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - - fd_vinyl_io_commit( io, FD_VINYL_IO_FLAG_BLOCKING ); - - } - - diag[ FD_VINYL_DIAG_DROP_LINK ] += accum_drop_link; accum_drop_link = 0UL; - diag[ FD_VINYL_DIAG_DROP_COMP ] += accum_drop_comp; accum_drop_comp = 0UL; - - /* Let the number of items of garbage generated since the last - compaction be accum_garbage_cnt and let the steady steady - average number of live / garbage items in the bstream's past be - L / G (i.e. L is the average value of pair_cnt). The average - number pieces of garbage collected per garbage collection round - is thus G / (L + G). If we do compact_max rounds garbage - collection this async handling, we expect to collect - - compact_max G / (L + G) - - items of garbage on average. To make sure we collect garbage - faster than we generate it on average, we then require: - - accum_garbage_cnt <~ compact_max G / (L + G) - -> compact_max >~ (L + G) accum_garbage_cnt / G - - Let the be 2^-gc_eager be the maximum fraction of items in the - bstream's past we are willing tolerate as garbage on average. - We then have G = 2^-gc_eager (L + G). This implies: - - -> compact_max >~ accum_garbage_cnt 2^gc_eager - - When accum_garbage_cnt is 0, we use a compact_max of 1 to do - compaction rounds at a minimum rate all the time. This allows - transients (e.g. a sudden change to new steady state - equilibrium, temporary disabling of garbage collection at key - times for highest performance, etc) and unaccounted zero - padding garbage to be absorbed when nothing else is going on. */ - - int gc_eager = vinyl->gc_eager; - if( FD_LIKELY( gc_eager>=0 ) ) { - - /* Saturating wide left shift */ - ulong overflow = (accum_garbage_cnt >> (63-gc_eager) >> 1); /* sigh ... avoid wide shift UB */ - ulong compact_max = fd_ulong_max( fd_ulong_if( !overflow, accum_garbage_cnt << gc_eager, ULONG_MAX ), 1UL ); - - /**/ accum_garbage_cnt = 0UL; - vinyl->garbage_sz += accum_garbage_sz; accum_garbage_sz = 0UL; - - fd_vinyl_compact( vinyl, compact_max ); - - } - - ulong signal = fd_cnc_signal_query( cnc ); - if( FD_UNLIKELY( signal!=FD_VINYL_CNC_SIGNAL_RUN ) ) { - if( FD_UNLIKELY( signal==FD_VINYL_CNC_SIGNAL_HALT ) ) break; - - switch( signal ) { - - case FD_VINYL_CNC_SIGNAL_SYNC: { - fd_vinyl_io_sync( io, FD_VINYL_IO_FLAG_BLOCKING ); - break; - } - - case FD_VINYL_CNC_SIGNAL_GET: { - ulong old; - int err = FD_VINYL_SUCCESS; - switch( cmd->get.opt ) { - case FD_VINYL_OPT_PART_THRESH: old = vinyl->part_thresh; break; - case FD_VINYL_OPT_GC_THRESH: old = vinyl->gc_thresh; break; - case FD_VINYL_OPT_GC_EAGER: old = (ulong)(long)vinyl->gc_eager; break; - case FD_VINYL_OPT_STYLE: old = (ulong)(uint)vinyl->style; break; - default: old = 0UL; err = FD_VINYL_ERR_INVAL; break; - } - cmd->get.val = old; - cmd->get.err = err; - break; - } - - case FD_VINYL_CNC_SIGNAL_SET: { /* FIXME: ADD VALIDATION TO SET VALUES FOR OPT_GC_EAGER AND OPT_STYLE */ - ulong new = cmd->set.val; - ulong old; - int err = FD_VINYL_SUCCESS; - switch( cmd->set.opt ) { - case FD_VINYL_OPT_PART_THRESH: old = vinyl->part_thresh; vinyl->part_thresh = new; break; - case FD_VINYL_OPT_GC_THRESH: old = vinyl->gc_thresh; vinyl->gc_thresh = new; break; - case FD_VINYL_OPT_GC_EAGER: old = (ulong)(long)vinyl->gc_eager; vinyl->gc_eager = (int)new; break; - case FD_VINYL_OPT_STYLE: old = (ulong)(uint)vinyl->style; vinyl->style = (int)new; break; - default: old = 0UL; err = FD_VINYL_ERR_INVAL; break; - } - cmd->set.val = old; - cmd->set.err = err; - break; - } - - case FD_VINYL_CNC_SIGNAL_CLIENT_JOIN: { - int err; - - ulong link_id = cmd->join.link_id; - ulong burst_max = cmd->join.burst_max; - ulong quota_max = cmd->join.quota_max; - char const * _rq = cmd->join.rq; - char const * _cq = cmd->join.cq; - char const * _wksp = cmd->join.wksp; - - if( FD_UNLIKELY( client_cnt>=FD_VINYL_CLIENT_MAX ) ) { - FD_LOG_WARNING(( "Too many clients (increase FD_VINYL_CLIENT_MAX)" )); - err = FD_VINYL_ERR_FULL; - goto join_done; - } - - if( FD_UNLIKELY( burst_max > burst_free ) ) { - FD_LOG_WARNING(( "Too large burst_max (increase FD_VINYL_REQ_MAX or decrease burst_max)" )); - err = FD_VINYL_ERR_FULL; - goto join_done; - } - - if( FD_UNLIKELY( quota_max > fd_ulong_min( quota_free, FD_VINYL_COMP_QUOTA_MAX ) ) ) { - FD_LOG_WARNING(( "Too large quota_max (increase line_cnt or decrease quota_max)" )); - err = FD_VINYL_ERR_FULL; - goto join_done; - } - - for( ulong client_idx=0UL; client_idxgaddr_hi; /* FIXME: HOW TO GET THIS CLEANLY */ - _client[ client_cnt ].quota_rem = quota_max; - _client[ client_cnt ].quota_max = quota_max; - client_cnt++; - - quota_free -= quota_max; - burst_free -= burst_max; - - /* Every client_cnt run loop iterations we receive at most: - - sum_clients recv_max = FD_VINYL_REQ_MAX - burst_free - - requests. To guarantee we processe requests fast enough - that we never overrun our receive queue, under maximum - client load, we need to process: - - sum_clients recv_max / client_cnt - - requests per run loop iteration. We thus set exec_max - to the ceil sum_clients recv_max / client_cnt. */ - - exec_max = (FD_VINYL_REQ_MAX - burst_free + client_cnt - 1UL) / client_cnt; - - err = FD_VINYL_SUCCESS; - - join_done: - cmd->join.err = err; - break; - } - - case FD_VINYL_CNC_SIGNAL_CLIENT_LEAVE: { - int err; - - ulong link_id = cmd->leave.link_id; - - for( ulong client_idx=0UL; client_idxleave.err = err; - break; - } - - default: { - FD_LOG_WARNING(( "unknown signal received (%lu); ignoring", signal )); - break; - } - - } - - fd_cnc_signal( cnc, FD_VINYL_CNC_SIGNAL_RUN ); - } - } - - /* Receive requests from clients */ - - if( FD_LIKELY( client_cnt ) ) { - - /* Select client to poll this run loop iteration */ - - client_idx = fd_ulong_if( client_idx+1ULrq; - ulong seq = client->seq; - ulong burst_max = client->burst_max; - ulong link_id = client->link_id; - - /* Enqueue up to burst_max requests from this client into the - local request queue. Using burst_max << FD_VINYL_REQ_MAX - allows applications to prevent a bursty client from starving - other clients of resources while preserving the spatial and - temporal locality of reasonably sized O(burst_max) bursts from - an individual client in processing below. Each run loop - iteration can enqueue up to burst_max requests per iterations. */ - - for( ulong recv_rem=fd_ulong_min( FD_VINYL_REQ_MAX-(req_tail-req_head), burst_max ); recv_rem; recv_rem-- ) { - fd_vinyl_req_t * req = _req + (req_tail & (FD_VINYL_REQ_MAX-1UL)); - - long diff = fd_vinyl_rq_recv( rq, seq, req ); - - if( FD_LIKELY( diff>0L ) ) break; /* No requests waiting in rq at this time */ - - if( FD_UNLIKELY( diff ) ) FD_LOG_CRIT(( "client overran request queue" )); - - seq++; - - /* We got the next request. Decide if we should accept it. - - Specifically, we ignore requests whose link_id don't match - link_id (e.g. an unknown link_id or matches a different - client's link_id ... don't know if it is where or even if it - is safe to the completion). Even if the request provided an - out-of-band location to send the completion (comp_gaddr!=0), - we have no reason to trust it given the mismatch. - - This also gives a mechanism for a client use a single rq to - send requests to multiple vinyl instances ... the client - should use a different link_id for each vinyl instance. Each - vinyl instance will quickly filter out the requests not - addressed to it. - - Since we know the client_idx at this point, given a matching - link_id, we stash the client_idx in the pending req link_id - to eliminate the need to maintain a link_id<>client_idx map - in the execution loop below. */ - - if( FD_UNLIKELY( req->link_id!=link_id ) ) { - accum_drop_link++; - continue; - } - - req->link_id = client_idx; - - req_tail++; - } - - client->seq = seq; - } - - /* Execute received requests */ - - for( ulong exec_rem=fd_ulong_min( req_tail-req_head, exec_max ); exec_rem; exec_rem-- ) { - fd_vinyl_req_t * req = _req + ((req_head++) & (FD_VINYL_REQ_MAX-1UL)); - - /* Determine the client that sent this request and unpack the - completion fields. We ignore requests with non-NULL but - unmappable out-of-band completion because we can't send the - completion in the expected manner and, in lieu of that, the - receivers aren't expecting any completion to come via the cq - (if any). Note that this implies requests that don't produce a - completion (e.g. FETCH and FLUSH) need to either provide NULL - or a valid non-NULL location for comp_gaddr to pass this - validation (this is not a burden practically). */ - - ulong req_id = req->req_id; - ulong client_idx = req->link_id; /* See note above about link_id / client_idx conversion */ - ulong batch_cnt = (ulong)req->batch_cnt; - ulong comp_gaddr = req->comp_gaddr; - - fd_vinyl_client_t * client = _client + client_idx; - - fd_vinyl_cq_t * cq = client->cq; - ulong link_id = client->link_id; - ulong client_laddr0 = client->laddr0; - ulong client_laddr1 = client->laddr1; - ulong quota_rem = client->quota_rem; - - FD_DCHECK_CRIT( quota_rem<=client->quota_max, "corruption detected" ); - - fd_vinyl_comp_t * comp = MAP_REQ_GADDR( comp_gaddr, fd_vinyl_comp_t, 1UL ); - if( FD_UNLIKELY( (!comp) & (!!comp_gaddr) ) ) { - accum_drop_comp++; - continue; - } - - int comp_err = 1; - ulong fail_cnt = 0UL; - - ulong read_cnt = 0UL; - ulong append_cnt = 0UL; - - switch( req->type ) { - -# include "fd_vinyl_case_acquire.c" -# include "fd_vinyl_case_release.c" -# include "fd_vinyl_case_erase.c" -# include "fd_vinyl_case_move.c" -# include "fd_vinyl_case_fetch.c" -# include "fd_vinyl_case_flush.c" -# include "fd_vinyl_case_try.c" -# include "fd_vinyl_case_test.c" - - default: - comp_err = FD_VINYL_ERR_INVAL; - break; - } - - for( ; read_cnt; read_cnt-- ) { - fd_vinyl_io_rd_t * _rd; /* avoid pointer escape */ - fd_vinyl_io_poll( io, &_rd, FD_VINYL_IO_FLAG_BLOCKING ); - fd_vinyl_io_rd_t * rd = _rd; - - fd_vinyl_data_obj_t * obj = (fd_vinyl_data_obj_t *) rd->ctx; - ulong seq = rd->seq; (void)seq; - fd_vinyl_bstream_phdr_t * cphdr = (fd_vinyl_bstream_phdr_t *)rd->dst; - ulong cpair_sz = rd->sz; (void)cpair_sz; - - fd_vinyl_data_obj_t * cobj = (fd_vinyl_data_obj_t *)fd_ulong_align_dn( (ulong)rd, FD_VINYL_BSTREAM_BLOCK_SZ ); - - FD_DCHECK_CRIT( cphdr==fd_vinyl_data_obj_phdr( cobj ), "corruption detected" ); - - ulong cpair_ctl = cphdr->ctl; - - int cpair_type = fd_vinyl_bstream_ctl_type ( cpair_ctl ); - int cpair_style = fd_vinyl_bstream_ctl_style( cpair_ctl ); - ulong cpair_val_esz = fd_vinyl_bstream_ctl_sz ( cpair_ctl ); - - FD_DCHECK_CRIT( cpair_type==FD_VINYL_BSTREAM_CTL_TYPE_PAIR, "corruption detected" ); - FD_DCHECK_CRIT( cpair_sz ==fd_vinyl_bstream_pair_sz( cpair_val_esz ), "corruption detected" ); - - schar * rd_err = cobj->rd_err; - - FD_DCHECK_CRIT ( rd_err, "corruption detected" ); - FD_DCHECK_ALERT( fd_vinyl_data_is_valid_obj( obj, vol, vol_cnt ), "corruption detected" ); - - ulong line_idx = obj->line_idx; - - FD_DCHECK_CRIT( line_idxinfo.val_sz; - - FD_DCHECK_CRIT( val_sz <= FD_VINYL_VAL_MAX, "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_data_obj_val_max( obj ) >= val_sz, "corruption detected" ); - - if( FD_LIKELY( cpair_style==FD_VINYL_BSTREAM_CTL_STYLE_RAW ) ) { - - FD_DCHECK_CRIT( obj==cobj, "corruption detected" ); - FD_DCHECK_CRIT( cpair_val_esz==val_sz, "corruption detected" ); - - } else { - - char const * cval = (char const *)fd_vinyl_data_obj_val( cobj ); - ulong cval_sz = fd_vinyl_bstream_ctl_sz( cpair_ctl ); - - ulong _val_sz = (ulong)LZ4_decompress_safe( cval, val, (int)cval_sz, (int)val_sz ); - if( FD_UNLIKELY( _val_sz!=val_sz ) ) FD_LOG_CRIT(( "LZ4_decompress_safe failed" )); - - fd_vinyl_data_free( data, cobj ); - - fd_vinyl_bstream_phdr_t * phdr = fd_vinyl_data_obj_phdr( obj ); - - phdr->ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ); - phdr->key = cphdr->key; - phdr->info = cphdr->info; - - } - - obj->rd_active = (short)0; - - /* Fill any trailing region with zeros (there is at least - FD_VINYL_BSTREAM_FTR_SZ) and tell the client the item was - successfully processed. */ - - memset( val + val_sz, 0, fd_vinyl_data_szc_obj_footprint( (ulong)obj->szc ) - - (sizeof(fd_vinyl_data_obj_t) + sizeof(fd_vinyl_bstream_phdr_t) + val_sz) ); - - FD_COMPILER_MFENCE(); - *rd_err = (schar)FD_VINYL_SUCCESS; - FD_COMPILER_MFENCE(); - - } - - if( FD_UNLIKELY( append_cnt ) ) fd_vinyl_io_commit( io, FD_VINYL_IO_FLAG_BLOCKING ); - - if( FD_LIKELY( comp_err<=0 ) ) fd_vinyl_cq_send( cq, comp, req_id, link_id, comp_err, batch_cnt, fail_cnt, quota_rem ); - - client->quota_rem = quota_rem; - - } - - } /* run loop */ - - ulong discard_cnt = req_tail - req_head; - - /* Append the final partition and sync so we can resume with a fast - parallel recovery */ - - fd_vinyl_io_append_part( io, seq_part, accum_dead_cnt, accum_move_cnt, NULL, 0UL ); - - accum_dead_cnt = 0UL; - accum_move_cnt = 0UL; - - accum_garbage_cnt++; - accum_garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - - fd_vinyl_io_commit( io, FD_VINYL_IO_FLAG_BLOCKING ); - - fd_vinyl_io_sync( io, FD_VINYL_IO_FLAG_BLOCKING ); - - /* Drain outstanding accumulators */ - - /**/ accum_garbage_cnt = 0UL; - vinyl->garbage_sz += accum_garbage_sz; accum_garbage_sz = 0UL; - - diag[ FD_VINYL_DIAG_DROP_LINK ] += accum_drop_link; accum_drop_link = 0UL; - diag[ FD_VINYL_DIAG_DROP_COMP ] += accum_drop_comp; accum_drop_comp = 0UL; - - /* Disconnect from the clients */ - - ulong released_cnt = 0UL; - for( ulong client_idx=0UL; client_idxpart.seq0; - seq0 = fd_vinyl_seq_gt( seq0, seq_past ) ? seq0 : seq_past; - -# if 0 - /* Compute the maximum number of deads the portion of this partition - in the bstream's past that could produce as the lesser the number - of deads reported in the partition and the number of blocks in - the partition. Similarly for move (note that each move makes two - tombstone but also requires at least two blocks ... so moves also - make, at most, 1 tombstone per block on average). */ - - ulong part_sz = seq1 - seq0 - FD_VINYL_BSTREAM_BLOCK_SZ; /* exclude trailing part block for below */ - - ulong dead_max = fd_ulong_min( block->part.dead_cnt, part_sz ); - ulong move_max = fd_ulong_min( block->part.move_cnt, part_sz ); - - tstone_req += fd_ulong_min( dead_max + 2UL*move_max, part_sz ); -# endif - - /* Move to the previous partition */ - - seq1 = seq0; - } - - /* We seem to have a valid partitioning for parallel recovery */ - -# if 0 - if( FD_UNLIKELY( tstone_req > tstone_max ) ) { - FD_LOG_WARNING(( "insufficient scratch space for parallel recovery" - "\n\tincrease data cache size" - "\n\tfalling back to serial recovery" )); - return FD_VINYL_ERR_FULL; - } -# endif - - return FD_VINYL_SUCCESS; -} - -/* fd_vinyl_recover_line_task tests parallel flushes all vinyl - lines and resets the evicition priority sequence. */ - -static FD_FOR_ALL_BEGIN( fd_vinyl_recover_line_task, 1L ) { - fd_vinyl_t * vinyl = (fd_vinyl_t *)arg[0]; - - fd_vinyl_line_t * line = vinyl->line; - ulong line_cnt = vinyl->line_cnt; - - ulong line0 = (ulong)block_i0; - ulong line1 = (ulong)block_i1; - - for( ulong line_idx=line0; line_idxmeta->lock; - - ulong reclaim_cnt = 0UL; - - for( long lock_idx=block_i0; lock_idxmeta->ele; - - fd_vinyl_meta_ele_t init_ele[1]; - memset( init_ele, 0, sizeof(fd_vinyl_meta_ele_t) ); - init_ele->line_idx = ULONG_MAX; - - for( long ele_idx=block_i0; ele_idxmeta->lock; - - for( long lock_idx=block_i0; lock_idxphdr.ctl ) ) { - - /* There is no version or tstone for pair key in the meta currently. - Insert a tstone at seq for key so any versions or tstone for pair - key encountered later in parallel recovery can tell if they are - before or after this tstone. Because we don't know if there will - version of key after this, we need to append key to the tstone - array. */ - - //pair_cnt unchanged - //garbage_sz unchanged - (*_tstone_cnt)++; - - ele->memo = fd_vinyl_meta_query_memo( query ); - ele->phdr.ctl = 1UL; - ele->phdr.key = *key; - //ele->phdr.info = d/c - ele->line_idx = ULONG_MAX - 1UL; // tstone - ele->seq = seq; - - fd_vinyl_meta_publish( query ); - - } else if( FD_LIKELY( fd_vinyl_seq_lt( ele->seq, seq ) ) ) { - - /* The version (or tstone) for pair key in the meta is older than - seq. We append a key to the tstone array if we haven't already. */ - - int old_ele_is_pair = (ele->line_idx==ULONG_MAX); - - (*_pair_cnt) -= (ulong)old_ele_is_pair; - (*_garbage_sz) += old_ele_is_pair ? fd_vinyl_bstream_pair_sz( fd_vinyl_bstream_ctl_sz( ele->phdr.ctl ) ) : 0UL; - (*_tstone_cnt) += (ulong)old_ele_is_pair; - - //ele->memo = already init - //ele->phdr.ctl = already init - //ele->phdr.key = already init - //ele->phdr.info = d/c - ele->line_idx = ULONG_MAX - 1UL; // tstone - ele->seq = seq; - - fd_vinyl_meta_publish( query ); - - } else { - - /* The meta entry (pair or tstone) for pair key in the meta is newer - than seq. We can skip this tstone. */ - - //pair_cnt unchanged - //garbage_sz unchanged - //tstone_cnt unchanged - - int corrupt = fd_vinyl_seq_eq( ele->seq, seq ); - - fd_vinyl_meta_cancel( query ); - - if( FD_UNLIKELY( corrupt ) ) { - FD_LOG_WARNING(( "%016lx: probable corruption detected", seq )); - return FD_VINYL_ERR_CORRUPT; - } - - } - - return FD_VINYL_SUCCESS; -} - -/* fd_vinyl_recover_part_task dynamically assigns the partitions of the - bstream's past to threads for recovery and then recovers them in - parallel. The bstream past partition iteration is near identical - to bstream past iteration in serial recovery. See - fd_vinyl_recover_serial.c for more details. */ - -/* FIXME: ADD MORE EXTENSIVE DATA INTEGRITY CHECKING LIKE SERIAL IMPL */ - -static FD_FN_UNUSED FD_MAP_REDUCE_BEGIN( fd_vinyl_recover_part_task, 1UL, alignof(ulong), sizeof(ulong), 4UL ) { - ulong * _rlocal = (ulong *) arg[0]; - fd_vinyl_t * vinyl = (fd_vinyl_t *) arg[1]; - ulong * _lock = (ulong *) arg[2]; - - fd_vinyl_io_t * io = vinyl->io; - fd_vinyl_meta_t * meta = vinyl->meta; - - ulong io_seed = fd_vinyl_io_seed ( io ); - ulong seq_past = fd_vinyl_io_seq_past( io ); - uchar * mmio = (uchar *)fd_vinyl_mmio ( io ); - ulong mmio_sz = fd_vinyl_mmio_sz ( io ); - - ulong fail = 1UL; - ulong pair_cnt = 0UL; - ulong garbage_sz = 0UL; - ulong tstone_cnt = 0UL; - - for(;;) { - - /* Determine the range of the bstream past we should process next. */ - - ulong seq0; - ulong seq1; - - /* Lock and fetch the task assignment cursor */ - - FD_COMPILER_MFENCE(); -# if FD_HAS_ATOMIC - while( FD_ATOMIC_CAS( _lock, 0UL, 1UL ) ) FD_SPIN_PAUSE(); -# else - *_lock = 1UL; -# endif - FD_COMPILER_MFENCE(); - - seq1 = _lock[1]; - - /* At this point, the bstream range [seq_past,seq1) has not been - assigned. If seq1 is at seq_past, everything has been assigned - already. Otherwise, the block before cursor is a valid partition - block (as per the test above) and we claim the range: - - [ the older of part_seq0 and seq_past, seq1 ) - - to process. */ - - if( FD_UNLIKELY( fd_vinyl_seq_le( seq1, seq_past ) ) ) seq0 = seq_past; - else { - fd_vinyl_bstream_block_t const * block = PEEK( seq1 - FD_VINYL_BSTREAM_BLOCK_SZ ); - seq0 = block->part.seq0; - if( fd_vinyl_seq_lt( seq0, seq_past ) ) seq0 = seq_past; - } - - /* Update and unlock the task assignment cursor */ - - _lock[1] = seq0; - FD_COMPILER_MFENCE(); - _lock[0] = 0UL; - FD_COMPILER_MFENCE(); - - if( FD_UNLIKELY( fd_vinyl_seq_le( seq1, seq_past ) ) ) break; - - /* At this point, we need to recover the range [seq0,seq1). */ - - ulong seq = seq0; - while( fd_vinyl_seq_lt( seq, seq1 ) ) { - - fd_vinyl_bstream_block_t block[1]; - - block[0] = *(fd_vinyl_bstream_block_t *)PEEK( seq ); /* testing is destructive */ - - ulong ctl = block->ctl; - - int type = fd_vinyl_bstream_ctl_type( ctl ); - - switch( type ) { - - case FD_VINYL_BSTREAM_CTL_TYPE_PAIR: { - - ulong pair_val_esz = fd_vinyl_bstream_ctl_sz( ctl ); - - ulong pair_sz = fd_vinyl_bstream_pair_sz( pair_val_esz ); - - if( FD_UNLIKELY( pair_sz > (seq1-seq) ) ) { /* Wrapping safe */ - FD_LOG_WARNING(( "%016lx: truncated", seq )); - goto done; - } - - fd_vinyl_bstream_block_t ftr[1]; - - ftr[0] = *PEEK( seq + pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ ); - - char const * _err = fd_vinyl_bstream_pair_test_fast( io_seed, seq, block, ftr ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - /* At this point, we appear to have valid completely written - pair. Prepare the meta to do an update for this key. */ - - fd_vinyl_meta_query_t query[1]; - - fd_vinyl_meta_prepare( meta, &block->phdr.key, NULL, query, FD_MAP_FLAG_BLOCKING ); - - fd_vinyl_meta_ele_t * ele = fd_vinyl_meta_query_ele( query ); - - if( FD_UNLIKELY( !ele ) ) { - FD_LOG_WARNING(( "%016lx: corruption detected or meta cache too small for parallel recovery", seq )); - goto done; - } - - if( FD_LIKELY( (!ele->phdr.ctl) | fd_vinyl_seq_gt( seq, ele->seq ) ) ) { - - pair_cnt++; - - /* At this point, this is the first time any thread has seen - pair key or this version of pair key is newer than the - version (or tstone) of pair key has been seed */ - - ele->memo = fd_vinyl_meta_query_memo( query ); - ele->phdr = block->phdr; - ele->line_idx = ULONG_MAX; // pair - ele->seq = seq; - - fd_vinyl_meta_publish( query ); - - } else { - - /* At this point, this version of pair key is older than the - version (or tstone) for pair key seen by all threads so - far. */ - - fd_vinyl_meta_cancel( query ); - - garbage_sz += pair_sz; - - } - - seq += pair_sz; - break; - } - - case FD_VINYL_BSTREAM_CTL_TYPE_DEAD: { - - char const * _err = fd_vinyl_bstream_dead_test( io_seed, seq, block ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - int err = fd_vinyl_recover_tstone( meta, &block->dead.phdr.key, seq, &pair_cnt, &garbage_sz, &tstone_cnt ); - if( FD_UNLIKELY( err ) ) goto done; /* logs details */ - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - } - - case FD_VINYL_BSTREAM_CTL_TYPE_MOVE: { - - if( FD_UNLIKELY( 2UL*FD_VINYL_BSTREAM_BLOCK_SZ > (seq1-seq) ) ) { /* Wrapping safe */ - FD_LOG_WARNING(( "%016lx: truncated", seq )); - goto done; - } - - fd_vinyl_bstream_block_t dst[1]; - - dst[0] = *PEEK( seq + FD_VINYL_BSTREAM_BLOCK_SZ ); - - char const * _err = fd_vinyl_bstream_move_test( io_seed, seq, block, dst ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - int err = fd_vinyl_recover_tstone( meta, &block->move.src.key, seq, &pair_cnt, &garbage_sz, &tstone_cnt ); - if( FD_UNLIKELY( err ) ) goto done; /* logs details */ - - /**/ err = fd_vinyl_recover_tstone( meta, &block->move.dst, seq, &pair_cnt, &garbage_sz, &tstone_cnt ); - if( FD_UNLIKELY( err ) ) goto done; /* logs details */ - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - } - - case FD_VINYL_BSTREAM_CTL_TYPE_PART: { - - char const * _err = fd_vinyl_bstream_part_test( io_seed, seq, block ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - } - - case FD_VINYL_BSTREAM_CTL_TYPE_ZPAD: { - - char const * _err = fd_vinyl_bstream_zpad_test( io_seed, seq, block ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - } - - default: - FD_LOG_WARNING(( "%016lx: unknown type (%x)", seq, (uint)type )); - goto done; - - } - } - - if( FD_UNLIKELY( fd_vinyl_seq_ne( seq, seq1 ) ) ) { - FD_LOG_WARNING(( "%016lx: bad partitioning", seq )); - goto done; - } - - } - - fail = 0UL; - -done: - - /* If we failed, tell all the other threads to not continue by - setting the task assignment cursor to seq_past. */ - - if( fail ) { - FD_COMPILER_MFENCE(); -# if FD_HAS_ATOMIC - while( FD_ATOMIC_CAS( _lock, 0UL, 1UL ) ) FD_SPIN_PAUSE(); -# else - *_lock = 1UL; -# endif - FD_COMPILER_MFENCE(); - _lock[1]= seq_past; - FD_COMPILER_MFENCE(); - _lock[0]= 0UL; - } - - _rlocal[0] = fail; - _rlocal[1] = pair_cnt; - _rlocal[2] = garbage_sz; - _rlocal[3] = tstone_cnt; - -} FD_MAP_END { - - ulong * _rlocal = (ulong *) arg[0]; - ulong const * _rremote = (ulong const *)_r1; - - _rlocal[0] |= _rremote[0]; - _rlocal[1] += _rremote[1]; - _rlocal[2] += _rremote[2]; - _rlocal[3] += _rremote[3]; - -} FD_REDUCE_END - -static FD_FN_UNUSED FD_MAP_REDUCE_BEGIN( fd_vinyl_recover_meta_cleanup_task, 1L, alignof(ulong), sizeof(ulong), 1UL ) { - ulong * _rlocal = (ulong *)arg[0]; - - fd_vinyl_t * vinyl = (fd_vinyl_t *)arg[1]; - - fd_vinyl_meta_t * meta = vinyl->meta; - - fd_vinyl_meta_ele_t * ele0 = meta->ele; - ulong const * lock = meta->lock; - int lock_shift = meta->lock_shift; - - ulong remove_cnt = 0UL; - - for( long ele_idx=block_i0; ele_idx> lock_shift; - - fd_vinyl_key_t key; - int try_remove; - - /* Do a non-blocking query by ele_idx (not be key). We have to do - this direct because this is no standard API for this. This is - highly unlikely to ever block (but theoretically could if the - remove in a different thread has locked a probe chain that - touches elements in this thread). */ - - for(;;) { - FD_COMPILER_MFENCE(); - ulong ver0 = lock[ lock_idx ]; - FD_COMPILER_MFENCE(); - if( FD_LIKELY( !(ver0 & 1UL) ) ) { - - try_remove = (!!ele0[ ele_idx ].phdr.ctl) & (ele0[ ele_idx ].line_idx==(ULONG_MAX-1UL)); - key = ele0[ ele_idx ].phdr.key; - - FD_COMPILER_MFENCE(); - ulong ver1 = lock[ lock_idx ]; - FD_COMPILER_MFENCE(); - if( FD_LIKELY( ver0==ver1 ) ) break; - } - FD_SPIN_PAUSE(); - } - - /* If try_remove is not set, ele_idx either had no key it in or - had a pair entry. So we continue to the next slot. */ - - if( FD_LIKELY( !try_remove ) ) continue; - - /* At this point, we observed key had a tstone in the meta above. - So we try to remove it. It is possible (though extremely - unlikely for big sparse maps and the vanilla thread partitioning - here) that a remove on another thread got key first. So it is - okay if this fails. We have to use the parallel version of this - (even if it is highly unlikely to interfere with other threads) - for the same reason we had to use a non-blocking query above. */ - - fd_vinyl_meta_query_t query[1]; - remove_cnt += (ulong)!fd_vinyl_meta_remove( meta, &key, query, FD_MAP_FLAG_BLOCKING ); - } - - *_rlocal = remove_cnt; - -} FD_MAP_END { - - ulong * _rlocal = (ulong *) arg[0]; - ulong const * _rremote = (ulong const *)_r1; - - *_rlocal += *_rremote; - -} FD_REDUCE_END - -ulong -fd_vinyl_recover( fd_tpool_t * tpool, ulong t0, ulong t1, int level, - fd_vinyl_t * vinyl ) { - - fd_vinyl_meta_t * meta = vinyl->meta; - ulong line_cnt = vinyl->line_cnt; - - ulong ele_max = meta->ele_max; - ulong lock_cnt = meta->lock_cnt; - - /* Using all available threads, flush the lines and meta cache. We do - the meta flush locked so we don't confuse any concurrent meta - readers. This will claim any existing locks (e.g. the previous - meta writer died while holding a lock and the user didn't clean it - up before calling this). */ - - ulong reclaim_cnt; - - FD_FOR_ALL ( fd_vinyl_recover_line_task, tpool,t0,t1, 0L,(long)line_cnt, vinyl ); - FD_MAP_REDUCE( fd_vinyl_recover_reclaim_task, tpool,t0,t1, 0L,(long)lock_cnt, &reclaim_cnt, vinyl ); - FD_FOR_ALL ( fd_vinyl_recover_meta_flush_task, tpool,t0,t1, 0L,(long)ele_max, vinyl ); - FD_FOR_ALL ( fd_vinyl_recover_unlock_task, tpool,t0,t1, 0L,(long)lock_cnt, vinyl ); - - if( FD_UNLIKELY( reclaim_cnt ) ) FD_LOG_WARNING(( "reclaimed %lu locks (dead writer?); attempting to continue", reclaim_cnt )); - - /* FIXME: should this fail if it detects in progress io? */ - - /* If there is only 1 thread provided or the bstream past doesn't - have a valid partitioning, use the serial recovery algorithm */ - -t1 = t0 + 1UL; /* Turn off parallel recovery while it is untested */ - - if( FD_UNLIKELY( (t1-t0)<=1UL ) || - FD_UNLIKELY( fd_vinyl_recover_test( vinyl->io ) || - !FD_HAS_ATOMIC ) ) { - fd_vinyl_data_reset( tpool,t0,t1, level, vinyl->data ); - return fd_vinyl_recover_serial( vinyl ); - } - -# if FD_HAS_ATOMIC - - /* The parallel recovery of bstream partition may leave tstones in the - meta elements. To clean this up, we have two options. - - Option 1 (simplest and most robust): we parallel scan all the meta - elements in parallel for tstones and remove them. We might have to - do more than one pass because the removal of elements could mean - some elements are not placed well. This requires no scratch (and - thus is more robust against arbitrary erase / move patterns in the - recovery region). While it isn't any less algo inefficient - (because we parallel scan all the elements already to clear them), - it is practically less efficient for applications access patterns - that don't generate many tombstones and/or have pair_cnt<io ); - - ulong rtmp[4]; - ulong lock[2]; - - lock[0] = 0UL; - lock[1] = seq; - - FD_MAP_REDUCE( fd_vinyl_recover_part_task, tpool,t0,t1, 0L,(long)(t1-t0), rtmp, vinyl, lock ); - - ulong fail = rtmp[0]; - if( FD_UNLIKELY( fail ) ) { - FD_LOG_WARNING(( "parallel recovery failed; attempting serial recovery" )); - - /* Reset the meta from whatever messy state failed parallel recovery - left it */ - - FD_MAP_REDUCE( fd_vinyl_recover_reclaim_task, tpool,t0,t1, 0L,(long)lock_cnt, &reclaim_cnt, vinyl ); - FD_FOR_ALL ( fd_vinyl_recover_meta_flush_task, tpool,t0,t1, 0L,(long)ele_max, vinyl ); - FD_FOR_ALL ( fd_vinyl_recover_unlock_task, tpool,t0,t1, 0L,(long)lock_cnt, vinyl ); - - fd_vinyl_data_reset( tpool,t0,t1, level, vinyl->data ); - - return fd_vinyl_recover_serial( vinyl ); - } - - vinyl->pair_cnt = rtmp[1]; - vinyl->garbage_sz = rtmp[2]; - - ulong tstone_rem = rtmp[3]; - - while( tstone_rem ) { - FD_FOR_ALL( fd_vinyl_recover_meta_cleanup_task, tpool,t0,t1, 0L,(long)ele_max, rtmp, vinyl ); - tstone_rem -= rtmp[0]; - } - - /* Reset the data cache to clean up any scratch usage (currently none - but no reason to do earlier) */ - - fd_vinyl_data_reset( tpool,t0,t1, level, vinyl->data ); - - return seq; - -# endif /* FD_HAS_ATOMIC */ -} diff --git a/src/vinyl/fd_vinyl_recover_serial.c b/src/vinyl/fd_vinyl_recover_serial.c deleted file mode 100644 index 80939c0a7fd..00000000000 --- a/src/vinyl/fd_vinyl_recover_serial.c +++ /dev/null @@ -1,351 +0,0 @@ -/* This is included directly by fd_vinyl_recover.c */ - -ulong -fd_vinyl_recover_serial( fd_vinyl_t * vinyl ) { - - /* Iterate over the bstream's past to populate the meta. Note that - our caller flushed the meta cache, data cache and reset the cache - line eviction priorities to their default. */ - - fd_vinyl_meta_t * meta = vinyl->meta; - fd_vinyl_line_t * line = vinyl->line; - fd_vinyl_io_t * io = vinyl->io; - - ulong line_cnt = vinyl->line_cnt; - ulong pair_max = vinyl->pair_max; - - ulong io_seed = fd_vinyl_io_seed ( io ); - ulong seq_past = fd_vinyl_io_seq_past ( io ); - ulong seq_present = fd_vinyl_io_seq_present( io ); - - fd_vinyl_meta_ele_t * ele0 = meta->ele; - ulong ele_max = meta->ele_max; - ulong meta_seed = meta->seed; - ulong * lock = meta->lock; - int lock_shift = meta->lock_shift; - - ulong seq = seq_past; - ulong pair_cnt = 0UL; - ulong garbage_sz = 0UL; - - while( fd_vinyl_seq_lt( seq, seq_present ) ) { - - /* At this point, we've recovered [seq_past,seq) and still need - recover [seq,seq_present) (non-empty). Read the block at seq. */ - - fd_vinyl_bstream_block_t block[1]; - - fd_vinyl_io_read_imm( io, seq, block, FD_VINYL_BSTREAM_BLOCK_SZ ); - - ulong ctl = block->ctl; - - int type = fd_vinyl_bstream_ctl_type( ctl ); - - switch( type ) { - - case FD_VINYL_BSTREAM_CTL_TYPE_PAIR: { - - /* Notes: - - - It is okay if we are in a move (move block processing the - previous iteration already confirmed this is the proper pair. - - - We could rewind the bstream to seq on truncation - automatically but then we might have failed to recover the - most recent pair and thus have recovered to a state that does - not correspond to the bstream's past. We instead kick this - to the user to decide if they want to discard an incompletely - written pair or not. */ - - ulong pair_val_esz = fd_vinyl_bstream_ctl_sz( ctl ); - - ulong pair_sz = fd_vinyl_bstream_pair_sz( pair_val_esz ); - - if( FD_UNLIKELY( pair_sz > (seq_present-seq) ) ) { /* Wrapping safe */ - FD_LOG_WARNING(( "%016lx: truncated", seq )); - goto done; - } - - fd_vinyl_bstream_block_t _ftr[1]; - fd_vinyl_bstream_block_t * ftr = _ftr; - - if( pair_sz <= FD_VINYL_BSTREAM_BLOCK_SZ ) ftr = block; - else fd_vinyl_io_read_imm( io, seq + pair_sz - FD_VINYL_BSTREAM_BLOCK_SZ, ftr, FD_VINYL_BSTREAM_BLOCK_SZ ); - - char const * _err = fd_vinyl_bstream_pair_test_fast( io_seed, seq, block, ftr ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - /* At this point, we appear to have valid completely written pair. - Extract the pair metadata and determine if this replaces a - version we've already seen. Since this single threaded, we can - use the single threaded optimized meta APIs here. */ - - fd_vinyl_key_t const * pair_key = &block->phdr.key; - - ulong pair_memo = fd_vinyl_key_memo( meta_seed, pair_key ); - - ulong _ele_idx; /* avoid pointer escape */ - int err = fd_vinyl_meta_query_fast( ele0, ele_max, pair_key, pair_memo, &_ele_idx ); - ulong ele_idx = _ele_idx; - - if( FD_LIKELY( err==FD_VINYL_ERR_KEY ) ) { - - /* This is the first time we've seen pair key or pair key was - erased in a previous iteration (e.g. we most recently - processed an erase for pair key or we are in a move). If we - have room for pair key, insert it into the meta at ele_idx. */ - - if( FD_UNLIKELY( pair_cnt>=pair_max ) ) { - FD_LOG_WARNING(( "%016lx: increase pair_max", seq )); - goto done; - } - - ele0[ ele_idx ].memo = pair_memo; - ele0[ ele_idx ].phdr = block->phdr; - ele0[ ele_idx ].seq = seq; - ele0[ ele_idx ].line_idx = ULONG_MAX; /* key-val not in cache */ - - pair_cnt++; - - } else if( FD_LIKELY( !err ) ) { - - /* This is a more recent version of a pair we saw previously and - meta element ele_idx currently maps pair key to this previous - version. Mark the old version as garbage to collect in the - future and update the mapping to this version. */ - - ulong old_pair_ctl = ele0[ ele_idx ].phdr.ctl; - - ulong old_pair_val_esz = fd_vinyl_bstream_ctl_sz( old_pair_ctl ); - - garbage_sz += fd_vinyl_bstream_pair_sz( old_pair_val_esz ); - - //ele0[ ele_idx ].memo = pair_memo; /* already current */ - ele0[ ele_idx ].phdr = block->phdr; - ele0[ ele_idx ].seq = seq; - //ele0[ ele_idx ].line_idx = ULONG_MAX; /* already current */ - - } else { - - FD_LOG_WARNING(( "%016lx: corrupt meta", seq )); - goto done; - - } - - seq += pair_sz; - break; - - } - - case FD_VINYL_BSTREAM_CTL_TYPE_DEAD: { - - char const * _err = fd_vinyl_bstream_dead_test( io_seed, seq, block ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - /* At this point, we appear to have a valid DEAD block. Look up - the pair it erases. */ - - ulong pair_val_esz = fd_vinyl_bstream_ctl_sz( block->dead.phdr.ctl ); - - fd_vinyl_key_t const * pair_key = &block->dead.phdr.key; - - ulong pair_memo = fd_vinyl_key_memo( meta_seed, pair_key ); - - ulong _ele_idx; /* avoid pointer escape */ - int err = fd_vinyl_meta_query_fast( ele0, ele_max, pair_key, pair_memo, &_ele_idx ); - ulong ele_idx = _ele_idx;; - - if( FD_LIKELY( err==FD_VINYL_ERR_KEY ) ) { - - /* This erases the most recent version of pair key in the - bstream's antiquity or is a redundant erase block (which is - arguably an error but, as we can't tell the difference at - this point, we assume the more likely antiquity case). In - short, there's nothing to do but mark this block as garbage - to collect in the future. */ - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - - } else { - - /* This erases the most recent version of pair key we've - processed. Validate the erasure target is correct. If so, - mark this block and that version of pair key as garbage for - future collection and remove pair key from the meta. */ - - int bad_order = fd_vinyl_seq_ge( ele0[ ele_idx ].seq, seq ); - int bad_phdr = !!memcmp( &ele0[ ele_idx ].phdr, &block->dead.phdr, sizeof(fd_vinyl_bstream_phdr_t) ); - - if( FD_UNLIKELY( bad_order | bad_phdr ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, bad_order ? "unordered sequence" : "mismatched dead pair metadata" )); - goto done; - } - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ + fd_vinyl_bstream_pair_sz( pair_val_esz ); - - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, line, line_cnt, ele_idx ); - - FD_DCHECK_CRIT( pair_cnt, "corruption detected" ); - pair_cnt--; - - } - - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - - } - - case FD_VINYL_BSTREAM_CTL_TYPE_MOVE: { - - if( FD_UNLIKELY( 2UL*FD_VINYL_BSTREAM_BLOCK_SZ > (seq_present-seq) ) ) { /* Wrapping safe */ - FD_LOG_WARNING(( "%016lx: truncated", seq )); - goto done; - } - - fd_vinyl_bstream_block_t dst[1]; - - fd_vinyl_io_read_imm( io, seq + FD_VINYL_BSTREAM_BLOCK_SZ, dst, FD_VINYL_BSTREAM_BLOCK_SZ ); - - char const * _err = fd_vinyl_bstream_move_test( io_seed, seq, block, dst ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - /* At this point, we appear to have a valid move. Technically, a - move is an atomic "erase pair src_key if any, erase pair - dst_key if any, insert pair dst_key with the info src_info_old - and val src_val_new" where src_val_new is typically the same as - src_val_old, but, strictly speaking, doesn't have to be. - - We do the "erase pair src_key if any" part of the move here. - The next iteration will handle rest naturally (including doing - more extensive validation on the new pair_dst). Note that if - the next iteration detects the new pair dst is invalid, it will - fail recovery in the middle of the move. So applications - should be very wary of using a partial recovery as such can - break move atomicity. */ - - ulong src_val_esz = fd_vinyl_bstream_ctl_sz( block->move.src.ctl ); - fd_vinyl_key_t const * src_key = &block->move.src.key; - - ulong src_memo = fd_vinyl_key_memo( meta_seed, src_key ); - - ulong _ele_idx; /* avoid pointer escape */ - int err = fd_vinyl_meta_query_fast( ele0, ele_max, src_key, src_memo, &_ele_idx ); - ulong ele_idx = _ele_idx; - - if( FD_LIKELY( err==FD_VINYL_ERR_KEY ) ) { - - /* This move erases the most recent version of pair src_key in - the bstream's antiquity or is a redundant move block (which - is arguably an error but, as we can't tell the difference at - this point, we assume the more likely antiquity case). In - short, there's nothing to do but mark this block as garbage - to collect in the future. */ - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - - } else { - - /* This move erases the most recent version of pair src_key - we've processed. Validate the erasure target is correct. If - so, mark this block and this version of pair src_key as - garbage for future collection and remove pair src_key from - the meta. */ - - int bad_order = fd_vinyl_seq_ge( ele0[ ele_idx ].seq, seq ); - int bad_cnt = !pair_cnt; - int bad_phdr = !!memcmp( &ele0[ ele_idx ].phdr, &block->move.src, sizeof(fd_vinyl_bstream_phdr_t) ); - - if( FD_UNLIKELY( bad_order | bad_cnt | bad_phdr ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, bad_order ? "unordered sequence" : - bad_cnt ? "corrupt meta" : - "mismatched move src metadata" )); - goto done; - } - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ + fd_vinyl_bstream_pair_sz( src_val_esz ); - - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, line, line_cnt, ele_idx ); - - pair_cnt--; - - } - - /* At this point, we've handled the "erase old src if any" part of - the move. The next iteration will handle the "erase old dst if - any" and the "insert new dst" part of the move. We know there - will be a next iteration for a type pair object with the - appropriate mojo because of the checks we've already done. So - moves behave atomically from the point of view of the - application when fully recovered. */ - - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - - } - - case FD_VINYL_BSTREAM_CTL_TYPE_PART: { - - if( FD_UNLIKELY( fd_vinyl_seq_ne( block->part.seq, seq ) ) ) { - FD_LOG_WARNING(( "%016lx: unexpected part seq", seq )); - goto done; - } - - char const * _err = fd_vinyl_bstream_part_test( io_seed, seq, block ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - garbage_sz += FD_VINYL_BSTREAM_BLOCK_SZ; - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - - } - - case FD_VINYL_BSTREAM_CTL_TYPE_ZPAD: { - - char const * _err = fd_vinyl_bstream_zpad_test( io_seed, seq, block ); - if( FD_UNLIKELY( _err ) ) { - FD_LOG_WARNING(( "%016lx: %s", seq, _err )); - goto done; - } - - /* Note: zpad blocks aren't included in garbage_sz because we - don't control when they get created (and thus can't easily - update garbage_sz to account for them when they are created). */ - - seq += FD_VINYL_BSTREAM_BLOCK_SZ; - break; - - } - - default: - FD_LOG_WARNING(( "%016lx: unknown type (%x)", seq, (uint)type )); - goto done; - } - } - -done: - - /* At this point, the meta is populated appropriately up to seq. - Update the vinyl state and return. If we did not get to - seq_present, we log a warning. */ - - vinyl->pair_cnt = pair_cnt; - vinyl->garbage_sz = garbage_sz; - - if( FD_UNLIKELY( fd_vinyl_seq_ne( seq, seq_present ) ) ) - FD_LOG_WARNING(( "recovery failed, recovered [%016lx,%016lx)/%lu, unrecovered [%016lx,%016lx)/%lu", - seq_past, seq, seq-seq_past, seq, seq_present, seq_present-seq )); - - return seq; -} diff --git a/src/vinyl/io/Local.mk b/src/vinyl/io/Local.mk deleted file mode 100644 index b112392beaf..00000000000 --- a/src/vinyl/io/Local.mk +++ /dev/null @@ -1,8 +0,0 @@ -ifdef FD_HAS_LZ4 -$(call add-hdrs,fd_vinyl_io.h) -$(call add-objs,fd_vinyl_io fd_vinyl_io_bd fd_vinyl_io_mm,fd_vinyl) -$(call make-unit-test,test_vinyl_io_bd,test_vinyl_io_bd,fd_vinyl fd_tango fd_util_test fd_util) -$(call make-unit-test,test_vinyl_io_mm,test_vinyl_io_mm,fd_vinyl fd_tango fd_util_test fd_util) -$(call run-unit-test,test_vinyl_io_bd) -$(call run-unit-test,test_vinyl_io_mm) -endif diff --git a/src/vinyl/io/fd_vinyl_io.c b/src/vinyl/io/fd_vinyl_io.c deleted file mode 100644 index 77e312edfb4..00000000000 --- a/src/vinyl/io/fd_vinyl_io.c +++ /dev/null @@ -1,229 +0,0 @@ -#include "fd_vinyl_io.h" -#include - -/* Using a separate translation unit for fini is a little silly but - compiler won't inline stuff with heavy operations like logging and - we don't want to give up logging edge cases with constructors. */ - -void * -fd_vinyl_io_fini( fd_vinyl_io_t * io ) { - if( !io ) { - FD_LOG_WARNING(( "NULL io" )); - return NULL; - } - return io->impl->fini( io ); -} - -ulong -fd_vinyl_io_spad_est( void ) { - return 2UL*fd_vinyl_bstream_pair_sz( (ulong)LZ4_COMPRESSBOUND( (int)FD_VINYL_VAL_MAX ) ); -} - -ulong -fd_vinyl_io_append_pair_raw( fd_vinyl_io_t * io, - fd_vinyl_key_t const * key, - fd_vinyl_info_t const * info, - void const * val ) { - - /* Allocate scratch to hold the formatted pair */ - - ulong val_sz = (ulong)info->val_sz; - FD_DCHECK_CRIT( val_sz<=FD_VINYL_VAL_MAX, "corruption detected" ); - - ulong pair_sz = fd_vinyl_bstream_pair_sz( val_sz ); - uchar * pair = (uchar *)fd_vinyl_io_alloc( io, pair_sz, FD_VINYL_IO_FLAG_BLOCKING ); - - uchar * dst = pair; - ulong dst_rem = pair_sz; (void)dst_rem; - - /* Gather the pair header */ - - fd_vinyl_bstream_phdr_t * phdr = (fd_vinyl_bstream_phdr_t *)dst; - - phdr->ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_RAW, val_sz ); - phdr->key = *key; - phdr->info = *info; - - dst += sizeof(fd_vinyl_bstream_phdr_t); - dst_rem -= sizeof(fd_vinyl_bstream_phdr_t); - - /* Gather the pair value */ - - if( val_sz ) memcpy( dst, val, val_sz ); - - dst += val_sz; - dst_rem -= val_sz; - - fd_vinyl_bstream_pair_hash( fd_vinyl_io_seed( io ), (fd_vinyl_bstream_block_t *)pair ); - - return fd_vinyl_io_append( io, pair, pair_sz ); -} - -ulong -fd_vinyl_io_append_dead( fd_vinyl_io_t * io, - fd_vinyl_bstream_phdr_t const * phdr, - void const * info, - ulong info_sz ) { - - if( !info ) info_sz = 0UL; - FD_DCHECK_CRIT( info_sz<=FD_VINYL_BSTREAM_DEAD_INFO_MAX, "corruption detected" ); - - fd_vinyl_bstream_block_t * block = (fd_vinyl_bstream_block_t *) - fd_vinyl_io_alloc( io, FD_VINYL_BSTREAM_BLOCK_SZ, FD_VINYL_IO_FLAG_BLOCKING ); - - memset( block, 0, FD_VINYL_BSTREAM_BLOCK_SZ ); /* bulk zero */ - - block->dead.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_DEAD, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - FD_VINYL_BSTREAM_BLOCK_SZ ); - block->dead.seq = io->seq_future; - block->dead.phdr = *phdr; - block->dead.info_sz = info_sz; - if( info_sz ) memcpy( block->dead.info, info, info_sz ); - - fd_vinyl_bstream_block_hash( fd_vinyl_io_seed( io ), block ); - - return fd_vinyl_io_append( io, block, FD_VINYL_BSTREAM_BLOCK_SZ ); -} - -ulong -fd_vinyl_io_append_move( fd_vinyl_io_t * io, - fd_vinyl_bstream_phdr_t const * src, - fd_vinyl_key_t const * dst, - void const * info, - ulong info_sz ) { - - if( !info ) info_sz = 0UL; - FD_DCHECK_CRIT( info_sz<=FD_VINYL_BSTREAM_MOVE_INFO_MAX, "corruption detected" ); - - fd_vinyl_bstream_block_t * block = (fd_vinyl_bstream_block_t *) - fd_vinyl_io_alloc( io, FD_VINYL_BSTREAM_BLOCK_SZ, FD_VINYL_IO_FLAG_BLOCKING ); - - memset( block, 0, FD_VINYL_BSTREAM_BLOCK_SZ ); /* bulk zero */ - - block->move.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_MOVE, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - FD_VINYL_BSTREAM_BLOCK_SZ ); - block->move.seq = io->seq_future; - block->move.src = *src; - block->move.dst = *dst; - block->move.info_sz = info_sz; - -# if 0 /* Note: with BLOCK_SZ==128, MOVE_INFO_MAX=0 so there's no move.info field due to language limitations */ - if( info_sz ) memcpy( block->move.info, info, info_sz ); -# endif - - fd_vinyl_bstream_block_hash( fd_vinyl_io_seed( io ), block ); - - return fd_vinyl_io_append( io, block, FD_VINYL_BSTREAM_BLOCK_SZ ); -} - -ulong -fd_vinyl_io_append_part( fd_vinyl_io_t * io, - ulong seq0, - ulong dead_cnt, - ulong move_cnt, - void const * info, - ulong info_sz ) { - - if( !info ) info_sz = 0UL; - FD_DCHECK_CRIT( info_sz<=FD_VINYL_BSTREAM_PART_INFO_MAX, "corruption detected" ); - - fd_vinyl_bstream_block_t * block = (fd_vinyl_bstream_block_t *) - fd_vinyl_io_alloc( io, FD_VINYL_BSTREAM_BLOCK_SZ, FD_VINYL_IO_FLAG_BLOCKING ); - - memset( block, 0, FD_VINYL_BSTREAM_BLOCK_SZ ); /* bulk zero */ - - block->part.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PART, - FD_VINYL_BSTREAM_CTL_STYLE_RAW, - FD_VINYL_BSTREAM_BLOCK_SZ ); - block->part.seq = io->seq_future; - block->part.seq0 = seq0; - block->part.dead_cnt = dead_cnt; - block->part.move_cnt = move_cnt; - block->part.info_sz = info_sz; - if( info_sz ) memcpy( block->part.info, info, info_sz ); - - fd_vinyl_bstream_block_hash( fd_vinyl_io_seed( io ), block ); - - return fd_vinyl_io_append( io, block, FD_VINYL_BSTREAM_BLOCK_SZ ); -} - -ulong -fd_vinyl_io_append_pair_inplace( fd_vinyl_io_t * io, - int style, - fd_vinyl_bstream_phdr_t * phdr, - int * _style, - ulong * _val_esz ) { - - ulong val_sz = (ulong)phdr->info.val_sz; - - FD_DCHECK_CRIT( val_sz <= FD_VINYL_VAL_MAX, "corruption detected" ); - - ulong pair_sz = fd_vinyl_bstream_pair_sz( val_sz ); - - ulong phdr_ctl = phdr->ctl; - - FD_DCHECK_CRIT( fd_vinyl_bstream_ctl_style( phdr_ctl )==FD_VINYL_BSTREAM_CTL_STYLE_RAW, "corruption detected" ); - FD_DCHECK_CRIT( fd_vinyl_bstream_ctl_sz ( phdr_ctl )==val_sz, "corruption detected" ); - - switch( style ) { - - case FD_VINYL_BSTREAM_CTL_STYLE_RAW: break; - - case FD_VINYL_BSTREAM_CTL_STYLE_LZ4: { - - /* If the pair is already small enough, no point in compressing. - Otherwise, allocate scratch from the io append spad for the worst - case compressed size and compress the pair val into it. If - compression fails (shouldn't given use of LZ4_COMPRESSBOUND) or - the value doesn't compress enough to make a difference in bstream - usage, free the scratch allocation and append the uncompressed - version. */ - - if( FD_UNLIKELY( val_sz<=FD_VINYL_BSTREAM_LZ4_VAL_THRESH ) ) break; - - ulong cval_max = (ulong)LZ4_COMPRESSBOUND( (int)val_sz ); - ulong cpair_max = fd_vinyl_bstream_pair_sz( cval_max ); - fd_vinyl_bstream_phdr_t * cphdr = (fd_vinyl_bstream_phdr_t *)fd_vinyl_io_alloc( io, cpair_max, FD_VINYL_IO_FLAG_BLOCKING ); - - ulong cval_sz = (ulong)LZ4_compress_default( (char const *)(phdr+1), (char *)(cphdr+1), (int)val_sz, (int)cval_max ); - ulong cpair_sz = fd_vinyl_bstream_pair_sz( cval_sz ); - - if( FD_UNLIKELY( (!cval_sz) | (cpair_sz>=pair_sz) ) ) { - fd_vinyl_io_trim( io, cpair_max ); - break; - } - - /* At this point, we usefully LZ4 compressed the pair val. Trim - the scratch allocation to compressed pair size, prepend the pair - header, clear any zero padding, append the pair footer and start - appending the compressed version to the bstream. */ - - fd_vinyl_io_trim( io, cpair_max - cpair_sz ); - - cphdr->ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_PAIR, FD_VINYL_BSTREAM_CTL_STYLE_LZ4, cval_sz ); - cphdr->key = phdr->key; - cphdr->info = phdr->info; - - fd_vinyl_bstream_pair_hash( fd_vinyl_io_seed( io ), (fd_vinyl_bstream_block_t *)cphdr ); - - *_style = FD_VINYL_BSTREAM_CTL_STYLE_LZ4; - *_val_esz = cval_sz; - return fd_vinyl_io_append( io, cphdr, cpair_sz ); - - } - - default: - FD_LOG_CRIT(( "unsupported style" )); - - } - - /* Append in place */ - - fd_vinyl_bstream_pair_hash( fd_vinyl_io_seed( io ), (fd_vinyl_bstream_block_t *)phdr ); - - *_style = FD_VINYL_BSTREAM_CTL_STYLE_RAW; - *_val_esz = val_sz; - return fd_vinyl_io_append( io, phdr, pair_sz ); -} diff --git a/src/vinyl/io/fd_vinyl_io.h b/src/vinyl/io/fd_vinyl_io.h deleted file mode 100644 index 4fbbc1f9777..00000000000 --- a/src/vinyl/io/fd_vinyl_io.h +++ /dev/null @@ -1,557 +0,0 @@ -#ifndef HEADER_fd_src_vinyl_io_fd_vinyl_io_h -#define HEADER_fd_src_vinyl_io_fd_vinyl_io_h - -/* A fd_vinyl_io_t reads from / appends to a bstream stored in some - physical layer (typically slow and non-volatile). Supports massive - numbers of async concurrent reads and appends and the ability to - recover from unexpected interrupts (Ctrl-C, power failures, etc). To - accommodate the myriad of different styles of physical layers and - interfaces, the API is run time plugin friendly. Summary of - operations: - - read_imm: blocking read a contiguous range of blocks in the - bstream's past. Mostly used for iterating over a bstream's past. - - read: start reading a contiguous range of blocks in the bstream's - past. The caller promises the range to read is contiguous in the - underlying physical storage. - - poll: finish an outstanding read. Outstanding reads can complete - in an arbitrary order. All reads must be finished by poll but note - that it is possible to detect a read is complete out-of-band too - (for speculative processing). - - append: start appending a set of blocks to the end of the bstream's - present (moving blocks from the bstream's future to the bstream's - present). The blocks will be contiguous in the underlying storage. - The blocks must be suitably aligned and with a lifetime until the - next commit. - - commit: finish all outstanding appends, moving all blocks in the - bstream's present to the bstream's past. This will empty the io's - append scratch pad. The underlying implementation is free to - process outstanding appends in any order (and free to interleave - them arbitrarily with outstanding reads). - - hint: indicates the next sz worth of blocks appended to the bstream - must be contiguous in the physical storage. - - alloc: allocate memory from the io's append scratch pad. These - allocations will have a suitable alignment for append and a - lifetime until the next commit. This may trigger a commit of - outstanding appends if there isn't enough scratch pad free. - - copy: append a contiguous range of blocks from the bstream's past - to the end of the bstream's present. May commit outstanding - appends. - - forget: forget all blocks before a given sequence number, moving - blocks from the bstream's past to the bstream's antiquity. The - caller can only forget up to the bstream's present. - - rewind: move blocks from the bstream's past (and potentially - antiquity) to the bstream's future. The bstream must have an empty - present (i.e. no appends in progress) and no reads in progress. - This allows, for example, on recovery, a multi-block pair that was - incompletely written to be cleaned up. - - sync: update the range for the bstream past where recovery will - resume. This moves all blocks in the bstream's antiquity to end of - the bstream's future. */ - -/* FIXME: consider a query to get how many reads are outstanding? (with - this, rewind and forget could be complete generic). */ - -#include "../bstream/fd_vinyl_bstream.h" - -/* FD_VINYL_IO_TYPE_* identifies which IO implementation is in use. */ - -#define FD_VINYL_IO_TYPE_MM (0) /* memory mapped */ -#define FD_VINYL_IO_TYPE_BD (1) /* synchronous blocking */ -#define FD_VINYL_IO_TYPE_WD (2) /* async O_DIRECT write (specialized) */ -#define FD_VINYL_IO_TYPE_UR (3) /* async io_uring */ - -/* FD_VINYL_IO_FLAG_* are flags used by various vinyl IO APIs */ - -#define FD_VINYL_IO_FLAG_BLOCKING (1) /* Okay to block the caller */ - -/* A fd_vinyl_io_rd_t describes a read request to the underlying I/O - implementation to read [seq,seq+sz) (cyclic) from the bstream's past - into dst. seq, dst and sz should be FD_VINYL_BSTREAM_BLOCK_SZ - aligned. Any failure encountered while reading should FD_LOG_CRIT - (just like reading an invalid memory address will seg fault). - Underlying I/O implementations can add other information to this - structure as necessary. ctx is an arbitrary user defined value. */ - -#define FD_VINYL_IO_READ_SZ (64UL) - -struct fd_vinyl_io_rd { - ulong ctx; - ulong seq; - void * dst; - ulong sz; - uchar _[ FD_VINYL_IO_READ_SZ - 32UL ]; -}; - -typedef struct fd_vinyl_io_rd fd_vinyl_io_rd_t; - -/* fd_vinyl_io_t is an opaque handle of a fd_vinyl_io instance. Some - details are exposed to facilitate inlining in high performance - contexts. */ - -struct fd_vinyl_io_private; -typedef struct fd_vinyl_io_private fd_vinyl_io_t; - -typedef void (*fd_vinyl_io_func_read_imm_t)( fd_vinyl_io_t * io, ulong seq, void * dst, ulong sz ); -typedef void (*fd_vinyl_io_func_read_t )( fd_vinyl_io_t * io, fd_vinyl_io_rd_t * rd ); -typedef int (*fd_vinyl_io_func_poll_t )( fd_vinyl_io_t * io, fd_vinyl_io_rd_t ** _rd, int flags ); -typedef ulong (*fd_vinyl_io_func_append_t )( fd_vinyl_io_t * io, void const * src, ulong sz ); -typedef int (*fd_vinyl_io_func_commit_t )( fd_vinyl_io_t * io, int flags ); -typedef ulong (*fd_vinyl_io_func_hint_t )( fd_vinyl_io_t * io, ulong sz ); -typedef void * (*fd_vinyl_io_func_alloc_t )( fd_vinyl_io_t * io, ulong sz, int flags ); -typedef ulong (*fd_vinyl_io_func_copy_t )( fd_vinyl_io_t * io, ulong seq, ulong sz ); -typedef void (*fd_vinyl_io_func_forget_t )( fd_vinyl_io_t * io, ulong seq ); -typedef void (*fd_vinyl_io_func_rewind_t )( fd_vinyl_io_t * io, ulong seq ); -typedef int (*fd_vinyl_io_func_sync_t )( fd_vinyl_io_t * io, int flags ); -typedef void * (*fd_vinyl_io_func_fini_t )( fd_vinyl_io_t * io ); - -struct fd_vinyl_io_impl { - fd_vinyl_io_func_read_imm_t read_imm; - fd_vinyl_io_func_read_t read; - fd_vinyl_io_func_poll_t poll; - fd_vinyl_io_func_append_t append; - fd_vinyl_io_func_commit_t commit; - fd_vinyl_io_func_hint_t hint; - fd_vinyl_io_func_alloc_t alloc; - fd_vinyl_io_func_copy_t copy; - fd_vinyl_io_func_forget_t forget; - fd_vinyl_io_func_rewind_t rewind; - fd_vinyl_io_func_sync_t sync; - fd_vinyl_io_func_fini_t fini; -}; - -typedef struct fd_vinyl_io_impl fd_vinyl_io_impl_t; - -struct fd_vinyl_io_private { - int type; - ulong seed; - ulong seq_ancient; /* FD_VINYL_BSTREAM_BLOCK_SZ multiple */ - ulong seq_past; /* " */ - ulong seq_present; /* " */ - ulong seq_future; /* " */ - ulong spad_max; /* " */ - ulong spad_used; /* " */ - fd_vinyl_io_impl_t * impl; /* implementation specific funcs */ - /* io implementation specific details follow */ -}; - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_io_* return the current value of the eponymous io field. - Assumes io is valid. For all but type and seed, the return value is - a FD_VINYL_BSTREAM_BLOCK_SZ multiple. Note that we don't have a - generic notion of dev_max or dev_free as such is not a well defined - concept. Individual IO implementations can provide them as - appropriate though. */ - -FD_FN_PURE static inline int fd_vinyl_io_type( fd_vinyl_io_t const * io ) { return io->type; } - -FD_FN_PURE static inline ulong fd_vinyl_io_seed( fd_vinyl_io_t const * io ) { return io->seed; } - -FD_FN_PURE static inline ulong fd_vinyl_io_seq_ancient( fd_vinyl_io_t const * io ) { return io->seq_ancient; } -FD_FN_PURE static inline ulong fd_vinyl_io_seq_past ( fd_vinyl_io_t const * io ) { return io->seq_past; } -FD_FN_PURE static inline ulong fd_vinyl_io_seq_present( fd_vinyl_io_t const * io ) { return io->seq_present; } -FD_FN_PURE static inline ulong fd_vinyl_io_seq_future ( fd_vinyl_io_t const * io ) { return io->seq_future; } - -FD_FN_PURE static inline ulong fd_vinyl_io_spad_max ( fd_vinyl_io_t const * io ) { return io->spad_max; } -FD_FN_PURE static inline ulong fd_vinyl_io_spad_used( fd_vinyl_io_t const * io ) { return io->spad_used; } -FD_FN_PURE static inline ulong fd_vinyl_io_spad_free( fd_vinyl_io_t const * io ) { return io->spad_max - io->spad_used; } - -FD_FN_PURE static inline ulong fd_vinyl_io_dev_used( fd_vinyl_io_t const * io ) { return io->seq_future - io->seq_ancient; } - -/* fd_vinyl_io_read_imm does an immediate (blocking) read of - [seq,seq+dst_sz) (cyclic) from io's bstream's past into dst. Assumes - there are no reads currently posted on io. Retains no interest in - dst. seq and sz should be FD_VINYL_BSTREAM_BLOCK_SZ aligned. This - is used mostly for sequential iterating over a bstream's past (i.e. - serial recovery and discovering partitions for parallel recovery). */ - -static inline void -fd_vinyl_io_read_imm( fd_vinyl_io_t * io, - ulong seq, - void * dst, - ulong sz ) { - io->impl->read_imm( io, seq, dst, sz ); -} - -/* fd_vinyl_io_read starts the executing the read command rd. That is, - start reading bstream bytes [seq,seq+sz) (cyclic) into dst. seq and - sz should be FD_VINYL_BSTREAM_BLOCK_SZ aligned. Further, - [seq,seq+sz) should be in the bstream's past and the region to read - should be stored contiguously in the underlying storage. - - On entry, the caller should have ownership of rd and rd->dst. The io - has ownership of these return and a read interest in bstream bytes - [seq,seq_sz) (cyclic). The ownership of these will be returned to - the caller and the read interest will end when poll returns the - request. */ - -static inline void -fd_vinyl_io_read( fd_vinyl_io_t * io, - fd_vinyl_io_rd_t * rd ) { - io->impl->read( io, rd ); -} - -/* fd_vinyl_io_poll checks if any outstanding reads are complete. Reads - can complete in any order by the I/O layer. flags is a bit-or of - FD_VINYL_IO_FLAGs. BLOCKING indicates the call is allowed to block - the caller (the io layer promises the call cannot fail from the - caller's point of view). Returns FD_VINYL_SUCCESS if a read complete - (*_rd will point to the read command ended with the ownership and - read interested as described above), FD_VINYL_ERR_EMPTY if there are - no commands pending (*_rd will be NULL) and FD_VINYL_ERR_AGAIN if - none of the posted commands are ready (*_rd will be NULL). AGAIN is - only possible for a non-blocking call). */ - -static inline int -fd_vinyl_io_poll( fd_vinyl_io_t * io, - fd_vinyl_io_rd_t ** _rd, - int flags ) { - return io->impl->poll( io, _rd, flags ); -} - -/* fd_vinyl_io_append starts appending sz bytes at src to the bstream. - src and sz should be FD_VINYL_BSTREAM_BLOCK_SZ aligned. Returns - bstream sequence number seq_append where the data is being appended. - io will have a read interest in src until the next commit. This - moves blocks from the bstream's future to the bstream's present. On - commit, the region [seq_future_before,seq_append) (cyclic) will be - filled with zero padding if the I/O implementation requires it to - keep the append contiguous in the physical store (this region will be - empty if covered by a previous hint or if this is an append of a - single block) and the region [seq_append,seq_future_after) (cyclic) - will be filled with the appended info. - - fd_vinyl_io_commit moves all blocks in the bstream's present to the - bstream's past (i.e. sets seq_present to seq_future). flags is a - bit-of FD_VINYL_IO_FLAGs. If BLOCKING is set, this is allowed to - block the caller. Returns FD_VINYL_SUCCESS (0) on success and - FD_VINYL_ERR_AGAIN (negative) if commit could not be completed - immediately (only possible for a non-blocking call). commit empties - the io append scratch pad on success. - - fd_vinyl_io_hint indicates the next sz bytes to append must be - contiguous in the bstream. This can move blocks from the bstream's - future to the bstream's present. Returns (the potentially updated) - seq_future. On commit, the region - [seq_future_before,seq_future_after) (cyclic) will be filled with - zero padding (this region will be empty if covered by a previous - hint) and the region [seq_future_after,seq_future_after+sz) (cyclic) - will contiguous in the physical storage. This is useful for grouping - sets of blocks from different memory regions on the host that must be - written contiguously from a protocol point of view (e.g. a move - control block and the pair that follows it). - - fd_vinyl_io_alloc returns a pointer to sz bytes of - FD_VINYL_BSTREAM_BLOCK_SZ aligned memory suitable allocated from io's - append scratch pad. flags is a bit-or FD_VINYL_IO_FLAG_*. BLOCKING - indicates the call is allowed to block the caller. If a non-blocking - call, will return NULL if there is no suitable memory at this time. - Will never return NULL for a blocking call. The lifetime of the - returned pointer is the lesser of the next append, next commit, the - next alloc or the io. sz should be FD_VINYL_BSTREAM_BLOCK_SZ aligned - and at most io's spad_max. This may do a commit to free up scratch - pad memory if necessary (moving blocks from the present to the past). - - fd_vinyl_io_trim trims sz bytes from the end of the most recent - fd_vinyl_io_alloc. sz should be FD_VINYL_BSTREAM_BLOCK_SZ aligned - and at most the size of the most recent alloc. - - fd_vinyl_io_copy starts appending a copy of the sz bytes at seq in - the bstream's past to the bstream. seq and sz should be - FD_VINYL_BSTREAM_BLOCK_SZ aligned. [seq,seq+sz) (cyclic) should be - in the bstream's past. io will have a read interest in this region - until the next commit. This will do a _blocking_ commit to free up - scratch pad memory if necessary (moving blocks from the present to - the past). FIXME: consider non-blocking copy support? (copy would - need a flags args). - - None of these can fail from the caller's perspective (they will all - FD_LOG_CRIT if anything goes wrong ... much like accessing invalid - memory will seg fault). */ - -static inline ulong -fd_vinyl_io_append( fd_vinyl_io_t * io, - void const * src, - ulong sz ) { - return io->impl->append( io, src, sz ); -} - -static inline int -fd_vinyl_io_commit( fd_vinyl_io_t * io, - int flags ) { - return io->impl->commit( io, flags ); -} - -static inline ulong -fd_vinyl_io_hint( fd_vinyl_io_t * io, - ulong sz ) { - return io->impl->hint( io, sz ); -} - -static inline void * -fd_vinyl_io_alloc( fd_vinyl_io_t * io, - ulong sz, - int flags ) { - return io->impl->alloc( io, sz, flags ); -} - -static inline void -fd_vinyl_io_trim( fd_vinyl_io_t * io, - ulong sz ) { - io->spad_used -= sz; -} - -static inline ulong -fd_vinyl_io_copy( fd_vinyl_io_t * io, - ulong seq, - ulong sz ) { - return io->impl->copy( io, seq, sz ); -} - -/* fd_vinyl_io_forget moves [seq_past,seq) (cyclic) from the bstream's - past to the bstream's antiquity, setting seq_past to seq. As such, - seq should be in [seq_past,seq_present] (cyclic) and - FD_VINYL_BSTREAM_BLOCK_SZ aligned. There should be no reads, copies - or appends in progress. Cannot fail from the caller's perspective - (will FD_LOG_CRIT if anything goes wrong). - - IMPORTANT SAFETY TIP! Though the bstream has been updated from the - caller's point of view, the bstream needs to be sync'd for recover to - start from the new seq_past. */ - -static inline void -fd_vinyl_io_forget( fd_vinyl_io_t * io, - ulong seq ) { - io->impl->forget( io, seq ); -} - -/* fd_vinyl_io_rewind moves blocks [seq,seq_present) (cyclic) from the - bstream's past to the bstream's future (updating seq_ancient and - seq_past as necessary). There should be no reads, copies or appends - in progress. seq should at most seq_present (cyclic) and - FD_VINYL_BSTREAM_BLOCK_SZ aligned. Cannot fail from the caller's - perspective (will FD_LOG_CRIT if anything goes wrong). - - IMPORTANT SAFETY TIP! Though the bstream has been updated from the - caller's point of view, the bstream needs to be sync'd for recovery - to account for the rewind (and this is probably more critical than - forget because appends will start modifying the bstream blocks that - recovery would be expecting to be in the pre-rewind state). */ - -static inline void -fd_vinyl_io_rewind( fd_vinyl_io_t * io, - ulong seq ) { - io->impl->rewind( io, seq ); -} - -/* fd_vinyl_io_sync moves [seq_ancient,seq_past) (cyclic) from the - bstream's antiquity to the end of the bstream's future, setting - seq_ancient to seq_past. It promises the caller the bstream's past - is fully written and that the bstream's past region is what recovery - will use to recover the bstream's key-val state at seq_present. - flags is a bit-or of FD_VINYL_IO_FLAGs. BLOCKING indicates the call - is allowed to block the caller. Returns FD_VINYL_SUCCESS (0) on - success and a FD_VINYL_ERR_AGAIN (negative) if the call would block - the caller (only possible for a non-blocking call). */ -/* FIXME: consider allowing new user info to be passed? */ - -static inline int -fd_vinyl_io_sync( fd_vinyl_io_t * io, - int flags ) { - return io->impl->sync( io, flags ); -} - -/* fd_vinyl_io_fini tears down io, returning the memory region used to - hold the I/O implementation state. Implicitly completes any - in-progress reads and cancels any in-progress appends (and thus can - block the caller). - - IMPORTANT SAFETY TIP! This does _not_ sync the bstream first (e.g. - if an application is tearing down due to an anomalous condition, it - may not want to sync on fini so that it can recover from a known good - point). */ - -void * -fd_vinyl_io_fini( fd_vinyl_io_t * io ); - -/* Helpers ************************************************************/ - -/* fd_vinyl_io_spad_est() returns estimate of the smallest scratch pad - size required for most applications. Specifically, this returns: - - 2 pair_sz( LZ4_COMPRESSBOUND( VAL_MAX ) ) - - so that it is possible to load a object footprint into the scratch - pad and then have a worst case scratch memory for compression to - re-encode the object. */ - -FD_FN_CONST ulong fd_vinyl_io_spad_est( void ); - -/* fd_vinyl_io_append_* are helper functions that start appending the - given info, appropriately formatted and hashed, to io's bstream. - There is no excess requirements for alignment. They do no input - argument checking. On return, io retains no interest in the given - info (that is, they use io's scratch memory and thus can trigger an - io commit to move blocks from the bstream's present to the bstream's - past if there isn't enough scratch pad free). They return the - bstream sequence number where the data is being appended. They - cannot fail from the caller's perspective (they will FD_LOG_CRIT if - anything goes awry). */ - -ulong -fd_vinyl_io_append_pair_raw( fd_vinyl_io_t * io, - fd_vinyl_key_t const * key, /* pair key */ - fd_vinyl_info_t const * info, /* pair info */ - void const * val ); /* contains info->val_sz bytes, in [0,FD_VINYL_VAL_MAX] */ - -ulong -fd_vinyl_io_append_dead( fd_vinyl_io_t * io, - fd_vinyl_bstream_phdr_t const * phdr, /* pair header of erased pair */ - void const * info, /* contains info_sz bytes, info_sz treated as 0 if NULL */ - ulong info_sz ); /* in [0,FD_VINYL_BSTREAM_DEAD_INFO_MAX] */ - -ulong -fd_vinyl_io_append_move( fd_vinyl_io_t * io, - fd_vinyl_bstream_phdr_t const * src, /* pair header of src pair */ - fd_vinyl_key_t const * dst, /* src pair getting renamed to dst or is replacing dst */ - void const * info, /* contains info_sz bytes, info_sz treated as 0 if NULL */ - ulong info_sz ); /* in [0,FD_VINYL_BSTREAM_MOVE_INFO_MAX] */ - -ulong -fd_vinyl_io_append_part( fd_vinyl_io_t * io, - ulong seq_prev, /* should be a part before seq or seq */ - ulong dead_cnt, /* number of dead blocks in the partition */ - ulong move_cnt, /* number of move blocks in the partition */ - void const * info, /* contains info_sz bytes, info_sz treated as 0 if NULL */ - ulong info_sz ); /* in [0,FD_VINYL_BSTREAM_PART_INFO_MAX] */ - -/* fd_vinyl_io_append_pair_inplace appends the style RAW pair at phdr - to the bstream. This will preferentially append the pair in the - given style. Returns the location where the pair was appended. On - return, *_style holds the actual style used and *_val_esz contains - the pair encoded value byte size. - - Note that if the requested style is RAW or if the pair could not be - usefully encoded in the requested style (e.g. the compressed size - ended up larger than the uncompressed size), this will append from - phdr in-place zero copy. When appending a pair in-place, this will - clear the zero padding region and insert the appropriate data - integrity footers at the end of the pair. On other cases, this will - append from the io append scratch memory the encoded pair and the - pair will be untouched. - - As such, the caller should assume the io has a read interest on the - pair's header region and value region and a write interest on the - pair zero padding region and footer region until the next append or - commit and the pair's zero padding and footer regions may be - clobbered by this call. */ - -ulong -fd_vinyl_io_append_pair_inplace( fd_vinyl_io_t * io, - int style, - fd_vinyl_bstream_phdr_t * phdr, - int * _style, - ulong * _val_esz ); - -/* fd_vinyl_io_bd *****************************************************/ - -/* fd_vinyl_io_bd_{align,footprint} specify the alignment and footprint - needed for a bstream stored on a block device / large file with a - spad_max append scratch pad. align will be a reasonable power-of-2 - and footprint will be a multiple of align. Returns 0 for an invalid - spad_max. - - fd_vinyl_io_bd_init starts using a file as a bstream store. lmem - points to a local memory region with suitable alignment and footprint - to hold the bstream's state. spad_max gives the size of the append - scratch pad (should be a FD_VINYL_BSTREAM_BLOCK_SZ multiple). dev_fd - is a file descriptor for the block device / large file. The file - should already exist and be sized to the appropriate capacity. - - FIXME: allow user to specify a subrange of dev_fd to use for the - store? - - If reset is non-zero, ignores any existing file contents and will - start a new bstream. The bstream metadata user info will be set to - the info_sz bytes at info and the bstream will use io_seed for its - data integrity hashing seed. - - Otherwise, this will attempt to resume at the point the bstream was - last synchronized. info, info_sz and io_seed will be ignored. - - IMPORTANT SAFETY TIP! The io_seed is the not same thing as the meta - seed. The io_seed is a property of the bstream (with a lifetime of - the bstream and is shared among all users of the bstream). The meta - seed is a property of the meta (and ideally uniquely and randomly set - per vinyl tile run). - - Returns a handle to the bstream on success (has ownership of lmem and - dev_fd, ownership returned on fini) and NULL on failure (logs - details, no ownership changed). Retains no interest in info. */ - -ulong fd_vinyl_io_bd_align ( void ); -ulong fd_vinyl_io_bd_footprint( ulong spad_max ); - -fd_vinyl_io_t * -fd_vinyl_io_bd_init( void * lmem, - ulong spad_max, - int dev_fd, - int reset, - void const * info, - ulong info_sz, - ulong io_seed ); - -/* fd_vinyl_io_mm *****************************************************/ - -/* fd_vinyl_io_mm_* is the same as fd_vinyl_io_bd_* but uses dev_sz byte - sized memory region dev as the "block device". The result is - bit-level identical to fd_vinyl_io_bd (and vice versa). This is - primarily for testing purposes but, as dev could also be a memory - mapped file / block device, this could be useful in general - (especially for concurrent read access, e.g. parallel recovery). - Note that "sync" only guarantees appends to the dev memory region - happened. If the memory region is backed by a file, when the actual - blocks are written to the physical storage is controlled by the - kernel / driver / physical device (it is up to the caller of sync to - do any additional context specific control here). */ - -ulong fd_vinyl_io_mm_align ( void ); -ulong fd_vinyl_io_mm_footprint( ulong spad_max ); - -fd_vinyl_io_t * -fd_vinyl_io_mm_init( void * lmem, - ulong spad_max, - void * dev, - ulong dev_sz, - int reset, - void const * info, - ulong info_sz, - ulong io_seed ); - -/* fd_vinyl_{mmio,mmio_sz} return {a pointer in the caller's address - space to the raw bstream storage,the raw bstream storage byte size). - These are a _subset_ of the dev / dev_sz region passed to mm_init and - these will be FD_VINYL_BSTREAM_BLOCK_SZ aligned. If a byte seq is in - the store, it will be at mmio[ seq % mmio_sz ]. Note that mmio_sz is - not necessarily a power of two. Note also that the bstream's past is - guaranteed to be in the store. The lifetime of the returned region - is the lifetime of the io. Returns NULL and 0 if io does not support - memory mapped io. These exist to support thread parallel recovery. */ - -void * fd_vinyl_mmio ( fd_vinyl_io_t * io ); -ulong fd_vinyl_mmio_sz( fd_vinyl_io_t * io ); - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_io_fd_vinyl_io_h */ diff --git a/src/vinyl/io/fd_vinyl_io_bd.c b/src/vinyl/io/fd_vinyl_io_bd.c deleted file mode 100644 index 0d87c93befb..00000000000 --- a/src/vinyl/io/fd_vinyl_io_bd.c +++ /dev/null @@ -1,665 +0,0 @@ -#include "fd_vinyl_io.h" - -#include -#include - -static inline void -bd_read( int fd, - ulong off, - void * buf, - ulong sz ) { - ssize_t ssz = pread( fd, buf, sz, (off_t)off ); - if( FD_LIKELY( ssz==(ssize_t)sz ) ) return; - if( ssz<(ssize_t)0 ) FD_LOG_CRIT(( "pread(fd %i,off %lu,sz %lu) failed (%i-%s)", fd, off, sz, errno, fd_io_strerror( errno ) )); - /**/ FD_LOG_CRIT(( "pread(fd %i,off %lu,sz %lu) failed (unexpected sz %li)", fd, off, sz, (long)ssz )); -} - -static inline void -bd_write( int fd, - ulong off, - void const * buf, - ulong sz ) { - ssize_t ssz = pwrite( fd, buf, sz, (off_t)off ); - if( FD_LIKELY( ssz==(ssize_t)sz ) ) return; - if( ssz<(ssize_t)0 ) FD_LOG_CRIT(( "pwrite(fd %i,off %lu,sz %lu) failed (%i-%s)", fd, off, sz, errno, fd_io_strerror( errno ) )); - else FD_LOG_CRIT(( "pwrite(fd %i,off %lu,sz %lu) failed (unexpected sz %li)", fd, off, sz, (long)ssz )); -} - -struct fd_vinyl_io_bd_rd; -typedef struct fd_vinyl_io_bd_rd fd_vinyl_io_bd_rd_t; - -struct fd_vinyl_io_bd_rd { - ulong ctx; /* Must mirror fd_vinyl_io_rd_t */ - ulong seq; /* " */ - void * dst; /* " */ - ulong sz; /* " */ - fd_vinyl_io_bd_rd_t * next; /* Next element in bd rd queue */ -}; - -struct fd_vinyl_io_bd { - fd_vinyl_io_t base[1]; - int dev_fd; /* File descriptor of block device */ - ulong dev_sync; /* Offset to block that holds bstream sync (BLOCK_SZ multiple) */ - ulong dev_base; /* Offset to first block (BLOCK_SZ multiple) */ - ulong dev_sz; /* Block store byte size (BLOCK_SZ multiple) */ - fd_vinyl_io_bd_rd_t * rd_head; /* Pointer to queue head */ - fd_vinyl_io_bd_rd_t ** rd_tail_next; /* Pointer to queue &tail->next or &rd_head if empty. */ - fd_vinyl_bstream_block_t sync[1]; - /* spad_max bytes follow */ -}; - -typedef struct fd_vinyl_io_bd fd_vinyl_io_bd_t; - -static void -fd_vinyl_io_bd_read_imm( fd_vinyl_io_t * io, - ulong seq0, - void * _dst, - ulong sz ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* If this is a request to read nothing, succeed immediately. If - this is a request to read outside the bstream's past, fail. */ - - if( FD_UNLIKELY( !sz ) ) return; - - uchar * dst = (uchar *)_dst; - ulong seq1 = seq0 + sz; - - ulong seq_past = bd->base->seq_past; - ulong seq_present = bd->base->seq_present; - - int bad_seq = !fd_ulong_is_aligned( seq0, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dst = !dst; - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_past = !(fd_vinyl_seq_le( seq_past, seq0 ) & fd_vinyl_seq_lt( seq0, seq1 ) & fd_vinyl_seq_le( seq1, seq_present )); - - if( FD_UNLIKELY( bad_seq | bad_dst | bad_sz | bad_past ) ) - FD_LOG_CRIT(( "bstream read_imm [%016lx,%016lx)/%lu failed (past [%016lx,%016lx)/%lu, %s)", - seq0, seq1, sz, seq_past, seq_present, seq_present-seq_past, - bad_seq ? "misaligned seq" : - bad_dst ? "NULL dst" : - bad_sz ? "misaligned sz" : - "not in past" )); - - /* At this point, we have a valid read request. Map seq0 into the - bstream store. Read the lesser of sz bytes or until the store end. - If we hit the store end with more to go, wrap around and finish the - read at the store start. */ - - int dev_fd = bd->dev_fd; - ulong dev_base = bd->dev_base; - ulong dev_sz = bd->dev_sz; - - ulong dev_off = seq0 % dev_sz; - - ulong rsz = fd_ulong_min( sz, dev_sz - dev_off ); - bd_read( dev_fd, dev_base + dev_off, dst, rsz ); - sz -= rsz; - - if( FD_UNLIKELY( sz ) ) bd_read( dev_fd, dev_base, dst + rsz, sz ); -} - -static void -fd_vinyl_io_bd_read( fd_vinyl_io_t * io, - fd_vinyl_io_rd_t * _rd ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *) io; /* Note: io must be non-NULL to have even been called */ - fd_vinyl_io_bd_rd_t * rd = (fd_vinyl_io_bd_rd_t *)_rd; - - rd->next = NULL; - *bd->rd_tail_next = rd; - bd->rd_tail_next = &rd->next; - - ulong seq0 = rd->seq; - uchar * dst = (uchar *)rd->dst; - ulong sz = rd->sz; - - /* If this is a request to read nothing, succeed immediately. If - this is a request to read outside the bstream's past, fail. */ - - if( FD_UNLIKELY( !sz ) ) return; - - ulong seq1 = seq0 + sz; - - ulong seq_past = bd->base->seq_past; - ulong seq_present = bd->base->seq_present; - - int bad_seq = !fd_ulong_is_aligned( seq0, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dst = !dst; - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_past = !(fd_vinyl_seq_le( seq_past, seq0 ) & fd_vinyl_seq_lt( seq0, seq1 ) & fd_vinyl_seq_le( seq1, seq_present )); - - if( FD_UNLIKELY( bad_seq | bad_dst | bad_sz | bad_past ) ) - FD_LOG_CRIT(( "bstream read [%016lx,%016lx)/%lu failed (past [%016lx,%016lx)/%lu, %s)", - seq0, seq1, sz, seq_past, seq_present, seq_present-seq_past, - bad_seq ? "misaligned seq" : - bad_dst ? "NULL dst" : - bad_sz ? "misaligned sz" : - "not in past" )); - - /* At this point, we have a valid read request. Map seq0 into the - bstream store. Read the lesser of sz bytes or until the store end. - If we hit the store end with more to go, wrap around and finish the - read at the store start. */ - - int dev_fd = bd->dev_fd; - ulong dev_base = bd->dev_base; - ulong dev_sz = bd->dev_sz; - - ulong dev_off = seq0 % dev_sz; - - ulong rsz = fd_ulong_min( sz, dev_sz - dev_off ); - bd_read( dev_fd, dev_base + dev_off, dst, rsz ); - sz -= rsz; - - if( FD_UNLIKELY( sz ) ) bd_read( dev_fd, dev_base, dst + rsz, sz ); -} - -static int -fd_vinyl_io_bd_poll( fd_vinyl_io_t * io, - fd_vinyl_io_rd_t ** _rd, - int flags ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t * )io; /* Note: io must be non-NULL to have even been called */ - (void)flags; - - fd_vinyl_io_bd_rd_t * rd = bd->rd_head; - - if( FD_UNLIKELY( !rd ) ) { - *_rd = NULL; - return FD_VINYL_ERR_EMPTY; - } - - fd_vinyl_io_bd_rd_t ** rd_tail_next = bd->rd_tail_next; - fd_vinyl_io_bd_rd_t * rd_next = rd->next; - - bd->rd_head = rd_next; - bd->rd_tail_next = fd_ptr_if( !!rd_next, rd_tail_next, &bd->rd_head ); - - *_rd = (fd_vinyl_io_rd_t *)rd; - return FD_VINYL_SUCCESS; -} - -static ulong -fd_vinyl_io_bd_append( fd_vinyl_io_t * io, - void const * _src, - ulong sz ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - uchar const * src = (uchar const *)_src; - - /* Validate the input args. */ - - ulong seq_future = bd->base->seq_future; if( FD_UNLIKELY( !sz ) ) return seq_future; - ulong seq_ancient = bd->base->seq_ancient; - int dev_fd = bd->dev_fd; - ulong dev_base = bd->dev_base; - ulong dev_sz = bd->dev_sz; - - int bad_src = !src; - int bad_align = !fd_ulong_is_aligned( (ulong)src, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_capacity = sz > (dev_sz - (seq_future-seq_ancient)); - - if( FD_UNLIKELY( bad_src | bad_align | bad_sz | bad_capacity ) ) - FD_LOG_CRIT(( bad_src ? "NULL src" : - bad_align ? "misaligned src" : - bad_sz ? "misaligned sz" : - "device full" )); - - /* At this point, we appear to have a valid append request. Map it to - the bstream (updating seq_future) and map it to the device. Then - write the lesser of sz bytes or until the store end. If we hit the - store end with more to go, wrap around and finish the write at the - store start. */ - - ulong seq = seq_future; - bd->base->seq_future = seq + sz; - - ulong dev_off = seq % dev_sz; - - ulong wsz = fd_ulong_min( sz, dev_sz - dev_off ); - bd_write( dev_fd, dev_base + dev_off, src, wsz ); - sz -= wsz; - if( sz ) bd_write( dev_fd, dev_base, src + wsz, sz ); - - return seq; -} - -static int -fd_vinyl_io_bd_commit( fd_vinyl_io_t * io, - int flags ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - (void)flags; - - bd->base->seq_present = bd->base->seq_future; - bd->base->spad_used = 0UL; - - return FD_VINYL_SUCCESS; -} - -static ulong -fd_vinyl_io_bd_hint( fd_vinyl_io_t * io, - ulong sz ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - - ulong seq_future = bd->base->seq_future; if( FD_UNLIKELY( !sz ) ) return seq_future; - ulong seq_ancient = bd->base->seq_ancient; - ulong dev_sz = bd->dev_sz; - - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_capacity = sz > (dev_sz - (seq_future-seq_ancient)); - - if( FD_UNLIKELY( bad_sz | bad_capacity ) ) FD_LOG_CRIT(( bad_sz ? "misaligned sz" : "device full" )); - - return bd->base->seq_future; -} - -static void * -fd_vinyl_io_bd_alloc( fd_vinyl_io_t * io, - ulong sz, - int flags ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - - ulong spad_max = bd->base->spad_max; - ulong spad_used = bd->base->spad_used; if( FD_UNLIKELY( !sz ) ) return ((uchar *)(bd+1)) + spad_used; - - int bad_align = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_sz = sz > spad_max; - - if( FD_UNLIKELY( bad_align | bad_sz ) ) FD_LOG_CRIT(( bad_align ? "misaligned sz" : "sz too large" )); - - if( FD_UNLIKELY( sz > (spad_max - spad_used ) ) ) { - if( FD_UNLIKELY( fd_vinyl_io_bd_commit( io, flags ) ) ) return NULL; - spad_used = 0UL; - } - - bd->base->spad_used = spad_used + sz; - - return ((uchar *)(bd+1)) + spad_used; -} - -static ulong -fd_vinyl_io_bd_copy( fd_vinyl_io_t * io, - ulong seq_src0, - ulong sz ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* Validate the input args */ - - ulong seq_ancient = bd->base->seq_ancient; - ulong seq_past = bd->base->seq_past; - ulong seq_present = bd->base->seq_present; - ulong seq_future = bd->base->seq_future; if( FD_UNLIKELY( !sz ) ) return seq_future; - ulong spad_max = bd->base->spad_max; - ulong spad_used = bd->base->spad_used; - int dev_fd = bd->dev_fd; - ulong dev_base = bd->dev_base; - ulong dev_sz = bd->dev_sz; - - ulong seq_src1 = seq_src0 + sz; - - int bad_past = !( fd_vinyl_seq_le( seq_past, seq_src0 ) & - fd_vinyl_seq_lt( seq_src0, seq_src1 ) & - fd_vinyl_seq_le( seq_src1, seq_present ) ); - int bad_src = !fd_ulong_is_aligned( seq_src0, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_capacity = sz > (dev_sz - (seq_future-seq_ancient)); - - if( FD_UNLIKELY( bad_past | bad_src | bad_sz | bad_capacity ) ) - FD_LOG_CRIT(( bad_past ? "src is not in the past" : - bad_src ? "misaligned src_seq" : - bad_sz ? "misaligned sz" : - "device full" )); - - /* At this point, we appear to have a valid copy request. Get - buffer space from the scratch pad (committing as necessary). */ - - if( FD_UNLIKELY( sz>(spad_max-spad_used) ) ) { - fd_vinyl_io_bd_commit( io, FD_VINYL_IO_FLAG_BLOCKING ); - spad_used = 0UL; - } - - uchar * buf = (uchar *)(bd+1) + spad_used; - ulong buf_max = spad_max - spad_used; - - /* Map the dst to the bstream (updating seq_future) and map the src - and dst regions onto the device. Then copy as much as we can at a - time, handling device wrap around and copy buffering space. */ - - ulong seq = seq_future; - bd->base->seq_future = seq + sz; - - ulong seq_dst0 = seq; - - for(;;) { - ulong src_off = seq_src0 % dev_sz; - ulong dst_off = seq_dst0 % dev_sz; - ulong csz = fd_ulong_min( fd_ulong_min( sz, buf_max ), fd_ulong_min( dev_sz - src_off, dev_sz - dst_off ) ); - - bd_read ( dev_fd, dev_base + src_off, buf, csz ); - bd_write( dev_fd, dev_base + dst_off, buf, csz ); - - sz -= csz; - if( !sz ) break; - - seq_src0 += csz; - seq_dst0 += csz; - } - - return seq; -} - -static void -fd_vinyl_io_bd_forget( fd_vinyl_io_t * io, - ulong seq ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* Validate input arguments. Note that we don't allow forgetting into - the future even when we have no uncommitted blocks because the - resulting [seq_ancient,seq_future) might contain blocks that were - never written (which might not be an issue practically but it would - be a bit strange for something to try to scan starting from - seq_ancient and discover unwritten blocks). */ - - ulong seq_past = bd->base->seq_past; - ulong seq_present = bd->base->seq_present; - ulong seq_future = bd->base->seq_future; - - int bad_seq = !fd_ulong_is_aligned( seq, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dir = !(fd_vinyl_seq_le( seq_past, seq ) & fd_vinyl_seq_le( seq, seq_present )); - int bad_read = !!bd->rd_head; - int bad_append = fd_vinyl_seq_ne( seq_present, seq_future ); - - if( FD_UNLIKELY( bad_seq | bad_dir | bad_read | bad_append ) ) - FD_LOG_CRIT(( "forget to seq %016lx failed (past [%016lx,%016lx)/%lu, %s)", - seq, seq_past, seq_present, seq_present-seq_past, - bad_seq ? "misaligned seq" : - bad_dir ? "seq out of bounds" : - bad_read ? "reads in progress" : - "appends/copies in progress" )); - - bd->base->seq_past = seq; -} - -static void -fd_vinyl_io_bd_rewind( fd_vinyl_io_t * io, - ulong seq ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* Validate input arguments. Unlike forgot, we do allow rewinding to - before seq_ancient as the region of sequence space reported to the - caller as written is still accurate. */ - - ulong seq_ancient = bd->base->seq_ancient; - ulong seq_past = bd->base->seq_past; - ulong seq_present = bd->base->seq_present; - ulong seq_future = bd->base->seq_future; - - int bad_seq = !fd_ulong_is_aligned( seq, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dir = fd_vinyl_seq_gt( seq, seq_present ); - int bad_read = !!bd->rd_head; - int bad_append = fd_vinyl_seq_ne( seq_present, seq_future ); - - if( FD_UNLIKELY( bad_seq | bad_dir | bad_read | bad_append ) ) - FD_LOG_CRIT(( "rewind to seq %016lx failed (present %016lx, %s)", seq, seq_present, - bad_seq ? "misaligned seq" : - bad_dir ? "seq after seq_present" : - bad_read ? "reads in progress" : - "appends/copies in progress" )); - - bd->base->seq_ancient = fd_ulong_if( fd_vinyl_seq_ge( seq, seq_ancient ), seq_ancient, seq ); - bd->base->seq_past = fd_ulong_if( fd_vinyl_seq_ge( seq, seq_past ), seq_past, seq ); - bd->base->seq_present = seq; - bd->base->seq_future = seq; -} - -static int -fd_vinyl_io_bd_sync( fd_vinyl_io_t * io, - int flags ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - (void)flags; - - ulong seed = bd->base->seed; - ulong seq_past = bd->base->seq_past; - ulong seq_present = bd->base->seq_present; - - int dev_fd = bd->dev_fd; - ulong dev_sync = bd->dev_sync; - - fd_vinyl_bstream_block_t * block = bd->sync; - - /* block->sync.ctl current (static) */ - block->sync.seq_past = seq_past; - block->sync.seq_present = seq_present; - /* block->sync.info_sz current (static) */ - /* block->sync.info current (static) */ - - block->sync.hash_trail = 0UL; - block->sync.hash_blocks = 0UL; - fd_vinyl_bstream_block_hash( seed, block ); /* sets hash_trail back to seed */ - - bd_write( dev_fd, dev_sync, block, FD_VINYL_BSTREAM_BLOCK_SZ ); - - bd->base->seq_ancient = seq_past; - - return FD_VINYL_SUCCESS; -} - -static void * -fd_vinyl_io_bd_fini( fd_vinyl_io_t * io ) { - fd_vinyl_io_bd_t * bd = (fd_vinyl_io_bd_t *)io; /* Note: io must be non-NULL to have even been called */ - - ulong seq_present = bd->base->seq_present; - ulong seq_future = bd->base->seq_future; - - if( FD_UNLIKELY( bd->rd_head ) ) FD_LOG_WARNING(( "fini completing outstanding reads" )); - if( FD_UNLIKELY( fd_vinyl_seq_ne( seq_present, seq_future ) ) ) FD_LOG_WARNING(( "fini discarding uncommited blocks" )); - - return io; -} - -static fd_vinyl_io_impl_t fd_vinyl_io_bd_impl[1] = { { - fd_vinyl_io_bd_read_imm, - fd_vinyl_io_bd_read, - fd_vinyl_io_bd_poll, - fd_vinyl_io_bd_append, - fd_vinyl_io_bd_commit, - fd_vinyl_io_bd_hint, - fd_vinyl_io_bd_alloc, - fd_vinyl_io_bd_copy, - fd_vinyl_io_bd_forget, - fd_vinyl_io_bd_rewind, - fd_vinyl_io_bd_sync, - fd_vinyl_io_bd_fini -} }; - -FD_STATIC_ASSERT( alignof(fd_vinyl_io_bd_t)==FD_VINYL_BSTREAM_BLOCK_SZ, layout ); - -ulong -fd_vinyl_io_bd_align( void ) { - return alignof(fd_vinyl_io_bd_t); -} - -ulong -fd_vinyl_io_bd_footprint( ulong spad_max ) { - if( FD_UNLIKELY( !((0UL (ulong)LONG_MAX; - int misaligned = !fd_ulong_is_aligned( dev_sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - - if( FD_UNLIKELY( too_small | too_large | misaligned ) ) { - FD_LOG_WARNING(( "bstream size %s", too_small ? "too small" : - too_large ? "too large" : - "not a block size multiple" )); - return NULL; - } - - if( reset ) { - if( FD_UNLIKELY( !info ) ) info_sz = 0UL; - if( FD_UNLIKELY( info_sz>FD_VINYL_BSTREAM_SYNC_INFO_MAX ) ) { - FD_LOG_WARNING(( "info_sz too large" )); - return NULL; - } - } - - memset( bd, 0, footprint ); - - bd->base->type = FD_VINYL_IO_TYPE_BD; - - /* io_seed, seq_ancient, seq_past, seq_present, seq_future are init - below */ - - bd->base->spad_max = spad_max; - bd->base->spad_used = 0UL; - bd->base->impl = fd_vinyl_io_bd_impl; - - bd->dev_fd = dev_fd; - bd->dev_sync = 0UL; /* Use the beginning of the file for the sync block */ - bd->dev_base = FD_VINYL_BSTREAM_BLOCK_SZ; /* Use the rest for the actual bstream store (at least 3.5 KiB) */ - bd->dev_sz = dev_sz - FD_VINYL_BSTREAM_BLOCK_SZ; - - bd->rd_head = NULL; - bd->rd_tail_next = &bd->rd_head; - - /* Note that [seq_ancient,seq_future) (cyclic) contains at most dev_sz - bytes, bstream's antiquity, past and present are subsets of this - range and dev_sz is less than 2^63 given the above (practically - much much less). As such, differences between two ordered bstream - sequence numbers (e.g. ulong sz = seq_a - seq_b where a is - logically not before b) will "just work" regardless of wrapping - and/or amount of data stored. */ - - /* FIXME: Consider having the sync block on a completely separate - device (to reduce seeking when syncing). */ - - fd_vinyl_bstream_block_t * block = bd->sync; - - if( reset ) { - - /* We are starting a new bstream. Write the initial sync block. */ - - bd->base->seed = io_seed; - bd->base->seq_ancient = 0UL; - bd->base->seq_past = 0UL; - bd->base->seq_present = 0UL; - bd->base->seq_future = 0UL; - - memset( block, 0, FD_VINYL_BSTREAM_BLOCK_SZ ); /* bulk zero */ - - block->sync.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_SYNC, 0, FD_VINYL_VAL_MAX ); - //block->sync.seq_past = ...; /* init by sync */ - //block->sync.seq_present = ...; /* init by sync */ - block->sync.info_sz = info_sz; - if( info_sz ) memcpy( block->sync.info, info, info_sz ); - //block->sync.hash_trail = ...; /* init by sync */ - //block->sync.hash_blocks = ...; /* init by sync */ - - int err = fd_vinyl_io_bd_sync( bd->base, FD_VINYL_IO_FLAG_BLOCKING ); /* logs details */ - if( FD_UNLIKELY( err ) ) { - FD_LOG_WARNING(( "sync block write failed (%i-%s)", err, fd_vinyl_strerror( err ) )); - return NULL; - } - - } else { - - /* We are resuming an existing bstream. Read and validate the - bstream's sync block. */ - - bd_read( dev_fd, bd->dev_sync, block, FD_VINYL_BSTREAM_BLOCK_SZ ); /* logs details */ - - int type = fd_vinyl_bstream_ctl_type ( block->sync.ctl ); - int version = fd_vinyl_bstream_ctl_style( block->sync.ctl ); - ulong val_max = fd_vinyl_bstream_ctl_sz ( block->sync.ctl ); - ulong seq_past = block->sync.seq_past; - ulong seq_present = block->sync.seq_present; - /**/ info_sz = block->sync.info_sz; // overrides user info_sz - /**/ info = block->sync.info; // overrides user info - /**/ io_seed = block->sync.hash_trail; // overrides user io_seed - - int bad_type = (type != FD_VINYL_BSTREAM_CTL_TYPE_SYNC); - int bad_version = (version != 0); - int bad_val_max = (val_max != FD_VINYL_VAL_MAX); - int bad_seq_past = !fd_ulong_is_aligned( seq_past, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_seq_present = !fd_ulong_is_aligned( seq_present, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_info_sz = (info_sz > FD_VINYL_BSTREAM_SYNC_INFO_MAX); - int bad_past_order = fd_vinyl_seq_gt( seq_past, seq_present ); - int bad_past_sz = ((seq_present-seq_past) > bd->dev_sz); - - if( FD_UNLIKELY( bad_type | bad_version | bad_val_max | bad_seq_past | bad_seq_present | bad_info_sz | - bad_past_order | bad_past_sz ) ) { - FD_LOG_WARNING(( "bad sync block when recovering bstream (%s)", - bad_type ? "unexpected type" : - bad_version ? "unexpected version" : - bad_val_max ? "unexpected max pair value decoded byte size" : - bad_seq_past ? "unaligned seq_past" : - bad_seq_present ? "unaligned seq_present" : - bad_info_sz ? "unexpected info size" : - bad_past_order ? "unordered seq_past and seq_present" : - "past size larger than bstream store" )); - return NULL; - } - - if( FD_UNLIKELY( fd_vinyl_bstream_block_test( io_seed, block ) ) ) { - FD_LOG_WARNING(( "corrupt sync block when recovering bstream" )); - return NULL; - } - - bd->base->seed = io_seed; - bd->base->seq_ancient = seq_past; - bd->base->seq_past = seq_past; - bd->base->seq_present = seq_present; - bd->base->seq_future = seq_present; - - } - - FD_LOG_NOTICE(( "IO config" - "\n\ttype bd" - "\n\tspad_max %lu bytes" - "\n\tdev_sz %lu bytes" - "\n\treset %i" - "\n\tinfo \"%s\" (info_sz %lu%s)" - "\n\tio_seed 0x%016lx%s", - spad_max, dev_sz, reset, - info ? (char const *)info : "", info_sz, reset ? "" : ", discovered", - io_seed, reset ? "" : " (discovered)" )); - - return bd->base; -} diff --git a/src/vinyl/io/fd_vinyl_io_mm.c b/src/vinyl/io/fd_vinyl_io_mm.c deleted file mode 100644 index 5b5744c290c..00000000000 --- a/src/vinyl/io/fd_vinyl_io_mm.c +++ /dev/null @@ -1,652 +0,0 @@ -#include "fd_vinyl_io.h" - -struct fd_vinyl_io_mm_rd; -typedef struct fd_vinyl_io_mm_rd fd_vinyl_io_mm_rd_t; - -struct fd_vinyl_io_mm_rd { - ulong ctx; /* Must mirror fd_vinyl_io_rd_t */ - ulong seq; /* " */ - void * dst; /* " */ - ulong sz; /* " */ - fd_vinyl_io_mm_rd_t * next; /* Next element in mm rd queue */ -}; - -struct fd_vinyl_io_mm { - fd_vinyl_io_t base[1]; - uchar * dev; /* Memory mapped I/O memory region */ - ulong dev_sync; /* Offset to the bstream's sync block (BLOCK_SZ multiple) */ - ulong dev_base; /* Offset to first block (BLOCK_SZ multiple) */ - ulong dev_sz; /* Block store byte size (BLOCK_SZ multiple) */ - fd_vinyl_io_mm_rd_t * rd_head; /* Pointer to queue head */ - fd_vinyl_io_mm_rd_t ** rd_tail_next; /* Pointer to queue &tail->next or &rd_head if empty. */ - fd_vinyl_bstream_block_t sync[1]; - /* spad_max bytes follow */ -}; - -typedef struct fd_vinyl_io_mm fd_vinyl_io_mm_t; - -static void -fd_vinyl_io_mm_read_imm( fd_vinyl_io_t * io, - ulong seq0, - void * _dst, - ulong sz ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* If this is a request to read nothing, succeed immediately. If - this is a request to read outside the bstream's past, fail. */ - - if( FD_UNLIKELY( !sz ) ) return; - - uchar * dst = (uchar *)_dst; - ulong seq1 = seq0 + sz; - - ulong seq_past = mm->base->seq_past; - ulong seq_present = mm->base->seq_present; - - int bad_seq = !fd_ulong_is_aligned( seq0, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dst = !dst; - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_past = !(fd_vinyl_seq_le( seq_past, seq0 ) & fd_vinyl_seq_lt( seq0, seq1 ) & fd_vinyl_seq_le( seq1, seq_present )); - - if( FD_UNLIKELY( bad_seq | bad_dst | bad_sz | bad_past ) ) - FD_LOG_CRIT(( "bstream read_imm [%016lx,%016lx)/%lu failed (past [%016lx,%016lx)/%lu, %s)", - seq0, seq1, sz, seq_past, seq_present, seq_present-seq_past, - bad_seq ? "misaligned seq" : - bad_dst ? "NULL dst" : - bad_sz ? "misaligned sz" : - "not in past" )); - - /* At this point, we have a valid read request. Map seq0 into the - bstream store. Read the lesser of sz bytes or until the store end. - If we hit the store end with more to go, wrap around and finish the - read at the store start. */ - - uchar * dev = mm->dev; - ulong dev_base = mm->dev_base; - ulong dev_sz = mm->dev_sz; - - ulong dev_off = seq0 % dev_sz; - - ulong rsz = fd_ulong_min( sz, dev_sz - dev_off ); - memcpy( dst, dev + dev_base + dev_off, rsz ); - sz -= rsz; - if( FD_UNLIKELY( sz ) ) memcpy( dst + rsz, dev + dev_base, sz ); -} - -static void -fd_vinyl_io_mm_read( fd_vinyl_io_t * io, - fd_vinyl_io_rd_t * _rd ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *) io; /* Note: io must be non-NULL to have even been called */ - fd_vinyl_io_mm_rd_t * rd = (fd_vinyl_io_mm_rd_t *)_rd; - - rd->next = NULL; - *mm->rd_tail_next = rd; - mm->rd_tail_next = &rd->next; - - ulong seq0 = rd->seq; - uchar * dst = (uchar *)rd->dst; - ulong sz = rd->sz; - - /* If this is a request to read nothing, succeed immediately. If - this is a request to read outside the bstream's past, fail. */ - - if( FD_UNLIKELY( !sz ) ) return; - - ulong seq1 = seq0 + sz; - - ulong seq_past = mm->base->seq_past; - ulong seq_present = mm->base->seq_present; - - int bad_seq = !fd_ulong_is_aligned( seq0, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dst = !dst; - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_past = !(fd_vinyl_seq_le( seq_past, seq0 ) & fd_vinyl_seq_lt( seq0, seq1 ) & fd_vinyl_seq_le( seq1, seq_present )); - - if( FD_UNLIKELY( bad_seq | bad_dst | bad_sz | bad_past ) ) - FD_LOG_CRIT(( "bstream read [%016lx,%016lx)/%lu failed (past [%016lx,%016lx)/%lu, %s)", - seq0, seq1, sz, seq_past, seq_present, seq_present-seq_past, - bad_seq ? "misaligned seq" : - bad_dst ? "NULL dst" : - bad_sz ? "misaligned sz" : - "not in past" )); - - /* At this point, we have a valid read request. Map seq0 into the - bstream store. Read the lesser of sz bytes or until the store end. - If we hit the store end with more to go, wrap around and finish the - read at the store start. */ - - uchar const * dev = mm->dev; - ulong dev_base = mm->dev_base; - ulong dev_sz = mm->dev_sz; - - ulong dev_off = seq0 % dev_sz; - - ulong rsz = fd_ulong_min( sz, dev_sz - dev_off ); - memcpy( dst, dev + dev_base + dev_off, rsz ); - sz -= rsz; - if( FD_UNLIKELY( sz ) ) memcpy( dst + rsz, dev + dev_base, sz ); -} - -static int -fd_vinyl_io_mm_poll( fd_vinyl_io_t * io, - fd_vinyl_io_rd_t ** _rd, - int flags ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t * )io; /* Note: io must be non-NULL to have even been called */ - (void)flags; - - fd_vinyl_io_mm_rd_t * rd = mm->rd_head; - - if( FD_UNLIKELY( !rd ) ) { - *_rd = NULL; - return FD_VINYL_ERR_EMPTY; - } - - fd_vinyl_io_mm_rd_t ** rd_tail_next = mm->rd_tail_next; - fd_vinyl_io_mm_rd_t * rd_next = rd->next; - - mm->rd_head = rd_next; - mm->rd_tail_next = fd_ptr_if( !!rd_next, rd_tail_next, &mm->rd_head ); - - *_rd = (fd_vinyl_io_rd_t *)rd; - return FD_VINYL_SUCCESS; -} - -static ulong -fd_vinyl_io_mm_append( fd_vinyl_io_t * io, - void const * _src, - ulong sz ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - uchar const * src = (uchar const *)_src; - - /* Validate the input args. */ - - ulong seq_future = mm->base->seq_future; if( FD_UNLIKELY( !sz ) ) return seq_future; - ulong seq_ancient = mm->base->seq_ancient; - uchar * dev = mm->dev; - ulong dev_base = mm->dev_base; - ulong dev_sz = mm->dev_sz; - - int bad_src = !src; - int bad_align = !fd_ulong_is_aligned( (ulong)src, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_capacity = sz > (dev_sz - (seq_future-seq_ancient)); - - if( FD_UNLIKELY( bad_src | bad_align | bad_sz | bad_capacity ) ) - FD_LOG_CRIT(( bad_src ? "NULL src" : - bad_align ? "misaligned src" : - bad_sz ? "misaligned sz" : - "device full" )); - - /* At this point, we appear to have a valid append request. Map it to - the bstream (updating seq_future) and map it to the device. Then - write the lesser of sz bytes or until the store end. If we hit the - store end with more to go, wrap around and finish the write at the - store start. */ - - ulong seq = seq_future; - mm->base->seq_future = seq + sz; - - ulong dev_off = seq % dev_sz; - - ulong wsz = fd_ulong_min( sz, dev_sz - dev_off ); - memcpy( dev + dev_base + dev_off, src, wsz ); - sz -= wsz; - if( sz ) memcpy( dev + dev_base, src + wsz, sz ); - - return seq; -} - -static int -fd_vinyl_io_mm_commit( fd_vinyl_io_t * io, - int flags ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - (void)flags; - - mm->base->seq_present = mm->base->seq_future; - mm->base->spad_used = 0UL; - - return FD_VINYL_SUCCESS; -} - -static ulong -fd_vinyl_io_mm_hint( fd_vinyl_io_t * io, - ulong sz ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - - ulong seq_future = mm->base->seq_future; if( FD_UNLIKELY( !sz ) ) return seq_future; - ulong seq_ancient = mm->base->seq_ancient; - ulong dev_sz = mm->dev_sz; - - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_capacity = sz > (dev_sz - (seq_future-seq_ancient)); - - if( FD_UNLIKELY( bad_sz | bad_capacity ) ) FD_LOG_CRIT(( bad_sz ? "misaligned sz" : "device full" )); - - return mm->base->seq_future; -} - -static void * -fd_vinyl_io_mm_alloc( fd_vinyl_io_t * io, - ulong sz, - int flags ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - - ulong spad_max = mm->base->spad_max; - ulong spad_used = mm->base->spad_used; if( FD_UNLIKELY( !sz ) ) return ((uchar *)(mm+1)) + spad_used; - - int bad_align = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_sz = sz > spad_max; - - if( FD_UNLIKELY( bad_align | bad_sz ) ) FD_LOG_CRIT(( bad_align ? "misaligned sz" : "sz too large" )); - - if( FD_UNLIKELY( sz > (spad_max - spad_used ) ) ) { - if( FD_UNLIKELY( fd_vinyl_io_mm_commit( io, flags ) ) ) return NULL; - spad_used = 0UL; - } - - mm->base->spad_used = spad_used + sz; - - return ((uchar *)(mm+1)) + spad_used; -} - -static ulong -fd_vinyl_io_mm_copy( fd_vinyl_io_t * io, - ulong seq_src0, - ulong sz ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* Validate the input args */ - - ulong seq_ancient = mm->base->seq_ancient; - ulong seq_past = mm->base->seq_past; - ulong seq_present = mm->base->seq_present; - ulong seq_future = mm->base->seq_future; if( FD_UNLIKELY( !sz ) ) return seq_future; - ulong spad_max = mm->base->spad_max; - ulong spad_used = mm->base->spad_used; - uchar * dev = mm->dev; - ulong dev_base = mm->dev_base; - ulong dev_sz = mm->dev_sz; - - ulong seq_src1 = seq_src0 + sz; - - int bad_past = !( fd_vinyl_seq_le( seq_past, seq_src0 ) & - fd_vinyl_seq_lt( seq_src0, seq_src1 ) & - fd_vinyl_seq_le( seq_src1, seq_present ) ); - int bad_src = !fd_ulong_is_aligned( seq_src0, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_sz = !fd_ulong_is_aligned( sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_capacity = sz > (dev_sz - (seq_future-seq_ancient)); - - if( FD_UNLIKELY( bad_past | bad_src | bad_sz | bad_capacity ) ) - FD_LOG_CRIT(( bad_past ? "src is not in the past" : - bad_src ? "misaligned src_seq" : - bad_sz ? "misaligned sz" : - "device full" )); - - /* At this point, we appear to have a valid copy request. Get - buffer space from the scratch pad (committing as necessary). */ - - if( FD_UNLIKELY( sz>(spad_max-spad_used) ) ) { - fd_vinyl_io_mm_commit( io, FD_VINYL_IO_FLAG_BLOCKING ); - spad_used = 0UL; - } - - uchar * buf = (uchar *)(mm+1) + spad_used; - ulong buf_max = spad_max - spad_used; - - /* Map the dst to the bstream (updating seq_future) and map the src - and dst regions onto the device. Then copy as much as we can at a - time, handling device wrap around and copy buffering space. */ - - ulong seq = seq_future; - mm->base->seq_future = seq + sz; - - ulong seq_dst0 = seq; - - for(;;) { - ulong src_off = seq_src0 % dev_sz; - ulong dst_off = seq_dst0 % dev_sz; - ulong csz = fd_ulong_min( fd_ulong_min( sz, buf_max ), fd_ulong_min( dev_sz - src_off, dev_sz - dst_off ) ); - - memcpy( buf, dev + dev_base + src_off, csz ); - memcpy( dev + dev_base + dst_off, buf, csz ); - - sz -= csz; - if( !sz ) break; - - seq_src0 += csz; - seq_dst0 += csz; - } - - return seq; -} - -static void -fd_vinyl_io_mm_forget( fd_vinyl_io_t * io, - ulong seq ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* Validate input arguments. Note that we don't allow forgetting into - the future even when we have no uncommitted blocks because the - resulting [seq_ancient,seq_future) might contain blocks that were - never written (which might not be an issue practically but it would - be a bit strange for something to try to scan starting from - seq_ancient and discover unwritten blocks). */ - - ulong seq_past = mm->base->seq_past; - ulong seq_present = mm->base->seq_present; - ulong seq_future = mm->base->seq_future; - - int bad_seq = !fd_ulong_is_aligned( seq, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dir = !(fd_vinyl_seq_le( seq_past, seq ) & fd_vinyl_seq_le( seq, seq_present )); - int bad_read = !!mm->rd_head; - int bad_append = fd_vinyl_seq_ne( seq_present, seq_future ); - - if( FD_UNLIKELY( bad_seq | bad_dir | bad_read | bad_append ) ) - FD_LOG_CRIT(( "forget to seq %016lx failed (past [%016lx,%016lx)/%lu, %s)", - seq, seq_past, seq_present, seq_present-seq_past, - bad_seq ? "misaligned seq" : - bad_dir ? "seq out of bounds" : - bad_read ? "reads in progress" : - "appends/copies in progress" )); - - mm->base->seq_past = seq; -} - -static void -fd_vinyl_io_mm_rewind( fd_vinyl_io_t * io, - ulong seq ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - - /* Validate input arguments. Unlike forgot, we do allow rewinding to - before seq_ancient as the region of sequence space reported to the - caller as written is still accurate. */ - - ulong seq_ancient = mm->base->seq_ancient; - ulong seq_past = mm->base->seq_past; - ulong seq_present = mm->base->seq_present; - ulong seq_future = mm->base->seq_future; - - int bad_seq = !fd_ulong_is_aligned( seq, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_dir = fd_vinyl_seq_gt( seq, seq_present ); - int bad_read = !!mm->rd_head; - int bad_append = fd_vinyl_seq_ne( seq_present, seq_future ); - - if( FD_UNLIKELY( bad_seq | bad_dir | bad_read | bad_append ) ) - FD_LOG_CRIT(( "rewind to seq %016lx failed (present %016lx, %s)", seq, seq_present, - bad_seq ? "misaligned seq" : - bad_dir ? "seq after seq_present" : - bad_read ? "reads in progress" : - "appends/copies in progress" )); - - mm->base->seq_ancient = fd_ulong_if( fd_vinyl_seq_ge( seq, seq_ancient ), seq_ancient, seq ); - mm->base->seq_past = fd_ulong_if( fd_vinyl_seq_ge( seq, seq_past ), seq_past, seq ); - mm->base->seq_present = seq; - mm->base->seq_future = seq; -} - -static int -fd_vinyl_io_mm_sync( fd_vinyl_io_t * io, - int flags ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - (void)flags; - - ulong seed = mm->base->seed; - ulong seq_past = mm->base->seq_past; - ulong seq_present = mm->base->seq_present; - uchar * dev = mm->dev; - ulong dev_sync = mm->dev_sync; - - fd_vinyl_bstream_block_t * block = mm->sync; - - /* block->sync.ctl current (static) */ - block->sync.seq_past = seq_past; - block->sync.seq_present = seq_present; - /* block->sync.info_sz current (static) */ - /* block->sync.info current (static) */ - - block->sync.hash_trail = 0UL; - block->sync.hash_blocks = 0UL; - fd_vinyl_bstream_block_hash( seed, block ); /* sets hash_trail back to seed */ - - memcpy( dev + dev_sync, block, FD_VINYL_BSTREAM_BLOCK_SZ ); - - mm->base->seq_ancient = seq_past; - - return FD_VINYL_SUCCESS; -} - -static void * -fd_vinyl_io_mm_fini( fd_vinyl_io_t * io ) { - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; /* Note: io must be non-NULL to have even been called */ - - ulong seq_present = mm->base->seq_present; - ulong seq_future = mm->base->seq_future; - - if( FD_UNLIKELY( mm->rd_head ) ) FD_LOG_WARNING(( "fini completing outstanding reads" )); - if( FD_UNLIKELY( fd_vinyl_seq_ne( seq_present, seq_future ) ) ) FD_LOG_WARNING(( "fini discarding uncommitted blocks" )); - - return io; -} - -static fd_vinyl_io_impl_t fd_vinyl_io_mm_impl[1] = { { - fd_vinyl_io_mm_read_imm, - fd_vinyl_io_mm_read, - fd_vinyl_io_mm_poll, - fd_vinyl_io_mm_append, - fd_vinyl_io_mm_commit, - fd_vinyl_io_mm_hint, - fd_vinyl_io_mm_alloc, - fd_vinyl_io_mm_copy, - fd_vinyl_io_mm_forget, - fd_vinyl_io_mm_rewind, - fd_vinyl_io_mm_sync, - fd_vinyl_io_mm_fini -} }; - -FD_STATIC_ASSERT( alignof(fd_vinyl_io_mm_t)==FD_VINYL_BSTREAM_BLOCK_SZ, layout ); - -ulong -fd_vinyl_io_mm_align( void ) { - return alignof(fd_vinyl_io_mm_t); -} - -ulong -fd_vinyl_io_mm_footprint( ulong spad_max ) { - if( FD_UNLIKELY( !((0UL (ulong)LONG_MAX; - int misaligned = !fd_ulong_is_aligned( dev_sz, FD_VINYL_BSTREAM_BLOCK_SZ ); - - if( FD_UNLIKELY( too_small | too_large | misaligned ) ) { - FD_LOG_WARNING(( "bstream size %s", too_small ? "too small" : - too_large ? "too large" : - "not a block size multiple" )); - return NULL; - } - - if( reset ) { - if( FD_UNLIKELY( !info ) ) info_sz = 0UL; - if( FD_UNLIKELY( info_sz>FD_VINYL_BSTREAM_SYNC_INFO_MAX ) ) { - FD_LOG_WARNING(( "info_sz too large" )); - return NULL; - } - } - - memset( mm, 0, footprint ); - - mm->base->type = FD_VINYL_IO_TYPE_MM; - - /* io_seed, seq_ancient, seq_past, seq_present, seq_future are init - below */ - - mm->base->spad_max = spad_max; - mm->base->spad_used = 0UL; - mm->base->impl = fd_vinyl_io_mm_impl; - - mm->dev = dev; - mm->dev_sync = 0UL; /* Use the beginning of the file for the sync block */ - mm->dev_base = FD_VINYL_BSTREAM_BLOCK_SZ; /* Use the rest for the actual bstream store (at least 4) */ - mm->dev_sz = dev_sz - FD_VINYL_BSTREAM_BLOCK_SZ; - - mm->rd_head = NULL; - mm->rd_tail_next = &mm->rd_head; - - /* Note that [seq_ancient,seq_future) (cyclic) contains at most dev_sz - bytes, bstream's antiquity, past and present are subsets of this - range and dev_sz is less than 2^63 given the above (practically - much much less). As such, differences between two ordered bstream - sequence numbers (e.g. ulong sz = seq_a - seq_b where a is - logically not before b) will "just work" regardless of wrapping - and/or amount of data stored. */ - - fd_vinyl_bstream_block_t * block = mm->sync; - - if( reset ) { - - /* We are starting a new bstream. Write the initial sync block. */ - - mm->base->seed = io_seed; - mm->base->seq_ancient = 0UL; - mm->base->seq_past = 0UL; - mm->base->seq_present = 0UL; - mm->base->seq_future = 0UL; - - memset( block, 0, FD_VINYL_BSTREAM_BLOCK_SZ ); /* bulk zero */ - - block->sync.ctl = fd_vinyl_bstream_ctl( FD_VINYL_BSTREAM_CTL_TYPE_SYNC, 0, FD_VINYL_VAL_MAX ); - //block->sync.seq_past = ...; /* init by sync */ - //block->sync.seq_present = ...; /* init by sync */ - block->sync.info_sz = info_sz; - if( info_sz ) memcpy( block->sync.info, info, info_sz ); - //block->sync.hash_trail = ...; /* init by sync */ - //block->sync.hash_blocks = ...; /* init by sync */ - - int err = fd_vinyl_io_mm_sync( mm->base, FD_VINYL_IO_FLAG_BLOCKING ); /* logs details */ - if( FD_UNLIKELY( err ) ) { - FD_LOG_WARNING(( "sync block write failed (%i-%s)", err, fd_vinyl_strerror( err ) )); - return NULL; - } - - } else { - - /* We are resuming an existing bstream. Read and validate the - bstream's sync block. */ - - memcpy( block, mm->dev + mm->dev_sync, FD_VINYL_BSTREAM_BLOCK_SZ ); /* logs details */ - - int type = fd_vinyl_bstream_ctl_type ( block->sync.ctl ); - int version = fd_vinyl_bstream_ctl_style( block->sync.ctl ); - ulong val_max = fd_vinyl_bstream_ctl_sz ( block->sync.ctl ); - ulong seq_past = block->sync.seq_past; - ulong seq_present = block->sync.seq_present; - /**/ info_sz = block->sync.info_sz; // overrides user info_sz - /**/ info = block->sync.info; // overrides user info - /**/ io_seed = block->sync.hash_trail; // overrides user io_seed - - int bad_type = (type != FD_VINYL_BSTREAM_CTL_TYPE_SYNC); - int bad_version = (version != 0); - int bad_val_max = (val_max != FD_VINYL_VAL_MAX); - int bad_seq_past = !fd_ulong_is_aligned( seq_past, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_seq_present = !fd_ulong_is_aligned( seq_present, FD_VINYL_BSTREAM_BLOCK_SZ ); - int bad_info_sz = (info_sz > FD_VINYL_BSTREAM_SYNC_INFO_MAX); - int bad_past_order = fd_vinyl_seq_gt( seq_past, seq_present ); - int bad_past_sz = ((seq_present-seq_past) > mm->dev_sz); - - if( FD_UNLIKELY( bad_type | bad_version | bad_val_max | bad_seq_past | bad_seq_present | bad_info_sz | - bad_past_order | bad_past_sz ) ) { - FD_LOG_WARNING(( "bad sync block when recovering (%s)", - bad_type ? "unexpected type" : - bad_version ? "unexpected version" : - bad_val_max ? "unexpected max pair value decoded byte size" : - bad_seq_past ? "unaligned seq_past" : - bad_seq_present ? "unaligned seq_present" : - bad_info_sz ? "unexpected info size" : - bad_past_order ? "unordered seq_past and seq_present" : - "past size larger than bstream store" )); - return NULL; - } - - if( FD_UNLIKELY( fd_vinyl_bstream_block_test( io_seed, block ) ) ) { - FD_LOG_WARNING(( "corrupt sync block when recovering bstream store" )); - return NULL; - } - - mm->base->seed = io_seed; - mm->base->seq_ancient = seq_past; - mm->base->seq_past = seq_past; - mm->base->seq_present = seq_present; - mm->base->seq_future = seq_present; - - } - - FD_LOG_NOTICE(( "IO config" - "\n\ttype mm" - "\n\tspad_max %lu bytes" - "\n\tdev_sz %lu bytes" - "\n\treset %i" - "\n\tinfo \"%s\" (info_sz %lu%s)" - "\n\tio_seed 0x%016lx%s", - spad_max, dev_sz, reset, - info ? (char const *)info : "", info_sz, reset ? "" : ", discovered", - io_seed, reset ? "" : " (discovered)" )); - - return mm->base; -} - -void * -fd_vinyl_mmio( fd_vinyl_io_t * io ) { - if( FD_UNLIKELY( io->type!=FD_VINYL_IO_TYPE_MM ) ) return NULL; - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; - return mm->dev + mm->dev_base; -} - -ulong -fd_vinyl_mmio_sz( fd_vinyl_io_t * io ) { - if( FD_UNLIKELY( io->type!=FD_VINYL_IO_TYPE_MM ) ) return 0UL; - fd_vinyl_io_mm_t * mm = (fd_vinyl_io_mm_t *)io; - return mm->dev_sz; -} diff --git a/src/vinyl/io/test_vinyl_io_bd.c b/src/vinyl/io/test_vinyl_io_bd.c deleted file mode 100644 index 8b8d31323fd..00000000000 --- a/src/vinyl/io/test_vinyl_io_bd.c +++ /dev/null @@ -1,175 +0,0 @@ -#include "../fd_vinyl.h" - -#include /* For mkstemp */ -#include /* For errno */ -#include /* For ftruncate */ -#include /* For open */ - -#include "test_vinyl_io_common.c" - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - ulong spad_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--spad-max", 0UL, 131072UL ); - char const * path = fd_env_strip_cmdline_cstr ( &argc, &argv, "--path", NULL, NULL ); - ulong seed = fd_env_strip_cmdline_ulong( &argc, &argv, "--seed", 0UL, 1234UL ); - - FD_LOG_NOTICE(( "Testing with --spad-max %lu --seed %lu", spad_max, seed )); - - fd_rng_t rng[1]; fd_rng_join( fd_rng_new( rng, 0U, 0UL ) ); - - ulong store_sz = (off_t)(FD_VINYL_BSTREAM_BLOCK_SZ + BCACHE_SZ); - - char _path[] = "/tmp/test_vinyl_io_bd.XXXXXX"; - - int fd; - if( FD_UNLIKELY( path ) ) { - FD_LOG_NOTICE(( "Using --path %s for the test storage", path )); - fd = open( path, O_RDWR | O_CREAT | O_EXCL, (mode_t)0644 ); - if( FD_UNLIKELY( fd==-1 ) ) FD_LOG_ERR(( "open failed (%i-%s)", errno, fd_io_strerror( errno ) )); - } else { - FD_LOG_NOTICE(( "--path not specified, using a temp file for test storage" )); - fd = mkstemp( _path ); - if( FD_UNLIKELY( fd==-1 ) ) FD_LOG_ERR(( "mkstemp failed (%i-%s)", errno, fd_io_strerror( errno ) )); - path = _path; - FD_LOG_NOTICE(( "temp file at %s", path )); - } - - if( FD_UNLIKELY( ftruncate( fd, (off_t)store_sz ) ) ) - FD_LOG_ERR(( "ftruncate failed (%i-%s)", errno, fd_io_strerror( errno ) )); - -# define MEM_MAX (1048576UL) - uchar mem[ MEM_MAX ] __attribute__((aligned(512))); - - FD_LOG_NOTICE(( "Testing construction" )); - - ulong align = fd_vinyl_io_bd_align(); - FD_TEST( fd_ulong_is_pow2( align ) ); - - FD_TEST( !fd_vinyl_io_bd_footprint( ULONG_MAX ) ); - - ulong footprint = fd_vinyl_io_bd_footprint( spad_max ); - FD_TEST( fd_ulong_is_aligned( footprint, align ) ); - if( FD_UNLIKELY( (footprint>MEM_MAX) | (align>512UL) ) ) FD_LOG_ERR(( "update mem for this test" )); - - char const * info = "info"; - ulong info_sz = strlen( info ) + 1UL; - ulong info_sz_bad = FD_VINYL_BSTREAM_SYNC_INFO_MAX + 1UL; - - FD_TEST( !fd_vinyl_io_bd_init( NULL, spad_max, fd, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_bd_init( (void *)1UL, spad_max, fd, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_bd_init( mem, 0UL, fd, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_bd_init( mem, 511UL, fd, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_bd_init( mem, 1UL<<63, fd, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_bd_init( mem, spad_max, -1, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_bd_init( mem, spad_max, fd, 1, info, info_sz_bad, seed ) ); - /* Note: info_sz, info and seed ignored with reset 0 */ - /* Note: info NULL implies info_sz zero */ - /* Note: seed arbitrary */ - - fd_vinyl_io_t * io = fd_vinyl_io_bd_init( mem, spad_max, fd, 1, info, info_sz, seed ); - FD_TEST( io ); - - FD_TEST( !fd_vinyl_mmio ( io ) ); - FD_TEST( !fd_vinyl_mmio_sz( io ) ); - - FD_LOG_NOTICE(( "Testing accessors" )); - - FD_TEST( fd_vinyl_io_type ( io )==FD_VINYL_IO_TYPE_BD ); - FD_TEST( fd_vinyl_io_seed ( io )==seed ); - FD_TEST( fd_vinyl_io_seq_ancient ( io )==seq_ancient ); - FD_TEST( fd_vinyl_io_seq_past ( io )==seq_past ); - FD_TEST( fd_vinyl_io_seq_present ( io )==seq_present ); - FD_TEST( fd_vinyl_io_seq_future ( io )==seq_future ); - - FD_LOG_NOTICE(( "Testing operations" )); - - test( io, rng ); - - FD_LOG_NOTICE(( "Aborting and resuming" )); - - FD_TEST( fd_vinyl_io_fini( io )==mem ); - - io = fd_vinyl_io_bd_init( mem, spad_max, fd, 0, (void *)1UL, ULONG_MAX, ~seed ); /* info_sz, info, seed ignored on resume */ - FD_TEST( io ); - - FD_LOG_NOTICE(( "Testing operations (after resume)" )); - - test( io, rng ); - - FD_TEST( fd_vinyl_io_type ( io )==FD_VINYL_IO_TYPE_BD ); - FD_TEST( fd_vinyl_io_seed ( io )==seed ); - FD_TEST( fd_vinyl_io_seq_ancient ( io )==seq_ancient ); - FD_TEST( fd_vinyl_io_seq_past ( io )==seq_past ); - FD_TEST( fd_vinyl_io_seq_present ( io )==seq_present ); - FD_TEST( fd_vinyl_io_seq_future ( io )==seq_future ); - - /* FIXME: TEST BSTREAM WRITE HELPERS */ - - FD_LOG_NOTICE(( "Testing scratch pad" )); - - FD_TEST( !fd_vinyl_io_commit( io, FD_VINYL_IO_FLAG_BLOCKING ) ); /* empty the spad */ - - void * smem = NULL; - ulong smem_sz = 0UL; - ulong spad_used = 0UL; - - while( spad_used=12 ) fd_vinyl_io_read_imm( io, seq, tst, sz ); - else { - fd_vinyl_io_rd_t rd[1]; - - rd->ctx = r; - rd->seq = seq; - rd->dst = tst; - rd->sz = sz; - - fd_vinyl_io_read( io, rd ); - - fd_vinyl_io_rd_t * _rd; - FD_TEST( !fd_vinyl_io_poll( io, &_rd, FD_VINYL_IO_FLAG_BLOCKING ) ); - - FD_TEST( _rd==rd ); - FD_TEST( rd->ctx==r ); - FD_TEST( rd->seq==seq ); - FD_TEST( rd->dst==tst ); - FD_TEST( rd->sz ==sz ); - - FD_TEST( fd_vinyl_io_poll( io, &_rd, FD_VINYL_IO_FLAG_BLOCKING )==FD_VINYL_ERR_EMPTY ); - FD_TEST( !_rd ); - } - FD_TEST( !memcmp( ref, tst, sz ) ); - break; - } - - } - } - - bcache_commit(); - FD_TEST( !fd_vinyl_io_commit( io, FD_VINYL_IO_FLAG_BLOCKING ) ); - - bcache_sync(); - FD_TEST( !fd_vinyl_io_sync( io, FD_VINYL_IO_FLAG_BLOCKING ) ); -} diff --git a/src/vinyl/io/test_vinyl_io_mm.c b/src/vinyl/io/test_vinyl_io_mm.c deleted file mode 100644 index dd129c83207..00000000000 --- a/src/vinyl/io/test_vinyl_io_mm.c +++ /dev/null @@ -1,141 +0,0 @@ -#include "../fd_vinyl.h" - -#include "test_vinyl_io_common.c" - -static ulong const dev_sz = FD_VINYL_BSTREAM_BLOCK_SZ + BCACHE_SZ; -static uchar dev[ FD_VINYL_BSTREAM_BLOCK_SZ + BCACHE_SZ ] __attribute__((aligned(FD_VINYL_BSTREAM_BLOCK_SZ))); - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - ulong spad_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--spad-max", 0UL, 131072UL ); - ulong seed = fd_env_strip_cmdline_ulong( &argc, &argv, "--seed", 0UL, 1234UL ); - - FD_LOG_NOTICE(( "Testing with --spad-max %lu --seed %lu", spad_max, seed )); - - fd_rng_t rng[1]; fd_rng_join( fd_rng_new( rng, 0U, 0UL ) ); - -# define MEM_MAX (1048576UL) - uchar mem[ MEM_MAX ] __attribute__((aligned(512))); - - FD_LOG_NOTICE(( "Testing construction" )); - - ulong align = fd_vinyl_io_mm_align(); - FD_TEST( fd_ulong_is_pow2( align ) ); - - FD_TEST( !fd_vinyl_io_mm_footprint( ULONG_MAX ) ); - - ulong footprint = fd_vinyl_io_mm_footprint( spad_max ); - FD_TEST( fd_ulong_is_aligned( footprint, align ) ); - if( FD_UNLIKELY( (footprint>MEM_MAX) | (align>512UL) ) ) FD_LOG_ERR(( "update mem for this test" )); - - char const * info = "info"; - ulong info_sz = strlen( info ) + 1UL; - ulong info_sz_bad = FD_VINYL_BSTREAM_SYNC_INFO_MAX + 1UL; - - FD_TEST( !fd_vinyl_io_mm_init( NULL, spad_max, dev, dev_sz, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( (void *)1UL, spad_max, dev, dev_sz, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, 0UL, dev, dev_sz, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, 1UL, dev, dev_sz, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, 1UL<<63, dev, dev_sz, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, spad_max, NULL, dev_sz, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, spad_max, (void *)1UL, dev_sz, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, spad_max, dev, 3584UL, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, spad_max, dev, dev_sz+1UL, 1, info, info_sz, seed ) ); - FD_TEST( !fd_vinyl_io_mm_init( mem, spad_max, dev, dev_sz, 1, info, info_sz_bad, seed ) ); - /* Note: info_sz, info and seed ignored with reset 0 */ - /* Note: info NULL implies info_sz zero */ - /* Note: seed arbitrary */ - - fd_vinyl_io_t * io = fd_vinyl_io_mm_init( mem, spad_max, dev, dev_sz, 1, info, info_sz, seed ); - FD_TEST( io ); - - FD_LOG_NOTICE(( "Testing accessors" )); - - FD_TEST( fd_vinyl_io_type ( io )==FD_VINYL_IO_TYPE_MM ); - FD_TEST( fd_vinyl_io_seed ( io )==seed ); - FD_TEST( fd_vinyl_io_seq_ancient ( io )==seq_ancient ); - FD_TEST( fd_vinyl_io_seq_past ( io )==seq_past ); - FD_TEST( fd_vinyl_io_seq_present ( io )==seq_present ); - FD_TEST( fd_vinyl_io_seq_future ( io )==seq_future ); - - FD_LOG_NOTICE(( "Testing operations" )); - - test( io, rng ); - - FD_LOG_NOTICE(( "Aborting and resuming" )); - - FD_TEST( fd_vinyl_io_fini( io )==mem ); - - io = fd_vinyl_io_mm_init( mem, spad_max, dev, dev_sz, 0, (void *)1UL, ULONG_MAX, ~seed ); /* info_sz, info, seed ignored on resume */ - FD_TEST( io ); - - void * mmio = fd_vinyl_mmio ( io ); FD_TEST( mmio && fd_ulong_is_aligned( (ulong)mmio, FD_VINYL_BSTREAM_BLOCK_SZ ) ); - ulong mmio_sz = fd_vinyl_mmio_sz( io ); FD_TEST( mmio_sz && fd_ulong_is_aligned( mmio_sz, FD_VINYL_BSTREAM_BLOCK_SZ ) ); - - FD_LOG_NOTICE(( "Testing operations (after resume)" )); - - test( io, rng ); - - FD_TEST( fd_vinyl_io_type ( io )==FD_VINYL_IO_TYPE_MM ); - FD_TEST( fd_vinyl_io_seed ( io )==seed ); - FD_TEST( fd_vinyl_io_seq_ancient ( io )==seq_ancient ); - FD_TEST( fd_vinyl_io_seq_past ( io )==seq_past ); - FD_TEST( fd_vinyl_io_seq_present ( io )==seq_present ); - FD_TEST( fd_vinyl_io_seq_future ( io )==seq_future ); - - FD_LOG_NOTICE(( "Testing scratch pad" )); - - FD_TEST( !fd_vinyl_io_commit( io, FD_VINYL_IO_FLAG_BLOCKING ) ); /* empty the spad */ - - void * smem = NULL; - ulong smem_sz = 0UL; - ulong spad_used = 0UL; - - while( spad_used=FD_VINYL_LINE_EVICT_PRIO_UNC ) ) return; /* no-op */ - - uint line_cnt = (uint)_line_cnt; - uint line_idx = (uint)_line_idx; - - /* Remove line_idx from its current position in the eviction sequence */ - - uint line_idx_lru = *_line_idx_lru; - uint line_idx_newer = line[ line_idx ].line_idx_newer; uint line_idx_older = line[ line_idx ].line_idx_older; - - FD_DCHECK_CRIT( line_idx_lru line_idx==line_idx, "corruption detected" ); - FD_DCHECK_CRIT( !obj->rd_active, "corruption detected" ); - fd_vinyl_data_free( data, obj ); - line[ line_idx ].obj = NULL; - } - - if( FD_LIKELY( ele_idx2 also treated as UNC */ - -/* FD_VINYL_LINE_MAX gives the maximum number of lines that can be - handled by a vinyl tile. This is a large power of 2 minus 1. The - below value ensures that a value in [0,line_cnt] can be represented - in 32-bits. Note that larger is possible but requires more DRAM - overhead. For typical applications (where pair vals sizes are - measured in KiB to MiB), a line_cnt at this LINE_MAX would require - impractically large amounts of DRAM for the vinyl cache ... TiB to - PiB). */ - -#define FD_VINYL_LINE_MAX ((1UL<<32)-1UL) - -/* FD_VINYL_LINE_REF_MAX gives the max acquires-for-read allowed on a - vinyl cache line. Like LINE_MAX, this could be made larger at the - expense of more DRAM overhead (and, more theoretical, more frequent - line version wrapping for speculative reads). The current value is - sufficient to support an impractical number of concurrent clients - acquiring the same pair for concurrently read. */ - -#define FD_VINYL_LINE_REF_MAX ((1L<<32)-2L) - -/* FD_VINYL_LINE_VER_MAX gives the maximum number of versions a line - can have before its version number gets reused. This is a large - power of 2 minus 1 such that version number reuse is impractical over - the duration of any speculative reads. */ - -#define FD_VINYL_LINE_VER_MAX ((1UL<<32)-1UL) - -/* A fd_vinyl_line_t stores information the vinyl tile uses to track - how a key-val pair has been cached in DRAM. If the val field is - NULL, there is no pair cached in that line and all other fields are - ignored. Practically speaking, the vinyl tile uses this (local) - structure to tie together the (shared) pair meta map and the (shared) - data cache region. - - ctl contains: - - ver: This is bumped every time the line is changed to allow clients - to do reliable lockfree speculative reads of the line under the - assumption speculative reader will complete and test their - speculative read before the version number can wrap. - - ref: -1 - the line is acquired for modify, 0 - the line is not - acquired for anything (and thus is evictable), >0 - the line is - acquired-for-read ref times. */ - -struct __attribute__((aligned(32))) fd_vinyl_line { - fd_vinyl_data_obj_t * obj; /* location in the data cache of the data_obj storing val, NULL if not caching a pair */ - ulong ele_idx; /* map element storing key and the pair metadata (app and key), in [0,map_cnt) */ - ulong ctl; /* packs the line version and line reference count */ - uint line_idx_older; /* older line in eviction sequence, in [0,line_cnt) */ - uint line_idx_newer; /* newer line in eviction sequence, in [0,line_cnt) */ -}; - -typedef struct fd_vinyl_line fd_vinyl_line_t; - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_line_ctl returns ver and ref encoded as a line ctl. ver is - wrapped to be in [0,FD_VINYL_LINE_VER_MAX]. ref is assumed to be in - [-1,FD_VINYL_LINE_REF_MAX]. - - fd_vinyl_line_ctl_{ver,ref} returns the decoded eponymous field from - a line ctl. The return will be in - {[0,FD_VINYL_LINE_VER_MAX]),[-1,FD_VINYL_LINE_REF_MAX]}. */ - -FD_FN_CONST static inline ulong -fd_vinyl_line_ctl( ulong ver, - long ref ) { - return (ver<<32) | ((ulong)(ref+1L)); -} - -FD_FN_CONST static inline ulong fd_vinyl_line_ctl_ver( ulong ctl ) { return ctl>>32; } -FD_FN_CONST static inline long fd_vinyl_line_ctl_ref( ulong ctl ) { return ((long)(ctl & ((1UL<<32)-1UL)))-1L; } - -/* fd_vinyl_line_evict_prio changes the eviction priority of line - line_idx to evict_prio. Cannot fail from the caller's perspective - (will FD_LOG_CRIT if line corruption was detected). */ - -void -fd_vinyl_line_evict_prio( uint * _line_idx_lru, /* Pointer to the LRU line idx */ - fd_vinyl_line_t * line, /* Indexed [0,line_cnt) */ - ulong _line_cnt, /* In [3,FD_VINYL_LINE_MAX] */ - ulong _line_idx, /* In [0,line_cnt) */ - int evict_prio ); /* FD_VINYL_LINE_EVICT_PRIO_* */ - -/* fd_vinyl_line_evict_lru finds the least recently used evictable line - and evicts it. Returns the line_idx of that line (will be free to - use). Cannot fail from the caller's perspective (will FD_LOG_CRIT if - corruption was detected or quotas were misconfigured). */ - -ulong -fd_vinyl_line_evict_lru( uint * _line_idx_lru, /* Pointer to the LRU line idx */ - fd_vinyl_line_t * line, /* Indexed [0,line_cnt) */ - ulong line_cnt, /* In [3,FD_VINYL_LINE_MAX] */ - fd_vinyl_meta_ele_t * ele0, /* Indexed [0,ele_max) */ - ulong ele_max, - fd_vinyl_data_t * data ); - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_line_fd_vinyl_line_h */ diff --git a/src/vinyl/line/test_vinyl_line.c b/src/vinyl/line/test_vinyl_line.c deleted file mode 100644 index 93ffe0479a8..00000000000 --- a/src/vinyl/line/test_vinyl_line.c +++ /dev/null @@ -1,92 +0,0 @@ -#include "../fd_vinyl.h" - -FD_STATIC_ASSERT( FD_VINYL_LINE_EVICT_PRIO_MRU==0, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_LINE_EVICT_PRIO_LRU==1, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_LINE_EVICT_PRIO_UNC==2, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_LINE_MAX ==(1UL<<32)-1UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_LINE_REF_MAX==(1L <<32)-2L, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_LINE_VER_MAX==(1UL<<32)-1UL, unit_test ); - -#define LINE_CNT (10UL) - -static uint ref_newer[ LINE_CNT ]; -static uint ref_older[ LINE_CNT ]; -static uint ref_lru; - -static void -ref_evict_prio( ulong _line_idx, - int evict_prio ) { - uint line_idx = (uint)_line_idx; - - if( evict_prio>=FD_VINYL_LINE_EVICT_PRIO_UNC ) return; - - uint newer = ref_newer[ line_idx ]; - uint older = ref_older[ line_idx ]; - - ref_older[ newer ] = older; - ref_newer[ older ] = newer; - - if( ref_lru==line_idx ) ref_lru = newer; - - newer = ref_lru; - older = ref_older[ ref_lru ]; - - ref_newer[ older ] = line_idx; ref_older[ line_idx ] = older; - ref_older[ newer ] = line_idx; ref_newer[ line_idx ] = newer; - - if( evict_prio==FD_VINYL_LINE_EVICT_PRIO_LRU ) ref_lru = line_idx; -} - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - fd_rng_t rng[1]; fd_rng_join( fd_rng_new( rng, 0U, 0UL ) ); - - fd_vinyl_line_t line[ LINE_CNT ]; - - for( ulong line_idx=0UL; line_idx>= 3; - ulong line_idx = r % LINE_CNT; - fd_vinyl_line_evict_prio( &line_idx_lru, line, LINE_CNT, line_idx, evict_prio ); - ref_evict_prio( line_idx, evict_prio ); - - FD_TEST( line_idx_lru==ref_lru ); - for( ulong line_idx=0UL; line_idxphdr.ctl) -#define MAP_ELE_FREE(ctx,ele) do { (ele)->phdr.ctl = 0UL; } while(0) -#define MAP_ELE_MOVE(ctx,dst,src) do { fd_vinyl_meta_ele_t * _src = (src); *(dst) = *_src; _src->phdr.ctl = 0UL; } while(0) -#define MAP_IMPL_STYLE 2 -#include "../../util/tmpl/fd_map_slot_para.c" - -int -fd_vinyl_meta_query_fast( fd_vinyl_meta_ele_t const * ele0, - ulong ele_max, - fd_vinyl_key_t const * key, - ulong memo, - ulong * _ele_idx ) { - - ulong ele_idx = memo & (ele_max-1UL); - - int err = FD_VINYL_ERR_CORRUPT; - - ulong rem; - - for( rem=ele_max; rem; rem-- ) { /* guarantee finite termination in face of corruption */ - fd_vinyl_meta_ele_t const * ele = ele0 + ele_idx; - - if( FD_UNLIKELY( !ele->phdr.ctl ) ) { /* not found */ - *_ele_idx = ele_idx; - err = FD_VINYL_ERR_KEY; - break; - } - - if( FD_LIKELY( ele->memo==memo ) && FD_LIKELY( fd_vinyl_key_eq( &ele->phdr.key, key ) ) ) { /* found */ - *_ele_idx = ele_idx; - err = FD_VINYL_SUCCESS; - break; - } - - ele_idx = (ele_idx+1UL) & (ele_max-1UL); /* collision, try next slot */ - } - - FD_DCHECK_CRIT( rem, "corruption detected" ); - - return err; -} - -#include "../line/fd_vinyl_line.h" /* FIXME: gross (maybe make line below meta in the API hierarchy?) */ - -void -fd_vinyl_meta_remove_fast( fd_vinyl_meta_ele_t * ele0, - ulong ele_max, - ulong * lock, - int lock_shift, - fd_vinyl_line_t * line, - ulong line_cnt, - ulong ele_idx ) { - - /* At this point, we know: - - - nobody will lock any elements behind our back (single writer) - - no elements are locked (not in a prepare) - - there is at least one unoccupied element in the meta (pair_max < ele_max) - - there is at least one occupied element in the meta (ele_idx) - - (thus ele_max is at least 2) - - When we remove element ele_idx, we might need to move elements in - the cyclic contiguously occupied range starting at ele_idx toward - ele_idx (but not before ele_idx) to keep their probe sequences - intact. This can interfere with lockfree non-blocking concurrent - reads whose probe sequences overlap this range. - - Thus, we determine the set of contiguously occupied elements - starting at ele_idx, determine the range of corresponding locks and - lock them before we remove ele_idx to protect concurrent readers. - If the map is not overfilled, contig_cnt is O(1) elements. - - If the ele is currently cached, we also need to update the - underlying line to it points to the moved location. */ - - ulong contig_cnt; - - for( contig_cnt=1UL; contig_cnt> lock_shift; /* lock that covers first element (wrapped) */ - ulong lock_hi = (ele_idx + contig_cnt - 1UL) >> lock_shift; /* lock that covers last element (unwrapped) */ - ulong lock_max = ele_max >> lock_shift; /* max locks */ - ulong lock_cnt = fd_ulong_min( lock_hi - lock_lo + 1UL, lock_max ); /* num locks required */ - - /* Lock the range */ - - for( ulong idx=0UL; idxele_idx) & ((hole_idxphdr.ctl) -#define MAP_ELE_FREE(ctx,ele) do { (ele)->phdr.ctl = 0UL; } while(0) -#define MAP_ELE_MOVE(ctx,dst,src) do { fd_vinyl_meta_ele_t * _src = (src); *(dst) = *_src; _src->phdr.ctl = 0UL; } while(0) -#define MAP_IMPL_STYLE 1 -#include "../../util/tmpl/fd_map_slot_para.c" - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_meta_ele_in_use returns 1 if the given ele is in use and 0 - otherwise. Assumes ele is valid. If the element is in use, - ele->phdr.key and ele->memo are valid. - - fd_vinyl_meta_ele_in_bstream returns 1 if the given ele is present at - bstream seq_present and 0 otherwise. Assumes ele valid and in use. - If ele is in bstream, ele->phdr exactly mirrors the pair header for - the version of the pair present at seq_present and ele->seq gives the - location in the bstream's past of this version of the pair. - - fd_vinyl_meta_ele_in_cache returns 1 if space has been allocated in - the vinyl's data cache for the pair. Assumes ele is valid and in - use. If ele is in cache, ele->line_idx gives vinyl cache line - assigned to the pair value. */ - -FD_FN_PURE static inline int fd_vinyl_meta_ele_in_use ( fd_vinyl_meta_ele_t const * ele ) { return !!ele->phdr.ctl; } -FD_FN_PURE static inline int fd_vinyl_meta_ele_in_bstream( fd_vinyl_meta_ele_t const * ele ) { return ele->phdr.ctl!=ULONG_MAX; } -FD_FN_PURE static inline int fd_vinyl_meta_ele_in_cache ( fd_vinyl_meta_ele_t const * ele ) { return ele->line_idx!=ULONG_MAX; } - -/* fd_vinyl_meta_prepare_fast prepares to modify meta element ele_idx - under the assumption the caller is the only active writer and there - are no meta prepares in progress. - - fd_vinyl_meta_publish_fast completes a prepare of meta element - ele_idx, making the modifications visible to concurrent readers. - - fd_vinyl_meta_cancel_fast completes a prepare of meta element - ele_idx, indicating to concurrent readers that no modifications were - made. The caller promises that shared fields of the meta element was - not modified at any point in time during the prepare. - - l is a pointer to the meta lock array, s is the meta lock shift and e - is the meta element index. Does no input argument checking. These - functions cannot fail. */ - -static inline void -fd_vinyl_meta_lock_update_fast( ulong * lock, - long dir ) { - ulong version = (*lock) + (ulong)dir; - FD_COMPILER_MFENCE(); - *lock = version; - FD_COMPILER_MFENCE(); -} - -#define fd_vinyl_meta_prepare_fast(l,s,e) fd_vinyl_meta_lock_update_fast( (l) + ((e)>>(s)), 1L ) -#define fd_vinyl_meta_publish_fast(l,s,e) fd_vinyl_meta_lock_update_fast( (l) + ((e)>>(s)), 1L ) -#define fd_vinyl_meta_cancel_fast( l,s,e) fd_vinyl_meta_lock_update_fast( (l) + ((e)>>(s)), -1L ) - -/* fd_vinyl_meta_query_fast queries a meta for key under the assumption - the caller is the only active meta writer and there are no meta - prepares in progress. Does no input arg checking. On success, - returns FD_VINYL_SUCCESS (0) (key was found, on return *_ele_idx - gives key's current meta element index). On failure, returns a - FD_VINYL_ERR code (negative). Reasons for failure include KEY (key - was not found, on return *_ele_idx gives the meta element index - suitable for storing pair metadata if the meta is not already at - capacity). Will FD_LOG_CRIT if anything wonky is detected. - - IMPORTANT SAFETY TIP! In general, meta element ele_idx should not be - modified by the writer unless it is protected by a prepare. - Inserting without doing a prepare is fine so long as phdr.ctl becomes - visible last. */ - -int -fd_vinyl_meta_query_fast( fd_vinyl_meta_ele_t const * ele0, /* indexed [0,ele_max) */ - ulong ele_max, /* power of 2 */ - fd_vinyl_key_t const * key, /* key to query */ - ulong memo, /* == fd_vinyl_key_memo( seed, key ) */ - ulong * _ele_idx ); /* will be in [0,ele_max) on return */ - -/* fd_vinyl_meta_remove_fast removes the key<>metadata mapping at meta - element ele_idx from the meta under the assumption the caller is the - only active writer to the meta and there are no meta prepares in - progress. This cannot fail from the caller's perspective - (FD_LOG_CRIT if meta or line is corruption detected during removal). */ - -void -fd_vinyl_meta_remove_fast( fd_vinyl_meta_ele_t * ele0, /* Assumes at least one unoccupied element in the meta */ - ulong ele_max, /* Assumes power of 2 (of at least 2 given 1 occupied and 1 hole) */ - ulong * lock, /* lock_max == ele_max >> lock_shift */ - int lock_shift, /* ==log2 ele_max / lock_cnt where lock_cnt is a power 2 <= ele_max */ - fd_vinyl_line_t * line, /* Indexed [0,line_cnt) */ - ulong line_cnt, /* In [3,FD_VINYL_LINE_MAX] */ - ulong ele_idx ); /* Assumes map element ele_idx is occupied */ - -/* fd_vinyl_meta_query is a simplified query for key for concurrent - readers. It handles out try / test and filters out keys that are in - the process of being created. Assumes meta is a current local join, - key and _info are valid for the duration of the call and _info is - a fd_vinyl_info_t compatible memory region. - - Returns FD_VINYL_SUCCESS (0) if pair key existed in the bstream's - seq_present when the bstream was observed during the call. On - return, *_info will be populated with the pair key's info at that - seq_present (including the val byte size). - - Returns FD_VINYL_ERR_KEY (negative) if pair key did not exist in the - bstream's seq_present when the bstream was observed. On return, - *_info will be clobbered. Retains no interest in key or _info. - - IMPORTANT SAFETY TIP! This can block the caller until the query - resolves. Concurrent readers wanting guaranteed non-blocking - behavior in the face of a vinyl tile dying in the middle of updating - conflicting ele meta, wanting to provide hints / extract the key memo - for reuse / etc, should use more general lockfree query API. */ - -static inline int -fd_vinyl_meta_query( fd_vinyl_meta_t * meta, - fd_vinyl_key_t const * key, - void * _info ) { - ulong ctl; - - for(;;) { - - fd_vinyl_meta_query_t query[1]; - int err = fd_vinyl_meta_query_try( meta, key, NULL, query, FD_MAP_FLAG_BLOCKING ); - if( FD_UNLIKELY( err ) ) return err; - fd_vinyl_meta_ele_t const * ele = fd_vinyl_meta_query_ele_const( query ); - - ctl = ele->phdr.ctl; - *(fd_vinyl_info_t *)_info = ele->phdr.info; - - if( FD_LIKELY( !fd_vinyl_meta_query_test( query ) ) ) break; - - FD_SPIN_PAUSE(); - - } - - return fd_int_if( ctl!=ULONG_MAX, FD_MAP_SUCCESS, FD_MAP_ERR_KEY ); -} - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_meta_fd_vinyl_meta_h */ diff --git a/src/vinyl/meta/test_vinyl_meta.c b/src/vinyl/meta/test_vinyl_meta.c deleted file mode 100644 index 46a03d93fa7..00000000000 --- a/src/vinyl/meta/test_vinyl_meta.c +++ /dev/null @@ -1,866 +0,0 @@ -#include "../fd_vinyl.h" - -#define SHMEM_MAX (1UL<<20) - -static FD_TL uchar shmem[ SHMEM_MAX ]; -static FD_TL ulong shmem_cnt = 0UL; - -static void * -shmem_alloc( ulong a, - ulong s ) { - uchar * m = (uchar *)fd_ulong_align_up( (ulong)(shmem + shmem_cnt), a ); - shmem_cnt = (ulong)((m + s) - shmem); - FD_TEST( shmem_cnt <= SHMEM_MAX ); - return (void *)m; -} - -static fd_vinyl_meta_t * tile_meta; -static ulong tile_iter_cnt; -static ulong tile_go; - -static int -tile_main( int argc, - char ** argv ) { - - /* Init local tile context */ - - fd_vinyl_meta_t * meta = tile_meta; - ulong iter_cnt = tile_iter_cnt; - ulong tile_idx = (ulong)(uint)argc; - ulong tile_cnt = (ulong)argv; - - fd_vinyl_meta_ele_t * ele0 = (fd_vinyl_meta_ele_t *)fd_vinyl_meta_shele( meta ); - ulong ele_max = fd_vinyl_meta_ele_max( meta ); - ulong seed = fd_vinyl_meta_seed( meta ); - - fd_vinyl_meta_ele_t sentinel[1]; - - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, (uint)tile_idx, fd_ulong_hash( tile_cnt ) ) ); - - ulong global_prefix = 0x0123456789abcdefUL; - ulong local_key = 0U; - - ulong save = shmem_cnt; - fd_vinyl_key_t * meta_key = shmem_alloc( alignof(fd_vinyl_key_t), ele_max*sizeof(fd_vinyl_key_t) ); - ulong meta_cnt = 0UL; - - ulong lock_max = fd_vinyl_meta_lock_max(); - ulong lock_cnt = fd_vinyl_meta_lock_cnt( meta ); - - /* Wait for the go code */ - - while( !FD_VOLATILE_CONST( tile_go ) ) FD_SPIN_PAUSE(); - - /* Hammer the meta with all manners of concurrent operations */ - - ulong diag_rem = 0UL; - for( ulong iter_idx=0UL; iter_idx>= 4; - int flags = (int)r; r >>= 32; - int blocking = !!(flags & FD_MAP_FLAG_BLOCKING); - int use_hint = !!(flags & FD_MAP_FLAG_USE_HINT); - int rdonly = !!(flags & FD_MAP_FLAG_RDONLY); - - fd_vinyl_meta_query_t query[1]; - - switch( op ) { - - case 0: { /* blocking read / bad insert (i.e. key already in meta) */ - if( FD_UNLIKELY( !meta_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, meta_cnt ); - fd_vinyl_key_t * key = meta_key + idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - - if( use_hint ) { - fd_vinyl_meta_hint( meta, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - } - - int err = fd_vinyl_meta_prepare( meta, key, sentinel, query, flags ); - fd_vinyl_meta_ele_t * ele = fd_vinyl_meta_query_ele( query ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - } else { - FD_TEST( (ulong)(ele-ele0)memo==memo ); - FD_TEST( fd_vinyl_key_eq( &ele->phdr.key, key ) ); - FD_TEST( fd_vinyl_meta_ele_in_use( ele ) ); - - ulong mod = ele->phdr.info.ul[0]; - - FD_TEST( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ); - FD_TEST( ele->seq == (key->ul[0] ^ mod) ); - FD_TEST( ele->line_idx == ULONG_MAX ); - - fd_vinyl_meta_cancel( query ); - } - - break; - } - - case 1: { /* good insert (i.e. key not already in meta) */ - fd_vinyl_key_t key[1]; fd_vinyl_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_vinyl_key_memo( seed, key ); - ulong mod = 0UL; - - if( use_hint ) { - fd_vinyl_meta_hint( meta, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - } - - int err = fd_vinyl_meta_prepare( meta, key, sentinel, query, flags ); - fd_vinyl_meta_ele_t * ele = fd_vinyl_meta_query_ele( query ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_FULL ); - } else { - FD_TEST( (ulong)(ele-ele0)memo = memo; - ele->phdr.ctl = 1UL; - ele->phdr.key = *key; - ele->phdr.info.ul[0] = mod; - ele->phdr.info.ul[1] = (key->ul[1] ^ mod); - ele->seq = (key->ul[0] ^ mod); - ele->line_idx = ULONG_MAX; - - fd_vinyl_meta_publish( query ); - - meta_key[ meta_cnt++ ] = *key; - local_key++; - } - - break; - } - - case 2: { /* bad remove (i.e. key not already in meta) */ - fd_vinyl_key_t key[1]; fd_vinyl_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_vinyl_key_memo( seed, key ); - - if( use_hint ) { - fd_vinyl_meta_hint( meta, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - } - - int err = fd_vinyl_meta_remove( meta, key, query, flags ); - - if( FD_LIKELY( err ) ) { - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_KEY ); - } - - break; - } - - case 3: { /* good remove (i.e. key already in meta) */ - if( FD_UNLIKELY( !meta_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, meta_cnt ); - fd_vinyl_key_t * key = meta_key + idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - - if( use_hint ) { - fd_vinyl_meta_hint( meta, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - } - - int err = fd_vinyl_meta_remove( meta, key, query, flags ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - } else { - meta_key[ idx ] = meta_key[ --meta_cnt ]; - } - - break; - } - - case 4: { /* bad modify (i.e. key not already in meta) */ - fd_vinyl_key_t key[1]; fd_vinyl_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_vinyl_key_memo( seed, key ); - - if( use_hint ) { - fd_vinyl_meta_hint( meta, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - } - - int err = fd_vinyl_meta_prepare( meta, key, sentinel, query, flags ); - fd_vinyl_meta_ele_t * ele = fd_vinyl_meta_query_ele( query ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_FULL ); - } else { - FD_TEST( (ulong)(ele-ele0)1UL ); - } else { - FD_TEST( (ulong)(ele-ele0)seq; - FD_TEST( ele->memo==memo ); - FD_TEST( fd_vinyl_meta_ele_in_use( ele ) ); - FD_TEST( fd_vinyl_key_eq( &ele->phdr.key, key ) ); - FD_TEST( ele->line_idx==ULONG_MAX ); - - mod++; - - ele->phdr.info.ul[0] = mod; - ele->phdr.info.ul[1] = (key->ul[1] ^ mod); - ele->seq = (key->ul[0] ^ mod); - ele->line_idx = ULONG_MAX; - - fd_vinyl_meta_publish( query ); - } - - break; - } - - case 6: { /* bad query (i.e. key not already in meta) */ - fd_vinyl_key_t key[1]; fd_vinyl_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_vinyl_key_memo( seed, key ); - - if( use_hint ) { - fd_vinyl_meta_hint( meta, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - } - - int err = fd_vinyl_meta_query_try( meta, key, sentinel, query, flags ); - fd_vinyl_meta_ele_t const * ele = fd_vinyl_meta_query_ele_const( query ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - - FD_TEST( ele==sentinel ); - if( err==FD_MAP_ERR_AGAIN ) { FD_TEST( !blocking ); FD_TEST( tile_cnt>1UL ); } - else FD_TEST( err==FD_MAP_ERR_KEY ); - - break; - } - - case 7: { /* good query */ - if( FD_UNLIKELY( !meta_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, meta_cnt ); - fd_vinyl_key_t * key = meta_key + idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - - if( use_hint ) { - fd_vinyl_meta_hint( meta, key, query, flags & (~FD_MAP_FLAG_USE_HINT) ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - } - - int err = fd_vinyl_meta_query_try( meta, key, sentinel, query, flags ); - fd_vinyl_meta_ele_t const * ele = fd_vinyl_meta_query_ele_const( query ); - FD_TEST( memo==fd_vinyl_meta_query_memo( query ) ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( ele==sentinel ); - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - } else { - FD_TEST( (ulong)(ele-ele0)phdr.info.ul[0]; - - int spec_good = ( ele->memo==memo ) - & ( fd_vinyl_meta_ele_in_use( ele ) ) - & ( fd_vinyl_key_eq( &ele->phdr.key, key ) ) - & ( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ) - & ( ele->seq == (key->ul[0] ^ mod) ) - & ( ele->line_idx == ULONG_MAX ); - - err = fd_vinyl_meta_query_test( query ); - - if( FD_UNLIKELY( err ) ) { - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( tile_cnt>1UL ); - } else { - FD_TEST( spec_good ); - } - } - - break; - } - - case 8: { /* parallel iteration */ - ulong version[ lock_max ]; - - ulong range_start = fd_rng_ulong_roll( rng, lock_cnt ); - ulong range_cnt = fd_ulong_min( fd_rng_coin_tosses( rng ), lock_cnt ); - - int err = fd_vinyl_meta_lock_range( meta, range_start, range_cnt, flags, version ); - - if( FD_UNLIKELY( err ) ) { - - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - - } else { - - ulong lock_idx = range_start; - for( ulong lock_rem=range_cnt; lock_rem; lock_rem-- ) { - ulong e0 = fd_vinyl_meta_lock_ele0( meta, lock_idx ); - ulong e1 = fd_vinyl_meta_lock_ele1( meta, lock_idx ); - - FD_TEST( e0< e1 ); - FD_TEST( e1<=ele_max ); - - for( ulong ele_idx=e0; ele_idxphdr.key; - ulong mod = ele->phdr.info.ul[0]; - - FD_TEST( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ); - FD_TEST( ele->seq == (key->ul[0] ^ mod) ); - FD_TEST( ele->line_idx == ULONG_MAX ); - - if( !rdonly ) { - mod++; - - ele->phdr.info.ul[0] = mod; - ele->phdr.info.ul[1] = (key->ul[1] ^ mod); - ele->seq = (key->ul[0] ^ mod); - ele->line_idx = ULONG_MAX; - - } - } - - lock_idx = (lock_idx+1UL) & (lock_cnt-1UL); - } - - fd_vinyl_meta_unlock_range( meta, range_start, range_cnt, version ); - - } - - break; - } - - case 9: { /* parallel memo iteration */ - ulong memo; - ulong iter_min; - if( FD_UNLIKELY( !meta_cnt ) ) { /* pick a memo not likely in meta */ - memo = fd_rng_ulong( rng ); - iter_min = 0UL; - } else { /* pick a memo in meta at least once */ - ulong idx = fd_rng_ulong_roll( rng, meta_cnt ); - fd_vinyl_key_t * key = meta_key + idx; - memo = fd_vinyl_key_memo( seed, key ); - iter_min = 1UL; - } - - fd_vinyl_meta_iter_t iter[1]; - int err = fd_vinyl_meta_iter_init( meta, memo, flags, iter ); - - if( FD_UNLIKELY( err ) ) { - - FD_TEST( err==FD_MAP_ERR_AGAIN ); - FD_TEST( !blocking ); - FD_TEST( tile_cnt>1UL ); - - } else { - - ulong iter_cnt = 0UL; - while( !fd_vinyl_meta_iter_done( iter ) ) { - fd_vinyl_meta_ele_t * ele = fd_vinyl_meta_iter_ele( iter ); - - FD_TEST( (ulong)(ele-ele0)phdr.key )==memo ); - FD_TEST( fd_vinyl_meta_ele_in_use( ele ) ); - - fd_vinyl_key_t const * key = &ele->phdr.key; - ulong mod = ele->phdr.info.ul[0]; - - FD_TEST( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ); - FD_TEST( ele->seq == (key->ul[0] ^ mod) ); - FD_TEST( ele->line_idx == ULONG_MAX ); - - if( !rdonly ) { - mod++; - - ele->phdr.info.ul[0] = mod; - ele->phdr.info.ul[1] = (key->ul[1] ^ mod); - ele->seq = (key->ul[0] ^ mod); - ele->line_idx = ULONG_MAX; - } - - iter_cnt++; - - FD_TEST( fd_vinyl_meta_iter_next( iter )==iter ); - } - - FD_TEST( fd_vinyl_meta_iter_fini( iter )==iter ); - - FD_TEST( iter_cnt>=iter_min ); - } - - break; - } - - default: - break; - } - } - - /* Clean up for the next text battery */ - - for( ulong meta_idx=0UL; meta_idxlock; - int lock_shift = meta->lock_shift; - - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, (uint)tile_idx, fd_ulong_hash( tile_cnt ) ) ); - - ulong global_prefix = 0x0123456789abcdefUL; - ulong local_key = 0U; - - /* Wait for the go code */ - - while( !FD_VOLATILE_CONST( tile_go ) ) FD_SPIN_PAUSE(); - - /* Hammer the meta with all manners of writer side operations */ - - ulong diag_rem = 0UL; - for( ulong iter_idx=0UL; iter_idx>= 4; - - switch( op ) { - - case 0: { /* single writer good query (i.e. key already in meta) */ - if( FD_UNLIKELY( !_meta_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, _meta_cnt ); - fd_vinyl_key_t * key = _meta_key + idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - - ulong ele_idx; - FD_TEST( !fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &ele_idx ) ); - FD_TEST( ele_idxmemo==memo ); - FD_TEST( fd_vinyl_meta_ele_in_use( ele ) ); - FD_TEST( fd_vinyl_key_eq( &ele->phdr.key, key ) ); - - ulong mod = ele->phdr.info.ul[0]; - - FD_TEST( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ); - FD_TEST( ele->seq == (key->ul[0] ^ mod) ); - FD_TEST( ele->line_idx == ULONG_MAX ); - - break; - } - - case 1: { /* single writer bad query (i.e. key not in meta) */ - fd_vinyl_key_t key[1]; fd_vinyl_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_vinyl_key_memo( seed, key ); - - ulong ele_idx; - int err = fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &ele_idx ); - FD_TEST( err==FD_VINYL_ERR_KEY ); - FD_TEST( ele_idx=(ele_max-1UL) ) ) break; - fd_vinyl_key_t key[1]; fd_vinyl_key_init_ulong( key, global_prefix, tile_cnt, tile_idx, local_key ); - ulong memo = fd_vinyl_key_memo( seed, key ); - ulong mod = 0UL; - - ulong ele_idx; - int err = fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &ele_idx ); - FD_TEST( err==FD_VINYL_ERR_KEY ); - FD_TEST( ele_idxmemo = memo; - ele->phdr.key = *key; - ele->phdr.info.ul[0] = mod; - ele->phdr.info.ul[1] = (key->ul[1] ^ mod); - ele->seq = (key->ul[0] ^ mod); - ele->line_idx = ULONG_MAX; - - FD_COMPILER_MFENCE(); - ele->phdr.ctl = 1UL; - FD_COMPILER_MFENCE(); - - _meta_key[ _meta_cnt++ ] = *key; - local_key++; - break; - } - - case 3: { /* single writer remove (i.e. key already in meta) */ - if( FD_UNLIKELY( !_meta_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, _meta_cnt ); - fd_vinyl_key_t * key = _meta_key + idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - - ulong ele_idx; - FD_TEST( !fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &ele_idx ) ); - FD_TEST( ele_idxmemo==memo ); - FD_TEST( fd_vinyl_meta_ele_in_use( ele ) ); - FD_TEST( fd_vinyl_key_eq( &ele->phdr.key, key ) ); - - ulong mod = ele->phdr.info.ul[0]; - - FD_TEST( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ); - FD_TEST( ele->seq == (key->ul[0] ^ mod) ); - FD_TEST( ele->line_idx == ULONG_MAX ); - - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, NULL, 0UL, ele_idx ); - - _meta_key[ idx ] = _meta_key[ --_meta_cnt ]; - break; - } - - case 4: { /* good modify (i.e. key already in meta) */ - if( FD_UNLIKELY( !_meta_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, _meta_cnt ); - fd_vinyl_key_t * key = _meta_key + idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - - ulong ele_idx; - FD_TEST( !fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &ele_idx ) ); - FD_TEST( ele_idxmemo==memo ); - FD_TEST( fd_vinyl_meta_ele_in_use( ele ) ); - FD_TEST( fd_vinyl_key_eq( &ele->phdr.key, key ) ); - - ulong mod = ele->phdr.info.ul[0]; - - FD_TEST( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ); - FD_TEST( ele->seq == (key->ul[0] ^ mod) ); - FD_TEST( ele->line_idx == ULONG_MAX ); - - fd_vinyl_meta_prepare_fast( lock, lock_shift, ele_idx ); - - mod++; - - ele->phdr.info.ul[0] = mod; - ele->phdr.info.ul[1] = (key->ul[1] ^ mod); - ele->seq = (key->ul[0] ^ mod); - ele->line_idx = ULONG_MAX; - - fd_vinyl_meta_publish_fast( lock, lock_shift, ele_idx ); - break; - } - - case 5: { /* bad modify (i.e. key already in meta) */ - if( FD_UNLIKELY( !_meta_cnt ) ) break; - ulong idx = fd_rng_ulong_roll( rng, _meta_cnt ); - fd_vinyl_key_t * key = _meta_key + idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - - ulong ele_idx; - FD_TEST( !fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &ele_idx ) ); - FD_TEST( ele_idxmemo==memo ); - FD_TEST( fd_vinyl_meta_ele_in_use( ele ) ); - FD_TEST( fd_vinyl_key_eq( &ele->phdr.key, key ) ); - - ulong mod = ele->phdr.info.ul[0]; - - FD_TEST( ele->phdr.info.ul[1] == (key->ul[1] ^ mod) ); - FD_TEST( ele->seq == (key->ul[0] ^ mod) ); - FD_TEST( ele->line_idx == ULONG_MAX ); - - fd_vinyl_meta_prepare_fast( lock, lock_shift, ele_idx ); - - mod++; - - fd_vinyl_meta_cancel_fast( lock, lock_shift, ele_idx ); - break; - } - - default: - break; - } - } - - /* Clean up for the next text battery */ - - for( ulong meta_idx=0UL; meta_idx<_meta_cnt; meta_idx++ ) { - fd_vinyl_key_t * key = _meta_key + meta_idx; - ulong memo = fd_vinyl_key_memo( seed, key ); - ulong ele_idx; - FD_TEST( !fd_vinyl_meta_query_fast( ele0, ele_max, key, memo, &ele_idx ) ); - fd_vinyl_meta_remove_fast( ele0, ele_max, lock, lock_shift, NULL, 0UL, ele_idx ); - } - - FD_COMPILER_MFENCE(); - tile_go = 2; - FD_COMPILER_MFENCE(); - - fd_rng_delete( fd_rng_leave( rng ) ); - return 0; -} - -static int -reader_main( int argc, - char ** argv ) { - (void)argc; (void)argv; - - /* Init local tile context */ - - fd_vinyl_meta_t * meta = tile_meta; - ulong tile_idx = (ulong)(uint)argc; - ulong tile_cnt = (ulong)argv; - - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, (uint)tile_idx, fd_ulong_hash( tile_cnt ) ) ); - - /* Wait for the go code */ - - while( !FD_VOLATILE_CONST( tile_go ) ) FD_SPIN_PAUSE(); - - /* Run while the writer is running */ - - for(;;) { - - /* Pick a key the writer has recent published. Since _meta_key is - allocated to largest possible _meta_cnt, we don't care if we do - this atomically accurate or not (if we get a botched read, the - query will fail with ERR_KEY). */ - - fd_vinyl_key_t key[1]; - fd_vinyl_info_t info[1]; - - FD_COMPILER_MFENCE(); - - if( FD_UNLIKELY( tile_go!=1 ) ) break; - - ulong meta_cnt = _meta_cnt; - - if( FD_UNLIKELY( !meta_cnt ) ) { - FD_SPIN_PAUSE(); - continue; - } - - *key = _meta_key[ fd_rng_ulong_roll( rng, meta_cnt ) ]; - - int err = fd_vinyl_meta_query( meta, key, info ); - if( FD_UNLIKELY( err ) ) FD_TEST( err==FD_MAP_ERR_KEY ); - else { - FD_TEST( !err ); - - ulong mod = info->ul[0]; - - FD_TEST( info->ul[1] == (key->ul[1] ^ mod) ); - } - - /* And try to query for a key we know isn't in the map */ - - key->ul[0] = ~0UL; - FD_TEST( fd_vinyl_meta_query( meta, key, info )==FD_VINYL_ERR_KEY ); - - } - - fd_rng_delete( fd_rng_leave( rng ) ); - - return 0; -} - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - ulong ele_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--ele-max", NULL, 4096UL ); - ulong lock_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--lock-cnt", NULL, fd_vinyl_meta_lock_cnt_est ( ele_max ) ); - ulong probe_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--probe-max", NULL, fd_vinyl_meta_probe_max_est( ele_max ) ); - ulong seed = fd_env_strip_cmdline_ulong( &argc, &argv, "--seed", NULL, 1234UL ); - ulong iter_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--iter-cnt", NULL, 10000000UL ); - - FD_LOG_NOTICE(( "Testing (--ele-max %lu --lock-cnt %lu --probe-max %lu --seed %lu --iter-cnt %lu)", - ele_max, lock_cnt, probe_max, seed, iter_cnt )); - - fd_rng_t rng[1]; fd_rng_join( fd_rng_new( rng, 0U, 0UL ) ); - - FD_LOG_NOTICE(( "Creating vinyl meta element" )); - - void * shele = shmem_alloc( alignof(fd_vinyl_meta_ele_t), ele_max*sizeof(fd_vinyl_meta_ele_t) ); - memset( shele, 0, ele_max*sizeof(fd_vinyl_meta_ele_t) ); - - FD_LOG_NOTICE(( "Creating vinyl meta" )); - - ulong align = fd_vinyl_meta_align(); - ulong footprint = fd_vinyl_meta_footprint( ele_max, lock_cnt, probe_max ); - if( FD_UNLIKELY( !footprint ) ) FD_LOG_ERR(( "Unsupported --ele-max / --lock-cnt / --probe-max" )); - void * shmeta = shmem_alloc( align, footprint ); - - FD_TEST( fd_vinyl_meta_new( shmeta, ele_max, lock_cnt, probe_max, seed )==shmeta ); - - FD_LOG_NOTICE(( "Joining vinyl meta" )); - - fd_vinyl_meta_t meta[1]; fd_vinyl_meta_join( meta, shmeta, shele ); - - tile_meta = meta; - tile_iter_cnt = iter_cnt; - - ulong tile_max = fd_tile_cnt(); - for( ulong tile_cnt=1UL; tile_cnt<=tile_max; tile_cnt++ ) { - - /* FIXME: use tpool here */ - - FD_LOG_NOTICE(( "Testing concurrent operation on %lu tiles", tile_cnt )); - - FD_COMPILER_MFENCE(); - tile_go = 0; - FD_COMPILER_MFENCE(); - - for( ulong tile_idx=1UL; tile_idxreq_cnt = req_cnt; - rq->seq = 0UL; - - fd_vinyl_req_t * req = fd_vinyl_rq_req( rq ); - - for( ulong seq=0UL; seqmagic = FD_VINYL_RQ_MAGIC; - FD_COMPILER_MFENCE(); - - return rq; -} - -fd_vinyl_rq_t * -fd_vinyl_rq_join( void * shrq ) { - fd_vinyl_rq_t * rq = (fd_vinyl_rq_t *)shrq; - - if( FD_UNLIKELY( !rq ) ) { - FD_LOG_WARNING(( "NULL shrq")); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)rq, fd_vinyl_rq_align() ) ) ) { - FD_LOG_WARNING(( "bad align")); - return NULL; - } - - if( FD_UNLIKELY( rq->magic!=FD_VINYL_RQ_MAGIC ) ) { - FD_LOG_WARNING(( "bad magic")); - return NULL; - } - - return (fd_vinyl_rq_t *)shrq; -} - -void * -fd_vinyl_rq_leave( fd_vinyl_rq_t * rq ) { - - if( FD_UNLIKELY( !rq ) ) { - FD_LOG_WARNING(( "NULL rq")); - return NULL; - } - - return rq; -} - -void * -fd_vinyl_rq_delete( void * shrq ) { - fd_vinyl_rq_t * rq = (fd_vinyl_rq_t *)shrq; - - if( FD_UNLIKELY( !rq ) ) { - FD_LOG_WARNING(( "NULL shrq")); - return NULL; - } - - if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)rq, fd_vinyl_rq_align() ) ) ) { - FD_LOG_WARNING(( "bad align")); - return NULL; - } - - if( FD_UNLIKELY( rq->magic!=FD_VINYL_RQ_MAGIC ) ) { - FD_LOG_WARNING(( "bad magic")); - return NULL; - } - - FD_COMPILER_MFENCE(); - rq->magic = 0UL; - FD_COMPILER_MFENCE(); - - return rq; -} diff --git a/src/vinyl/rq/fd_vinyl_rq.h b/src/vinyl/rq/fd_vinyl_rq.h deleted file mode 100644 index ca29746eb57..00000000000 --- a/src/vinyl/rq/fd_vinyl_rq.h +++ /dev/null @@ -1,540 +0,0 @@ -#ifndef HEADER_fd_src_vinyl_rq_fd_vinyl_rq_h -#define HEADER_fd_src_vinyl_rq_fd_vinyl_rq_h - -/* A fd_vinyl_req_t describes a batch request to a vinyl tile. Batch - requests from a client are processed in order from that client. The - individual items in a batch request can be processed in _any_ order - (including concurrently). Thus ... - - IMPORTANT SAFETY TIP! The client promises that all items in a - request are logically independent such that they can be safely - processed in an arbitrary order / in parallel. - - The simplest way to ensure this is to keep all keys in a request - unique. That being said, repeated keys in a request will generally - "do the right thing" in the sense that the result can be interpreted - as though each item in the request executed in some order (with some - technical caveats for the advanced optimization of processing batch - items before receiving a completion). Batch requests from different - clients can be arbitrarily interleaved at batch request granularity. - - As such, a batch request executes _atomically_ relative to other - requests. - - Note though that concurrent meta readers see individual batch items - complete as they happen. Specifically, except for "move", individual - items in a batch complete atomically in whatever order they were - processed. (During the move of an individual item for the current - implementation, there is a brief moment where a concurrent meta - reader could observe both key_src and key_dst as not present.) - Similarly clients doing speculative processing will see individual - batch items complete atomically (including move) in whatever order - they are being processed (which is the point of speculative - processing). */ - -#include "../fd_vinyl_base.h" - -/* FD_VINYL_REQ_{ALIGN,FOOTPRINT} give the byte alignment and footprint - of a fd_vinyl_req_t. ALIGN is a reasonable power-of-2. FOOTPRINT is - a multiple of ALIGN. */ - -#define FD_VINYL_REQ_ALIGN (64UL) -#define FD_VINYL_REQ_FOOTPRINT (64UL) - -/* FD_VINYL_REQ_TYPE_* give supported request types. Note that - requests like JOIN are handled out of band because the client doesn't - have a rq to send the request until it is joined. (LEAVE could be - handled in band though but is handle via the same mechanism as JOIN - for symmetry. SYNC and PART requests are also handled out of band as - they don't pertain to individual clients.) */ - -#define FD_VINYL_REQ_TYPE_ACQUIRE (0) /* Acquire the requested pairs */ -#define FD_VINYL_REQ_TYPE_RELEASE (1) /* Release the requested pairs */ -#define FD_VINYL_REQ_TYPE_ERASE (2) /* Erase the requested pairs */ -#define FD_VINYL_REQ_TYPE_MOVE (3) /* Move the requested pairs (replace if dst exists, rename if not) */ -#define FD_VINYL_REQ_TYPE_FETCH (4) /* Fetch the requested pairs into cache (does not generate a completion) */ -#define FD_VINYL_REQ_TYPE_FLUSH (5) /* Flush the requested pairs from cache (does not generate a completion) */ -#define FD_VINYL_REQ_TYPE_TRY (6) /* Start to speculatively read (non-blocking) the requested pairs */ -#define FD_VINYL_REQ_TYPE_TEST (7) /* Test for speculation success */ - -/* FD_VINL_REQ_FLAG_* give flags that specify options for the above - request types. */ - -#define FD_VINYL_REQ_FLAG_MODIFY (1UL<<0) /* (Acquire) The client intends to modify the given pairs (info and/or val) - (Release) The client modified the given pairs (info and/or val) */ -#define FD_VINYL_REQ_FLAG_IGNORE (1UL<<1) /* (Acquire) Ignore the existing pair val (ignored if not MODIFY), - (Release) Cached pair val and/or info were clobbered (ignored if MODIFY or an acquire-for-read FIXME: have acquire-for-read with ignore set flush?) */ -#define FD_VINYL_REQ_FLAG_CREATE (1UL<<2) /* (Acquire) Create any pairs that do not already exist (ignored if not MODIFY) */ -#define FD_VINYL_REQ_FLAG_EXCL (1UL<<3) /* (Acquire) Do not modify any pairs that already exist (ignored if not MODIFY) */ -#define FD_VINYL_REQ_FLAG_ERASE (1UL<<4) /* (Release) Erase given pairs (ignored if not MODIFY) */ -#define FD_VINYL_REQ_FLAG_BY_KEY (1UL<<5) /* (Release) Use keys instead of cache val gaddrs to specify pairs to release */ -#define FD_VINYL_REQ_FLAG_MRU (0UL<<6) /* Make given pairs MRU (note: arb order from MRU->LRU for individual items in batch) */ -#define FD_VINYL_REQ_FLAG_LRU (1UL<<6) /* Make given pairs LRU (note: arb order from LRU->MRU for individual items in batch) */ -#define FD_VINYL_REQ_FLAG_UNC (2UL<<6) /* Do not change eviction priorities for the given pairs */ - -/* A fd_vinyl_req_t gives the layout of a request */ - -struct __attribute__((aligned(FD_VINYL_REQ_ALIGN))) fd_vinyl_req { - - ulong seq; /* Message sequence number */ - ulong req_id; /* Request id (identifies the request for the completion recipients) */ - ulong link_id; /* Link id (identifies client -> vinyl tile for request and completion recipients) */ - schar type; /* == FD_VINYL_REQ_TYPE_* */ - uchar flags; /* Bit-or of FD_VINYL_REQ_FLAG flags */ - ushort batch_cnt; /* Num key-val pairs in request */ - uint reserved; - - /* key_gaddr is typically the shared client request global address of - a: - - fd_vinyl_key_t key[ batch_cnt ] - - array that contains the set of keys for this batch request. The - vinyl tile will typically have a read interest in this region until - the request is completed. In particular: - - ACQUIRE - keys to acquire - RELEASE - (BY_KEY) keys to release, (~BY_KEY) ignored (release ~BY_KEY faster) - ERASE - keys to erase - FETCH - keys to fetch - FLUSH - keys to flush - MOVE - src keys to move - TRY - keys to speculatively read - TEST - ignored - - If there are redundant keys in a batch request, from the caller's - perspective, the items will appear to executed in some serial - order. E.g. in an acquire for modify, the first batch item - processed for a key will result in either success or a failure and - the remaining batch items for key will result in either AGAIN - (because the first item acquired) or failure (because it couldn't - acquire key for the same reason as first item). - - IMPORTANT SAFETY TIP! When doing the advanced optimization of - processing batch items before receiving a completion in a batch - acquire-for-read or a try, the remaining batch items for key can - report success before the first batch item for key has finished - reading, validating and decoding the pair. See case_acquire for - more details. - - Note that, for FETCH, evict prio flags are ignored. The keys will - always be fetched at MRU priority. */ - - ulong key_gaddr; - - /* val_gaddr_gaddr is typically the shared client request global - address of a: - - ulong val_gaddr[ batch_cnt ] - - array where shared data cache global addresses for pair vals should - be stored on completion. The current pair info is also available - to the client at this location for read / modification. The vinyl - tile will have a write interest in this region until the request is - complete. On receipt of a successful completion, batch items that - were successfully processed will have valid entries and batch items - that failed will have unchanged entries. On receipt of a failed - completion, this array will be unchanged. In particular: - - ACQUIRE - in: (MODIFY) requested val_max for val, (~MODIFY) ignored - out: pair val (and pair info) data cache global address - RELEASE - (BY_KEY) ignored, (~BY_KEY) data cache global addresses - of pairs to release (i.e. echo back the array returned - by acquire), (release ~BY_KEY faster) - ERASE - ignored - MOVE - this field is repurposed to point to the batch_cnt dst - keys for the move (vinyl tile will have a read interest - in this region until the request is complete) - (FIXME: consider having key_gaddr be 2*batch_cnt for - move, store all the keys there, and ignore this - entirely ... would make sense if it is easier for - store move src/dst keys interleaved) - FETCH - ignored - FLUSH - ignored - TRY - pair val (and pair info) data cache global addresses - must be a 2*batch_cnt array for try (the second half is - for internal use). Note that the info region - (including the val_sz for the try) might be corrupt. - The pointer returned though is guaranteed to be - readable out to FD_VINYL_VAL_MAX regardless val_sz. - TEST - echo back the same array (with an untouched second - half) from the corresponding try */ - - ulong val_gaddr_gaddr; - - /* err_gaddr is typically the shared client request global address of - a: - - schar err[ batch_cnt ] - - array where the results of individual batch item executions will be - stored by the vinyl tile. The vinyl tile will have a write - interest in this region until the request is completed. On receipt - of a successful completion, the individual entries will be valid. - Entries will be FD_VINYL_SUCCESS (zero) or a FD_VINYL_ERR code - (negative). On receipt of a failed completion, this array is - unchanged. In particular: - - ACQUIRE (for read) - - SUCCESS - acquired the corresponding pair key. The pair val - and pair info can be found via the corresponding - val_gaddr in the shared data cache. This location - will be stable until the pair is released. info will - have the current pair metadata (including the val - byte size), val will have the current pair val. - - AGAIN - there was a conflicting acquire on pair key, try - again after the conflicting acquire has been - released. That is, pair key is currently - acquired-for-modify (this includes pair key in the - process of being created). - - KEY - pair key did not exist at bstream seq_present and - is not in the process of being created. - - ACQUIRE (for modify) - - SUCCESS - acquired the corresponding pair key. The pair val - and pair info can be found via the corresponding - val_gaddr in the shared data cache. This location - will be stable until the pair is released. - - If pair key exists and ignore was _not_ set, info - will contain the current pair metadata (including the - val byte size), val contain the current pair val and - the val region will be sized to the greater of the - current val byte size and the requested max val byte - size. - - If pair key exists and ignore _was_ set, info will - contain the _current_ pair metadata (_but_ with a val - byte size of 0), val will be empty and the region - will be sized to the requested max val byte size. - - If pair key did not exist, the pair info will be all - zeros, val will be empty and the val region will be - sized to the requested max val byte size. - - INVAL - pair key existed and exclusive was set, or the - requested val_max was larger than FD_VINYL_VAL_MAX - - KEY - pair key did not exist and create was not set - - AGAIN - there was a conflicting acquire on pair, try again - after conflicting acquire(s) have been released. - That is, pair key is currently acquired-for-read one - or more times or acquired-for-modify. - - RELEASE - - SUCCESS - released the corresponding pair key. The key-val - store was updated appropriately. In particular: - - When releasing an acquire-for-modify _with_ modify - set, the modification "happened" (i.e. is now part of - the bstream's past). This includes creating or - erasing a pair. The cached pair info gave the - modified pair info (including the updated pair val - byte size). - - When releasing an acquire-for-modify _without_ modify - set, the modification was cancelled. If the ignore - flag _was_set, the vinyl tile assumed the cached info - and/or val were clobbered by the client during the - acquire. If the ignore flag was _not_ set, the vinyl - tile assumed the client did not alter any pair value - or pair info _at_ _any_ _time_ _during_ _the_ - _acquire_ (such that the cached values and any - speculations on those cached values during the - acquire were valid). Note that when releasing an - acquire-for-modify-with-ignore, the implicily - clobbered the cached pair info val byte size and - cached pair val for pairs that had a non-zero val - size immediately before the acquire. Since the - client probably didn't know if the val size was zero - beforehand, it is strongly recommended that - acquire-for-modify-with-ignore only be canceled via a - release-with-ignore. - - TL;DR If client never wrote to the cached pair info - or cached pair val on an - acquire-for-modify-without-ignore, the modification - can be cancelled fast with a release-without-modify. - In all other cases, a modification should be - cancelled via a release-without-modify-with-ignore. - - Note that release-with-modify of an acquire-for-read - is considered a catastrophe (the client modified data - it promised not to change and may have corrupted - state of themselves, other clients and the store - itself). Overrunning pair cache storage (the client - set the pair info val_sz to something larger than the - acquire-with-modify val_max) is also treated as a - catastrophe for similar reasons. - - INVAL - the corresponding key (BY_KEY) / val (~BY_KEY) did - not appear to be to an acquired pair - - ERASE - - SUCCESS - the corresponding key was erased - - KEY - the corresponding key did not exist - - AGAIN - corresponding key is currently acquired for something - (including read, modify or create), try again after - conflicting acquires have been released - - MOVE - - SUCCESS - the corresponding src_key was renamed to the - corresponding dst_key. If dst_key existed before the - move, it was atomically erased before renaming pair - src_key. The new pair dst_key has the pair info and - pair val of the old pair src_key. - - Note that a move from src_key to src_key is treated - as a no-op that immediately succeeds (with no - checking, for example, whether or not src_key even - exists). - - KEY - the corresponding src_key did not exist - - AGAIN - there was at least one conflicting acquire on src_key - and/or dst_key. Try again after the conflicting - acquires have been released. - - FETCH - ignored - - FLUSH - ignored - - TRY - - SUCCESS - the client is clear to try a speculative - (non-blocking) read of the corresponding key. The - key was located in the data cache region at the given - val_gaddr at the start of the try. - - KEY - the corresponding key did not exist to try - - AGAIN - the corresponding key was acquired-for-modify - (includes create), try again after conflicting - acquires have been released - - TEST - - SUCCESS - the speculative read was successful - - INVAL - the corresponding try never started (i.e. try failed - with KEY or AGAIN) - - CORRUPT - the corresponding try failed (i.e. the pair was - potentially changed during the speculation) - - If these are set to a positive number before sending the request, - the caller can detect individual items as they finish processing - (and then access the pair val and pair info via the corresponding - val_gaddr on success) before receiving the completion (with some - caveats for requests with redundant keys described in - case_acquire). */ - - ulong err_gaddr; - - /* comp_gaddr gives the shared client request gaddr of a: - - fd_vinyl_comp_t comp[ 1 ] - - or zero. - - If non-zero, the completion information will be written into comp - (with comp->seq set to 1 last). If not, the completion will be - sent to the completion queue registered to the client for this - vinyl tile (and if no completion queue was registered, no - completion will be sent ... which is not a recommended mode of - operation). - - If a completion was successful, err will be FD_VINYL_SUCCESS (0), - batch_cnt will match the request batch_cnt, fail_cnt will give the - number of individual items in the batch that failed (in - [0,batch_cnt]), and quota_rem will give the remaining client - acquire quota (each successful acquire/release - decrements/increments the client's remaining quota by 1 all other - operations do not impact the client's quota). - - If a completion has an error, err will be a FD_VINYL_ERR code - (negative). No processing of any items in the batch was done, - batch_cnt will match the request batch_cnt, fail_cnt will be zero - and quota_rem will give the remaining client acquire quota - (unchanged). Reasons for a completion error are: - - INVAL - one or more input arrays were unmappable (i.e. not a - valid global address in the shared client request memory - region). - - FULL - client acquire quota remaining is too low to process this - request fully (ACQUIRE only) - - FETCH and FLUSH requests do not produce completions. */ - - ulong comp_gaddr; -}; - -typedef struct fd_vinyl_req fd_vinyl_req_t; - -/* A fd_vinyl_rq_t is an interprocess shared persistent SPMC queue - used to communicate fd_vinyl_req_t requests from a client to vinyl - tiles. It is virtually identically to fd_vinyl_cq_t but holds - fd_vinyl_req_t instead of fd_vinyl_comp_t. See fd_vinyl_cq_t for - concurrency and flow control details. */ - -#define FD_VINYL_RQ_MAGIC (0xfd3a7352d703a6c0UL) /* fd warm snd rq magc version 0 */ - -struct __attribute__((aligned(128))) fd_vinyl_rq_private { - - ulong magic; /* ==FD_VINYL_RQ_MAGIC */ - ulong req_cnt; /* Number of requests that can be in flight on this cq at any given time, power of 2 of least 4 */ - uchar _[ 112 ]; /* Padding to put seq on a separate cache line pair */ - ulong seq; /* Request sequence number to publish next */ - - /* padding to 128 alignment */ - - /* fd_vinyl_req_t req[ req_cnt ] here, seq number at idx = seq & (req_cnt-1UL) when available */ - - /* padding to 128 alignment */ - -}; - -typedef struct fd_vinyl_rq_private fd_vinyl_rq_t; - -FD_PROTOTYPES_BEGIN - -/* fd_vinyl_req_{flag_*,evict_prio} extract the {given flag,eviction - priority} from a fd_vinyl_req_t flags field. */ - -FD_FN_CONST static inline int fd_vinyl_req_flag_modify( ulong flags ) { return !!(flags & FD_VINYL_REQ_FLAG_MODIFY); } -FD_FN_CONST static inline int fd_vinyl_req_flag_ignore( ulong flags ) { return !!(flags & FD_VINYL_REQ_FLAG_IGNORE); } -FD_FN_CONST static inline int fd_vinyl_req_flag_create( ulong flags ) { return !!(flags & FD_VINYL_REQ_FLAG_CREATE); } -FD_FN_CONST static inline int fd_vinyl_req_flag_excl ( ulong flags ) { return !!(flags & FD_VINYL_REQ_FLAG_EXCL ); } -FD_FN_CONST static inline int fd_vinyl_req_flag_erase ( ulong flags ) { return !!(flags & FD_VINYL_REQ_FLAG_ERASE); } -FD_FN_CONST static inline int fd_vinyl_req_flag_by_key( ulong flags ) { return !!(flags & FD_VINYL_REQ_FLAG_BY_KEY); } -FD_FN_CONST static inline int fd_vinyl_req_evict_prio ( ulong flags ) { return (int)((flags >> 6) & 3UL); } - -/* fd_vinyl_rq_{align,footprint,new,join,leave_delete} have the usual - interprocess shared persistent object semantics. req_cnt is a - power-of-2 of at least 4 that gives the number requests that can be - in flight on this rq. */ - -FD_FN_CONST ulong fd_vinyl_rq_align ( void ); -FD_FN_CONST ulong fd_vinyl_rq_footprint( ulong req_cnt ); -void * fd_vinyl_rq_new ( void * shmem, ulong req_cnt ); -fd_vinyl_rq_t * fd_vinyl_rq_join ( void * shrq ); -void * fd_vinyl_rq_leave ( fd_vinyl_rq_t * rq ); -void * fd_vinyl_rq_delete ( void * shrq ); - -/* fd_vinyl_rq_req returns the location in the caller's address space of - the rq's request array. fd_vinyl_rq_req_const is a const correct - version. fd_vinyl_rq_req_cnt is the size of this array. The - lifetime of the returned array is the lifetime of the local join. - These assume rq is a current local join. - - fd_vinyl_rq_req_idx gives the array index that will cache request seq - in a rq request array with req_cnt elements. */ - -FD_FN_CONST static inline fd_vinyl_req_t * -fd_vinyl_rq_req( fd_vinyl_rq_t * rq ) { - return (fd_vinyl_req_t *)(rq+1); -} - -FD_FN_CONST static inline fd_vinyl_req_t const * -fd_vinyl_rq_req_const( fd_vinyl_rq_t const * rq ) { - return (fd_vinyl_req_t const *)(rq+1); -} - -FD_FN_PURE static inline ulong fd_vinyl_rq_req_cnt( fd_vinyl_rq_t const * rq ) { return rq->req_cnt; } - -FD_FN_CONST static inline ulong fd_vinyl_rq_req_idx( ulong seq, ulong req_cnt ) { return seq & (req_cnt-1UL); } - -/* fd_vinyl_rq_seq returns the position of the request producer's - sequence number cursor. Specifically, at some point during the call, - requests [0,seq) were published, requests (seq,ULONG_MAX] were not - published, and request seq was either published, being published or - not published. This is used for initial synchronization between - producer and consumers. This is a compiler fence. */ - -static inline ulong -fd_vinyl_rq_seq( fd_vinyl_rq_t const * rq ) { - FD_COMPILER_MFENCE(); - ulong seq = rq->seq; - FD_COMPILER_MFENCE(); - return seq; -} - -/* fd_vinyl_rq_send enqueues the given request into the given rq. - Assumes rq is a current local join. This is a compiler fence. */ - -/* FIXME: consider SIMD accelerating */ - -static inline void -fd_vinyl_rq_send( fd_vinyl_rq_t * rq, - ulong req_id, - ulong link_id, - int type, /* In [-2^7,2^7) */ - ulong flags, /* In [0,2^8) */ - ulong batch_cnt, /* In [0,2^16) */ - ulong key_gaddr, - ulong val_gaddr_gaddr, - ulong err_gaddr, - ulong comp_gaddr ) { - ulong seq = rq->seq; - fd_vinyl_req_t * req = fd_vinyl_rq_req( rq ) + fd_vinyl_rq_req_idx( seq, rq->req_cnt ); - FD_COMPILER_MFENCE(); - req->seq = seq - 1UL; /* Mark request seq being written */ - FD_COMPILER_MFENCE(); - req->req_id = req_id; - req->link_id = link_id; - req->type = (schar)type; - req->flags = (uchar)flags; - req->batch_cnt = (ushort)batch_cnt; - req->key_gaddr = key_gaddr; - req->val_gaddr_gaddr = val_gaddr_gaddr; - req->err_gaddr = err_gaddr; - req->comp_gaddr = comp_gaddr; - FD_COMPILER_MFENCE(); - req->seq = seq; /* Mark request seq as written */ - FD_COMPILER_MFENCE(); - rq->seq = seq + 1UL; /* Record that requests [0,seq) are published */ - FD_COMPILER_MFENCE(); -} - -/* fd_vinyl_rq_recv receives request seq from the given rq. Returns 0 - on success, positive if request seq has not been published yet and - negative if the consumer has been overrun by the producer. On - return, *dst will contain the desired request on success and was - clobbered otherwise. Assumes rq is a current local join and dst is - valid. This is a compiler fence. */ - -/* FIXME: consider SIMD accelerating */ - -static inline long -fd_vinyl_rq_recv( fd_vinyl_rq_t const * rq, - ulong seq, - fd_vinyl_req_t * dst ) { - fd_vinyl_req_t const * src = fd_vinyl_rq_req_const( rq ) + fd_vinyl_rq_req_idx( seq, rq->req_cnt ); - - FD_COMPILER_MFENCE(); - ulong seq0 = src->seq; - FD_COMPILER_MFENCE(); - *dst = *src; - FD_COMPILER_MFENCE(); - ulong seq1 = src->seq; - FD_COMPILER_MFENCE(); - - long diff0 = (long)(seq-seq0); - long diff1 = (long)(seq-seq1); - return fd_long_if( !diff0, diff1, diff0 ); -} - -FD_PROTOTYPES_END - -#endif /* HEADER_fd_src_vinyl_rq_fd_vinyl_rq_h */ diff --git a/src/vinyl/rq/test_vinyl_rq.c b/src/vinyl/rq/test_vinyl_rq.c deleted file mode 100644 index 7f1e13e22d0..00000000000 --- a/src/vinyl/rq/test_vinyl_rq.c +++ /dev/null @@ -1,207 +0,0 @@ -#include "../fd_vinyl.h" - -FD_STATIC_ASSERT( FD_VINYL_REQ_ALIGN ==64UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FOOTPRINT==64UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_REQ_ALIGN ==alignof(fd_vinyl_req_t), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FOOTPRINT==sizeof (fd_vinyl_req_t), unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_ACQUIRE==0, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_RELEASE==1, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_ERASE ==2, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_MOVE ==3, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_FETCH ==4, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_FLUSH ==5, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_TRY ==6, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_TYPE_TEST ==7, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_MODIFY==(1UL<<0), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_IGNORE==(1UL<<1), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_CREATE==(1UL<<2), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_EXCL ==(1UL<<3), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_ERASE ==(1UL<<4), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_BY_KEY==(1UL<<5), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_MRU ==(0UL<<6), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_LRU ==(1UL<<6), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_REQ_FLAG_UNC ==(2UL<<6), unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_RQ_MAGIC==0xfd3a7352d703a6c0UL, unit_test ); - -static FD_FOR_ALL_BEGIN( test_tile, 1L ) { - fd_vinyl_rq_t * rq = (fd_vinyl_rq_t *)arg[0]; - long t_stop = (long) arg[1]; - - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, (uint)(tpool_t0+1UL), 0UL ) ); - - if( !tpool_t0 ) { /* Req sender */ - - FD_LOG_NOTICE(( "Starting transmitter" )); - - for(;;) { - - if( FD_UNLIKELY( fd_log_wallclock() > t_stop ) ) break; - - ulong val = fd_rng_ulong( rng ); - - val += (ulong)!val; /* Use req_id==0 as termination for receivers in this test */ - - ulong req_id = val++; - ulong link_id = val++; - int type = (int)(val++); - ulong flags = val++; - ulong batch_cnt = val++; - ulong key_gaddr = val++; - ulong val_gaddr_gaddr = val++; - ulong err_gaddr = val++; - ulong comp_gaddr = val++; - - fd_vinyl_rq_send( rq, req_id, link_id, type, flags, batch_cnt, key_gaddr, val_gaddr_gaddr, err_gaddr, comp_gaddr ); - - } - - /* Send terminate */ - - fd_vinyl_rq_send( rq, 0UL, 0UL, 0, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL ); - - } else { - - FD_LOG_NOTICE(( "Starting receiver %lu", tpool_t0-1UL )); - - ulong seq = fd_vinyl_rq_seq( rq ); - - for(;;) { - - fd_vinyl_req_t req[1]; - long diff = fd_vinyl_rq_recv( rq, seq, req ); - - if( FD_LIKELY( diff ) ) { - if( FD_UNLIKELY( diff<0L ) ) { - FD_LOG_WARNING(( "Receiver %lu overrun; resynchronizing", tpool_t0-1UL )); - seq = fd_vinyl_rq_seq( rq ); - } - FD_SPIN_PAUSE(); - continue; - } - - ulong val = req->req_id; - if( FD_UNLIKELY( !val ) ) break; - - FD_TEST( req->req_id == val ); val++; - FD_TEST( req->link_id == val ); val++; - FD_TEST( req->type == (schar) val ); val++; - FD_TEST( req->flags == (uchar) val ); val++; - FD_TEST( req->batch_cnt == (ushort)val ); val++; - FD_TEST( req->key_gaddr == val ); val++; - FD_TEST( req->val_gaddr_gaddr == val ); val++; - FD_TEST( req->err_gaddr == val ); val++; - FD_TEST( req->comp_gaddr == val ); val++; - - seq++; - } - - } - - fd_rng_delete( fd_rng_leave( rng ) ); - -} FD_FOR_ALL_END - -#define SHMEM_ALIGN (128) -#define SHMEM_FOOTPRINT (1UL<<20) - -static uchar shmem[ SHMEM_FOOTPRINT ] __attribute__((aligned(SHMEM_ALIGN))); - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - ulong req_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--req-cnt", NULL, 8192UL ); - - FD_LOG_NOTICE(( "Testing (--req-cnt %lu)", req_cnt )); - - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, 0U, 0UL ) ); - - for( ulong rem=1000000UL; rem; rem-- ) { - ulong flags = fd_rng_ulong( rng ); - FD_TEST( fd_vinyl_req_flag_modify( flags )==!!(flags & FD_VINYL_REQ_FLAG_MODIFY) ); - FD_TEST( fd_vinyl_req_flag_ignore( flags )==!!(flags & FD_VINYL_REQ_FLAG_IGNORE) ); - FD_TEST( fd_vinyl_req_flag_create( flags )==!!(flags & FD_VINYL_REQ_FLAG_CREATE) ); - FD_TEST( fd_vinyl_req_flag_excl ( flags )==!!(flags & FD_VINYL_REQ_FLAG_EXCL ) ); - FD_TEST( fd_vinyl_req_flag_erase ( flags )==!!(flags & FD_VINYL_REQ_FLAG_ERASE ) ); - FD_TEST( fd_vinyl_req_flag_by_key( flags )==!!(flags & FD_VINYL_REQ_FLAG_BY_KEY) ); - FD_TEST( fd_vinyl_req_evict_prio ( flags )==(int)((flags>>6) & 3UL) ); - } - - ulong thread_cnt = fd_tile_cnt(); - - FD_LOG_NOTICE(( "Creating tpool from all %lu tiles", thread_cnt )); - - static uchar _tpool[ FD_TPOOL_FOOTPRINT( FD_TILE_MAX ) ] __attribute__((aligned(FD_TPOOL_ALIGN))); - - fd_tpool_t * tpool = fd_tpool_init( _tpool, thread_cnt, 0UL ); /* logs details */ - if( FD_UNLIKELY( !tpool ) ) FD_LOG_ERR(( "fd_tpool_init failed" )); - - for( ulong thread_idx=1UL; thread_idx SHMEM_ALIGN) | (footprint > SHMEM_FOOTPRINT) ) ) - FD_LOG_ERR(( "Update SHMEM_ALIGN and/or SHMEM_FOOTPRINT for this req_cnt" )); - - FD_TEST( !fd_vinyl_rq_new( NULL, req_cnt ) ); /* NULL shmem */ - FD_TEST( !fd_vinyl_rq_new( (void *)1UL, req_cnt ) ); /* misaligned shmem */ - FD_TEST( !fd_vinyl_rq_new( shmem, 0UL ) ); /* bad req_cnt */ - void * shrq = fd_vinyl_rq_new( shmem, req_cnt ); FD_TEST( !!shrq ); - - FD_TEST( !fd_vinyl_rq_join( NULL ) ); /* NULL shmem */ - FD_TEST( !fd_vinyl_rq_join( (void *)1UL ) ); /* misaligned shmem */ - fd_vinyl_rq_t * rq = fd_vinyl_rq_join( shrq ); FD_TEST( !!rq ); - - fd_vinyl_req_t * req = fd_vinyl_rq_req( rq ); FD_TEST( !!req ); - FD_TEST( fd_vinyl_rq_req_const( rq )==req ); - - FD_TEST( fd_vinyl_rq_req_cnt( rq )==req_cnt ); - - for( ulong rem=10000UL; rem; rem-- ) { - ulong seq = fd_rng_ulong( rng ); - FD_TEST( fd_vinyl_rq_req_idx( seq, req_cnt )==(seq & (req_cnt-1UL)) ); - } - - FD_TEST( !fd_vinyl_rq_seq( rq ) ); /* Initial sequence number at zero */ - - long t_stop = fd_log_wallclock() + (long)1e9; - - FD_FOR_ALL( test_tile, tpool,0UL,thread_cnt, 0L,(long)thread_cnt, rq, t_stop ); - - FD_TEST( !fd_vinyl_rq_leave( NULL ) ); /* NULL rq */ - FD_TEST( fd_vinyl_rq_leave( rq )==shrq ); - - FD_TEST( !fd_vinyl_rq_delete( NULL ) ); /* NULL shmem */ - FD_TEST( !fd_vinyl_rq_delete( (void *)1UL ) ); /* misaligned shmem */ - FD_TEST( fd_vinyl_rq_delete( shrq )==shmem ); - - FD_TEST( !fd_vinyl_rq_join ( shrq ) ); /* bad magic */ - FD_TEST( !fd_vinyl_rq_delete( shrq ) ); /* bad magic */ - - FD_LOG_NOTICE(( "Cleaning up" )); - - /* Note: fini automatically pops all worker threads */ - - fd_tpool_fini( tpool ); /* logs details */ - - fd_rng_delete( fd_rng_leave( rng ) ); - - FD_LOG_NOTICE(( "pass" )); - fd_halt(); - return 0; -} diff --git a/src/vinyl/test_vinyl_base.c b/src/vinyl/test_vinyl_base.c deleted file mode 100644 index a7cd98b0f46..00000000000 --- a/src/vinyl/test_vinyl_base.c +++ /dev/null @@ -1,98 +0,0 @@ -#include "fd_vinyl.h" - -FD_STATIC_ASSERT( FD_VINYL_SUCCESS == 0, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_ERR_INVAL ==-1, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_ERR_AGAIN ==-2, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_ERR_CORRUPT==-3, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_ERR_EMPTY ==-4, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_ERR_FULL ==-5, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_ERR_KEY ==-6, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_KEY_ALIGN == 8UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_KEY_FOOTPRINT==32UL, unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_KEY_ALIGN ==alignof(fd_vinyl_key_t), unit_test ); -FD_STATIC_ASSERT( FD_VINYL_KEY_FOOTPRINT==sizeof (fd_vinyl_key_t), unit_test ); - -FD_STATIC_ASSERT( FD_VINYL_VAL_MAX==10485816UL, unit_test ); -FD_STATIC_ASSERT( FD_VINYL_INFO_SZ==16UL, unit_test ); - -FD_STATIC_ASSERT( alignof(fd_vinyl_info_t)== 8UL, unit_test ); -FD_STATIC_ASSERT( sizeof (fd_vinyl_info_t)==16UL, unit_test ); - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - fd_rng_t rng[1]; fd_rng_join( fd_rng_new( rng, 0U, 0UL ) ); - - FD_LOG_NOTICE(( "bad error code (%i-%s)", 1, fd_vinyl_strerror( 1 ) )); - FD_LOG_NOTICE(( "FD_VINYL_SUCCESS (%i-%s)", FD_VINYL_SUCCESS, fd_vinyl_strerror( FD_VINYL_SUCCESS ) )); - FD_LOG_NOTICE(( "FD_VINYL_ERR_INVAL (%i-%s)", FD_VINYL_ERR_INVAL, fd_vinyl_strerror( FD_VINYL_ERR_INVAL ) )); - FD_LOG_NOTICE(( "FD_VINYL_ERR_AGAIN (%i-%s)", FD_VINYL_ERR_AGAIN, fd_vinyl_strerror( FD_VINYL_ERR_AGAIN ) )); - FD_LOG_NOTICE(( "FD_VINYL_ERR_CORRUPT (%i-%s)", FD_VINYL_ERR_CORRUPT, fd_vinyl_strerror( FD_VINYL_ERR_CORRUPT ) )); - FD_LOG_NOTICE(( "FD_VINYL_ERR_EMPTY (%i-%s)", FD_VINYL_ERR_EMPTY, fd_vinyl_strerror( FD_VINYL_ERR_EMPTY ) )); - FD_LOG_NOTICE(( "FD_VINYL_ERR_FULL (%i-%s)", FD_VINYL_ERR_FULL, fd_vinyl_strerror( FD_VINYL_ERR_FULL ) )); - FD_LOG_NOTICE(( "FD_VINYL_ERR_KEY (%i-%s)", FD_VINYL_ERR_KEY, fd_vinyl_strerror( FD_VINYL_ERR_KEY ) )); - - fd_rng_delete( fd_rng_leave( rng ) ); - - for( ulong rem=1000000UL; rem; rem-- ) { - ulong seed = fd_rng_ulong( rng ); - ulong limb[8]; for( ulong idx=0UL; idx<8UL; idx++ ) limb[idx] = fd_rng_ulong( rng ); - - fd_vinyl_key_t ka[1]; FD_TEST( fd_vinyl_key_init_ulong( ka, limb[0], limb[1], limb[2], limb[3] )==ka ); - fd_vinyl_key_t kb[1]; FD_TEST( fd_vinyl_key_init_ulong( kb, limb[4], limb[5], limb[6], limb[7] )==kb ); - - FD_TEST( ka->ul[0]==limb[0] && ka->ul[1]==limb[1] && ka->ul[2]==limb[2] && ka->ul[3]==limb[3] && - kb->ul[0]==limb[4] && kb->ul[1]==limb[5] && kb->ul[2]==limb[6] && kb->ul[3]==limb[7] ); - - int eq = (ka->ul[0]==kb->ul[0]) && (ka->ul[1]==kb->ul[1]) && - (ka->ul[2]==kb->ul[2]) && (ka->ul[3]==kb->ul[3]); - - ulong ma = fd_vinyl_key_memo( seed, ka ); - ulong mb = fd_vinyl_key_memo( seed, kb ); - - FD_TEST( fd_vinyl_key_eq( ka, ka )== 1 ); FD_TEST( fd_vinyl_key_eq( ka, kb )==eq ); - FD_TEST( fd_vinyl_key_eq( kb, ka )==eq ); FD_TEST( fd_vinyl_key_eq( kb, kb )== 1 ); - - if( eq ) FD_TEST( ma==mb ); - - /* zero padding copy */ - - ulong csz = seed % FD_VINYL_KEY_FOOTPRINT; - - FD_TEST( fd_vinyl_key_init( kb, ka, csz )==kb ); - - eq = (ka->ul[0]==kb->ul[0]) && (ka->ul[1]==kb->ul[1]) && - (ka->ul[2]==kb->ul[2]) && (ka->ul[3]==kb->ul[3]); - - mb = fd_vinyl_key_memo( seed, kb ); - - for( ulong idx=csz; idxc[idx] ); - - FD_TEST( fd_vinyl_key_eq( ka, ka )== 1 ); FD_TEST( fd_vinyl_key_eq( ka, kb )==eq ); - FD_TEST( fd_vinyl_key_eq( kb, ka )==eq ); FD_TEST( fd_vinyl_key_eq( kb, kb )== 1 ); - if( eq ) FD_TEST( ma==mb ); - - /* normal copy and truncating copy */ - - csz += FD_VINYL_KEY_FOOTPRINT; - - FD_TEST( fd_vinyl_key_init( kb, ka, csz )==kb ); - - eq = 1; - mb = fd_vinyl_key_memo( seed, kb ); - - FD_TEST( fd_vinyl_key_eq( ka, ka )== 1 ); FD_TEST( fd_vinyl_key_eq( ka, kb )==eq ); - FD_TEST( fd_vinyl_key_eq( kb, ka )==eq ); FD_TEST( fd_vinyl_key_eq( kb, kb )== 1 ); - if( eq ) FD_TEST( ma==mb ); - } - - fd_rng_delete( fd_rng_leave( rng ) ); - - FD_LOG_NOTICE(( "pass" )); - fd_halt(); - return 0; -} diff --git a/src/vinyl/test_vinyl_req.c b/src/vinyl/test_vinyl_req.c deleted file mode 100644 index 5fc56dd8094..00000000000 --- a/src/vinyl/test_vinyl_req.c +++ /dev/null @@ -1,1101 +0,0 @@ -#include "fd_vinyl.h" - -#define PAIR_MAX (4UL) - -struct pair { - int creating; - long acq; - ulong ver; - ulong val_max; - fd_vinyl_key_t key [ 1 ]; - fd_vinyl_info_t info [ 1 ]; - fd_vinyl_info_t backup_info[ 1 ]; - uchar val [ FD_VINYL_VAL_MAX ]; - uchar backup_val [ FD_VINYL_VAL_MAX ]; -}; - -typedef struct pair pair_t; - -static struct { - ulong quota_rem; - ulong used; - pair_t pair[ PAIR_MAX ]; -} ref; - -static int -req( int type, /* request type */ - ulong flags, /* request flags */ - ulong val_max, /* for acquire-with-modify */ - fd_vinyl_key_t const * key, /* Key for req */ - pair_t ** _pair, /* Location of pair on successful acquire and/or successful try */ - ulong * _ver ) { /* Version of pair on try, has try version on test, ignored otherwise */ - - switch( type ) { - - case FD_VINYL_REQ_TYPE_ACQUIRE: { - if( !ref.quota_rem ) return FD_VINYL_ERR_FULL; /* (comp err) client quota exhausted */ - - if( fd_vinyl_req_flag_modify( flags ) && (val_max>FD_VINYL_VAL_MAX) ) return FD_VINYL_ERR_INVAL; /* bad req val_max */ - - ulong idx = 0UL; - pair_t * pair = NULL; - for( ; idx> idx) & 1UL) && fd_vinyl_key_eq( key, ref.pair[ idx ].key ) ) { - pair = &ref.pair[ idx ]; - break; - } - } - - if( pair ) { - - if( !fd_vinyl_req_flag_modify( flags ) ) { - - /* start blocking read of an existing pair */ - - if( pair->acq < 0L ) return FD_VINYL_ERR_AGAIN; /* key acquired for modify */ - - ulong ref_szc = fd_vinyl_data_szc( (ulong)pair->info->val_sz ); - pair->val_max = fd_vinyl_data_szc_val_max( ref_szc ); - - FD_TEST( !pair->creating ); - pair->acq++; - //pair->ver unchanged - - ref.quota_rem--; - - *_pair = pair; - return FD_VINYL_SUCCESS; - - } - - /* start modify of an existing pair */ - - if( pair->acq ) return FD_VINYL_ERR_AGAIN; /* key acquired at least once */ - if( fd_vinyl_req_flag_excl( flags ) ) return FD_VINYL_ERR_INVAL; /* not allowed to modify existing */ - - pair->backup_info[0] = pair->info[0]; - if( pair->info->val_sz ) memcpy( pair->backup_val, pair->val, (ulong)pair->info->val_sz ); - - if( fd_vinyl_req_flag_ignore( flags ) ) pair->info->val_sz = 0U; - - ulong ref_szc = fd_vinyl_data_szc( fd_ulong_max( val_max, (ulong)pair->info->val_sz ) ); - pair->val_max = fd_vinyl_data_szc_val_max( ref_szc ); - - FD_TEST( !pair->creating ); - pair->acq = -1L; - pair->ver++; - - ref.quota_rem--; - - *_pair = pair; - return FD_VINYL_SUCCESS; - - } - - /* start creating a pair */ - - if( !(fd_vinyl_req_flag_modify( flags ) && fd_vinyl_req_flag_create( flags )) ) return FD_VINYL_ERR_KEY; - - FD_TEST( (ulong)fd_ulong_popcnt( ref.used ) < PAIR_MAX ); - idx = (ulong)fd_ulong_find_lsb( ~ref.used ); - pair = &ref.pair[ idx ]; - ref.used |= (1UL << idx); - - pair->key[0] = key[0]; - memset( pair->info, 0UL, sizeof(fd_vinyl_info_t) ); - - ulong ref_szc = fd_vinyl_data_szc( val_max ); - pair->val_max = fd_vinyl_data_szc_val_max( ref_szc ); - - pair->creating = 1; - pair->acq = -1L; - pair->ver++; - - ref.quota_rem--; - - *_pair = pair; - return FD_VINYL_SUCCESS; - } - - case FD_VINYL_REQ_TYPE_RELEASE: { - ulong idx = 0UL; - pair_t * pair = NULL; - for( ; idx> idx) & 1UL) && fd_vinyl_key_eq( key, ref.pair[ idx ].key ) ) { - pair = &ref.pair[ idx ]; - break; - } - } - - if( !pair ) return FD_VINYL_ERR_INVAL; /* Key does not exist and is not being created (cannot have been acquired) */ - if( !pair->acq ) return FD_VINYL_ERR_INVAL; /* Key is not acquired */ - - if( pair->acq > 0L ) { - - /* finish blocking read */ - - if( fd_vinyl_req_flag_modify( flags ) ) FD_LOG_CRIT(( "modify read only" )); - - FD_TEST( !pair->creating ); - - pair->acq--; - - ref.quota_rem++; - return FD_VINYL_SUCCESS; - - } - - if( pair->creating ) { - - if( ((!fd_vinyl_req_flag_modify( flags )) | fd_vinyl_req_flag_erase( flags )) ) { - - /* cancel / erase a create */ - - pair->ver++; - - ref.used &= ~(1UL << idx); - - ref.quota_rem++; - return FD_VINYL_SUCCESS; - - } - - /* finish a create */ - - if( (ulong)pair->info->val_sz > pair->val_max ) FD_LOG_CRIT(( "val buffer overrun" )); - - ulong ref_szc = fd_vinyl_data_szc( (ulong)pair->info->val_sz ); - pair->val_max = fd_vinyl_data_szc_val_max( ref_szc ); - - pair->creating = 0; - pair->acq = 0L; - pair->ver++; - - ref.quota_rem++; - return FD_VINYL_SUCCESS; - - } - - if( !fd_vinyl_req_flag_modify( flags ) ) { - - if( !fd_vinyl_req_flag_ignore( flags ) ) { - - /* cancel modify existing (info/val were not clobbered during the modify attempt) */ - - pair->acq = 0L; - pair->ver--; - - ref.quota_rem++; - return FD_VINYL_SUCCESS; - - } - - /* cancel modify existing (info/val were potentially clobbered during the modify attempt) */ - - pair->info[0] = pair->backup_info[0]; - if( pair->backup_info->val_sz ) memcpy( pair->val, pair->backup_val, (ulong)pair->backup_info->val_sz ); - - ulong ref_szc = fd_vinyl_data_szc( (ulong)pair->info->val_sz ); - pair->val_max = fd_vinyl_data_szc_val_max( ref_szc ); - - pair->acq = 0L; - pair->ver++; - - ref.quota_rem++; - return FD_VINYL_SUCCESS; - - } - - if( fd_vinyl_req_flag_erase( flags ) ) { - - /* erase existing */ - - pair->ver++; - - ref.used &= ~(1UL<info->val_sz > pair->val_max ) FD_LOG_CRIT(( "val buffer overrun" )); - - ulong ref_szc = fd_vinyl_data_szc( (ulong)pair->info->val_sz ); - pair->val_max = fd_vinyl_data_szc_val_max( ref_szc ); - - pair->acq = 0L; - pair->ver++; - - ref.quota_rem++; - return FD_VINYL_SUCCESS; - } - - case FD_VINYL_REQ_TYPE_ERASE: { - ulong idx = 0UL; - pair_t * pair = NULL; - for( ; idx> idx) & 1UL) && fd_vinyl_key_eq( key, ref.pair[ idx ].key ) ) { - pair = &ref.pair[ idx ]; - break; - } - } - - if( !pair ) return FD_VINYL_ERR_KEY; /* Key does not exist */ - if( pair->acq ) return FD_VINYL_ERR_AGAIN; /* Key acquired at least once */ - - pair->ver++; - - ref.used &= ~(1UL<> dst_idx) & 1UL) && fd_vinyl_key_eq( dst_key, ref.pair[ dst_idx ].key ) ) { - dst_pair = &ref.pair[ dst_idx ]; - break; - } - } - - if( dst_pair && dst_pair->acq ) return FD_VINYL_ERR_AGAIN; - - /* Lookup pair src_key. If it doesn't exist, fail with KEY. If it - is acquired, fail with AGAIN. */ - - ulong src_idx = 0UL; - pair_t * src_pair = NULL; - for( ; src_idx> src_idx) & 1UL) && fd_vinyl_key_eq( src_key, ref.pair[ src_idx ].key ) ) { - src_pair = &ref.pair[ src_idx ]; - break; - } - } - - if( !src_pair ) return FD_VINYL_ERR_KEY; - if( src_pair->acq ) return FD_VINYL_ERR_AGAIN; - - /* At this point: - - pair dst_key may or may not exist. If it exists it is not - acquired. - - pair src_key exists and is not acquired. - Thus we are clear to move. Erase pair dst_key if it exists. Then - rename pair src_key to pair dst_key. */ - - if( dst_pair ) { - dst_pair->ver++; - ref.used &= ~(1UL<key[0] = dst_key[0]; - src_pair->ver++; - - return FD_VINYL_SUCCESS; - } - - case FD_VINYL_REQ_TYPE_FETCH: { - return FD_VINYL_SUCCESS; - } - - case FD_VINYL_REQ_TYPE_FLUSH: { - return FD_VINYL_SUCCESS; - } - - case FD_VINYL_REQ_TYPE_TRY: { - ulong idx = 0UL; - pair_t * pair = NULL; - for( ; idx> idx) & 1UL) && fd_vinyl_key_eq( key, ref.pair[ idx ].key ) ) { - pair = &ref.pair[ idx ]; - break; - } - } - - if( !pair ) return FD_VINYL_ERR_KEY; /* Key does not exist */ - if( pair->acq<0L ) return FD_VINYL_ERR_AGAIN; /* Key acquired-for-modify */ - - *_pair = pair; - *_ver = pair->ver; - - return FD_VINYL_SUCCESS; - } - - case FD_VINYL_REQ_TYPE_TEST: { - - if( (*_pair)->ver != (*_ver) ) return FD_VINYL_ERR_CORRUPT; /* Key modified during the try */ - - return FD_VINYL_SUCCESS; - } - - default: - break; - } - - return FD_VINYL_ERR_INVAL; -} - -static int -fd_vinyl_tile( int argc, - char ** argv ) { - (void)argc; - fd_vinyl_exec( (fd_vinyl_t *)argv ); - return 0; -} - -static void -client_tile( ulong iter_max, - fd_cnc_t * cnc, - ulong link_id, - fd_vinyl_rq_t * rq, - fd_vinyl_cq_t * cq, - fd_wksp_t * wksp, - void * _scratch ) { - fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, 0U, 0UL ) ); - - uchar * top = (uchar *)_scratch; - - fd_vinyl_key_t * _key = (fd_vinyl_key_t *) top; top += sizeof(fd_vinyl_key_t)*PAIR_MAX; - ulong * _try_gaddr = (ulong *) top; top += sizeof(ulong)*2UL *PAIR_MAX; - - fd_vinyl_comp_t * comp = (fd_vinyl_comp_t *)top; top += sizeof(fd_vinyl_comp_t); - ulong * val_gaddr = (ulong *) top; top += sizeof(ulong); - schar * err = (schar *) top; top += sizeof(schar); - - ulong comp_gaddr = fd_wksp_gaddr( wksp, comp ); - ulong val_gaddr_gaddr = fd_wksp_gaddr( wksp, val_gaddr ); - ulong err_gaddr = fd_wksp_gaddr( wksp, err ); - - ulong cq_seq = fd_vinyl_cq_seq( cq ); - -# define WAIT do { \ - if( oob ) { \ - while( !FD_VOLATILE_CONST( comp->seq ) ) FD_SPIN_PAUSE(); \ - FD_TEST( comp->seq==1UL ); \ - } else { \ - while( fd_vinyl_cq_recv( cq, cq_seq, comp ) ) FD_SPIN_PAUSE(); \ - FD_TEST( comp->seq==cq_seq ); \ - cq_seq++; \ - } \ - FD_TEST( comp->req_id ==req_id ); \ - FD_TEST( comp->link_id==link_id ); \ - } while(0) - - ulong val_max_bad = FD_VINYL_VAL_MAX+1UL; - - long acq [ PAIR_MAX ]; for( ulong idx=0UL; idx>= 2; - ulong dst_idx = (r & 3UL); r >>= 2; - int op = (int)(r & 63UL); r >>= 6; - int by_key = (int)(r & 1UL); r >>= 1; - int do_mod = (int)(r & 1UL); r >>= 1; - ulong oob = (int)(r & 1UL) ? comp_gaddr : 0UL; r >>= 1; - ulong flags = (r & 255UL); r >>= 8; - ulong val_max = (r & (ulong)UINT_MAX) % (FD_VINYL_VAL_MAX+1UL); r >>= 32; - int pat = (int)(r & 255UL); r >>= 8; - - fd_vinyl_key_t * src_key = _key + src_idx; - fd_vinyl_key_t * dst_key = _key + dst_idx; - ulong * try_gaddr = _try_gaddr + 2UL*src_idx; - - ulong src_key_gaddr = fd_wksp_gaddr( wksp, src_key ); - ulong dst_key_gaddr = fd_wksp_gaddr( wksp, dst_key ); - ulong try_gaddr_gaddr = fd_wksp_gaddr( wksp, try_gaddr ); - - comp->seq = 0UL; - - switch( op ) { - - case 0: /* mismatched link id (dropped and ticks the DROP_LINK counter) */ - val_gaddr[0] = val_max; - fd_vinyl_rq_send( rq, req_id, ~link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags, 1UL, - src_key_gaddr, val_gaddr_gaddr, err_gaddr, oob ); - break; - - case 1: /* unmappable oob completion (dropped and ticks the DROP_COMP counter) */ - val_gaddr[0] = val_max; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags, 1UL, - src_key_gaddr, val_gaddr_gaddr, err_gaddr, ULONG_MAX ); - break; - - case 2: /* bad request type */ - fd_vinyl_rq_send( rq, req_id, link_id, -1, flags, 1UL, - src_key_gaddr, val_gaddr_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - /* acquire tests */ - - case 3: /* acquire with unmappable key */ - val_gaddr[0] = val_max; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags, 1UL, - 0UL, val_gaddr_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 4: /* acquire with unmappable val */ - val_gaddr[0] = val_max; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags, 1UL, - src_key_gaddr, 0UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 5: /* acquire with unmappable err */ - val_gaddr[0] = val_max; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags, 1UL, - src_key_gaddr, val_gaddr_gaddr, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 6: /* acquire with zero batch */ - val_gaddr[0] = val_max; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags, 0UL, - 0UL, 0UL, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)0 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 7: { /* acquire with bad val_max */ - pair_t * pair; - int ref_err = req( FD_VINYL_REQ_TYPE_ACQUIRE, flags | FD_VINYL_REQ_FLAG_MODIFY, val_max_bad, src_key, &pair, NULL ); - val_gaddr[0] = val_max_bad; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags | FD_VINYL_REQ_FLAG_MODIFY, 1UL, - src_key_gaddr, val_gaddr_gaddr, err_gaddr, oob ); WAIT; - if( ref_err==FD_VINYL_ERR_FULL ) { - FD_TEST( comp->err ==FD_VINYL_ERR_FULL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - } - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)1 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - FD_TEST( err[0]==(schar)FD_VINYL_ERR_INVAL ); - break; - } - - case 8: { /* acquire */ - pair_t * pair; - int ref_err = req( FD_VINYL_REQ_TYPE_ACQUIRE, flags, val_max, src_key, &pair, NULL ); - val_gaddr[0] = val_max; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ACQUIRE, flags, 1UL, - src_key_gaddr, val_gaddr_gaddr, err_gaddr, oob ); WAIT; - if( ref_err==FD_VINYL_ERR_FULL ) { - FD_TEST( comp->err ==FD_VINYL_ERR_FULL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - } - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)!!ref_err ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - FD_TEST( err[0]==(schar)ref_err ); - - if( !ref_err ) { - acq_gaddr[ src_idx ] = val_gaddr[0]; - - void * val = fd_wksp_laddr_fast( wksp, val_gaddr[0] ); - fd_vinyl_info_t * info = fd_vinyl_data_info( val ); - ulong val_sz = (ulong)info->val_sz; - ulong val_max = fd_vinyl_data_val_max( val ); - - FD_TEST( val_max==pair->val_max ); - - FD_TEST( !memcmp( info, pair->info, sizeof(fd_vinyl_info_t) ) ); - - if( val_sz ) FD_TEST( !memcmp( val, pair->val, val_sz ) ); - - /* FIXME: TEST [VAL_SZ,VAL_MAX) ZPAD? */ - - if( fd_vinyl_req_flag_modify( flags ) ) { - acq [ src_idx ] = -1L; - acq_modified[ src_idx ] = fd_vinyl_req_flag_ignore( flags ); - if( do_mod ) { - val_sz = fd_rng_ulong_roll( rng, val_max + 1UL ); - memset( info, pat, sizeof(fd_vinyl_info_t) ); memset( pair->info, pat, sizeof(fd_vinyl_info_t) ); - memset( val, pat, val_sz ); memset( pair->val, pat, val_sz ); - info->val_sz = (uint)val_sz; pair->info->val_sz = (uint)val_sz; - acq_modified[ src_idx ] |= 1; - } - } else { - FD_TEST( !memcmp( info, pair->info, sizeof(fd_vinyl_info_t) ) ); - FD_TEST( !memcmp( val, pair->val, val_sz ) ); - acq [ src_idx ]++; - acq_modified[ src_idx ] = 0; - } - } - break; - } - - /* release tests */ - - case 9: /* release with unmappable key */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_RELEASE, flags | FD_VINYL_REQ_FLAG_BY_KEY, 1UL, - 0UL, val_gaddr_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 10: /* release with unmappable val */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_RELEASE, flags & ~FD_VINYL_REQ_FLAG_BY_KEY, 1UL, - src_key_gaddr, 0UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 11: /* release with unmappable err */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_RELEASE, flags, 1UL, - src_key_gaddr, val_gaddr_gaddr, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 12: /* release with zero batch */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_RELEASE, flags, 0UL, - 0UL, 0UL, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)0 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 13: { /* release */ - if( acq[ src_idx ] > 0L ) flags &= ~FD_VINYL_REQ_FLAG_MODIFY; /* can't say modify on an acquire-for-read */ - if( ((acq[ src_idx ] < 0L) & (!fd_vinyl_req_flag_modify( flags )) & acq_modified[ src_idx ]) ) - flags |= FD_VINYL_REQ_FLAG_IGNORE; - int ref_err = req( FD_VINYL_REQ_TYPE_RELEASE, flags, val_max_bad, src_key, NULL, NULL ); - if( by_key || !acq[ src_idx ] ) { - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_RELEASE, flags | FD_VINYL_REQ_FLAG_BY_KEY, 1UL, - src_key_gaddr, 0UL, err_gaddr, oob ); - } else { - val_gaddr[0] = acq_gaddr[ src_idx ]; - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_RELEASE, flags & ~FD_VINYL_REQ_FLAG_BY_KEY, 1UL, - 0UL, val_gaddr_gaddr, err_gaddr, oob ); - } - WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)!!ref_err ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - FD_TEST( err[0]==(schar)ref_err ); - if( !ref_err ) acq[ src_idx ] = (acq[ src_idx ] > 0L) ? (acq[ src_idx ]-1L) : 0L; - break; - } - - /* erase tests */ - - case 14: /* erase with unmappable key */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ERASE, flags, 1UL, - 0UL, 0UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 15: /* erase with unmappable err */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ERASE, flags, 1UL, - src_key_gaddr, 0UL, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 16: /* erase with zero batch */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ERASE, flags, 0UL, - src_key_gaddr, 0UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)0 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 17: { /* erase */ - int ref_err = req( FD_VINYL_REQ_TYPE_ERASE, flags, val_max_bad, src_key, NULL, NULL ); - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_ERASE, flags, 1UL, - src_key_gaddr, 1UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)!!ref_err ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - FD_TEST( err[0]==(schar)ref_err ); - break; - } - - /* move tests */ - - case 18: /* move with unmappable src */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_MOVE, flags, 1UL, - 0UL, src_key_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 19: /* move with unmappable dst */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_MOVE, flags, 1UL, - src_key_gaddr, 0UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 20: /* move with unmappable err */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_MOVE, flags, 1UL, - src_key_gaddr, dst_key_gaddr, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 21: /* move with zero batch */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_MOVE, flags, 0UL, - src_key_gaddr, dst_key_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)0 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 22: { /* move */ - int ref_err = req( FD_VINYL_REQ_TYPE_MOVE, flags, val_max_bad, src_key, (pair_t **)dst_key, NULL ); - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_MOVE, flags, 1UL, - src_key_gaddr, dst_key_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)!!ref_err ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - FD_TEST( err[0]==(schar)ref_err ); - break; - } - - /* fetch tests (these are logical no-op / hints and don't generate completions) */ - - case 23: /* fetch with unmappable key */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_FETCH, flags, 1UL, 0UL, 0UL, 0UL, oob ); - break; - - case 24: /* fetch with zero batch cnt */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_FETCH, flags, 0UL, 0UL, 0UL, 0UL, oob ); - break; - - case 25: /* fetch */ - FD_TEST( !req( FD_VINYL_REQ_TYPE_FETCH, 0UL, 0UL, src_key, NULL, NULL ) ); - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_FETCH, flags, 1UL, src_key_gaddr, 0UL, 0UL, oob ); - break; - - /* flush tests (these are logical no-ops / hints and don't generate completions) */ - - case 26: /* flush with unmappable key */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_FLUSH, flags, 1UL, 0UL, 0UL, 0UL, oob ); - break; - - case 27: /* flush with zero batch cnt */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_FLUSH, flags, 0UL, 0UL, 0UL, 0UL, oob ); - break; - - case 28: /* flush */ - FD_TEST( !req( FD_VINYL_REQ_TYPE_FLUSH, 0UL, 0UL, src_key, NULL, NULL ) ); - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_FLUSH, flags, 1UL, src_key_gaddr, 0UL, 0UL, oob ); - break; - - /* try tests */ - - case 29: /* try with unmappable key */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TRY, flags, 1UL, - 0UL, try_gaddr_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 30: /* try with unmappable try */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TRY, flags, 1UL, - src_key_gaddr, 0UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 31: /* try with unmappable err */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TRY, flags, 1UL, - src_key_gaddr, try_gaddr_gaddr, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 32: /* try with zero batch */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TRY, flags, 0UL, - src_key_gaddr, try_gaddr_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)0 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 33: { /* try */ - int ref_err = req( FD_VINYL_REQ_TYPE_TRY, flags, val_max_bad, src_key, &try_pair[ src_idx ], &try_ver[ src_idx ] ); - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TRY, flags, 1UL, - src_key_gaddr, try_gaddr_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)!!ref_err ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - FD_TEST( err[0]==(schar)ref_err ); - if( !ref_err ) try_live[ src_idx ] = 1; - break; - } - - /* test tests */ - - case 34: /* test with unmappable try */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TEST, flags, 1UL, - 0UL, 0UL, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 35: /* test with unmappable err */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TEST, flags, 1UL, - 0UL, try_gaddr_gaddr, 0UL, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_ERR_INVAL ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 36: /* test with zero batch */ - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TEST, flags, 0UL, - 0UL, try_gaddr_gaddr, err_gaddr, oob ); WAIT; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)0 ); - FD_TEST( comp->fail_cnt ==(ushort)0 ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - break; - - case 37: { /* test */ - if( !try_live[ src_idx ] ) break; - void * try_val = fd_wksp_laddr_fast( wksp, try_gaddr[0] ); - fd_vinyl_info_t * try_info = fd_vinyl_data_info( try_val ); - ulong try_val_sz = fd_ulong_min( (ulong)try_info->val_sz, FD_VINYL_VAL_MAX ); - int try_cmp = (!memcmp( try_info, try_pair[ src_idx ]->info, sizeof(fd_vinyl_info_t) )) && - (!memcmp( try_val, try_pair[ src_idx ]->val, try_val_sz )); - int ref_err = req( FD_VINYL_REQ_TYPE_TEST, flags, val_max_bad, NULL, &try_pair[ src_idx ], &try_ver[ src_idx ] ); - fd_vinyl_rq_send( rq, req_id, link_id, FD_VINYL_REQ_TYPE_TEST, flags, 1UL, - 0UL, try_gaddr_gaddr, err_gaddr, oob ); WAIT; - try_live[ src_idx ] = 0; - FD_TEST( comp->err ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - /**/ FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - /* Note that it is possible for a try to work in the ref version - and fail in the full version (e.g. cache line flushing, val - resizing in the background, etc). */ - if( ((!ref_err) & (!!err[0])) ) break; - FD_TEST( comp->fail_cnt ==(ushort)!!ref_err ); - FD_TEST( (!!err[0]) | try_cmp ); - break; - } - - case 38: { /* sync */ - FD_TEST( !fd_vinyl_sync( cnc ) ); - break; - } - - case 39: { /* randomly toggle data compression on and off */ - int new_style = (r & 1UL) ? (ulong)FD_VINYL_BSTREAM_CTL_STYLE_RAW : (ulong)FD_VINYL_BSTREAM_CTL_STYLE_LZ4; - FD_TEST( !fd_vinyl_set( cnc, FD_VINYL_OPT_STYLE, (ulong)new_style, NULL ) ); - break; - } - - default: break; - } - } - - /* Clean up */ - - for( ulong src_idx=0UL; src_idxerr ==FD_VINYL_SUCCESS ); FD_TEST( comp->batch_cnt==(ushort)1 ); - FD_TEST( comp->fail_cnt ==(ushort)!!ref_err ); FD_TEST( comp->quota_rem==(ushort)ref.quota_rem ); - FD_TEST( err[0]==(schar)ref_err ); - } - - } - - fd_rng_delete( fd_rng_leave( rng ) ); -} - -int -main( int argc, - char ** argv ) { - fd_boot( &argc, &argv ); - - if( FD_UNLIKELY( fd_tile_cnt() < 2UL ) ) FD_LOG_ERR(( "This test requires at least 2 tiles" )); - - char const * _wksp = fd_env_strip_cmdline_cstr ( &argc, &argv, "--wksp", NULL, NULL ); - char const * _page_sz = fd_env_strip_cmdline_cstr ( &argc, &argv, "--page-sz", NULL, "gigantic" ); - ulong page_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--page-cnt", NULL, 8UL ); - ulong near_cpu = fd_env_strip_cmdline_ulong( &argc, &argv, "--near-cpu", NULL, fd_log_cpu_id() ); - ulong tag = fd_env_strip_cmdline_ulong( &argc, &argv, "--tag", NULL, 1UL ); - - ulong spad_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--spad-max", NULL, fd_vinyl_io_spad_est() ); - ulong dev_sz = fd_env_strip_cmdline_ulong( &argc, &argv, "--dev-sz", NULL, 1UL << 30 ); - ulong io_seed = fd_env_strip_cmdline_ulong( &argc, &argv, "--io-seed", NULL, 1234UL ); - - ulong line_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--line-cnt", NULL, 7UL ); - - ulong ele_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--ele-max", NULL, 8UL ); - ulong lock_cnt = fd_env_strip_cmdline_ulong( &argc, &argv, "--lock_cnt", NULL, 8UL ); - ulong probe_max = ele_max; - ulong seed = fd_env_strip_cmdline_ulong( &argc, &argv, "--seed", NULL, 5678UL ); - - ulong obj_sz = fd_env_strip_cmdline_ulong( &argc, &argv, "--obj-sz", NULL, 6UL << 30 ); - - ulong async_min = fd_env_strip_cmdline_ulong( &argc, &argv, "--async-min", NULL, 5UL ); - ulong async_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--async-max", NULL, 2UL*async_min ); - ulong part_thresh = fd_env_strip_cmdline_ulong( &argc, &argv, "--part-thresh", NULL, 64UL << 20 ); - ulong gc_thresh = fd_env_strip_cmdline_ulong( &argc, &argv, "--gc-thresh", NULL, 128UL << 20 ); - int gc_eager = fd_env_strip_cmdline_int ( &argc, &argv, "--gc-eager", NULL, 2 ); - char const * _style = fd_env_strip_cmdline_cstr ( &argc, &argv, "--style", NULL, "lz4" ); - int level = fd_env_strip_cmdline_int ( &argc, &argv, "--level", NULL, 0 ); - - ulong rq_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--rq-max", NULL, 32UL ); - ulong cq_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--cq-max", NULL, 32UL ); - ulong link_id = fd_env_strip_cmdline_ulong( &argc, &argv, "--link-id", NULL, 2345UL ); - ulong burst_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--burst-max", NULL, 1UL ); - ulong quota_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--quota-max", NULL, 2UL ); - ulong scratch_sz = fd_env_strip_cmdline_ulong( &argc, &argv, "--scratch-sz", NULL, 4096UL ); - - ulong iter_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--iter-max", NULL, (ulong)1e7 ); - - int style = fd_cstr_to_vinyl_bstream_ctl_style( _style ); - - fd_wksp_t * wksp; - if( _wksp ) { - FD_LOG_NOTICE(( "Attaching to --wksp %s", _wksp )); - wksp = fd_wksp_attach( _wksp ); - } else { - FD_LOG_NOTICE(( "--wksp not specified, using an anonymous local workspace (--page-sz %s --page-cnt %lu --near-cpu %lu)", - _page_sz, page_cnt, near_cpu )); - wksp = fd_wksp_new_anonymous( fd_cstr_to_shmem_page_sz( _page_sz ), page_cnt, near_cpu, "wksp", 0UL ); - } - FD_TEST( wksp ); - - FD_LOG_NOTICE(( "Creating vinyl tile" )); - - ulong io_footprint = fd_vinyl_io_mm_footprint( spad_max ); FD_TEST( io_footprint ); - ulong dev_footprint = fd_ulong_align_dn( dev_sz, FD_VINYL_BSTREAM_BLOCK_SZ ); FD_TEST( dev_footprint ); - ulong vinyl_footprint = fd_vinyl_footprint(); FD_TEST( vinyl_footprint ); - ulong cnc_footprint = fd_cnc_footprint( FD_VINYL_CNC_APP_SZ ); FD_TEST( cnc_footprint ); - ulong meta_footprint = fd_vinyl_meta_footprint( ele_max, lock_cnt, probe_max ); FD_TEST( meta_footprint ); - ulong line_footprint = sizeof(fd_vinyl_line_t) * line_cnt; FD_TEST( line_footprint ); - ulong ele_footprint = sizeof(fd_vinyl_meta_ele_t) * ele_max; FD_TEST( ele_footprint ); - ulong obj_footprint = fd_ulong_align_dn( obj_sz, alignof(fd_vinyl_data_obj_t) ); FD_TEST( obj_footprint ); - ulong rq_footprint = fd_vinyl_rq_footprint( rq_max ); FD_TEST( rq_footprint ); - ulong cq_footprint = fd_vinyl_cq_footprint( cq_max ); FD_TEST( cq_footprint ); - - void * _io = fd_wksp_alloc_laddr( wksp, fd_vinyl_io_mm_align(), io_footprint, tag ); FD_TEST( _io ); - void * _dev = fd_wksp_alloc_laddr( wksp, FD_VINYL_BSTREAM_BLOCK_SZ, dev_footprint, tag ); FD_TEST( _dev ); - void * _vinyl = fd_wksp_alloc_laddr( wksp, fd_vinyl_align(), vinyl_footprint, tag ); FD_TEST( _vinyl ); - void * _cnc = fd_wksp_alloc_laddr( wksp, fd_cnc_align(), cnc_footprint, tag ); FD_TEST( _cnc ); - void * _meta = fd_wksp_alloc_laddr( wksp, fd_vinyl_meta_align(), meta_footprint, tag ); FD_TEST( _meta ); - void * _line = fd_wksp_alloc_laddr( wksp, alignof(fd_vinyl_line_t), line_footprint, tag ); FD_TEST( _line ); - void * _ele = fd_wksp_alloc_laddr( wksp, alignof(fd_vinyl_meta_ele_t), ele_footprint, tag ); FD_TEST( _ele ); - void * _obj = fd_wksp_alloc_laddr( wksp, alignof(fd_vinyl_data_obj_t), obj_footprint, tag ); FD_TEST( _obj ); - void * _rq = fd_wksp_alloc_laddr( wksp, fd_vinyl_rq_align(), rq_footprint, tag ); FD_TEST( _rq ); - void * _cq = fd_wksp_alloc_laddr( wksp, fd_vinyl_cq_align(), cq_footprint, tag ); FD_TEST( _cq ); - void * _scratch = fd_wksp_alloc_laddr( wksp, 4096UL, scratch_sz, tag ); FD_TEST( _scratch ); - - fd_vinyl_io_t * io = fd_vinyl_io_mm_init( _io, spad_max, _dev, dev_footprint, 1, "test", 5UL, io_seed ); FD_TEST( io ); - - fd_tpool_t * tpool = NULL; - - ulong thread_cnt = fd_tile_cnt(); - - if( FD_LIKELY( thread_cnt>1UL ) ) { - FD_LOG_NOTICE(( "Creating temporary tpool from all %lu tiles for thread parallel init", thread_cnt )); - - static uchar _tpool[ FD_TPOOL_FOOTPRINT( FD_TILE_MAX ) ] __attribute__((aligned(FD_TPOOL_ALIGN))); - - tpool = fd_tpool_init( _tpool, thread_cnt, 0UL ); /* logs details */ - if( FD_UNLIKELY( !tpool ) ) FD_LOG_ERR(( "fd_tpool_init failed" )); - - for( ulong thread_idx=1UL; thread_idx Date: Fri, 23 Oct 2026 21:44:09 +0000 Subject: [PATCH 4/5] Remove groove --- contrib/test/gen_unit_test_pages.sh | 10 ---------- contrib/test/run_unit_tests.sh | 1 - 2 files changed, 11 deletions(-) diff --git a/contrib/test/gen_unit_test_pages.sh b/contrib/test/gen_unit_test_pages.sh index 333b2a883ff..8980a46bd87 100755 --- a/contrib/test/gen_unit_test_pages.sh +++ b/contrib/test/gen_unit_test_pages.sh @@ -55,14 +55,4 @@ grep -rln '"--page-cnt"' src/ | while read -r f; do printf '%s %s\n' "$(basename "$f" .c)" "$pages" fi done -# Groove tests size their shmem by --volume-cnt * FD_GROOVE_VOLUME_FOOTPRINT -# (1 GiB per volume), not --page-cnt. -grep -rln '"--volume-cnt"' src/ | while read -r f; do - cnt=$( { grep -oE '"--volume-cnt"[^;]*NULL,[[:space:]]*[0-9]+UL' "$f" || true; } \ - | grep -oE '[0-9]+UL' | grep -oE '[0-9]+' | head -1 ) - cnt=${cnt:-1} - if [[ "$cnt" -gt 1 ]]; then - printf '%s %s\n' "$(basename "$f" .c)" "$cnt" - fi -done } | sort -k2 -rn -k1 | awk '{ printf " [%s]=%s\n", $1, $2 }' diff --git a/contrib/test/run_unit_tests.sh b/contrib/test/run_unit_tests.sh index c5e9cf64bd4..88df30c4109 100755 --- a/contrib/test/run_unit_tests.sh +++ b/contrib/test/run_unit_tests.sh @@ -48,7 +48,6 @@ declare -A TEST_GIGANTIC_PAGES=( [test_quic_concurrency]=2 [test_quic_bw]=2 [test_progcache]=2 - [test_groove_volume]=2 [test_snapin_tile]=3 ) From 9eadab0626ec4cf705ad736d40a35dbac8c319c9 Mon Sep 17 00:00:00 2001 From: Richard Patel Date: Fri, 23 Oct 2026 21:44:17 +0000 Subject: [PATCH 5/5] Remove vinyl --- contrib/test/run_unit_tests.sh | 1 - src/discof/replay/Local.mk | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/test/run_unit_tests.sh b/contrib/test/run_unit_tests.sh index 88df30c4109..6e96c8e7780 100755 --- a/contrib/test/run_unit_tests.sh +++ b/contrib/test/run_unit_tests.sh @@ -27,7 +27,6 @@ VERBOSE=0 # Keep this list in sync with the sources; # contrib/test/gen_unit_test_pages.sh regenerates it. declare -A TEST_GIGANTIC_PAGES=( - [test_vinyl_req]=8 [test_vat_refresh_vote_accounts]=6 [test_vm_syscalls]=5 [test_vm_syscall_hash]=5 diff --git a/src/discof/replay/Local.mk b/src/discof/replay/Local.mk index 143394ca799..be254a6585c 100644 --- a/src/discof/replay/Local.mk +++ b/src/discof/replay/Local.mk @@ -13,7 +13,7 @@ endif ifdef FD_HAS_HOSTED $(call add-objs,fd_replay_tile,fd_discof) -$(call make-unit-test,test_replay_tile,test_replay_tile,fd_discof fd_choreo fd_disco fd_flamenco fd_vinyl fd_tango fd_ballet fd_util_extra fd_util) +$(call make-unit-test,test_replay_tile,test_replay_tile,fd_discof fd_choreo fd_disco fd_flamenco fd_tango fd_ballet fd_util_extra fd_util) $(call add-hdrs,fd_vote_tracker.h) $(call add-objs,fd_vote_tracker,fd_discof) endif