diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 52db284..f6a3d77 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -53,6 +53,13 @@ jobs: - name: Build release run: cargo build --release + - name: Prepare CLI compatibility alias + run: | + # wpadm 是项目管理工具的新名称;集成用例仍使用旧入口 wproj。 + ln -sf wpadm target/release/wproj + test -x target/release/wpadm + test -x target/release/wproj + - name: Determine wp-examples branch id: examples-branch run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 90f2a74..686224f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Release on: push: - tags: ["v*.*.*"] + tags: ["exp-v*.*.*"] env: REGISTRY: ghcr.io @@ -179,6 +179,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | + type=raw,value={{tag}} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} @@ -281,7 +282,7 @@ jobs: ### Quick Install Script ```bash - curl -sSf https://get.warpparse.ai/inst-x.sh | bash -s -- wparse ${{ steps.branch.outputs.branch }} + curl -fsSL https://raw.githubusercontent.com/wp-labs/warp-parse/${{ steps.branch.outputs.branch }}/scripts/install.sh | bash ``` ### Docker Image @@ -363,9 +364,11 @@ jobs: - name: Build assets map and validate run: | set -euo pipefail - jq -c ' + REPO="${{ github.event.repository.name }}" + jq -c --arg prefix "${REPO}-" ' [ .assets[] - | select(.name | test("^warp-parse-v[^/]+-(aarch64-apple-darwin|aarch64-unknown-linux-gnu|x86_64-unknown-linux-gnu)\\.tar\\.gz$")) + | select(.name | startswith($prefix)) + | select(.name | test("-(aarch64-apple-darwin|aarch64-unknown-linux-gnu|x86_64-unknown-linux-gnu)\\.tar\\.gz$")) | { key: (.name | capture("-(?aarch64-apple-darwin|aarch64-unknown-linux-gnu|x86_64-unknown-linux-gnu)\\.tar\\.gz$").target), value: { @@ -376,9 +379,9 @@ jobs: ] | from_entries ' /tmp/release.json > /tmp/assets.json - jq -e ' + jq -e --arg release_url_prefix "https://github.com/${GITHUB_REPOSITORY}/releases/download/" ' (keys | sort) == ["aarch64-apple-darwin","aarch64-unknown-linux-gnu","x86_64-unknown-linux-gnu"] and - (to_entries | all(.value.url | startswith("https://github.com/wp-labs/warp-parse/releases/download/"))) and + (to_entries | all(.value.url | startswith($release_url_prefix))) and (to_entries | all(.value.sha256 | test("^[0-9a-f]{64}$"))) ' /tmp/assets.json >/dev/null @@ -400,8 +403,8 @@ jobs: fi cd wp-install - MANIFEST="updates/${CHANNEL}/manifest.json" - HISTORY="updates/${CHANNEL}/versions/${TAG}.json" + MANIFEST="updates-exp/${CHANNEL}/manifest.json" + HISTORY="updates-exp/${CHANNEL}/versions/${TAG}.json" mkdir -p "$(dirname "$MANIFEST")" "$(dirname "$HISTORY")" jq -n \ @@ -428,7 +431,7 @@ jobs: cd wp-install git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add updates + git add updates-exp if git diff --cached --quiet; then echo "no manifest changes" exit 0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f463048..b957867 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ cargo build --no-default-features --features core --release ./target/release/wpgen --help # Run the project manager -./target/release/wproj --help +./target/release/wpadm --help # Run the recovery tool ./target/release/wprescue --help @@ -475,7 +475,7 @@ cargo build --no-default-features --features core --release ./target/release/wpgen --help # 运行项目管理工具 -./target/release/wproj --help +./target/release/wpadm --help # 运行恢复工具 ./target/release/wprescue --help diff --git a/Cargo.lock b/Cargo.lock index 621acd4..740f7d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.18.13" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f290edc569ad10c07287eebb96629686b37f1e55283a093569151d3adde0c7" +checksum = "b796ba3c974e5133f6ca1a2c4e97c5941ea1a46d202df83344afaa18219f1a62" dependencies = [ "actix-codec", "actix-service", @@ -122,9 +122,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.16.0" +version = "4.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824c7250d7e468feb7903e41efc5d6883cb1f3957e8db2c3d2402a9ff430b886" +checksum = "67756b0fc99e07b8702b9a856e3d6ec036387243c2ce6310070d1e8f02680bb0" dependencies = [ "actix-codec", "actix-http", @@ -323,9 +323,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" dependencies = [ "rustversion", ] @@ -413,7 +413,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-select", - "atoi 3.1.0", + "atoi 3.1.1", "base64 0.23.1", "chrono", "half", @@ -674,9 +674,9 @@ dependencies = [ [[package]] name = "atoi" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" +checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993" dependencies = [ "num-traits", ] @@ -858,9 +858,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" dependencies = [ "find-msvc-tools", "jobserver", @@ -1040,9 +1040,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.8" +version = "4.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" dependencies = [ "bytes", "futures-core", @@ -1218,9 +1218,9 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -1379,9 +1379,9 @@ checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" [[package]] name = "defmt" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -1389,21 +1389,21 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "defmt-parser" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" dependencies = [ "thiserror 2.0.21", ] @@ -1674,9 +1674,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -1685,7 +1685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1764,9 +1764,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flatbuffers" @@ -2869,9 +2869,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.190" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libflate" @@ -2911,6 +2911,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -2926,7 +2936,7 @@ dependencies = [ "bitflags 2.13.2", "libc", "plain", - "redox_syscall 0.9.4", + "redox_syscall 0.9.5", ] [[package]] @@ -4147,7 +4157,7 @@ dependencies = [ "once_cell", "socket2 0.6.5", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4358,9 +4368,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" dependencies = [ "bitflags 2.13.2", ] @@ -4519,14 +4529,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -4578,7 +4588,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4589,9 +4599,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -4622,9 +4632,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" dependencies = [ "windows-sys 0.61.2", ] @@ -5016,9 +5026,9 @@ dependencies = [ [[package]] name = "smol_str" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +checksum = "eeb7109d23f319619d4cc0de5464613c8b7b1a09db2774a58101459cf6be2f10" dependencies = [ "borsh", "serde_core", @@ -5576,7 +5586,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5712,9 +5722,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.2" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" dependencies = [ "bytes", "libc", @@ -5968,9 +5978,9 @@ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" dependencies = [ "tinyvec", ] @@ -6105,7 +6115,7 @@ dependencies = [ [[package]] name = "warp-parse" -version = "0.31.1" +version = "0.31.1-E4" dependencies = [ "anyhow", "chrono", @@ -6143,6 +6153,7 @@ dependencies = [ "wp-config", "wp-connector-api", "wp-connectors", + "wp-connectors-labs", "wp-engine", "wp-error", "wp-knowledge", @@ -6250,9 +6261,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -6263,14 +6274,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -6309,9 +6320,9 @@ dependencies = [ [[package]] name = "wildmatch" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29333c3ea1ba8b17211763463ff24ee84e41c78224c16b001cd907e663a38c68" +checksum = "f18219fbd41a807dec044965d4106bb4fdb1d3a4b70aac4e5510aacc80a9f140" dependencies = [ "serde", ] @@ -6338,7 +6349,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -6820,6 +6831,32 @@ dependencies = [ "wp-specs", ] +[[package]] +name = "wp-connectors-labs" +version = "0.1.14" +source = "git+https://github.com/wp-labs/wp-connectors-labs.git?tag=v0.1.14#4706316888cf38ba672d04126c8b165eb83da5a2" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "educe 0.6.0", + "hex", + "libloading", + "log", + "orion-error", + "orion_conf", + "serde", + "serde_json", + "socket2 0.6.5", + "thiserror 2.0.21", + "tokio", + "wp-conf-base", + "wp-connector-api", + "wp-data-fmt", + "wp-log", + "wp-model-core", +] + [[package]] name = "wp-core-connectors" version = "0.10.2" @@ -7340,18 +7377,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.62" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.62" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -7450,9 +7487,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.1.1+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 9c0352a..60d1a84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ # Binary applications for data parsing and processing: # - wparse: 主解析引擎 # - wpgen: 规则生成工具 -# - wproj: 项目管理工具 +# - wpadm: 项目管理工具 # - wprescue: 数据恢复工具 # # Feature flags: @@ -16,7 +16,7 @@ [package] name = "warp-parse" -version = "0.31.1" # @gxl:set(version) +version = "0.31.1-E4" # @gxl:set(version) edition = "2021" rust-version = "1.75" authors = ["Warp Parse Team"] @@ -92,12 +92,15 @@ default = ["community"] core = [] # --- Community Edition (default) --- -community = ["core", "wp-connectors"] +community = ["core", "wp-connectors", "labs"] + +# --- Labs (experimental connectors) --- +labs = ["wp-connectors-labs"] # --- Enterprise --- # --- Convenience --- -all = ["community"] +all = ["community", "labs"] # --- Testing (cfg gates) --- sys_test = [] @@ -198,12 +201,13 @@ wp-proj = { package = "wp-proj", git = "https://github.com/wp-labs/w wp-self-update = "0.4" wp-connectors = { git ="https://github.com/wp-labs/wp-connectors",tag = "v0.23.0", optional = true } - +wp-connectors-labs = { git = "https://github.com/wp-labs/wp-connectors-labs.git", tag = "v0.1.14", optional = true, features = ["dmdb", "udp"] } +# wp-connectors-labs = {package = "wp-connectors-labs", path = "../wp-connectors-exp", optional = true, features = ["dmdb", "udp"] } # @gxl:block(local_motor) -#wp-engine = { path = "../wp-motor" } -#wp-config = { path = "../wp-motor/crates/wp-config" } -#wp-cli-core = { path = "../wp-motor/crates/wp-cli-core" } -#wp-proj = { path = "../wp-motor/crates/wp-proj" } +# wp-engine = { path = "../wp-motor" } +# wp-config = { path = "../wp-motor/crates/wp-config" } +# wp-cli-core = { path = "../wp-motor/crates/wp-cli-core" } +# wp-proj = { path = "../wp-motor/crates/wp-proj" } # @gxl:end(local_motor) # wp-enterprise = { path = "../wp-enterprise" } diff --git a/README.md b/README.md index 0e9f49f..447bde6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Warp Parse is a high-performance Rust ETL engine built for observability, securi - **🚀 Extreme throughput:** EPS (Events Per Second) significantly surpasses Vector across multiple scenarios, with 2~6x performance advantages in core scenarios like fixed rate and large logs (see [Performance Report](https://github.com/wp-labs/wp-examples/tree/main/benchmark/report)). - **📝 Readable rules:** Self-developed WPL (Parse DSL) + OML (Transform DSL) offer far superior readability and maintainability compared to regular expressions and Lua scripts. - **🔌 Unified connectors:** Built on standardized `wp-connector-api` interface design, enabling community developers to rapidly extend multi-source log connector ecosystem. -- **🛠️ Ops friendly:** Single binary deployment with full configuration-based management; includes `wproj`, `wpgen`, `wprescue` tool suite to reduce operational costs. +- **🛠️ Ops friendly:** Single binary deployment with full configuration-based management; includes `wpadm`, `wpgen`, `wprescue` tool suite to reduce operational costs. - **🧠 Knowledge transformation:** Built-in in-memory database supports real-time SQL queries for log data field enrichment and correlation analysis. - **🎯 Data routing:** Flexible routing based on rule engine and transformation models, supporting multi-path data replication, precise filtering, and target distribution. @@ -54,8 +54,8 @@ curl -sSf https://get.warpparse.ai/inst-x.sh | bash -s -- wp-skills Warp Parse currently supports runtime admin operations in daemon mode only. - Usage guide: [docs/ENGINE_ADMIN_USAGE.md](docs/ENGINE_ADMIN_USAGE.md) -- Status query: `cargo run --bin wproj -- engine status --work-root .` -- Reload trigger: `cargo run --bin wproj -- engine reload --work-root . --reason "manual reload"` +- Status query: `cargo run --bin wpadm -- engine status --work-root .` +- Reload trigger: `cargo run --bin wpadm -- engine reload --work-root . --reason "manual reload"` To enable the admin API, configure `[admin_api]` in `conf/wparse.toml`, create `${HOME}/.warp_parse/admin_api.token`, and start: @@ -102,7 +102,7 @@ For details, please refer to the [LICENSE](LICENSE) file in the repository root. - **🚀 极致吞吐:** 多场景下 EPS(事件处理速率)全面超越 Vector,固定速率/大日志等核心场景性能优势达 2~6 倍(详见[性能测试报告](https://github.com/wp-labs/wp-examples/tree/main/benchmark/report))。 - **📝 规则易编写:** 自研 WPL(解析 DSL)+ OML(转换 DSL),可读性、可维护性远超正则表达式与 Lua 脚本。 - **🔌 连接器统一:** 基于 `wp-connector-api` 标准化接口设计,支持社区开发者快速扩展多源日志连接器生态。 -- **🛠️ 运维友好:** 单二进制文件部署,全配置化管理;配套 `wproj`、`wpgen`、`wprescue` 工具套件,降低运维成本。 +- **🛠️ 运维友好:** 单二进制文件部署,全配置化管理;配套 `wpadm`、`wpgen`、`wprescue` 工具套件,降低运维成本。 - **🧠 知识转换:** 内置内存数据库支持 SQL 实时查询,实现日志数据字段富化与关联分析。 - **🎯 数据路由:** 基于规则引擎与转换模型的灵活路由,支持多路数据复制、精准过滤与目标分发。 diff --git a/_gal/adm.gxl b/_gal/adm.gxl index 57797ff..18cbc50 100644 --- a/_gal/adm.gxl +++ b/_gal/adm.gxl @@ -8,11 +8,18 @@ mod envs { mod main : ver_adm { #[auto_load(entry)] - flow _entry : ver.use { + flow _entry | ver.use { + gx.sn( file: "./exp_sn.txt"); } #[auto_load(exit)] flow _set_ver | ver.syn_file { + VERSION = "${VERSION}-E${SN}"; SYN_FILE= "${ENV_ROOT}/Cargo.toml"; + gx.echo ("syn cargo file"); + } + + flow up_sn { + gx.sn( file: "./exp_sn.txt", action : "add"); } } diff --git a/_gal/vfm.gxl b/_gal/vfm.gxl index 95a7c5c..1c2ce48 100644 --- a/_gal/vfm.gxl +++ b/_gal/vfm.gxl @@ -10,18 +10,21 @@ mod sys_ver_adm { #[usage(desp="add tag by version ")] flow tag_stable { gx.ver ( file : "${GXL_START_ROOT}/version.txt" , inc : "null" ); - gx.cmd ( "git tag v${VERSION}" ); + gx.sn( file: "./exp_sn.txt"); + gx.cmd ( "git tag exp-v${VERSION}-${SN}" ); gx.cmd ( "git push --tags" ); } flow tag_beta{ gx.ver ( file : "${GXL_START_ROOT}/version.txt" , inc : "null" ); - gx.cmd ( "git tag v${VERSION}-beta" ); + gx.sn( file: "./exp_sn.txt"); + gx.cmd ( "git tag exp-v${VERSION}-${SN}-beta" ); gx.cmd ( "git push --tags" ); } flow tag_alpha{ gx.ver ( file : "${GXL_START_ROOT}/version.txt" , inc : "null" ); - gx.cmd ( "git tag v${VERSION}-alpha" ); + gx.sn( file: "./exp_sn.txt"); + gx.cmd ( "git tag exp-v${VERSION}-${SN}-alpha" ); gx.cmd ( "git push --tags" ); } } diff --git a/docker/Dockerfile b/docker/Dockerfile index 57fde0f..3810771 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,9 @@ LABEL org.opencontainers.image.source="https://github.com/wp-labs/warp-parse" LABEL org.opencontainers.image.description="High-performance flow data parsing and processing system" LABEL org.opencontainers.image.licenses="ELv2" -# Install runtime dependencies +# 安装运行时依赖。 +# 其中 unixODBC 运行时用于 DMDB connector 在启动时通过 libloading 加载 ODBC Driver Manager。 +# 达梦运行时还会按库名查找 `libodbc.so` / `libodbcinst.so`,这里在镜像内补齐无版本软链。 RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ libssl3t64 \ @@ -18,6 +20,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zlib1g \ libsasl2-2 \ libzstd1 \ + libodbc2 \ + libodbcinst2 \ + unixodbc \ + odbcinst \ + libltdl7 \ + && ODBC_SO="$(find /usr/lib -path '*/libodbc.so.2' | head -n1)" \ + && test -n "$ODBC_SO" \ + && ln -sf "$(basename "$ODBC_SO")" "$(dirname "$ODBC_SO")/libodbc.so" \ + && ODBCINST_SO="$(find /usr/lib -path '*/libodbcinst.so.2' | head -n1)" \ + && test -n "$ODBCINST_SO" \ + && ln -sf "$(basename "$ODBCINST_SO")" "$(dirname "$ODBCINST_SO")/libodbcinst.so" \ && rm -rf /var/lib/apt/lists/* \ && useradd -r -s /bin/false wparse diff --git a/exp_sn.txt b/exp_sn.txt new file mode 100644 index 0000000..bf0d87a --- /dev/null +++ b/exp_sn.txt @@ -0,0 +1 @@ +4 \ No newline at end of file diff --git a/src/admin_api.rs b/src/admin_api.rs index 9a19ab3..9ff8125 100644 --- a/src/admin_api.rs +++ b/src/admin_api.rs @@ -191,10 +191,10 @@ fn load_config(work_root: &Path, dict: &EnvDict) -> RunResult &'static str { +pub fn features_list() -> String { + let mut features = String::new(); if cfg!(feature = "wp-connectors") { - "community (kafka,mysql,postgres,clickhouse,elasticsearch,prometheus,victorialogs,victoriametrics,doris,http,count)" + features.push_str("community (kafka,mysql,postgres,clickhouse,elasticsearch,prometheus,victorialogs,victoriametrics,doris,http,count,udp) "); } else { - "core" + features.push_str("core "); + } + + if cfg!(feature = "wp-connectors-labs") { + features.push_str("labs (dmdb)"); } + features } diff --git a/version.txt b/version.txt index f176c94..8314545 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.31.1 +0.31.1 \ No newline at end of file