From 2fa05962681e66426b2a9e5b130f169ac7bed03e Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Sun, 28 Jun 2026 06:27:43 +0300 Subject: [PATCH] add clickhouse-backed oteldb variant + --only fast-loop mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit oteldb-ch: same binary without --embedded, serving all three query APIs from chstorage on a dedicated clickhouse-oteldb server (isolated footprint). Wired into the ingest fan-out (vmagent remote-write, otelcol logs+traces) and systems.yml with a per-system volume override for footprint accounting. --only (or BENCH_ONLY): restrict a run to a subset of engines, bringing up just those plus the lane's ingest driver and querying/collecting/reporting only them — a fast feedback loop (e.g. --only oteldb,oteldb-ch for a head-to- head of the embedded vs ClickHouse storage). --- .gitignore | 1 + README.md | 22 +++++++++-- cmd/benchctl/main.go | 32 ++++++++++++++++ compose/metrics.yml | 1 + compose/shared.yml | 41 ++++++++++++++++++++ config/otelcol/logs.yml | 4 ++ config/otelcol/traces.yml | 4 ++ config/oteldb/oteldb-ch.yml | 45 ++++++++++++++++++++++ internal/bench/collect.go | 8 +++- internal/bench/compose.go | 76 ++++++++++++++++++++++++++++++++++++- internal/bench/config.go | 43 +++++++++++++++++++-- internal/bench/query.go | 2 +- internal/bench/report.go | 2 +- systems.yml | 3 ++ 14 files changed, 272 insertions(+), 12 deletions(-) create mode 100644 config/oteldb/oteldb-ch.yml diff --git a/.gitignore b/.gitignore index 688138b..c053887 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .idea/ results/*/ results/*.csv +results/*.log results/REPORT.md !results/.gitkeep !results/pprof/ diff --git a/README.md b/README.md index 3862ba4..6fca979 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,15 @@ languages: | Signal | Reference language | Systems under test | |---------|--------------------|--------------------| -| Metrics | **PromQL** | oteldb · Prometheus · VictoriaMetrics · Mimir · GreptimeDB · gigapipe | -| Logs | **LogQL** | oteldb · Loki · VictoriaLogs\* · gigapipe | -| Traces | **TraceQL** | oteldb · Tempo · VictoriaTraces\* · gigapipe | +| Metrics | **PromQL** | oteldb · oteldb-ch · Prometheus · VictoriaMetrics · Mimir · GreptimeDB · gigapipe | +| Logs | **LogQL** | oteldb · oteldb-ch · Loki · VictoriaLogs\* · gigapipe | +| Traces | **TraceQL** | oteldb · oteldb-ch · Tempo · VictoriaTraces\* · gigapipe | + +**oteldb** is benched in two configurations from the *same binary*: `oteldb` is +the embedded go-faster/storage engine (`--embedded`, `file` backend), and +`oteldb-ch` serves the same three query APIs from oteldb's original ClickHouse +storage (chstorage) on a dedicated `clickhouse-oteldb` server. Identical query +semantics and ingest path — the comparison isolates the storage layer. \* VictoriaLogs and VictoriaTraces do not implement LogQL/TraceQL; they are benched on their native dialect (LogsQL / Jaeger query) over the *same dataset @@ -109,6 +115,16 @@ go build -o benchctl ./cmd/benchctl && ./benchctl bench-all ./benchctl report ``` +**Faster feedback — only some engines.** `--only ` (or `BENCH_ONLY` in +`.env`) restricts a run to a subset of systems: it brings up just those engines +plus the lane's ingest driver (skipping the competitors entirely) and +queries/collects/reports only them. + +```bash +./benchctl --only oteldb bench-all # just the embedded engine +./benchctl --only oteldb,oteldb-ch bench-all # embedded vs ClickHouse, head-to-head +``` + Live dashboards while it runs: **Grafana** http://localhost:3000 (every engine pre-provisioned as a datasource), **cAdvisor** http://localhost:8085. diff --git a/cmd/benchctl/main.go b/cmd/benchctl/main.go index 8ee0399..8a85629 100644 --- a/cmd/benchctl/main.go +++ b/cmd/benchctl/main.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "strconv" + "strings" "time" "github.com/oteldb/benchmark/internal/bench" @@ -31,6 +32,7 @@ func main() { } func run(args []string) error { + args, only := stripOnly(args) if len(args) == 0 { usage() return nil @@ -39,6 +41,9 @@ func run(args []string) error { if err != nil { return err } + if only != "" { + e.Only = bench.ParseOnly(only) + } cmd, rest := args[0], args[1:] switch cmd { case "up": @@ -101,9 +106,36 @@ func usage() { bench [runs] up→ingest→settle→query→collect bench-all [runs] all lanes, then report down stop + wipe volumes + +global flags: + --only restrict the run to these engines (e.g. --only oteldb) for a + faster feedback loop: brings up only those engines + the lane's + ingest driver, and queries/collects/reports just them. Also set + via BENCH_ONLY in .env. `) } +// stripOnly extracts a global `--only ` / `--only=` flag from anywhere +// in args (last value wins) and returns the remaining args plus the value. It +// restricts the run to a subset of engines — e.g. `--only oteldb` for a fast +// oteldb-only feedback loop. Empty when the flag is absent. +func stripOnly(args []string) (rest []string, only string) { + for i := 0; i < len(args); i++ { + switch a := args[i]; { + case a == "--only": + if i+1 < len(args) { + only = args[i+1] + i++ + } + case strings.HasPrefix(a, "--only="): + only = strings.TrimPrefix(a, "--only=") + default: + rest = append(rest, a) + } + } + return rest, only +} + func arg(a []string, i int, def string) string { if i < len(a) { return a[i] diff --git a/compose/metrics.yml b/compose/metrics.yml index 4eb961d..bd4e856 100644 --- a/compose/metrics.yml +++ b/compose/metrics.yml @@ -27,6 +27,7 @@ services: command: - -promscrape.config=/etc/vmagent/scrape.yml - -remoteWrite.url=http://oteldb:19291/api/v1/write + - -remoteWrite.url=http://oteldb-ch:19291/api/v1/write - -remoteWrite.url=http://prometheus:9090/api/v1/write - -remoteWrite.url=http://victoriametrics:8428/api/v1/write - -remoteWrite.url=http://mimir:9009/api/v1/push diff --git a/compose/shared.yml b/compose/shared.yml index ec1151d..2b5d48c 100644 --- a/compose/shared.yml +++ b/compose/shared.yml @@ -8,6 +8,7 @@ volumes: oteldb-data: clickhouse-data: + clickhouse-oteldb-data: services: oteldb: @@ -37,6 +38,46 @@ services: - "127.0.0.1:9010:9010" # pprof - "127.0.0.1:19291:19291" # prometheus remote-write ingest + # oteldb, ClickHouse-backed variant: same image WITHOUT --embedded, serving all + # three signals from oteldb's chstorage on a dedicated ClickHouse (so its disk + # footprint is isolated from gigapipe's). Distinct host ports from the embedded + # oteldb; internal ports are identical (separate container). + oteldb-ch: + image: ${OTELDB_IMAGE:-ghcr.io/oteldb/oteldb:v0.41.0} + command: ["--config=/etc/otel/cfg.yml"] + networks: [bench] + depends_on: + clickhouse-oteldb: { condition: service_healthy } + volumes: + - ../config/oteldb/oteldb-ch.yml:/etc/otel/cfg.yml:ro + environment: + - OTEL_LOG_LEVEL=info + - OTEL_METRICS_EXPORTER=none + - OTEL_LOGS_EXPORTER=none + - OTEL_TRACES_EXPORTER=none + - PPROF_ADDR=0.0.0.0:9010 + - GOMAXPROCS=${BENCH_CPUS:-} + - GOMEMLIMIT=${BENCH_MEMLIMIT:-} + ports: + - "127.0.0.1:9092:9090" # promql + - "127.0.0.1:3102:3100" # logql + - "127.0.0.1:3202:3200" # traceql + - "127.0.0.1:9011:9010" # pprof + + # Dedicated ClickHouse for the oteldb-ch variant (separate from gigapipe's). + clickhouse-oteldb: + image: clickhouse/clickhouse-server:25.9 + networks: [bench] + environment: + - CLICKHOUSE_PASSWORD=default + volumes: + - clickhouse-oteldb-data:/var/lib/clickhouse + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "127.0.0.1:8123/ping"] + interval: 2s + timeout: 2s + retries: 30 + # gigapipe/qryn — Loki+Prometheus+Tempo APIs over ClickHouse. clickhouse: image: clickhouse/clickhouse-server:25.9 diff --git a/config/otelcol/logs.yml b/config/otelcol/logs.yml index f8d89b6..36d2fae 100644 --- a/config/otelcol/logs.yml +++ b/config/otelcol/logs.yml @@ -14,6 +14,9 @@ exporters: otlp/oteldb: endpoint: oteldb:4317 tls: { insecure: true } + otlp/oteldb-ch: + endpoint: oteldb-ch:4317 + tls: { insecure: true } otlphttp/victorialogs: logs_endpoint: http://victorialogs:9428/insert/opentelemetry/v1/logs tls: { insecure: true } @@ -31,6 +34,7 @@ service: processors: [batch] exporters: - otlp/oteldb + - otlp/oteldb-ch - otlphttp/victorialogs - loki/loki - loki/gigapipe diff --git a/config/otelcol/traces.yml b/config/otelcol/traces.yml index 957640d..22c2133 100644 --- a/config/otelcol/traces.yml +++ b/config/otelcol/traces.yml @@ -12,6 +12,9 @@ exporters: otlp/oteldb: endpoint: oteldb:4317 tls: { insecure: true } + otlp/oteldb-ch: + endpoint: oteldb-ch:4317 + tls: { insecure: true } otlp/tempo: endpoint: tempo:4317 tls: { insecure: true } @@ -29,6 +32,7 @@ service: processors: [batch] exporters: - otlp/oteldb + - otlp/oteldb-ch - otlp/tempo - otlphttp/victoriatraces - otlp/gigapipe diff --git a/config/oteldb/oteldb-ch.yml b/config/oteldb/oteldb-ch.yml new file mode 100644 index 0000000..2482e81 --- /dev/null +++ b/config/oteldb/oteldb-ch.yml @@ -0,0 +1,45 @@ +# oteldb under test, ClickHouse-backed variant. Same binary and same three query +# APIs as the embedded variant (config/oteldb/oteldb.yml), but WITHOUT --embedded: +# every signal is served from oteldb's original ClickHouse storage (chstorage) +# instead of the go-faster/storage engine. This benches oteldb's columnar layout +# on ClickHouse against the embedded engine on the same dataset/hardware/ingest. +# +# Points at a DEDICATED clickhouse-oteldb server (not gigapipe's) so the on-disk +# footprint is attributable to oteldb alone. `dsn` is the query side; the otelcol +# oteldbexporter dsn is the write side — both must name the same ClickHouse. +dsn: clickhouse://default:default@clickhouse-oteldb:9000 + +prometheus: # PromQL HTTP API (Prometheus-compatible) + bind: 0.0.0.0:9090 + max_samples: 1_000_000 + timeout: 1m + enable_negative_offset: true +loki: # LogQL HTTP API (Loki-compatible) + bind: 0.0.0.0:3100 +tempo: # TraceQL HTTP API (Tempo-compatible) + bind: 0.0.0.0:3200 +health_check: + bind: 0.0.0.0:13133 + +# Ingest receivers: Prometheus remote-write for metrics, OTLP for logs + traces +# (fanned out by config/otelcol/{logs,traces}.yml). The oteldbexporter writes to +# ClickHouse via the dsn below. +otelcol: + receivers: + prometheusremotewrite: + endpoint: 0.0.0.0:19291 + time_threshold: 100000h + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + exporters: + oteldbexporter: + dsn: clickhouse://default:default@clickhouse-oteldb:9000 + service: + pipelines: + metrics: { receivers: [prometheusremotewrite], exporters: [oteldbexporter] } + logs: { receivers: [otlp], exporters: [oteldbexporter] } + traces: { receivers: [otlp], exporters: [oteldbexporter] } diff --git a/internal/bench/collect.go b/internal/bench/collect.go index 84cbcc8..ee31648 100644 --- a/internal/bench/collect.go +++ b/internal/bench/collect.go @@ -26,7 +26,7 @@ func (e *Env) Collect() error { seen := map[string]bool{} for _, signal := range []string{"metrics", "logs", "traces"} { - for _, sys := range systems.For(signal) { + for _, sys := range e.selected(systems.For(signal)) { if seen[sys.Name] { continue } @@ -39,7 +39,11 @@ func (e *Env) Collect() error { case "local": disk = volBytes(sys.Name + "-data") case "clickhouse": - disk = volBytes("clickhouse-data") + vol := sys.Volume + if vol == "" { + vol = "clickhouse-data" + } + disk = volBytes(vol) } mem := memMB(sys.Name) rows = append(rows, []string{sys.Name, sys.Storage, strconv.FormatInt(disk, 10), strconv.FormatInt(mem, 10)}) diff --git a/internal/bench/compose.go b/internal/bench/compose.go index e5c847f..65518d4 100644 --- a/internal/bench/compose.go +++ b/internal/bench/compose.go @@ -46,11 +46,36 @@ func (e *Env) compose(lane string, args ...string) error { return cmd.Run() } -// Up builds and starts a lane (or "all"). +// laneDrivers are the ingest-side services every engine in a lane shares (the +// fan-out collector / scraper). They feed the same data to all backends, so the +// Only mode keeps just these plus the selected engines. +var laneDrivers = map[string][]string{ + "metrics": {"node-exporter", "vmagent"}, + "logs": {"otelcol-logs"}, + "traces": {"otelcol-traces"}, +} + +// engineDeps are non-engine services a given engine needs beyond the lane +// driver. oteldb's file backend is self-contained, so it has none. +var engineDeps = map[string][]string{ + "oteldb-ch": {"clickhouse-oteldb"}, + "gigapipe": {"clickhouse"}, + "mimir": {"fs", "fs-init"}, + "greptimedb": {"fs", "fs-init"}, + "loki": {"fs", "fs-init"}, + "tempo": {"fs", "fs-init"}, +} + +// Up builds and starts a lane (or "all"). With Env.Only set it brings up just +// the selected engines plus the lane's ingest driver — a much faster loop than +// the full matrix (the driver still fans data out to whatever is up). func (e *Env) Up(lane string) error { if profilesFor(lane) == nil { return fmt.Errorf("usage: up ") } + if len(e.Only) > 0 { + return e.upOnly(lane) + } fmt.Printf(">> building + starting lane=%s\n", lane) if err := e.compose(lane, "up", "-d", "--build", "--remove-orphans"); err != nil { return err @@ -59,6 +84,55 @@ func (e *Env) Up(lane string) error { return e.compose(lane, "ps") } +// upOnly starts only the selected engines and the lane's ingest driver, naming +// services explicitly so profile-gated services start without pulling in the +// other engines. +func (e *Env) upOnly(lane string) error { + lanes := []string{"metrics", "logs", "traces"} + if lane != "all" { + lanes = []string{lane} + } + set := map[string]bool{} + var services []string + add := func(name string) { + if !set[name] { + set[name] = true + services = append(services, name) + } + } + for name := range e.Only { + add(name) + for _, dep := range engineDeps[name] { + add(dep) + } + } + for _, l := range lanes { + for _, d := range laneDrivers[l] { + add(d) + } + } + + fmt.Printf(">> building + starting only=%v lane=%s services=%v\n", keys(e.Only), lane, services) + // No profiles: explicitly named services start regardless of profile gating. + args := append(e.composeFiles(), "up", "-d", "--build", "--remove-orphans") + args = append(args, services...) + cmd := exec.Command("docker", append([]string{"compose"}, args...)...) + cmd.Dir = e.Dir + cmd.Stdout, cmd.Stderr, cmd.Stdin = os.Stdout, os.Stderr, os.Stdin + if err := cmd.Run(); err != nil { + return err + } + return e.compose(lane, "ps") +} + +func keys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + // Down stops everything and wipes volumes. func (e *Env) Down() error { return e.compose("all", "down", "-v", "--remove-orphans") diff --git a/internal/bench/config.go b/internal/bench/config.go index b588966..b73512f 100644 --- a/internal/bench/config.go +++ b/internal/bench/config.go @@ -16,10 +16,11 @@ import ( // Env holds resolved paths and .env-derived settings shared by every command. type Env struct { - Dir string // benchmark root (holds systems.yml, compose/, config/, ...) - OteldbSrc string // sibling oteldb checkout used to run otelbench during ingest - Runs int // benchmark runs per query (after warmup) - LoghubDir string // optional dir of loghub .log files for the LogQL suite + Dir string // benchmark root (holds systems.yml, compose/, config/, ...) + OteldbSrc string // sibling oteldb checkout used to run otelbench during ingest + Runs int // benchmark runs per query (after warmup) + LoghubDir string // optional dir of loghub .log files for the LogQL suite + Only map[string]bool // if non-empty, restrict the run to these systems (e.g. {"oteldb"}) } // LoadEnv finds the benchmark root (the dir containing systems.yml, walking up @@ -52,9 +53,38 @@ func LoadEnv() (*Env, error) { fmt.Sscanf(v, "%d", &e.Runs) } e.LoghubDir = get("LOGHUB_DIR", "") + e.Only = ParseOnly(get("BENCH_ONLY", "")) return e, nil } +// ParseOnly turns a comma-separated system list into a set (empty == all +// systems). Used by the --only flag and the BENCH_ONLY env var to restrict a +// run to a subset of engines (e.g. just oteldb) for a faster feedback loop. +func ParseOnly(csv string) map[string]bool { + set := map[string]bool{} + for _, name := range strings.Split(csv, ",") { + if name = strings.TrimSpace(name); name != "" { + set[name] = true + } + } + return set +} + +// selected filters a signal's systems to the Only set, preserving order. With no +// Only set it returns the list unchanged. +func (e *Env) selected(list []System) []System { + if len(e.Only) == 0 { + return list + } + out := list[:0:0] + for _, s := range list { + if e.Only[s.Name] { + out = append(out, s) + } + } + return out +} + func findRoot() (string, error) { d, err := os.Getwd() if err != nil { @@ -101,6 +131,11 @@ type System struct { Lang string `yaml:"lang"` Apples bool `yaml:"apples"` Storage string `yaml:"storage"` + // Volume overrides the named docker volume measured for the on-disk + // footprint. Defaults to "-data"; set it when a system stores data in a + // differently-named volume (e.g. the clickhouse-backed oteldb-ch variant uses + // its dedicated clickhouse-oteldb-data volume, not gigapipe's clickhouse-data). + Volume string `yaml:"volume"` } // Systems is the parsed matrix, preserving file order within each signal so the diff --git a/internal/bench/query.go b/internal/bench/query.go index 68f2c25..2b488de 100644 --- a/internal/bench/query.go +++ b/internal/bench/query.go @@ -32,7 +32,7 @@ func (e *Env) Query(signal string, count, warmup int) error { } client := &http.Client{Timeout: 120 * time.Second} - for _, sys := range systems.For(signal) { + for _, sys := range e.selected(systems.For(signal)) { native, err := LoadNative(e.Dir, sys.Name) if err != nil { return err diff --git a/internal/bench/report.go b/internal/bench/report.go index 04d7ecc..1804db2 100644 --- a/internal/bench/report.go +++ b/internal/bench/report.go @@ -36,7 +36,7 @@ func (e *Env) Report() error { // Engines with a results file, in matrix order. var cols []System p90 := map[string]map[string]string{} // system -> id -> p90 - for _, sys := range systems.For(signal) { + for _, sys := range e.selected(systems.For(signal)) { data, ok := readP90(filepath.Join(dir, sys.Name+".csv")) if !ok { continue diff --git a/systems.yml b/systems.yml index 7d2729b..ae0cb26 100644 --- a/systems.yml +++ b/systems.yml @@ -17,6 +17,7 @@ metrics: # reference language: PromQL (Prometheus HTTP API /api/v1) oteldb: { addr: http://127.0.0.1:9090, lang: promql, apples: true, storage: local } + oteldb-ch: { addr: http://127.0.0.1:9092, lang: promql, apples: true, storage: clickhouse, volume: clickhouse-oteldb-data } prometheus: { addr: http://127.0.0.1:9091, lang: promql, apples: true, storage: local } victoriametrics: { addr: http://127.0.0.1:8428/prometheus, lang: promql, apples: true, storage: local } mimir: { addr: http://127.0.0.1:9009/prometheus, lang: promql, apples: true, storage: fs } @@ -25,12 +26,14 @@ metrics: # reference language: PromQL (Prometheus HTTP A logs: # reference language: LogQL (Loki HTTP API /loki/api/v1) oteldb: { addr: http://127.0.0.1:3100, lang: logql, apples: true, storage: local } + oteldb-ch: { addr: http://127.0.0.1:3102, lang: logql, apples: true, storage: clickhouse, volume: clickhouse-oteldb-data } loki: { addr: http://127.0.0.1:3101, lang: logql, apples: true, storage: fs } victorialogs: { addr: http://127.0.0.1:9428, lang: logsql, apples: false, storage: local } # native LogsQL gigapipe: { addr: http://127.0.0.1:3110, lang: logql, apples: true, storage: clickhouse } traces: # reference language: TraceQL (Tempo HTTP API /api/search) oteldb: { addr: http://127.0.0.1:3200, lang: traceql, apples: true, storage: local } + oteldb-ch: { addr: http://127.0.0.1:3202, lang: traceql, apples: true, storage: clickhouse, volume: clickhouse-oteldb-data } tempo: { addr: http://127.0.0.1:3201, lang: traceql, apples: true, storage: fs } victoriatraces: { addr: http://127.0.0.1:10428, lang: native, apples: false, storage: local } # Jaeger query API gigapipe: { addr: http://127.0.0.1:3110, lang: traceql, apples: true, storage: clickhouse }