add clickhouse-backed oteldb variant + --only fast-loop mode

.gitignore

@@ -4,6 +4,7 @@
 .idea/
 results/*/
 results/*.csv
+results/*.log
 results/REPORT.md
 !results/.gitkeep
 !results/pprof/

README.md

@@ -7,9 +7,15 @@ languages:
 
 | Signal  | Reference language | Systems under test |
 |---------|--------------------|--------------------|
-| Metrics | **PromQL**  | oteldb · Prometheus · VictoriaMetrics · Mimir · GreptimeDB · gigapipe |
-| Logs    | **LogQL**   | oteldb · Loki · VictoriaLogs\* · gigapipe |
-| Traces  | **TraceQL** | oteldb · Tempo · VictoriaTraces\* · gigapipe |
+| Metrics | **PromQL**  | oteldb · oteldb-ch · Prometheus · VictoriaMetrics · Mimir · GreptimeDB · gigapipe |
+| Logs    | **LogQL**   | oteldb · oteldb-ch · Loki · VictoriaLogs\* · gigapipe |
+| Traces  | **TraceQL** | oteldb · oteldb-ch · Tempo · VictoriaTraces\* · gigapipe |
+
+**oteldb** is benched in two configurations from the *same binary*: `oteldb` is
+the embedded go-faster/storage engine (`--embedded`, `file` backend), and
+`oteldb-ch` serves the same three query APIs from oteldb's original ClickHouse
+storage (chstorage) on a dedicated `clickhouse-oteldb` server. Identical query
+semantics and ingest path — the comparison isolates the storage layer.
 
 \* VictoriaLogs and VictoriaTraces do not implement LogQL/TraceQL; they are
 benched on their native dialect (LogsQL / Jaeger query) over the *same dataset
@@ -109,6 +115,16 @@ go build -o benchctl ./cmd/benchctl && ./benchctl bench-all
 ./benchctl report
 ```
 
+**Faster feedback — only some engines.** `--only <csv>` (or `BENCH_ONLY` in
+`.env`) restricts a run to a subset of systems: it brings up just those engines
+plus the lane's ingest driver (skipping the competitors entirely) and
+queries/collects/reports only them.
+
+```bash
+./benchctl --only oteldb bench-all            # just the embedded engine
+./benchctl --only oteldb,oteldb-ch bench-all  # embedded vs ClickHouse, head-to-head
+```
+
 Live dashboards while it runs: **Grafana** http://localhost:3000 (every engine
 pre-provisioned as a datasource), **cAdvisor** http://localhost:8085.
 

cmd/benchctl/main.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/oteldb/benchmark/internal/bench"
@@ -31,6 +32,7 @@ func main() {
 }
 
 func run(args []string) error {
+	args, only := stripOnly(args)
 	if len(args) == 0 {
 		usage()
 		return nil
@@ -39,6 +41,9 @@ func run(args []string) error {
 	if err != nil {
 		return err
 	}
+	if only != "" {
+		e.Only = bench.ParseOnly(only)
+	}
 	cmd, rest := args[0], args[1:]
 	switch cmd {
 	case "up":
@@ -101,9 +106,36 @@ func usage() {
   bench   <metrics|logs|traces> [runs] up→ingest→settle→query→collect
   bench-all [runs]                     all lanes, then report
   down                                 stop + wipe volumes
+
+global flags:
+  --only <csv>   restrict the run to these engines (e.g. --only oteldb) for a
+                 faster feedback loop: brings up only those engines + the lane's
+                 ingest driver, and queries/collects/reports just them. Also set
+                 via BENCH_ONLY in .env.
 `)
 }
 
+// stripOnly extracts a global `--only <csv>` / `--only=<csv>` flag from anywhere
+// in args (last value wins) and returns the remaining args plus the value. It
+// restricts the run to a subset of engines — e.g. `--only oteldb` for a fast
+// oteldb-only feedback loop. Empty when the flag is absent.
+func stripOnly(args []string) (rest []string, only string) {
+	for i := 0; i < len(args); i++ {
+		switch a := args[i]; {
+		case a == "--only":
+			if i+1 < len(args) {
+				only = args[i+1]
+				i++
+			}
+		case strings.HasPrefix(a, "--only="):
+			only = strings.TrimPrefix(a, "--only=")
+		default:
+			rest = append(rest, a)
+		}
+	}
+	return rest, only
+}
+
 func arg(a []string, i int, def string) string {
 	if i < len(a) {
 		return a[i]

compose/metrics.yml

@@ -27,6 +27,7 @@ services:
     command:
       - -promscrape.config=/etc/vmagent/scrape.yml
       - -remoteWrite.url=http://oteldb:19291/api/v1/write
+      - -remoteWrite.url=http://oteldb-ch:19291/api/v1/write
       - -remoteWrite.url=http://prometheus:9090/api/v1/write
       - -remoteWrite.url=http://victoriametrics:8428/api/v1/write
       - -remoteWrite.url=http://mimir:9009/api/v1/push

compose/shared.yml

@@ -8,6 +8,7 @@
 volumes:
   oteldb-data:
   clickhouse-data:
+  clickhouse-oteldb-data:
 
 services:
   oteldb:
@@ -37,6 +38,46 @@ services:
       - "127.0.0.1:9010:9010"   # pprof
       - "127.0.0.1:19291:19291" # prometheus remote-write ingest
 
+  # oteldb, ClickHouse-backed variant: same image WITHOUT --embedded, serving all
+  # three signals from oteldb's chstorage on a dedicated ClickHouse (so its disk
+  # footprint is isolated from gigapipe's). Distinct host ports from the embedded
+  # oteldb; internal ports are identical (separate container).
+  oteldb-ch:
+    image: ${OTELDB_IMAGE:-ghcr.io/oteldb/oteldb:v0.41.0}
+    command: ["--config=/etc/otel/cfg.yml"]
+    networks: [bench]
+    depends_on:
+      clickhouse-oteldb: { condition: service_healthy }
+    volumes:
+      - ../config/oteldb/oteldb-ch.yml:/etc/otel/cfg.yml:ro
+    environment:
+      - OTEL_LOG_LEVEL=info
+      - OTEL_METRICS_EXPORTER=none
+      - OTEL_LOGS_EXPORTER=none
+      - OTEL_TRACES_EXPORTER=none
+      - PPROF_ADDR=0.0.0.0:9010
+      - GOMAXPROCS=${BENCH_CPUS:-}
+      - GOMEMLIMIT=${BENCH_MEMLIMIT:-}
+    ports:
+      - "127.0.0.1:9092:9090"   # promql
+      - "127.0.0.1:3102:3100"   # logql
+      - "127.0.0.1:3202:3200"   # traceql
+      - "127.0.0.1:9011:9010"   # pprof
+
+  # Dedicated ClickHouse for the oteldb-ch variant (separate from gigapipe's).
+  clickhouse-oteldb:
+    image: clickhouse/clickhouse-server:25.9
+    networks: [bench]
+    environment:
+      - CLICKHOUSE_PASSWORD=default
+    volumes:
+      - clickhouse-oteldb-data:/var/lib/clickhouse
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "127.0.0.1:8123/ping"]
+      interval: 2s
+      timeout: 2s
+      retries: 30
+
   # gigapipe/qryn — Loki+Prometheus+Tempo APIs over ClickHouse.
   clickhouse:
     image: clickhouse/clickhouse-server:25.9

config/otelcol/logs.yml

@@ -14,6 +14,9 @@ exporters:
   otlp/oteldb:
     endpoint: oteldb:4317
     tls: { insecure: true }
+  otlp/oteldb-ch:
+    endpoint: oteldb-ch:4317
+    tls: { insecure: true }
   otlphttp/victorialogs:
     logs_endpoint: http://victorialogs:9428/insert/opentelemetry/v1/logs
     tls: { insecure: true }
@@ -31,6 +34,7 @@ service:
       processors: [batch]
       exporters:
         - otlp/oteldb
+        - otlp/oteldb-ch
         - otlphttp/victorialogs
         - loki/loki
         - loki/gigapipe

config/otelcol/traces.yml

@@ -12,6 +12,9 @@ exporters:
   otlp/oteldb:
     endpoint: oteldb:4317
     tls: { insecure: true }
+  otlp/oteldb-ch:
+    endpoint: oteldb-ch:4317
+    tls: { insecure: true }
   otlp/tempo:
     endpoint: tempo:4317
     tls: { insecure: true }
@@ -29,6 +32,7 @@ service:
       processors: [batch]
       exporters:
         - otlp/oteldb
+        - otlp/oteldb-ch
         - otlp/tempo
         - otlphttp/victoriatraces
         - otlp/gigapipe

config/oteldb/oteldb-ch.yml

@@ -0,0 +1,45 @@
+# oteldb under test, ClickHouse-backed variant. Same binary and same three query
+# APIs as the embedded variant (config/oteldb/oteldb.yml), but WITHOUT --embedded:
+# every signal is served from oteldb's original ClickHouse storage (chstorage)
+# instead of the go-faster/storage engine. This benches oteldb's columnar layout
+# on ClickHouse against the embedded engine on the same dataset/hardware/ingest.
+#
+# Points at a DEDICATED clickhouse-oteldb server (not gigapipe's) so the on-disk
+# footprint is attributable to oteldb alone. `dsn` is the query side; the otelcol
+# oteldbexporter dsn is the write side — both must name the same ClickHouse.
+dsn: clickhouse://default:default@clickhouse-oteldb:9000
+
+prometheus:                     # PromQL HTTP API (Prometheus-compatible)
+  bind: 0.0.0.0:9090
+  max_samples: 1_000_000
+  timeout: 1m
+  enable_negative_offset: true
+loki:                           # LogQL HTTP API (Loki-compatible)
+  bind: 0.0.0.0:3100
+tempo:                          # TraceQL HTTP API (Tempo-compatible)
+  bind: 0.0.0.0:3200
+health_check:
+  bind: 0.0.0.0:13133
+
+# Ingest receivers: Prometheus remote-write for metrics, OTLP for logs + traces
+# (fanned out by config/otelcol/{logs,traces}.yml). The oteldbexporter writes to
+# ClickHouse via the dsn below.
+otelcol:
+  receivers:
+    prometheusremotewrite:
+      endpoint: 0.0.0.0:19291
+      time_threshold: 100000h
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+  exporters:
+    oteldbexporter:
+      dsn: clickhouse://default:default@clickhouse-oteldb:9000
+  service:
+    pipelines:
+      metrics: { receivers: [prometheusremotewrite], exporters: [oteldbexporter] }
+      logs:    { receivers: [otlp], exporters: [oteldbexporter] }
+      traces:  { receivers: [otlp], exporters: [oteldbexporter] }

internal/bench/collect.go

@@ -26,7 +26,7 @@ func (e *Env) Collect() error {
 
 	seen := map[string]bool{}
 	for _, signal := range []string{"metrics", "logs", "traces"} {
-		for _, sys := range systems.For(signal) {
+		for _, sys := range e.selected(systems.For(signal)) {
 			if seen[sys.Name] {
 				continue
 			}
@@ -39,7 +39,11 @@ func (e *Env) Collect() error {
 			case "local":
 				disk = volBytes(sys.Name + "-data")
 			case "clickhouse":
-				disk = volBytes("clickhouse-data")
+				vol := sys.Volume
+				if vol == "" {
+					vol = "clickhouse-data"
+				}
+				disk = volBytes(vol)
 			}
 			mem := memMB(sys.Name)
 			rows = append(rows, []string{sys.Name, sys.Storage, strconv.FormatInt(disk, 10), strconv.FormatInt(mem, 10)})

internal/bench/compose.go

@@ -46,11 +46,36 @@ func (e *Env) compose(lane string, args ...string) error {
 	return cmd.Run()
 }
 
-// Up builds and starts a lane (or "all").
+// laneDrivers are the ingest-side services every engine in a lane shares (the
+// fan-out collector / scraper). They feed the same data to all backends, so the
+// Only mode keeps just these plus the selected engines.
+var laneDrivers = map[string][]string{
+	"metrics": {"node-exporter", "vmagent"},
+	"logs":    {"otelcol-logs"},
+	"traces":  {"otelcol-traces"},
+}
+
+// engineDeps are non-engine services a given engine needs beyond the lane
+// driver. oteldb's file backend is self-contained, so it has none.
+var engineDeps = map[string][]string{
+	"oteldb-ch":  {"clickhouse-oteldb"},
+	"gigapipe":   {"clickhouse"},
+	"mimir":      {"fs", "fs-init"},
+	"greptimedb": {"fs", "fs-init"},
+	"loki":       {"fs", "fs-init"},
+	"tempo":      {"fs", "fs-init"},
+}
+
+// Up builds and starts a lane (or "all"). With Env.Only set it brings up just
+// the selected engines plus the lane's ingest driver — a much faster loop than
+// the full matrix (the driver still fans data out to whatever is up).
 func (e *Env) Up(lane string) error {
 	if profilesFor(lane) == nil {
 		return fmt.Errorf("usage: up <metrics|logs|traces|all>")
 	}
+	if len(e.Only) > 0 {
+		return e.upOnly(lane)
+	}
 	fmt.Printf(">> building + starting lane=%s\n", lane)
 	if err := e.compose(lane, "up", "-d", "--build", "--remove-orphans"); err != nil {
 		return err
@@ -59,6 +84,55 @@ func (e *Env) Up(lane string) error {
 	return e.compose(lane, "ps")
 }
 
+// upOnly starts only the selected engines and the lane's ingest driver, naming
+// services explicitly so profile-gated services start without pulling in the
+// other engines.
+func (e *Env) upOnly(lane string) error {
+	lanes := []string{"metrics", "logs", "traces"}
+	if lane != "all" {
+		lanes = []string{lane}
+	}
+	set := map[string]bool{}
+	var services []string
+	add := func(name string) {
+		if !set[name] {
+			set[name] = true
+			services = append(services, name)
+		}
+	}
+	for name := range e.Only {
+		add(name)
+		for _, dep := range engineDeps[name] {
+			add(dep)
+		}
+	}
+	for _, l := range lanes {
+		for _, d := range laneDrivers[l] {
+			add(d)
+		}
+	}
+
+	fmt.Printf(">> building + starting only=%v lane=%s services=%v\n", keys(e.Only), lane, services)
+	// No profiles: explicitly named services start regardless of profile gating.
+	args := append(e.composeFiles(), "up", "-d", "--build", "--remove-orphans")
+	args = append(args, services...)
+	cmd := exec.Command("docker", append([]string{"compose"}, args...)...)
+	cmd.Dir = e.Dir
+	cmd.Stdout, cmd.Stderr, cmd.Stdin = os.Stdout, os.Stderr, os.Stdin
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+	return e.compose(lane, "ps")
+}
+
+func keys(m map[string]bool) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	return out
+}
+
 // Down stops everything and wipes volumes.
 func (e *Env) Down() error {
 	return e.compose("all", "down", "-v", "--remove-orphans")