diff --git a/_data/production-test-metrics/autoscaling-container.csv b/_data/production-test-metrics/autoscaling-container.csv new file mode 100644 index 0000000000..af068f9446 --- /dev/null +++ b/_data/production-test-metrics/autoscaling-container.csv @@ -0,0 +1,5 @@ +family,unit,description,suffixes +active-nodes,count,Maximum number of active container nodes in the cluster during the test window.,-container +non-active-node-fraction,fraction,"Peak fraction of non-active (mid-transition) nodes in the container cluster during the test window, non-zero while a resize is in progress.",-container +query-growth-rate-vs-3h-average,ratio,"Ratio of the container cluster's query rate during the test window to its trailing 3-hour average, a proxy for the autoscaler's growth-headroom signal (values above 1.0 indicate rising traffic).", +query-vs-write-rate-cpu-mix-driver,operations/second,Average query rate across the container cluster during the test window; one side of the query/write mix that shapes the autoscaler's ideal CPU target.,"-query, -write" diff --git a/_data/production-test-metrics/autoscaling-content.csv b/_data/production-test-metrics/autoscaling-content.csv new file mode 100644 index 0000000000..085ae35e63 --- /dev/null +++ b/_data/production-test-metrics/autoscaling-content.csv @@ -0,0 +1,9 @@ +family,unit,description,suffixes +active-nodes-count,count,Maximum number of active nodes in the content cluster during the test window., +active-nodes-non-active-fraction,fraction (0-1),"Fraction of content-cluster nodes that are not yet active, indicating an in-progress resize or data redistribution.", +autoscaling-actions,count,Number of autoscaler rescale actions taken for the content cluster during the test window.,-sum +cpu-load-adjustment,"ratio (peak / ideal, 1.0 = ideally scaled)",Ratio of peak to ideal CPU load for the content cluster; above 1.0 signals scale-up pressure and below 1.0 signals the cluster is over-provisioned for CPU.,-ratio +cpu-load-peak-vs-ideal,percentage (0-100),"The autoscaler's ideal CPU load target for the content cluster, as a percentage (already includes growth and failover headroom, so normally below 100).","-ideal, -peak" +query-growth-rate-vs-3h-average-content,ratio (test-window average / trailing 3h average),"Ratio of the content cluster's query rate during the test window to its own trailing 3-hour average, a proxy for the autoscaler's internal query-growth headroom factor.", +query-vs-write-rate-cpu-mix-driver-query-content,operations/second,"Query rate for the content cluster, one half of the query/write mix that determines the autoscaler's ideal CPU target.", +query-vs-write-rate-cpu-mix-driver-write-content,operations/second,"Write rate (put + update + remove operations per second) for the content cluster, the other half of the query/write mix that determines the autoscaler's ideal CPU target.", diff --git a/_data/production-test-metrics/content-node.csv b/_data/production-test-metrics/content-node.csv new file mode 100644 index 0000000000..d038ad0b19 --- /dev/null +++ b/_data/production-test-metrics/content-node.csv @@ -0,0 +1,16 @@ +family,unit,description,suffixes +attribute-feeding-blocked,percentage (0-100),Percentage of the time window during which feeding was blocked due to attribute resource usage limits on any content node.,-pct +bucket-move-pending,count,Maximum number of buckets pending a move operation (e.g. during cluster resizing or redistribution) on any content node.,-max +document-store-cache-hit-rate,percentage (0-100),"Cache hit rate for the ready document store's summary-field cache, as a percentage; low values on quiescent clusters often show up as elevated CPU IOWait from disk-backed docsum reads.",-max +document-store-disk-usage,bytes,Maximum disk space used by the ready document store on any content node.,-max +documents-count,count,"Number of active/searchable document copies, summed across content clusters after taking the max across replica groups within each cluster to avoid double counting.","-active, -ready, -removed, -total" +field-writer-saturation,percentage (0-100),Maximum observed saturation of Proton's field writer executor thread pool; high saturation indicates a write bottleneck.,-max +field-writer-utilization,percentage (0-100),Average utilization of Proton's field writer executor thread pool across content nodes.,-avg +filestor-average-queue-wait,milliseconds,Average time operations spent waiting in the filestor queue across content nodes.,-avg +filestor-queue-size,count,Maximum filestor queue size observed on any content node; a growing queue indicates storage cannot keep up with incoming operations.,-max +maintenance-job-activity,percentage (0-100),"Activity level of the attribute-flush maintenance job (0% = idle, 100% = fully active), maximum across content nodes.","-attr-flush, -disk-idx-fusion, -docstore-compact, -lid-compact, -mem-idx-flush" +proton-disk-usage,percentage (0-100),"Relative disk usage of the content node's Proton process, as a percentage of the available disk; 100% means feeding is blocked.",-avg +proton-executor-utilization,percentage (0-100),Average utilization of Proton's own executor thread pool across content nodes.,"-avg, -max" +proton-memory-usage,percentage (0-100),"Relative memory usage of the content node's Proton process, as a percentage of the available memory.",-avg +shared-executor-queue-size,count,Maximum shared executor queue size observed on any content node during the time window.,-max +shared-executor-utilization,percentage (0-100),Average utilization of Proton's shared executor thread pool across content nodes.,"-avg, -max" diff --git a/_data/production-test-metrics/feed.csv b/_data/production-test-metrics/feed.csv new file mode 100644 index 0000000000..714c8691df --- /dev/null +++ b/_data/production-test-metrics/feed.csv @@ -0,0 +1,35 @@ +family,unit,description,suffixes +container-feed-latency,milliseconds,"Maximum feed request latency observed at the container, across all operation types and clusters.",-max +container-feed-operations,operations/second,"Total feed operation rate (puts, updates, removes) handled by container clusters, summed across operation types.",-sum +content-commit-latency,milliseconds,Average commit latency (time to commit fed changes to the index/store) on content nodes.,-avg +content-commit-operations,operations/second,Rate of commit operations (batched writes applied to the index/store) on content nodes.,-sum +content-storage-feed-rates,operations/second,"Rate of PUT operations processed by the storage/persistence-engine layer on content nodes, counted per document replica.","-put, -remove, -update" +content-storage-put-latency,milliseconds,Maximum PUT operation latency observed at the storage/persistence-engine layer on content nodes.,-max +content-storage-update-latency,milliseconds,Average UPDATE operation latency at the storage/persistence-engine layer on content nodes.,"-avg, -max" +distributor-latency-gets,milliseconds,Maximum get operation latency observed at the distributor level.,-max +distributor-latency-puts,milliseconds,Maximum put operation latency observed at the distributor level.,-max +distributor-latency-removes,milliseconds,Maximum remove operation latency observed at the distributor level.,-max +distributor-latency-updates,milliseconds,Maximum update operation latency observed at the distributor level.,-max +distributor-latency-visitor,milliseconds,Maximum visitor operation latency observed at the distributor level.,-max +distributor-operation-gets-failures,operations/second,Rate of failed get operations at the distributor level; non-zero values indicate issues with writes.,-sum +distributor-operation-puts-failures,operations/second,Rate of failed put operations at the distributor level; non-zero values indicate issues with writes.,-sum +distributor-operation-rates-gets-ok,operations/second,Rate of successfully completed (OK) get operations at the distributor level.,-sum +distributor-operation-rates-puts-ok,operations/second,Rate of successfully completed (OK) put operations at the distributor level.,-sum +distributor-operation-rates-removes-ok,operations/second,Rate of successfully completed (OK) remove operations at the distributor level.,-sum +distributor-operation-rates-updates-ok,operations/second,Rate of successfully completed (OK) update operations at the distributor level.,-sum +distributor-operation-rates-visitor-ok,operations/second,Rate of successfully completed (OK) visitor operations at the distributor level.,-sum +distributor-operation-removes-failures,operations/second,Rate of failed remove operations at the distributor level; non-zero values indicate issues with writes.,-sum +distributor-operation-updates-failures,operations/second,Rate of failed update operations at the distributor level; non-zero values indicate issues with writes.,-sum +distributor-operation-visitor-failures,operations/second,Rate of failed visitor operations at the distributor level; non-zero values indicate issues with writes.,-sum +document-processing-latency,milliseconds,Average document processing (docproc chain) latency.,-avg +document-processing-rate,operations/second,"Rate of documents processed through document processing (docproc) chains, summed across all chains and clusters.",-sum +feed-blocked-nodes-above-resource-limit,count,"Number of content nodes currently exceeding a resource usage limit; when greater than zero, feeding is blocked for that cluster.", +feed-latency-avg-ms,milliseconds,"Average feed latency, in milliseconds, for successful feed operations.", +http-api-feed-rates,operations/second,Rate of PUT operations received via the Document/HTTP API.,"-put, -remove, -update" +http-api-latency,milliseconds,"Average latency of Document/HTTP API requests, distinct from the generic jdisc HTTP layer latency.",-avg +http-api-pending-requests,count,"Maximum number of Document/HTTP API requests pending (in-flight, not yet completed) on the container.",-max +http-api-success-vs-failures,operations/second,Rate of failed Document/HTTP API operations.,"-failed, -parse-error, -success" +http-write-latency,milliseconds,Average end-to-end HTTP write request latency at the container's HTTP layer.,"-avg, -p95, -p99" +memory-index-docs-in-memory,count,"Maximum number of documents held in the in-memory index on content nodes, summed per content group (to combine per-node/stripe values) and maxed across groups (since each group holds a full replica) to avoid double-counting.",-max +persistence-engine-input-queue,count,Average depth of the per-node persistence engine's input operation queue (operations waiting to be dispatched to Proton) over the time window.,"-avg, -max" +persistence-engine-throttle-saturation,count,Maximum number of concurrently active (in-flight) operations in the content node's persistence engine.,"-active, -throttle" diff --git a/_data/production-test-metrics/health.csv b/_data/production-test-metrics/health.csv new file mode 100644 index 0000000000..e7bff7aaff --- /dev/null +++ b/_data/production-test-metrics/health.csv @@ -0,0 +1,19 @@ +family,unit,description,suffixes +cluster-state-changes,count,Number of cluster state changes observed in the given time window. Frequent changes are most often caused by Vespa upgrades or required node restarts.,-sum +core-dumps-processed,count,Number of core dumps processed across all nodes in the given time window; should be 0.,-max +deactivated-containers,count,"Average number of deactivated container instances per sample, over the given time window. Brief spikes during deployments are normal; persistently high values may indicate resource leaks.", +estimated-time-to-in-sync,seconds,"Rough estimate, from the recent (fixed 30-minute) convergence rate, of the time remaining until all buckets are back in sync; capped at 7 days.", +failed-component-graphs,operations/second,"Rate of failed application component-graph reconfigurations, averaged over the given time window; nonzero after a deployment may indicate a configuration or component issue.", +http-requests-prematurely-closed,operations/second,"Rate of HTTP requests closed by the client before the server finished responding, averaged over the given time window.", +merge-bucket-pending,count,"Number of pending bucket merge operations across all distributors, summed per application.", +node-count,count,Total number of nodes currently in the active state for the application.,-max +node-events,count,Number of node events reported by the cluster controller in the given time window.,-sum +node-state-down,count,Number of content nodes in the 'down' state per cluster (nodes considered unavailable by the cluster controller).,-max +node-state-initializing,count,Number of content nodes in the 'initializing' state per cluster (nodes currently starting up and not yet serving).,-max +node-state-maintenance,count,Number of content nodes in the 'maintenance' state per cluster (nodes intentionally taken down for maintenance).,-max +node-state-retired,count,Number of content nodes in the 'retired' state per cluster (nodes being decommissioned).,-max +node-state-up,count,Number of content nodes in the 'up' state per cluster.,-max +re-indexing-remaining,percentage (0-100),"Worst-case document reindexing progress remaining across document types, as a percentage (100 = not started, 0 = complete or no reindex running).", +resource-usage-attribute-address-space,percentage (0-100),Worst-case attribute address-space utilization across content nodes; feed is blocked when this approaches 100%., +resource-usage-max-disk-utilization-vs-limit,percentage (0-100),Maximum disk utilization vs the configured limit; feed is blocked when utilization exceeds the limit., +service-restarts,count,"Total service restarts across all nodes, as tracked by the config sentinel. Increases indicate services crashing or being restarted.",-total diff --git a/_data/production-test-metrics/nearest-neighbor-search.csv b/_data/production-test-metrics/nearest-neighbor-search.csv new file mode 100644 index 0000000000..a6d784715c --- /dev/null +++ b/_data/production-test-metrics/nearest-neighbor-search.csv @@ -0,0 +1,21 @@ +family,unit,description,suffixes +ann-timeout-rate,requests/second,Rate of queries per second that terminated an approximate NNS search early after hitting the ANN timeout.,-sum +approximate-nns-distances-computed,operations/second,Rate of distance computations performed during approximate NNS across the content cluster.,-rate +approximate-nns-nodes-visited,operations/second,Rate of HNSW graph nodes visited during approximate NNS across the content cluster.,-rate +approximate-nns-query-rate,requests/second,Rate of incoming queries per second that perform an approximate nearest-neighbor search., +approximate-nns-time,milliseconds,"Average time spent performing approximate NNS, measured only across queries that actually perform it.","-avg, -max" +approximate-nns-visit-efficiency,ratio (dimensionless),Ratio of HNSW graph nodes visited to distances computed during approximate NNS; values above 1.0 indicate the filter-first (Acorn-1) heuristic is active., +buckets-pending-merge,count,"Number of buckets currently pending merge across distributors, which rises during data redistribution or node recovery.",-sum +documents-active-total,count,Total number of active documents (primary copies) across all document types and content nodes in the cluster., +documents-ready-total,count,Total number of ready (searchable) documents across all document types and content nodes in the cluster., +exact-nns-distance-ratio,percentage (0-100),Share of NNS distance computations done via exact (brute-force) search rather than approximate search; high values may indicate the approximate-threshold is set too high.,-pct +exact-nns-distances-computed,operations/second,Rate of distance computations performed during exact (brute-force) NNS across the content cluster.,-rate +nns-query-latency,milliseconds,"Average total time (matching plus ranking) spent per query on the content nodes, specific to the NNS/matching pipeline.","-avg, -max" +non-approximate-nns-query-rate,requests/second,"Rate of incoming queries per second that do not perform approximate NNS, including queries that fall back to exact search.", +query-matching-time,milliseconds,"Average time spent on the core query matching process, computed as query latency minus query setup time; includes exact NNS.","-avg, -max" +query-rate,requests/second,Rate of incoming queries per second against the content cluster's document matching pipeline (equal to approximate-nns-query-rate plus non-approximate-nns-query-rate)., +query-setup-time,milliseconds,"Average time spent setting up and tearing down queries, including filter computation and approximate NNS overhead.","-avg, -max" +query-setup-time-excl-ann,milliseconds,"Average time spent on query setup and teardown excluding approximate NNS time, isolating filter computation and other overhead.",-avg +soft-timeout-rate,requests/second,"Rate of queries per second terminated early due to hitting the soft timeout, aggregated across rank profiles.",-sum +thread-pool-match-utilization,percentage (0-100),Average utilization of the match thread pool across content nodes.,"-avg, -max" +total-distances-computed,operations/second,"Total NNS distance computations per second, combining exact and approximate search, as an overall indicator of NNS-related CPU cost.",-sum diff --git a/_data/production-test-metrics/overview.csv b/_data/production-test-metrics/overview.csv new file mode 100644 index 0000000000..3ff6098a3e --- /dev/null +++ b/_data/production-test-metrics/overview.csv @@ -0,0 +1,26 @@ +family,unit,description,suffixes +buckets-out-of-sync,percentage (0-100),"Percentage of buckets currently out of sync, fleet-wide.", +container-nodes-down,percentage (0-100),"Percentage of active container nodes with at least one service down, fleet-wide.", +container-thread-saturation-document-api-only,percentage (0-100),"Worst-case container thread-pool utilization, fleet-wide, restricted to deployments running document-api only (no search).", +container-thread-saturation-search-document-api,percentage (0-100),"Worst-case container thread-pool utilization, fleet-wide, restricted to deployments running both search and document-api.", +container-thread-saturation-search-only,percentage (0-100),"Worst-case container thread-pool utilization, fleet-wide, restricted to deployments running search only (no document-api).", +content-executor-saturation,percentage (0-100),"Worst-case content-node executor utilization/saturation across the match, docsum, and field-writer executors, fleet-wide.", +content-groups-nodes-down,count,Fleet-wide count of cluster groups currently reporting at least one down node., +core-dumps,count,Total number of core dumps processed across content clusters during the time window., +disk-utilization,percentage (0-100),"Maximum disk utilization across nodes in container clusters (logs, working files).","-container, -content" +documents-per-content-cluster,count,"Maximum number of unique documents stored, aggregated across content clusters.",-max +feed-blocked,count,"Total number of nodes currently above the feed-block resource limit, across all content clusters.", +feed-latency,milliseconds,Maximum feed latency observed (OK status only).,-max +headroom-to-feed-block-per-content-cluster,percentage (0-100),"Smallest remaining headroom before the feed-block limit, across memory and disk, fleet-wide.", +http-2xx-responses,requests/second,Rate of successful (2xx) HTTP responses across the container cluster.,-sum +http-4xx-responses,requests/second,Rate of client error (4xx) HTTP responses across the container cluster.,-sum +http-5xx-responses,requests/second,Rate of server error (5xx) HTTP responses across the container cluster.,-sum +http-read-latency,milliseconds,"Average HTTP-level read latency, including time to receive the request and send the response.","-avg, -max, -p95, -p99" +http-write-latency,milliseconds,Maximum HTTP-level write latency observed.,-max +jvm-heap-pressure,percentage (0-100),Fleet-wide average JVM heap utilization (heap used / heap capacity) across container clusters., +memory-utilization-node,percentage (0-100),Maximum OS-level memory utilization across nodes in container clusters.,"-container, -content" +nodes-alive,boolean (0 or 1),"Whether every node in the deployment is currently reporting a heartbeat (1 if all nodes are alive, 0 if at least one is down or not reporting).",-min +qos-read-success,percentage (0-100),Percentage of read requests that did not return a 5xx server error., +qos-write-success,percentage (0-100),Percentage of write requests that did not return a 5xx server error., +query-latency,milliseconds,Average query latency across the container cluster (total latency divided by request count).,"-avg, -max" +restarts,count,Total number of Vespa service restarts across the fleet during the time window., diff --git a/_data/production-test-metrics/query.csv b/_data/production-test-metrics/query.csv new file mode 100644 index 0000000000..a32df86971 --- /dev/null +++ b/_data/production-test-metrics/query.csv @@ -0,0 +1,33 @@ +family,unit,description,suffixes +degraded-queries,requests/second,"Rate of queries that were degraded (soft-doomed or match-limited), summed across the container cluster.",-sum +docs-matched-per-query-per-rank-profile,count,"Average documents matched per query, aggregated across all rank profiles - a proxy for query selectivity.", +docsum-executor-accepted,operations/second,Average rate of docsum tasks accepted by the per-node docsum executor across content nodes.,-rate +docsum-executor-queue-size,count,Peak length of the per-node docsum thread-pool queue on any single content node.,-max +docsum-latency,seconds,"Average summary-rendering (docsum) latency across content nodes, in seconds.","-avg, -max" +document-summaries-requested,requests/second,"Document summaries requested per second across content nodes, derived from the docsum-latency sample count.",-rate +documents-coverage,percentage (0-100),Percentage of the document corpus actually covered by search (documents covered vs. documents total)., +documents-matched,operations/second,"Documents matched per second across the content cluster, summed across all content nodes and rank profiles.",-rate +empty-results,requests/second,"Rate of queries returning zero results, summed across the container cluster.",-sum +error-rate-pct,percentage (0-100),"Percentage of HTTP requests returning a 5xx response, of all HTTP requests.", +failed-queries,requests/second,"Rate of queries that failed, summed across the container cluster.",-sum +grouping-time-per-rank-profile,milliseconds,"Average post-match time per query (result construction, grouping, packing), aggregated across all rank profiles and content nodes.","-avg, -peak" +hits-per-query,count,"Average number of hits returned per query, weighted across the container cluster.",-avg +match-executor-accepted,operations/second,Average rate of tasks accepted into the match executor across content nodes.,-rate +match-executor-queue-size,count,Peak match thread-pool queue depth on any single content node.,-max +match-executor-utilization,percentage (0-100),"Average match-thread-pool utilization across content nodes, as a percentage.","-avg, -max" +matching-queries,requests/second,Per-node-average rate of queries being matched on content nodes.,-rate +matching-query-latency,milliseconds,Average matching-phase query latency across all content nodes and rank profiles.,-avg +matching-query-setup-time,milliseconds,"Average per-query setup overhead (query decode, blueprint build, rank setup) on content nodes.",-avg +queries-per-second-by-content-group,requests/second,"Lowest per-content-group query rate observed, deliberately using min to catch a failed node emitting stale/frozen metric values.",-min +queries-per-second-per-rank-profile,requests/second,"Average per-(node, rank-profile) query rate observed at the content nodes.", +query-container-latency,milliseconds,"Time spent inside the container processing the query, excluding backend dispatch.",-avg +query-error-breakdown,requests/second,Rate of queries that failed due to backend (content-node) communication errors.,"-backend-comm, -empty-docsums, -invalid-param, -timeout, -unhandled" +query-latency,milliseconds,Average 95th percentile query latency.,"-p95, -p99" +query-rate-qps,requests/second,"Total application-wide query rate, summed across all container instances.","-avg, -peak" +query-timeout,milliseconds,99th percentile of the configured per-query timeout values actually applied by clients.,-p99 +rank-profile-query-latency,milliseconds,"Peak per-query matching latency on content nodes, taken across all rank profiles combined.",-max +rerank-time-per-rank-profile,milliseconds,"Average second-phase rerank time per query, aggregated across all rank profiles and content nodes (no non-rank-profile-labeled equivalent metric exists to prefer instead).","-avg, -peak" +search-handler-utilization,percentage (0-100),Average utilization of the container's search-handler thread pool.,"-avg, -max" +search-protocol-query-latency,milliseconds,Average query latency measured at the search-protocol level on content nodes.,"-avg, -max" +soft-doom-factor-per-rank-profile,count,"Average adaptive soft-timeout multiplier per query, aggregated across all rank profiles.","-avg, -max, -min" +soft-doomed-queries-per-rank-profile,requests/second,"Rate of queries that hit the soft timeout, summed across all rank profiles and content nodes.",-sum diff --git a/_data/production-test-metrics/resources.csv b/_data/production-test-metrics/resources.csv new file mode 100644 index 0000000000..e1a9fdfdc6 --- /dev/null +++ b/_data/production-test-metrics/resources.csv @@ -0,0 +1,26 @@ +family,unit,description,suffixes +cpu-iowait,percentage (0-100),"Average per-host CPU I/O-wait percentage across the cluster over the time window, indicating time spent waiting on disk I/O rather than doing CPU work.","-avg, -max, -min" +cpu-throttled-time,count,Sum of per-node CPU cgroup-throttling rate across the cluster over the time window; non-zero values indicate nodes are hitting their CPU resource limits.,-sum +cpu-utilization,percentage (0-100),Maximum CPU utilization across the nodes of the container cluster.,"-container, -content" +default-handler-common-utilization,percentage (0-100),Average active threads as a percentage of pool size for the default-handler-common thread pool over the time window.,"-avg, -max" +default-handler-common-work-queue-size,count,Average number of pending tasks queued for the default-handler-common thread pool over the time window.,"-avg, -max" +default-handler-common-work-queue-utilization,percentage (0-100),Average queue fill percentage (size / capacity) for the default-handler-common thread pool over the time window.,"-avg, -max" +disk-utilization,percentage (0-100),Average OS-level disk utilization percentage across the cluster's nodes.,"-avg, -max, -min" +feedapi-handler-utilization,percentage (0-100),Average active threads as a percentage of pool size for the feedapi-handler thread pool over the time window.,"-avg, -max" +feedapi-handler-work-queue-size,count,Average number of pending tasks queued for the feedapi-handler thread pool over the time window.,"-avg, -max" +feedapi-handler-work-queue-utilization,percentage (0-100),Average queue fill percentage (size / capacity) for the feedapi-handler thread pool over the time window.,"-avg, -max" +gpu-memory-utilization,percentage (0-100),"Maximum GPU memory utilization (used/total) across GPU-equipped nodes, averaged over the time window.",-max +gpu-utilization-container,percentage (0-100),"Maximum GPU utilization across container-cluster nodes with GPUs attached, averaged over the time window (only produces data for GPU-equipped nodes).",-max +jvm-direct-memory,bytes,Average configured JVM direct (off-heap) memory capacity on container nodes.,"-capacity, -used" +jvm-gc-overhead,percentage (0-100),"Approximate percentage of CPU time spent in JVM garbage collection on container nodes, averaged over the time window.",-max +jvm-gc-pause-duration,seconds,Average stop-the-world JVM GC pause duration on container nodes over the time window.,"-avg, -max" +jvm-heap-usage,bytes,Average configured JVM heap memory capacity (max heap size) on container nodes.,"-capacity, -used" +jvm-native-memory,bytes,"Average native (off-JVM) memory used by C++ libraries loaded via JNI (e.g. ONNX Runtime for embedders, linguistics) on container nodes.",-avg +memory-node-utilization,percentage (0-100),"Average OS-level memory utilization across the cluster's nodes (physical RAM usage), distinct from the Vespa-computed content memory-utilization metric.","-avg, -max, -min" +memory-utilization-content,percentage (0-100),Maximum memory utilization of the content cluster's cluster controller., +network-throughput-bytes-received-sent,bytes,"Average per-interval network bytes received by container nodes, summed across the cluster over the time window.","-received, -sent" +node-liveness,"boolean (1 = alive, 0 = down)",Minimum node-liveness value across all nodes in the cluster over the time window; drops to 0 if any node stops reporting a heartbeat.,-min +open-server-connections,count,Peak number of open HTTP server connections on container nodes observed during the time window.,-max +requests-per-http-connection,count,Average number of HTTP requests served per connection over its lifetime on container nodes during the time window.,"-avg, -max" +search-handler-work-queue-size,count,Average number of pending tasks queued for the search-handler thread pool over the time window.,"-avg, -max" +search-handler-work-queue-utilization,percentage (0-100),Average queue fill percentage (size / capacity) for the search-handler thread pool over the time window.,"-avg, -max" diff --git a/_data/sidebar.yml b/_data/sidebar.yml index 33fd223346..c61c0a243c 100644 --- a/_data/sidebar.yml +++ b/_data/sidebar.yml @@ -572,6 +572,8 @@ docs: url: /en/reference/applications/testing.html - page: System test (Java) url: /en/reference/applications/testing-java.html + - page: Production test + url: /en/reference/applications/testing-production.html - title: Schemas and documents sub: - page: Schemas diff --git a/en/applications/testing.html b/en/applications/testing.html index 184bf7b524..84e340437e 100644 --- a/en/applications/testing.html +++ b/en/applications/testing.html @@ -24,11 +24,16 @@

These two frameworks also includes an upgrade—or staging—test construct for scenarios where the application - is upgraded, and state in the backend depends on the old application configuration; - as well as a production verification test—basically a health check for production deployments. + is upgraded, and state in the backend depends on the old application configuration. For system and staging tests, the frameworks provide an easy way to perform HTTP request against a designated test deployment, separating the tests from the deployment and configuration of the test clusters.

+

+ There is also a production test + construct, essentially a health check for a production deployment. Production tests are configured + as YAML or JSON files, and are evaluated directly by the controller against the metrics of the + production deployment, rather than run against a dedicated test deployment. +

This document describes how each of these test categories can be run as part of an imagined CI/CD system for safely deploying changes to a Vespa application in a continuous manner. diff --git a/en/operations/automated-deployments.html b/en/operations/automated-deployments.html index 796f8b2d91..f466d6ff56 100644 --- a/en/operations/automated-deployments.html +++ b/en/operations/automated-deployments.html @@ -519,6 +519,11 @@

Production tests

This is useful in multi-zone deployments where the first zone serves as a canary. Production tests run against the endpoints of the preceding production region in the pipeline.

+

+ Production tests are written as YAML or JSON files. See the + production test reference for the + file format and the list of available metric presets. +

Picture of production test hover card with version or build tested diff --git a/en/reference/applications/testing-java.html b/en/reference/applications/testing-java.html index 068f4a4e8a..b392f64a1c 100644 --- a/en/reference/applications/testing-java.html +++ b/en/reference/applications/testing-java.html @@ -57,7 +57,7 @@

Test suites

Production test @ProductionTest production - Verify domain specific metrics + Verify domain specific metrics (deprecated, see below)

@@ -65,6 +65,14 @@

Test suites

sample application test suite.

+

+ New applications should not write production tests in Java: use the + declarative production test format, based on YAML or JSON, + instead. The @ProductionTest category above is kept only for backward compatibility + with applications that already use it — if a test package includes both a declarative production + test and a Java @ProductionTest bundle, the declarative test is used, and the Java one + is ignored. +

TestNG

diff --git a/en/reference/applications/testing-production.html b/en/reference/applications/testing-production.html new file mode 100644 index 0000000000..5b0431be8c --- /dev/null +++ b/en/reference/applications/testing-production.html @@ -0,0 +1,264 @@ +--- +# Copyright Vespa.ai. All rights reserved. +title: Production tests +applies_to: cloud +--- + +

+ This is the Vespa Testing reference for production tests. +

+

+ Production tests verify domain specific metrics for a production deployment, such as query latency + or error rate. Unlike system and staging + tests, production tests check a named metric preset against expected + bounds, using Grafana metrics for the application deployed to that zone. +

+

+ A production test is placed after a deployment zone in deployment.xml. + It acts as a gate for the rollout: if it fails, the rollout stops and subsequent zones will not + receive the new version. This is useful in multi-zone deployments where the first zone serves as a + canary, but a production test can validate any zone before the rollout proceeds to the next, + as described in + production tests. +

+ + + +

Production test setup

+

+ To use production tests, first deploy to a zone, use + <delay> for some time, and then + run the production tests against that zone. This pattern can be repeated for each zone in the + pipeline, to validate it before the rollout proceeds to the next. In the following example, + the zone deployment runs for 10 minutes before the tests run against it: +

+
{% highlight xml %}
+
+  aws-us-east-1c
+  
+  aws-us-east-1c
+  
+
+{% endhighlight %}
+

+ The time field should not be longer than the delay before the test: +

+
{% highlight yaml %}
+name: cpu check
+metric: cpu-utilization-container
+time: 10
+max: 85
+{% endhighlight %}
+

+ This matters because the production test only starts once the preceding <delay> + has elapsed. If the test's time window reaches further back than that, it averages in metrics from + before the zone was deployed, which dilutes the result and defeats the purpose of the gate. +

+{% include note.html content="Tests based on traffic metrics, such as query or feed latency, need +real queries or feeding against the zone during the test window, since without traffic there are no +metrics to evaluate and the test may fail or be inconclusive." %} + +

Test suites

+

+ Production tests are placed under tests/production-test/ in the application package, + as one or more .yaml or .json files: +

+
+$ ls tests/production-test/
+
+tests/production-test/metrics-test.yaml
+
+

+ Each file describes one or more tests. A file may contain: +

+ + + + +

Test file structure

+

+ Each test checks a named metric preset, over a time window, against a + min and/or max bound. A simple example: +

+
{% highlight yaml %}
+name: cpu check
+metric: cpu-utilization-container
+time: 5
+max: 85
+{% endhighlight %}
+

+ This test fails the production test step if the average CPU utilization of the container cluster, + over the last 5 minutes, exceeds 85%. +

+

+ To place multiple tests in one file, wrap them in a top-level tests field, containing a + list of test objects. This works the same way for both YAML and JSON: +

+
{% highlight yaml %}
+tests:
+  - name: container cpu check
+    metric: cpu-utilization-container
+    time: 5
+    max: 35
+
+  - name: content cpu check
+    metric: cpu-utilization-content
+    time: 5
+    max: 85
+{% endhighlight %}
+ +

Test specification

+

A full list of fields, with description:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDefaultDescription
namestring"<unnamed test>"Name used for display purposes in logs and failure messages.
metricstringRequired. The name of one of the metric presets. +
timenumberRequired. The size, in minutes, of the time window the metric is averaged, or otherwise + aggregated, over. Must be a positive whole number.
minnumberThe inclusive lower bound the query result must satisfy. At least one of min and + max is required. The unit depends on the metric preset + used, see its Unit column. +
maxnumberThe inclusive upper bound the query result must satisfy. At least one of min and + max is required. The unit depends on the metric preset + used, see its Unit column. +
+

+ A test that is outside its min/max bounds fails the production test step + immediately — production tests are not retried. +

+ +

Examples

+

A single test, in YAML:

+
{% highlight yaml %}
+name: error rate check
+metric: error-rate-pct
+time: 5
+max: 1
+{% endhighlight %}
+ +

Multiple tests in a single YAML file:

+
{% highlight yaml %}
+tests:
+  - name: node count check
+    metric: node-count-max
+    time: 5
+    max: 10
+
+  - name: documents count check
+    metric: documents-count-active
+    time: 5
+    min: 1
+{% endhighlight %}
+ +

A single test, in JSON:

+
{% highlight json %}
+{
+    "name": "query latency check",
+    "metric": "query-latency-p95",
+    "time": 5,
+    "max": 200
+}
+{% endhighlight %}
+ +

Multiple tests in a single JSON file:

+
{% highlight json %}
+{
+    "tests": [
+        {
+            "name": "container feed latency check",
+            "metric": "container-feed-latency-max",
+            "time": 5,
+            "max": 100
+        },
+        {
+            "name": "content commit latency check",
+            "metric": "content-commit-latency-avg",
+            "time": 5,
+            "max": 50
+        }
+    ]
+}
+{% endhighlight %}
+ + +

Metric presets

+

+ Every test checks one of the following named presets, in its metric field. Each preset + queries the same metrics that back the application's overview dashboard, reduced to a single number + for the given production deployment. Presets are grouped below by the dashboard tab they come from. + Where a preset comes in several variants of the same underlying metric - e.g. an average and a p95 - + they're listed once, as a family name plus its available suffixes: append a suffix to the + family name to get the actual metric value, e.g. the family query-latency with + suffix -p95 is used as metric: query-latency-p95. A family with no suffixes + listed is used as-is, e.g. metric: error-rate-pct. +

+ +{% assign metric_categories = "Overview,Query,Feed,Nearest Neighbor Search,Content Node,Resources,Health,Autoscaling: Content,Autoscaling: Container" | split: "," %} +{% assign metric_category_slugs = "overview,query,feed,nearest-neighbor-search,content-node,resources,health,autoscaling-content,autoscaling-container" | split: "," %} +{% for category in metric_categories %} + {% assign slug = metric_category_slugs[forloop.index0] %} + {% assign rows = site.data.production-test-metrics[slug] %} +

{{ category }}

+ + + + + + + + + + + {% for row in rows %} + + + + + + + {% endfor %} + +
FamilyUnitDescriptionSuffixes
{{ row.family }}{{ row.unit }}{{ row.description }}{{ row.suffixes }}
+{% endfor %} + + diff --git a/en/reference/applications/testing.html b/en/reference/applications/testing.html index e95fae2750..99d60ab46b 100644 --- a/en/reference/applications/testing.html +++ b/en/reference/applications/testing.html @@ -21,7 +21,7 @@

Test suites

The testing documentation defines three test scenarios, - comprised of four test code categories. For basic HTTP tests, the category of a test is defined + comprised of three test code categories. For basic HTTP tests, the category of a test is defined by its placement in the application tests directory:

@@ -45,21 +45,20 @@

Test suites

- - - -
Staging test tests/staging-test/ Verify state after upgrade
Production testtests/production-test/Verify domain specific metrics
{% include note.html content="If the application package has Java code, the tests directory is src/test/application/tests"%} +

+ See the production test reference for the + tests/production-test/ category, which uses a different, YAML or JSON based, format. +

Each test is described by a JSON file, and may include other files using relative paths: 

 $ ls -1 tests/*/*
 
-tests/production-test/metrics-test.json
 tests/staging-setup/set-up-old-documents.json
 tests/staging-test/verify-search-still-works.json
 tests/system-test/data/document.json
@@ -69,19 +68,7 @@ 
Test suites

 

   For an example with actual system and staging tests, check out a Vespa Cloud
   sample test suite.
-  Since production tests are highly application-specific, this suite has none, but such a test could be:
 

-
{% highlight json %}
-{
-    "steps": [
-        {
-            "request": {
-                "uri": "https://my.external.service/metrics/?query=customer-engagement"
-            }
-        }
-    ]
-}
-{% endhighlight %}