From 8bc98b46ded09a9b8effeaebe436c3d309564dca Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 28 Jul 2026 16:21:20 -0700 Subject: [PATCH 01/14] Adjust kafka configs to make connection more resilient --- config/sample_webconfig.conf | 166 ++++++++++++++++++++++++---------- db/sqlite/sqlite_client.go | 4 + go.mod | 10 +- go.sum | 28 +++--- http/webconfig_server.go | 69 +++++++++++++- kafka/kafka_consumer_group.go | 50 ++++++++++ main.go | 78 ++++++++++------ 7 files changed, 309 insertions(+), 96 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index c8b4417..075dc47 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -3,7 +3,9 @@ webconfig { encryption_key_env_name = "WEBCONFIG_KEY" } + app_name = "webconfig" panic_exit_enabled = false + factory_reset_enabled = false tracing { moracide_tag_prefix = "X-Cl-Experiment" otel { @@ -36,6 +38,8 @@ webconfig { level = "info" file = "/tmp/webconfig.log" sarama_logger_enabled = false + // List of request header names to suppress from logs (case-insensitive) + ignored_headers = [] } metrics { @@ -104,6 +108,8 @@ webconfig { // Defaults to false. Setting this to true disables server certificate // verification for all outbound HTTPS connectors and is insecure. tls_insecure_skip_verify = false + // Optional User-Agent header sent on all outbound HTTP requests. + user_agent = "" } jwt { @@ -112,7 +118,6 @@ webconfig { // north bound APIs from orc/customers, no mac embedded in tokens api_token { - enabled = true kids = [ "sat-prod-k1-1024", ] @@ -126,7 +131,6 @@ webconfig { // south bound APIs from CPEs, macs embedded in tokens cpe_token { - enabled = true kids = [ "webconfig_key", ] @@ -244,20 +248,58 @@ webconfig { } // TLS configuration for secure Kafka connections - tls { - enabled = false - // Path to client certificate file (PEM format) - // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) - // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) - cert_file = "/etc/webconfig/kafka/client-cert.pem" - // Path to client private key file (PEM format) - key_file = "/etc/webconfig/kafka/client-key.pem" - // Path to CA certificate file for broker verification (PEM format) - ca_cert_file = "/etc/webconfig/kafka/ca-cert.pem" - // Skip certificate verification (WARNING: use ONLY in dev/test environments, NEVER in production) - // When true, certificate files become optional (TLS without client authentication) - insecure_skip_verify = false - } + // NOTE: These are flat keys read by LoadKafkaTLSConfig at prefix "webconfig.kafka" + // Certificate files (tls_cert_file, tls_key_file, tls_ca_cert_file) are REQUIRED when + // tls_insecure_skip_verify=false (default). They are OPTIONAL when tls_insecure_skip_verify=true. + tls_enabled = false + tls_cert_file = "/etc/webconfig/kafka/client-cert.pem" + tls_key_file = "/etc/webconfig/kafka/client-key.pem" + tls_ca_cert_file = "/etc/webconfig/kafka/ca-cert.pem" + // WARNING: set true only in dev/test environments, NEVER in production + tls_insecure_skip_verify = false + + // Reconnect resilience — tuned for broker restart recovery. + + // metadata.retry_max: max metadata fetch retries before giving up. + // lib default: 3 | configured: 10 + // Extends the retry window to 10×2s = 20s, covering the cluster + // stabilisation window during leader election after a broker restart. + metadata.retry_max = 10 + + // metadata.retry_backoff_sec: seconds between metadata fetch retries. + // lib default: 0.25s | configured: 2s + // Prevents rapid-fire retries against a restarting broker; gives the + // cluster time to stabilise between attempts. + metadata.retry_backoff_sec = 2 + + // metadata.refresh_frequency_sec: background metadata refresh interval in seconds. + // lib default: 600s (10min) | configured: 300s (5min) + // Proactive background refresh catches stale partition leaders up to 5min + // sooner than the library default after a broker restart. + metadata.refresh_frequency_sec = 300 + + // consumer.session_timeout_sec: consumer group session timeout in seconds. + // lib default: 10s | configured: 30s + // Must be within broker group.min.session.timeout.ms (6s) / + // group.max.session.timeout.ms (300s). 30s gives the consumer group + // time to survive a restarting group coordinator without triggering + // an unnecessary rebalance. + // Heartbeat interval is derived as session_timeout_sec / 6 (= 5s for + // the default 30s). The Kafka protocol ceiling is session_timeout / 3 + // (10s); using /6 gives double the margin against transient jitter. + // lib heartbeat default: 3s | configured: 5s + consumer.session_timeout_sec = 30 + + // consumer.retry_backoff_sec: seconds a partition reader waits before retrying a failed fetch. + // lib default: 2s | configured: 2s + // Set explicitly to make the value visible and tunable. + consumer.retry_backoff_sec = 2 + + // net.timeout_sec: dial/read/write timeout for broker connections in seconds. + // lib default: 30s | configured: 10s + // Tighter timeout lets the retry loop engage within 10s per attempt + // instead of blocking for 30s (fail-fast on broken broker connections). + net.timeout_sec = 10 // if we want to use more than 1 cluster clusters { @@ -274,16 +316,12 @@ webconfig { messages_per_second = 10 } - // TLS configuration for mesh cluster - tls { - enabled = false - // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) - // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) - cert_file = "/etc/webconfig/kafka/mesh-client-cert.pem" - key_file = "/etc/webconfig/kafka/mesh-client-key.pem" - ca_cert_file = "/etc/webconfig/kafka/mesh-ca-cert.pem" - insecure_skip_verify = false - } + // TLS configuration for mesh cluster (flat keys at prefix "webconfig.kafka.clusters.mesh") + tls_enabled = false + tls_cert_file = "/etc/webconfig/kafka/mesh-client-cert.pem" + tls_key_file = "/etc/webconfig/kafka/mesh-client-key.pem" + tls_ca_cert_file = "/etc/webconfig/kafka/mesh-ca-cert.pem" + tls_insecure_skip_verify = false } east { enabled = false @@ -298,16 +336,12 @@ webconfig { messages_per_second = 10 } - // TLS configuration for east cluster - tls { - enabled = false - // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) - // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) - cert_file = "/etc/webconfig/kafka/east-client-cert.pem" - key_file = "/etc/webconfig/kafka/east-client-key.pem" - ca_cert_file = "/etc/webconfig/kafka/east-ca-cert.pem" - insecure_skip_verify = false - } + // TLS configuration for east cluster (flat keys at prefix "webconfig.kafka.clusters.east") + tls_enabled = false + tls_cert_file = "/etc/webconfig/kafka/east-client-cert.pem" + tls_key_file = "/etc/webconfig/kafka/east-client-key.pem" + tls_ca_cert_file = "/etc/webconfig/kafka/east-ca-cert.pem" + tls_insecure_skip_verify = false } } } @@ -327,22 +361,59 @@ webconfig { // correct subdoc states if versions match but not "deployed" state_correction_enabled= false + // pre-cook supplementary subdocs into the database + supplementary_precook_enabled = false + // number of days to retain supplementary precook state (default 7) + supplementary_precook_state_ttl_days = 7 + // forward kafka messages if needed kafka_producer { enabled = false brokers = "localhost:9092" topic = "webconfig_downstream" - // TLS configuration for Kafka producer - tls { - enabled = false - // NOTE: Certificate files (cert, key, ca) are REQUIRED when insecure_skip_verify=false (default) - // Certificate files are OPTIONAL when insecure_skip_verify=true (insecure mode) - cert_file = "/etc/webconfig/kafka/producer-client-cert.pem" - key_file = "/etc/webconfig/kafka/producer-client-key.pem" - ca_cert_file = "/etc/webconfig/kafka/producer-ca-cert.pem" - insecure_skip_verify = false - } + // TLS configuration for Kafka producer (flat keys at prefix "webconfig.kafka_producer") + tls_enabled = false + tls_cert_file = "/etc/webconfig/kafka/producer-client-cert.pem" + tls_key_file = "/etc/webconfig/kafka/producer-client-key.pem" + tls_ca_cert_file = "/etc/webconfig/kafka/producer-ca-cert.pem" + tls_insecure_skip_verify = false + + // Reconnect resilience — tuned for broker restart recovery. + + // metadata.retry_max: max metadata fetch retries before giving up. + // lib default: 3 | configured: 10 + // Extends the retry window to 10×2s = 20s, covering the cluster + // stabilisation window during leader election after a broker restart. + metadata.retry_max = 10 + + // metadata.retry_backoff_sec: seconds between metadata fetch retries. + // lib default: 0.25s | configured: 2s + // Prevents rapid-fire retries against a restarting broker. + metadata.retry_backoff_sec = 2 + + // metadata.refresh_frequency_sec: background metadata refresh interval in seconds. + // lib default: 600s (10min) | configured: 300s (5min) + // Proactive background refresh catches stale partition leaders up to 5min + // sooner than the library default after a broker restart. + metadata.refresh_frequency_sec = 300 + + // producer.retry_max: max internal send retries before a message is put on the error channel. + // lib default: 3 | configured: 10 + // Mirrors metadata.retry_max — gives the producer the same tolerance for + // transient broker unavailability as the consumer has for metadata fetches. + producer.retry_max = 10 + + // producer.retry_backoff_sec: seconds between internal send retries. + // lib default: 0.1s | configured: 2s + // Matches metadata.retry_backoff_sec cadence; avoids hammering a restarting broker. + producer.retry_backoff_sec = 2 + + // net.timeout_sec: dial/read/write timeout for broker connections in seconds. + // lib default: 30s | configured: 10s + // Tighter timeout lets the retry loop engage within 10s per attempt + // instead of blocking for 30s (fail-fast on broken broker connections). + net.timeout_sec = 10 } // this allows the root document locked if needed @@ -363,4 +434,7 @@ webconfig { filter_output_by_bitmap_enabled = false bitmap_filter_exempt_subdoc_ids = [] + + // return an empty profile document when no upstream data is available + default_empty_profile_enabled = false } diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 01b1e1e..7b1b5d9 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -21,6 +21,7 @@ import ( "database/sql" "errors" "fmt" + "os" "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" @@ -56,6 +57,9 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, var dbfile string if testOnly { dbfile = conf.GetString("webconfig.database.sqlite.unittest_db_file", defaultSqliteTestDbFile) + if x := os.Getenv("WEBCONFIG_TESTDB_SQLITE_FILE"); len(x) > 0 { + dbfile = x + } } else { dbfile = conf.GetString("webconfig.database.sqlite.db_file", defaultSqliteDbFile) } diff --git a/go.mod b/go.mod index abe8783..1a7ca88 100644 --- a/go.mod +++ b/go.mod @@ -66,14 +66,14 @@ require ( go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 // indirect - google.golang.org/grpc v1.81.0 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/libc v1.72.2 // indirect diff --git a/go.sum b/go.sum index 1adcd3b..6af8102 100644 --- a/go.sum +++ b/go.sum @@ -169,11 +169,11 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -182,8 +182,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= @@ -194,8 +194,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -205,13 +205,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -223,8 +223,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348/go.mod h1:Yzdzr5OOZFgSsEV2D/Xi9NL3bszpXFAg0hFJiRohcD8= google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 h1:pfIbyB44sWzHiCpRqIen67ZQnVXSfIxWrqUMk1qwODE= google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw= -google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 633201b..9851957 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -287,7 +287,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaEnabled := conf.GetBoolean("webconfig.kafka.enabled") upstreamEnabled := conf.GetBoolean("webconfig.upstream.enabled") - appName := conf.GetString("webconfig.app_name") + appName := conf.GetString("webconfig.app_name", "webconfig") validateMacEnabled := conf.GetBoolean("webconfig.validate_device_id_as_mac_address", tokenApiEnabledDefault) configValidPartners := conf.GetStringList("webconfig.valid_partners") validPartners := []string{} @@ -314,6 +314,41 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer saramaConfig := sarama.NewConfig() saramaConfig.Producer.Return.Errors = true + // Resilience tuning — survive Kafka broker restarts without restarting the app. + // + // Metadata.Retry.Max: lib default 3 → now 10 (configurable) + // Extends retry window to 10×2s = 20s, covering the cluster stabilisation + // window during leader election after a broker restart. + saramaConfig.Metadata.Retry.Max = int(conf.GetInt32("webconfig.kafka_producer.metadata.retry_max", 10)) + + // Metadata.Retry.Backoff: lib default 250ms → now 2s (configurable) + // Prevents rapid-fire retries against a restarting broker; paced to give + // the cluster time to stabilise between attempts. + saramaConfig.Metadata.Retry.Backoff = time.Duration(conf.GetInt32("webconfig.kafka_producer.metadata.retry_backoff_sec", 2)) * time.Second + + // Metadata.RefreshFrequency: lib default 10min → now 5min (configurable) + // Proactive background refresh. Catches stale partition leaders (caused by + // broker restarts) up to 5min sooner than the library default. + saramaConfig.Metadata.RefreshFrequency = time.Duration(conf.GetInt32("webconfig.kafka_producer.metadata.refresh_frequency_sec", 300)) * time.Second + + // Producer.Retry.Max: lib default 3 → now 10 (configurable) + // Internal send retries before a message is put on the Errors() channel. + // Mirrors Metadata.Retry.Max — gives the producer the same tolerance for + // transient broker unavailability as the consumer has for metadata fetches. + saramaConfig.Producer.Retry.Max = int(conf.GetInt32("webconfig.kafka_producer.producer.retry_max", 10)) + + // Producer.Retry.Backoff: lib default 100ms → now 2s (configurable) + // Matches Metadata.Retry.Backoff cadence; avoids hammering a restarting broker. + saramaConfig.Producer.Retry.Backoff = time.Duration(conf.GetInt32("webconfig.kafka_producer.producer.retry_backoff_sec", 2)) * time.Second + + // Net.DialTimeout / ReadTimeout / WriteTimeout: lib default 30s → now 10s (configurable) + // Tighter timeouts let the retry loop engage within 10s per attempt instead + // of blocking for 30s. Fail-fast on broken broker connections. + producerNetTimeout := time.Duration(conf.GetInt32("webconfig.kafka_producer.net.timeout_sec", 10)) * time.Second + saramaConfig.Net.DialTimeout = producerNetTimeout + saramaConfig.Net.ReadTimeout = producerNetTimeout + saramaConfig.Net.WriteTimeout = producerNetTimeout + // Load TLS configuration for producer tlsConfig, err := common.LoadKafkaTLSConfig(conf, "webconfig.kafka_producer") if err != nil { @@ -1085,6 +1120,18 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess Key: sarama.ByteEncoder(kbytes), Value: sarama.ByteEncoder(bbytes), } + + defer func() { + if r := recover(); r != nil { + if m := s.Metrics(); m != nil { + m.ObserveKafkaProducerErr(s.KafkaProducerTopic(), -1) + } + tfields["logger"] = "kafkaproducer" + tfields["error"] = r + log.WithFields(tfields).Warn("dropped: producer closed during shutdown") + } + }() + s.Input() <- outMessage tfields["logger"] = "kafkaproducer" @@ -1119,7 +1166,21 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes Key: sarama.ByteEncoder(strings.ToLower(mac)), Value: sarama.ByteEncoder(bbytes), } - s.Input() <- outMessage + + sent := func() (ok bool) { + defer func() { + if r := recover(); r != nil { + tfields["error"] = r + log.WithFields(tfields).Warn("dropped: producer closed during shutdown") + ok = false + } + }() + s.Input() <- outMessage + return true + }() + if !sent { + return + } tfields["output_key"] = mac tfields["output_body"] = m @@ -1165,13 +1226,15 @@ func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token str log.WithFields(tfields).Debug(tokenErr) } -func (s *WebconfigServer) HandleKafkaProducerResults() { +func (s *WebconfigServer) HandleKafkaProducerResults(ctx context.Context) { if s.AsyncProducer == nil { return } for { select { + case <-ctx.Done(): + return case success := <-s.Successes(): if success == nil { continue diff --git a/kafka/kafka_consumer_group.go b/kafka/kafka_consumer_group.go index 9cc24e9..cb915fe 100644 --- a/kafka/kafka_consumer_group.go +++ b/kafka/kafka_consumer_group.go @@ -29,6 +29,14 @@ import ( wchttp "github.com/rdkcentral/webconfig/http" ) +const ( + // Resilience defaults — configurable via webconfig.kafka.* keys. + defaultMetadataRetryMax = 10 + defaultMetadataRetryBackoffSec = 2 + defaultNetTimeoutSec = 10 + defaultSessionTimeoutSec = 30 +) + type KafkaConsumerGroup struct { sarama.ConsumerGroup db.DatabaseClient @@ -103,6 +111,48 @@ func NewKafkaConsumerGroup(conf *configuration.Config, s *wchttp.WebconfigServer } } + // Resilience tuning — survive Kafka broker restarts without restarting the app. + // + // Metadata.Retry.Max: lib default 3 → now 10 (configurable) + // Extends retry window to 10×2s = 20s, covering the cluster stabilisation + // window during leader election after a broker restart. + sconfig.Metadata.Retry.Max = int(conf.GetInt32(prefix+".metadata.retry_max", defaultMetadataRetryMax)) + + // Metadata.Retry.Backoff: lib default 250ms → now 2s (configurable) + // Prevents rapid-fire retries against a restarting broker; paced to give + // the cluster time to stabilise between attempts. + sconfig.Metadata.Retry.Backoff = time.Duration(conf.GetInt32(prefix+".metadata.retry_backoff_sec", defaultMetadataRetryBackoffSec)) * time.Second + + // Metadata.RefreshFrequency: lib default 10min → now 5min (configurable) + // Proactive background refresh. Catches stale partition leaders (caused by + // broker restarts) up to 5min sooner than the library default. + sconfig.Metadata.RefreshFrequency = time.Duration(conf.GetInt32(prefix+".metadata.refresh_frequency_sec", 300)) * time.Second + + // Consumer.Group.Session.Timeout: lib default 10s → now 30s (configurable) + // Time before the broker evicts a consumer whose heartbeats have stopped. + // 30s gives the consumer group time to survive a restarting group coordinator + // without triggering an unnecessary rebalance. + // + // Consumer.Group.Heartbeat.Interval: lib default 3s → now Session.Timeout/6 = 5s + // The Kafka protocol ceiling is Session.Timeout/3 (10s). Using /6 keeps the + // interval at half the ceiling, providing headroom for transient network jitter + // during broker restarts without risking false session expiry. + sessionTimeout := time.Duration(conf.GetInt32(prefix+".consumer.session_timeout_sec", defaultSessionTimeoutSec)) * time.Second + sconfig.Consumer.Group.Session.Timeout = sessionTimeout + sconfig.Consumer.Group.Heartbeat.Interval = sessionTimeout / 6 + + // Consumer.Retry.Backoff: lib default 2s (configurable) + // How long a partition reader waits before retrying a failed fetch. + sconfig.Consumer.Retry.Backoff = time.Duration(conf.GetInt32(prefix+".consumer.retry_backoff_sec", 2)) * time.Second + + // Net.DialTimeout / ReadTimeout / WriteTimeout: lib default 30s → now 10s (configurable) + // Tighter timeouts let the retry loop engage within 10s per attempt instead + // of blocking for 30s. Fail-fast on broken broker connections. + netTimeout := time.Duration(conf.GetInt32(prefix+".net.timeout_sec", defaultNetTimeoutSec)) * time.Second + sconfig.Net.DialTimeout = netTimeout + sconfig.Net.ReadTimeout = netTimeout + sconfig.Net.WriteTimeout = netTimeout + // Load TLS configuration tlsConfig, err := common.LoadKafkaTLSConfig(conf, prefix) if err != nil { diff --git a/main.go b/main.go index a6e5f80..1b3d6a9 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "os/signal" + "sync" "syscall" "time" @@ -107,9 +108,6 @@ func main() { router.Handle("/metrics", promhttp.Handler()) metrics = common.NewMetrics(sc.Config) server.SetMetrics(metrics) - if server.KafkaProducerEnabled() { - go server.HandleKafkaProducerResults() - } handler := metrics.WebMetrics(router) server.Handler = handler } else { @@ -119,6 +117,10 @@ func main() { // setup contexts groups g, gCtx := errgroup.WithContext(mainCtx) + if server.KafkaProducerEnabled() { + go server.HandleKafkaProducerResults(gCtx) + } + // setup http server g.Go( func() error { @@ -126,35 +128,37 @@ func main() { }, ) - g.Go( - func() error { - <-gCtx.Done() - fmt.Printf("HTTP server shutdown NOW !!\n") - if server.KafkaProducerEnabled() { - if err := server.AsyncProducer.Close(); err != nil { - fmt.Fprintf(os.Stderr, "%v AsyncProducer.Close() err=%v\n", time.Now().Format(common.LoggingTimeFormat), err) - } - } - return server.Shutdown(context.Background()) - }, - ) - // setup kafka consumer, if config kafka.enabled=false, then kcgroup=nil, err=nil kcgroups, err := kafka.NewKafkaConsumerGroups(sc, server, metrics) if err != nil { panic(err) } + // consumeWg tracks active consume loops so shutdown can wait for them to + // finish before closing the AsyncProducer (prevents "send on closed channel"). + var consumeWg sync.WaitGroup + for _, kcgroup := range kcgroups { - consumer := *(kcgroup.Consumer()) + consumer := kcgroup.Consumer() topics := kcgroup.Topics() + consumeWg.Add(1) g.Go( func() error { + defer consumeWg.Done() for { - if err := kcgroup.Consume(gCtx, topics, &consumer); err != nil { - fmt.Printf("kcgroup.Consumer: err=%v\n", err) - return err + if err := kcgroup.Consume(gCtx, topics, consumer); err != nil { + select { + case <-gCtx.Done(): + fmt.Printf("kcgroup.Consumer: topics=|%v|, shutdown gracefully\n", topics) + return nil + default: + fmt.Printf("kcgroup.Consumer: topics=|%v|, err=%v, retrying in 2s\n", topics, err) + time.Sleep(2 * time.Second) + if gCtx.Err() != nil { + return nil + } + } } consumer.Ready = make(chan bool) } @@ -164,16 +168,34 @@ func main() { // it is more or less optional, without this reading from the chan, // the consumer runs anyway. <-consumer.Ready - - g.Go( - func() error { - <-gCtx.Done() - fmt.Printf("SARAMA shutdown NOW !!\n") - return kcgroup.Close() - }, - ) } + // Single shutdown goroutine with ordered cleanup: + // 1. close consumer groups (signals ConsumeClaim goroutines to stop) + // 2. wait for all consume loops to drain (ensures no goroutine is mid-send) + // 3. close AsyncProducer (safe now that no one sends to its Input channel) + // 4. shut down HTTP server + g.Go( + func() error { + <-gCtx.Done() + fmt.Printf("shutdown: initiating consumer group shutdown\n") + for _, kcgroup := range kcgroups { + if err := kcgroup.Close(); err != nil { + fmt.Fprintf(os.Stderr, "kcgroup.Close() err=%v\n", err) + } + } + consumeWg.Wait() + fmt.Printf("shutdown: all consumer loops drained\n") + if server.KafkaProducerEnabled() { + if err := server.AsyncProducer.Close(); err != nil { + fmt.Fprintf(os.Stderr, "%v AsyncProducer.Close() err=%v\n", time.Now().Format(common.LoggingTimeFormat), err) + } + } + fmt.Printf("shutdown: stopping HTTP server\n") + return server.Shutdown(context.Background()) + }, + ) + if err := g.Wait(); err != nil { fmt.Printf("exit reason: %s \n", err) } From 8ef2f7fc514d4813dfb311ccfbd0c44d3da3ae9c Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 28 Jul 2026 16:46:41 -0700 Subject: [PATCH 02/14] kafka changes based on copilot review --- http/webconfig_server.go | 16 ++++++++++++++-- main.go | 17 +++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 9851957..dbbbe0b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1123,6 +1123,9 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess defer func() { if r := recover(); r != nil { + if fmt.Sprint(r) != "send on closed channel" { + panic(r) + } if m := s.Metrics(); m != nil { m.ObserveKafkaProducerErr(s.KafkaProducerTopic(), -1) } @@ -1170,6 +1173,9 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes sent := func() (ok bool) { defer func() { if r := recover(); r != nil { + if fmt.Sprint(r) != "send on closed channel" { + panic(r) + } tfields["error"] = r log.WithFields(tfields).Warn("dropped: producer closed during shutdown") ok = false @@ -1235,7 +1241,10 @@ func (s *WebconfigServer) HandleKafkaProducerResults(ctx context.Context) { select { case <-ctx.Done(): return - case success := <-s.Successes(): + case success, ok := <-s.Successes(): + if !ok { + return + } if success == nil { continue } @@ -1245,7 +1254,10 @@ func (s *WebconfigServer) HandleKafkaProducerResults(ctx context.Context) { fields["output_partition"] = success.Partition fields["output_offset"] = success.Offset log.WithFields(fields).Debug("sent") - case pErr := <-s.Errors(): + case pErr, ok := <-s.Errors(): + if !ok { + return + } if pErr == nil || pErr.Msg == nil { continue } diff --git a/main.go b/main.go index 1b3d6a9..ced6179 100644 --- a/main.go +++ b/main.go @@ -154,8 +154,9 @@ func main() { return nil default: fmt.Printf("kcgroup.Consumer: topics=|%v|, err=%v, retrying in 2s\n", topics, err) - time.Sleep(2 * time.Second) - if gCtx.Err() != nil { + select { + case <-time.After(2 * time.Second): + case <-gCtx.Done(): return nil } } @@ -164,10 +165,14 @@ func main() { } }, ) - // This is to setup notify AFTER the sarama is running - // it is more or less optional, without this reading from the chan, - // the consumer runs anyway. - <-consumer.Ready + // Wait for the first session to be established before starting the next + // consumer group. Context-aware: if Kafka is unreachable on startup and + // the context is cancelled (e.g. SIGTERM), we stop waiting rather than + // deadlocking until Setup() is eventually called. + select { + case <-consumer.Ready: + case <-gCtx.Done(): + } } // Single shutdown goroutine with ordered cleanup: From 2eb03ca34d1e0e32cf901b1c3c559bed218136c9 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 31 Jul 2026 14:26:43 -0700 Subject: [PATCH 03/14] Improve Cassandra connection resilience and configurability - Make disable_initial_host_lookup configurable via config key; replace hardcoded constant with dbconf.GetBoolean - Add exponential reconnection policy with configurable initial interval, max retries (default 10), and max interval - Add socket keepalive support via configurable socket_keepalive_sec - Upgrade host selection to TokenAwareHostPolicy wrapping DCAwareRoundRobinPolicy when local_dc is set - Fix TLS host verification: derive EnableHostVerification from insecure_skip_verify instead of hardcoding false - Read insecure_skip_verify once and pass to loadCassandraTLSConfig to eliminate duplicate config reads - Add tls.server_name support for SNI override - Raise default timeouts from 1s to 10s; align sample config - Document all new config keys in sample_webconfig.conf for both cassandra and yugabyte blocks --- .gitignore | 2 ++ config/sample_webconfig.conf | 48 +++++++++++++++++++++++++---- db/cassandra/cassandra_client.go | 52 +++++++++++++++++++++----------- 3 files changed, 79 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index dd4f44c..5125374 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,6 @@ bin/* openspec/ .github/ .claude/ +.opencode/ AGENTS.md +CLAUDE.md diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 075dc47..249ec43 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -173,16 +173,29 @@ webconfig { "127.0.0.1" ] keyspace = "webconfig" - timeout_in_sec = 5 - connect_timeout_in_sec = 5 + timeout_in_sec = 10 + connect_timeout_in_sec = 10 concurrent_queries = 5 - connections = 5 + connections = 2 local_dc = "" page_size = 50 user = "dbuser" test_keyspace = "test_webconfig" is_ssl_enabled = true + // TCP keepalive interval in seconds (default: 30) + socket_keepalive_sec = 30 + + // Exponential reconnection policy settings. + // reconnect_max_retries: number of attempts before giving up (0 = never retry). + reconnect_initial_interval_ms = 2000 + reconnect_max_retries = 10 + reconnect_max_interval_sec = 60 + + // Disable initial host lookup (default: false). + // Set to true when Cassandra hosts are only accessible by IP and DNS is unavailable. + disable_initial_host_lookup = false + // TLS/SSL configuration (optional when is_ssl_enabled = true) // If tls block is not provided or incomplete, will use insecure TLS tls { @@ -196,6 +209,11 @@ webconfig { // Skip certificate verification (INSECURE - for testing only) // When true, allows TLS without certificates or CA validation insecure_skip_verify = false + + // Override SNI hostname sent during TLS handshake. + // Required when the certificate uses DNS SANs only (no IP SANs). + // Leave empty to use the host IP address (gocql default). + server_name = "" } } @@ -205,15 +223,28 @@ webconfig { "192.168.1.114" ] keyspace = "yugabytedb" - timeout_in_sec = 5 - connect_timeout_in_sec = 5 + timeout_in_sec = 10 + connect_timeout_in_sec = 10 concurrent_queries = 5 - connections = 5 + connections = 2 local_dc = "" page_size = 50 user = "dbuser" test_keyspace = "test_yugabytedb" + // TCP keepalive interval in seconds (default: 30) + socket_keepalive_sec = 30 + + // Exponential reconnection policy settings. + // reconnect_max_retries: number of attempts before giving up (0 = never retry). + reconnect_initial_interval_ms = 2000 + reconnect_max_retries = 10 + reconnect_max_interval_sec = 60 + + // Disable initial host lookup (default: false). + // Set to true when Cassandra hosts are only accessible by IP and DNS is unavailable. + disable_initial_host_lookup = false + // TODO change to false for CI/CD is_ssl_enabled = true @@ -230,6 +261,11 @@ webconfig { // Skip certificate verification (INSECURE - for testing only) // When true, allows TLS without certificates or CA validation insecure_skip_verify = false + + // Override SNI hostname sent during TLS handshake. + // Required when the certificate uses DNS SANs only (no IP SANs). + // Leave empty to use the host IP address (gocql default). + server_name = "" } } } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 675f388..0145d6e 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -36,13 +36,19 @@ import ( ) const ( - ProtocolVersion = 4 - DefaultKeyspace = "webconfig" - DefaultTestKeyspace = "test_webconfig" - DisableInitialHostLookup = false - DefaultSleepTimeInMillisecond = 10 - DefaultConnections = 2 - DefaultPageSize = 50 + ProtocolVersion = 4 + DefaultKeyspace = "webconfig" + DefaultTestKeyspace = "test_webconfig" + DefaultDisableInitialHostLookup = false + DefaultSleepTimeInMillisecond = 10 + DefaultConnections = 2 + DefaultPageSize = 50 + DefaultTimeoutSec = 10 + DefaultConnectTimeoutSec = 10 + DefaultSocketKeepaliveSec = 30 + DefaultReconnectInitialIntervalMs = 2000 + DefaultReconnectMaxRetries = 10 + DefaultReconnectMaxIntervalSec = 60 ) // if 'wifi_schema_v2_enabled'=true, v1.3 is also supported @@ -103,10 +109,11 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl cluster.Consistency = gocql.LocalQuorum cluster.ProtoVersion = ProtocolVersion - cluster.DisableInitialHostLookup = DisableInitialHostLookup - cluster.Timeout = time.Duration(dbconf.GetInt32("timeout_in_sec", 1)) * time.Second - cluster.ConnectTimeout = time.Duration(dbconf.GetInt32("connect_timeout_in_sec", 1)) * time.Second + cluster.DisableInitialHostLookup = dbconf.GetBoolean("disable_initial_host_lookup", DefaultDisableInitialHostLookup) + cluster.Timeout = time.Duration(dbconf.GetInt32("timeout_in_sec", DefaultTimeoutSec)) * time.Second + cluster.ConnectTimeout = time.Duration(dbconf.GetInt32("connect_timeout_in_sec", DefaultConnectTimeoutSec)) * time.Second cluster.NumConns = int(dbconf.GetInt32("connections", DefaultConnections)) + cluster.SocketKeepalive = time.Duration(dbconf.GetInt32("socket_keepalive_sec", DefaultSocketKeepaliveSec)) * time.Second cluster.RetryPolicy = &gocql.DowngradingConsistencyRetryPolicy{ ConsistencyLevelsToTry: []gocql.Consistency{ @@ -116,9 +123,20 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl }, } + reconnectIntervalMs := dbconf.GetInt32("reconnect_initial_interval_ms", DefaultReconnectInitialIntervalMs) + reconnectMaxRetries := int(dbconf.GetInt32("reconnect_max_retries", DefaultReconnectMaxRetries)) + reconnectMaxIntervalSec := dbconf.GetInt32("reconnect_max_interval_sec", DefaultReconnectMaxIntervalSec) + cluster.ReconnectionPolicy = &gocql.ExponentialReconnectionPolicy{ + InitialInterval: time.Duration(reconnectIntervalMs) * time.Millisecond, + MaxRetries: reconnectMaxRetries, + MaxInterval: time.Duration(reconnectMaxIntervalSec) * time.Second, + } + localDc := dbconf.GetString("local_dc") if len(localDc) > 0 { - cluster.PoolConfig.HostSelectionPolicy = gocql.DCAwareRoundRobinPolicy(localDc) + cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy( + gocql.DCAwareRoundRobinPolicy(localDc), + ) } var password string @@ -145,14 +163,15 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl } if isSslEnabled { - tlsConfig, err := loadCassandraTLSConfig(dbconf, dbdriver) + insecureSkipVerify := dbconf.GetBoolean("tls.insecure_skip_verify") + tlsConfig, err := loadCassandraTLSConfig(dbconf, dbdriver, insecureSkipVerify) if err != nil { return nil, common.NewError(err) } sslOpts := &gocql.SslOptions{ Config: tlsConfig, - EnableHostVerification: false, + EnableHostVerification: !insecureSkipVerify, } cluster.SslOpts = sslOpts @@ -197,20 +216,19 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl // loadCassandraTLSConfig loads TLS configuration for Cassandra connection. // Returns a tls.Config with certificates loaded from the configuration. // The function expects tls.{} block under the database driver config (cassandra or yugabyte). -func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string) (*tls.Config, error) { - // Check insecure_skip_verify flag first - insecureSkipVerify := dbconf.GetBoolean("tls.insecure_skip_verify") - +func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string, insecureSkipVerify bool) (*tls.Config, error) { // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) certFile := dbconf.GetString("tls.cert_file") keyFile := dbconf.GetString("tls.key_file") caCertFile := dbconf.GetString("tls.ca_cert_file") + serverName := dbconf.GetString("tls.server_name") // Create TLS config for Cassandra connection. // Prefer modern ECDHE+AEAD suites for forward secrecy; keep TLS_RSA_WITH_AES_128_CBC_SHA // last as a fallback for legacy Cassandra 3.11.x nodes that only negotiate that suite. tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, + ServerName: serverName, CipherSuites: []uint16{ tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, From 0fa473c1f1c6ad69687a46d51c2e271139507659 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 10 Aug 2026 15:46:13 -0700 Subject: [PATCH 04/14] replace the tls layer in config key path with flat key names --- config/sample_webconfig.conf | 66 +++++++++++++++----------------- db/cassandra/cassandra_client.go | 12 +++--- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 249ec43..c336ac4 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -197,24 +197,21 @@ webconfig { disable_initial_host_lookup = false // TLS/SSL configuration (optional when is_ssl_enabled = true) - // If tls block is not provided or incomplete, will use insecure TLS - tls { - // Client certificate for mutual TLS (mTLS) - optional - cert_file = "/path/to/client-cert.pem" - key_file = "/path/to/client-key.pem" - - // CA certificate for server verification - optional - ca_cert_file = "/path/to/ca-cert.pem" - - // Skip certificate verification (INSECURE - for testing only) - // When true, allows TLS without certificates or CA validation - insecure_skip_verify = false - - // Override SNI hostname sent during TLS handshake. - // Required when the certificate uses DNS SANs only (no IP SANs). - // Leave empty to use the host IP address (gocql default). - server_name = "" - } + // Client certificate for mutual TLS (mTLS) - optional + tls_cert_file = "/path/to/client-cert.pem" + tls_key_file = "/path/to/client-key.pem" + + // CA certificate for server verification - optional + tls_ca_cert_file = "/path/to/ca-cert.pem" + + // Skip certificate verification (INSECURE - for testing only) + // When true, allows TLS without certificates or CA validation + tls_insecure_skip_verify = false + + // Override SNI hostname sent during TLS handshake. + // Required when the certificate uses DNS SANs only (no IP SANs). + // Leave empty to use the host IP address (gocql default). + tls_server_name = "" } yugabyte { @@ -249,24 +246,21 @@ webconfig { is_ssl_enabled = true // TLS/SSL configuration (optional when is_ssl_enabled = true) - // If tls block is not provided or incomplete, will use insecure TLS - tls { - // Client certificate for mutual TLS (mTLS) - optional - cert_file = "/path/to/client-cert.pem" - key_file = "/path/to/client-key.pem" - - // CA certificate for server verification - optional - ca_cert_file = "/path/to/ca-cert.pem" - - // Skip certificate verification (INSECURE - for testing only) - // When true, allows TLS without certificates or CA validation - insecure_skip_verify = false - - // Override SNI hostname sent during TLS handshake. - // Required when the certificate uses DNS SANs only (no IP SANs). - // Leave empty to use the host IP address (gocql default). - server_name = "" - } + // Client certificate for mutual TLS (mTLS) - optional + tls_cert_file = "/path/to/client-cert.pem" + tls_key_file = "/path/to/client-key.pem" + + // CA certificate for server verification - optional + tls_ca_cert_file = "/path/to/ca-cert.pem" + + // Skip certificate verification (INSECURE - for testing only) + // When true, allows TLS without certificates or CA validation + tls_insecure_skip_verify = false + + // Override SNI hostname sent during TLS handshake. + // Required when the certificate uses DNS SANs only (no IP SANs). + // Leave empty to use the host IP address (gocql default). + tls_server_name = "" } } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 0145d6e..057f898 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -163,7 +163,7 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl } if isSslEnabled { - insecureSkipVerify := dbconf.GetBoolean("tls.insecure_skip_verify") + insecureSkipVerify := dbconf.GetBoolean("tls_insecure_skip_verify") tlsConfig, err := loadCassandraTLSConfig(dbconf, dbdriver, insecureSkipVerify) if err != nil { return nil, common.NewError(err) @@ -215,13 +215,13 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl // loadCassandraTLSConfig loads TLS configuration for Cassandra connection. // Returns a tls.Config with certificates loaded from the configuration. -// The function expects tls.{} block under the database driver config (cassandra or yugabyte). +// Reads flat tls_* keys directly under the database driver config (cassandra or yugabyte). func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string, insecureSkipVerify bool) (*tls.Config, error) { // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) - certFile := dbconf.GetString("tls.cert_file") - keyFile := dbconf.GetString("tls.key_file") - caCertFile := dbconf.GetString("tls.ca_cert_file") - serverName := dbconf.GetString("tls.server_name") + certFile := dbconf.GetString("tls_cert_file") + keyFile := dbconf.GetString("tls_key_file") + caCertFile := dbconf.GetString("tls_ca_cert_file") + serverName := dbconf.GetString("tls_server_name") // Create TLS config for Cassandra connection. // Prefer modern ECDHE+AEAD suites for forward secrecy; keep TLS_RSA_WITH_AES_128_CBC_SHA From 5f371603758c3d417e74b75f5d9ee8ca0ffa11d7 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 12 Aug 2026 23:03:40 -0700 Subject: [PATCH 05/14] Change the http status code from 500 to 504 when cassandra timeout --- db/cassandra/cassandra_client.go | 28 ++++ db/database_client.go | 3 + db/sqlite/sqlite_client.go | 4 + http/cassandra_timeout_test.go | 241 +++++++++++++++++++++++++++++++ http/document_handler.go | 28 ++-- http/multipart.go | 30 ++-- http/poke_handler.go | 6 +- http/refsubdocument_handler.go | 6 +- http/rootdocument_handler.go | 4 +- http/supplementary_handler.go | 4 +- http/webconfig_server.go | 9 ++ 11 files changed, 325 insertions(+), 38 deletions(-) create mode 100644 http/cassandra_timeout_test.go diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 057f898..fe2b218 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -18,6 +18,7 @@ package cassandra import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -343,6 +344,33 @@ func (c *CassandraClient) IsDbNotFound(err error) bool { return errors.Is(err, gocql.ErrNotFound) } +func (c *CassandraClient) IsDbTimeout(err error) bool { + if err == nil { + return false + } + if errors.Is(err, gocql.ErrTimeoutNoResponse) { + return true + } + if errors.Is(err, gocql.ErrConnectionClosed) { + return true + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + if errors.Is(err, context.Canceled) { + return true + } + var readTimeout *gocql.RequestErrReadTimeout + if errors.As(err, &readTimeout) { + return true + } + var writeTimeout *gocql.RequestErrWriteTimeout + if errors.As(err, &writeTimeout) { + return true + } + return false +} + func (c *CassandraClient) Close() error { c.Session.Close() return nil diff --git a/db/database_client.go b/db/database_client.go index 0548a81..f9d45ef 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -48,6 +48,9 @@ type DatabaseClient interface { // not found IsDbNotFound(error) bool + // timeout + IsDbTimeout(error) bool + // set metrics Metrics() *common.AppMetrics SetMetrics(*common.AppMetrics) diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 7b1b5d9..c425057 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -130,6 +130,10 @@ func (c *SqliteClient) IsDbNotFound(err error) bool { return false } +func (c *SqliteClient) IsDbTimeout(err error) bool { + return false +} + func (c *SqliteClient) Metrics() *common.AppMetrics { return c.AppMetrics } diff --git a/http/cassandra_timeout_test.go b/http/cassandra_timeout_test.go new file mode 100644 index 0000000..a02a433 --- /dev/null +++ b/http/cassandra_timeout_test.go @@ -0,0 +1,241 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/gocql/gocql" + "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/db/cassandra" + "github.com/rdkcentral/webconfig/util" + "gotest.tools/assert" +) + +// errSimulatedTimeout is a sentinel used by timeoutMockClient to trigger the 504 path. +var errSimulatedTimeout = errors.New("simulated cassandra timeout") + +// timeoutMockClient wraps the real (SQLite) DatabaseClient and overrides IsDbTimeout plus +// selected read methods to return errSimulatedTimeout, simulating a Cassandra timeout +// without a live Cassandra instance. Overridden methods: +// - IsDbTimeout — recognises errSimulatedTimeout as a timeout +// - GetSubDocument — used by GetSubDocumentHandler (GET /document/{id}) +// - GetRootDocumentLabels — used by PostSubDocumentHandler (POST /document/{id}) +// - GetRootDocument — used by BuildGetDocument (GET /config) +// - GetDocument — used by BuildGetDocument fallback paths +// +// All other interface methods (SetSubDocument, DeleteDocument, etc.) fall through to the +// embedded SQLite client and execute normally. +type timeoutMockClient struct { + db.DatabaseClient +} + +func (m *timeoutMockClient) IsDbTimeout(err error) bool { + return errors.Is(err, errSimulatedTimeout) +} + +func (m *timeoutMockClient) GetSubDocument(mac, subdocId string) (*common.SubDocument, error) { + return nil, errSimulatedTimeout +} + +func (m *timeoutMockClient) GetRootDocumentLabels(mac string) (prometheus.Labels, error) { + return nil, errSimulatedTimeout +} + +func (m *timeoutMockClient) GetRootDocument(mac string) (*common.RootDocument, error) { + return nil, errSimulatedTimeout +} + +func (m *timeoutMockClient) GetDocument(mac string, args ...interface{}) (*common.Document, error) { + return nil, errSimulatedTimeout +} + +// TestIsDbTimeout tests CassandraClient.IsDbTimeout against all gocql timeout error +// variants — including connection-closed and context deadline — as well as direct errors, +// wrapped errors, and multi-layer chains. Non-timeout errors must return false. +// No live Cassandra connection is required because IsDbTimeout is a pure error-inspection +// function. +func TestIsDbTimeout(t *testing.T) { + c := &cassandra.CassandraClient{} + + cases := []struct { + name string + err error + want bool + }{ + { + name: "ErrTimeoutNoResponse direct", + err: gocql.ErrTimeoutNoResponse, + want: true, + }, + { + name: "ErrTimeoutNoResponse wrapped once", + err: fmt.Errorf("layer: %w", gocql.ErrTimeoutNoResponse), + want: true, + }, + { + name: "ErrTimeoutNoResponse wrapped via common.NewError", + err: common.NewError(gocql.ErrTimeoutNoResponse), + want: true, + }, + { + name: "ErrTimeoutNoResponse wrapped multiple times", + err: fmt.Errorf("outer: %w", fmt.Errorf("inner: %w", gocql.ErrTimeoutNoResponse)), + want: true, + }, + { + name: "RequestErrReadTimeout direct", + err: &gocql.RequestErrReadTimeout{}, + want: true, + }, + { + name: "RequestErrReadTimeout wrapped", + err: fmt.Errorf("layer: %w", &gocql.RequestErrReadTimeout{}), + want: true, + }, + { + name: "RequestErrWriteTimeout direct", + err: &gocql.RequestErrWriteTimeout{}, + want: true, + }, + { + name: "RequestErrWriteTimeout wrapped", + err: fmt.Errorf("layer: %w", &gocql.RequestErrWriteTimeout{}), + want: true, + }, + { + name: "ErrConnectionClosed direct", + err: gocql.ErrConnectionClosed, + want: true, + }, + { + name: "ErrConnectionClosed wrapped", + err: fmt.Errorf("layer: %w", gocql.ErrConnectionClosed), + want: true, + }, + { + name: "context.DeadlineExceeded direct", + err: context.DeadlineExceeded, + want: true, + }, + { + name: "context.DeadlineExceeded wrapped", + err: fmt.Errorf("layer: %w", context.DeadlineExceeded), + want: true, + }, + { + name: "context.Canceled direct", + err: context.Canceled, + want: true, + }, + { + name: "context.Canceled wrapped", + err: fmt.Errorf("layer: %w", context.Canceled), + want: true, + }, + { + name: "ErrNotFound is not a timeout", + err: gocql.ErrNotFound, + want: false, + }, + { + name: "generic error is not a timeout", + err: errors.New("some db error"), + want: false, + }, + { + name: "nil is not a timeout", + err: nil, + want: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := c.IsDbTimeout(tc.err) + assert.Equal(t, got, tc.want) + }) + } +} + +// TestDbErrToStatus verifies that dbErrToStatus maps a timeout error to 504 and a +// non-timeout error to 500, using the mock client to drive IsDbTimeout. +func TestDbErrToStatus(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.DatabaseClient = &timeoutMockClient{DatabaseClient: server.DatabaseClient} + + assert.Equal(t, server.dbErrToStatus(errSimulatedTimeout), http.StatusGatewayTimeout) + assert.Equal(t, server.dbErrToStatus(errors.New("other error")), http.StatusInternalServerError) +} + +// TestGetSubDocumentHandlerCassandraTimeout verifies that GET /document/{id} returns 504 +// when the database layer reports a Cassandra timeout on GetSubDocument. +func TestGetSubDocumentHandlerCassandraTimeout(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.DatabaseClient = &timeoutMockClient{DatabaseClient: server.DatabaseClient} + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + url := fmt.Sprintf("/api/v1/device/%v/document/lan", cpeMac) + req, err := http.NewRequest("GET", url, nil) + assert.NilError(t, err) + + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} + +// TestPostSubDocumentHandlerCassandraTimeout verifies that POST /document/{id} returns 504 +// when the database layer reports a Cassandra timeout on GetRootDocumentLabels. +func TestPostSubDocumentHandlerCassandraTimeout(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.DatabaseClient = &timeoutMockClient{DatabaseClient: server.DatabaseClient} + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + url := fmt.Sprintf("/api/v1/device/%v/document/lan", cpeMac) + req, err := http.NewRequest("POST", url, bytes.NewReader([]byte{0x80})) + assert.NilError(t, err) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} + +// TestMultipartConfigHandlerCassandraTimeout verifies that GET /config returns 504 when +// the database layer reports a Cassandra timeout during document retrieval. +func TestMultipartConfigHandlerCassandraTimeout(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.DatabaseClient = &timeoutMockClient{DatabaseClient: server.DatabaseClient} + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + url := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", url, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderSchemaVersion, "none") + + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} diff --git a/http/document_handler.go b/http/document_handler.go index e833c8a..210e15c 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -79,7 +79,7 @@ func (s *WebconfigServer) GetSubDocumentHandler(w http.ResponseWriter, r *http.R Error(w, http.StatusNotFound, nil) } else { LogError(w, err) - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) } return } @@ -169,14 +169,14 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. labels, err := s.GetRootDocumentLabels(deviceId) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } labels["client"] = metricsAgent err = s.SetSubDocument(deviceId, subdocId, subdoc, oldState, labels, fields) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } @@ -187,7 +187,7 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. if s.IsDbNotFound(err) { doc = common.NewDocument(nil) } else { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } } @@ -196,7 +196,7 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. newRootVersion = db.HashRootVersion(doc.VersionMap()) err = s.SetRootDocumentVersion(deviceId, newRootVersion) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } rootVersionMap[deviceId] = newRootVersion @@ -233,7 +233,7 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt if s.IsDbNotFound(err) { Error(w, http.StatusNotFound, nil) } else { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) } return } @@ -245,11 +245,12 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt if s.IsDbNotFound(err) { err := s.DeleteRootDocumentVersion(mac) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) + return } WriteOkResponse(w, nil) } else { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) } return } @@ -257,7 +258,7 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt newRootVersion := db.HashRootVersion(doc.VersionMap()) err = s.SetRootDocumentVersion(mac, newRootVersion) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } @@ -290,7 +291,7 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R if s.IsDbNotFound(err) { Error(w, http.StatusNotFound, nil) } else { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) } return } @@ -302,11 +303,12 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R if s.IsDbNotFound(err) { err := s.DeleteRootDocumentVersion(mac) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) + return } WriteOkResponse(w, nil) } else { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) } return } @@ -314,7 +316,7 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R newRootVersion := db.HashRootVersion(doc.VersionMap()) err = s.SetRootDocumentVersion(mac, newRootVersion) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } diff --git a/http/multipart.go b/http/multipart.go index bd154ba..d431139 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -147,7 +147,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } if err != nil { if !s.IsDbNotFound(err) { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } return http.StatusNotFound, respHeader, nil, common.NewError(err) } @@ -164,7 +164,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document, err = db.LoadRefSubDocuments(c, document, fields) if err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } if s.FilterOutputByBitmapEnabled() { @@ -183,7 +183,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // skip updating states if userAgent != "mget" { if err := db.UpdateDocumentStateIndeployment(c, mac, document, fields); err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } } @@ -194,7 +194,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if err != nil { if !s.IsDbNotFound(err) { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } // 404 if !postUpstream { @@ -216,7 +216,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if !postUpstream { document, err = db.LoadRefSubDocuments(c, document, fields) if err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } } @@ -243,7 +243,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if !postUpstream { // update states to InDeployment before the final response if err := db.UpdateDocumentStateIndeployment(c, mac, document, fields); err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } // 304 @@ -305,7 +305,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if errors.As(err, &rherr) { return rherr.StatusCode, respHeader, nil, common.NewError(err) } - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } // ==== parse the upstreamRespBytes and store them ==== @@ -331,7 +331,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // update states based on the final document err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + return s.dbErrToStatus(err), upstreamRespHeader, upstreamRespBytes, common.NewError(err) } } @@ -351,7 +351,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalFilteredDocument, err = db.LoadRefSubDocuments(c, finalFilteredDocument, fields) if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), upstreamRespHeader, nil, common.NewError(err) } finalFilteredBytes, err := finalFilteredDocument.Bytes() if err != nil { @@ -375,7 +375,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l rootDocument, err := db.PreprocessRootDocument(c, rHeader, mac, partnerId, fields) if err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } document, err := c.GetDocument(mac, fields) @@ -383,7 +383,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l if s.IsDbNotFound(err) { return http.StatusNotFound, respHeader, nil, nil } else { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } } if document == nil { @@ -400,7 +400,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l if !s.UpstreamEnabled() { err := c.DeleteDocument(mac) if err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } return http.StatusNotFound, respHeader, nil, nil } @@ -437,7 +437,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l if errors.As(err, &rherr) { return rherr.StatusCode, respHeader, nil, common.NewError(err) } - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), respHeader, nil, common.NewError(err) } // ==== parse the upstreamRespBytes and store them ==== @@ -460,7 +460,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // update states based on the final document err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + return s.dbErrToStatus(err), upstreamRespHeader, upstreamRespBytes, common.NewError(err) } } @@ -470,7 +470,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l finalDocument, err = db.LoadRefSubDocuments(c, finalDocument, fields) if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + return s.dbErrToStatus(err), upstreamRespHeader, nil, common.NewError(err) } // filter by bitmaps and blockedIds diff --git a/http/poke_handler.go b/http/poke_handler.go index 266023d..7ab2a5a 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -89,7 +89,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { Error(w, http.StatusNotFound, nil) return } - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } if document.Length() == 0 { @@ -118,7 +118,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { err = db.UpdateStatesInBatch(s.DatabaseClient, deviceId, metricsAgent, fields, document.StateMap()) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } } @@ -134,7 +134,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { Error(w, http.StatusNoContent, nil) return } - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } if document.Length() == 0 { diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go index cd8743d..1cd0ed0 100644 --- a/http/refsubdocument_handler.go +++ b/http/refsubdocument_handler.go @@ -48,7 +48,7 @@ func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *htt Error(w, http.StatusNotFound, nil) } else { LogError(w, err) - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) } return } @@ -88,7 +88,7 @@ func (s *WebconfigServer) PostRefSubDocumentHandler(w http.ResponseWriter, r *ht err = s.SetRefSubDocument(refId, refsubdoc) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } @@ -117,7 +117,7 @@ func (s *WebconfigServer) DeleteRefSubDocumentHandler(w http.ResponseWriter, r * if s.IsDbNotFound(err) { Error(w, http.StatusNotFound, nil) } else { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) } return } diff --git a/http/rootdocument_handler.go b/http/rootdocument_handler.go index 2d4a5e3..8d3b396 100644 --- a/http/rootdocument_handler.go +++ b/http/rootdocument_handler.go @@ -48,7 +48,7 @@ func (s *WebconfigServer) GetRootDocumentHandler(w http.ResponseWriter, r *http. Error(w, http.StatusNotFound, nil) return } - Error(w, http.StatusInternalServerError, err) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } @@ -98,7 +98,7 @@ func (s *WebconfigServer) PostRootDocumentHandler(w http.ResponseWriter, r *http } err = s.SetRootDocument(mac, rootdoc) if err != nil { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 5561b85..079d7fe 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -59,7 +59,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r // Check state from xpc_group_config by cpe_mac and group_id=telemetry telemetrySubdoc, err := s.GetSubDocument(mac, "telemetry") if err != nil && !s.IsDbNotFound(err) { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } @@ -156,7 +156,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r rootdoc, err = s.GetRootDocument(mac) if err != nil { if !s.IsDbNotFound(err) { - Error(w, http.StatusInternalServerError, common.NewError(err)) + Error(w, s.dbErrToStatus(err), common.NewError(err)) return } } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index dbbbe0b..5ba1d9f 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1311,3 +1311,12 @@ func (s *WebconfigServer) SpanMiddleware(next http.Handler) http.Handler { next.ServeHTTP(w, r) }) } + +// dbErrToStatus maps a database error to its HTTP status code. +// Cassandra timeouts become 504; all other errors become 500. +func (s *WebconfigServer) dbErrToStatus(err error) int { + if s.IsDbTimeout(err) { + return http.StatusGatewayTimeout + } + return http.StatusInternalServerError +} From 987f0e9b1b5a87c59f8c13a767fb76fcb34fe0eb Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 12 Aug 2026 23:31:18 -0700 Subject: [PATCH 06/14] context cancelled should not be considered as timeout-like error and return 504 --- db/cassandra/cassandra_client.go | 5 ++--- http/cassandra_timeout_test.go | 4 ++-- http/webconfig_server.go | 5 +++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index fe2b218..8ceed76 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -357,9 +357,8 @@ func (c *CassandraClient) IsDbTimeout(err error) bool { if errors.Is(err, context.DeadlineExceeded) { return true } - if errors.Is(err, context.Canceled) { - return true - } + // context.Canceled is excluded: it indicates caller cancellation (client disconnect), + // not a DB/network timeout. Mapping it to 504 would misrepresent the failure cause. var readTimeout *gocql.RequestErrReadTimeout if errors.As(err, &readTimeout) { return true diff --git a/http/cassandra_timeout_test.go b/http/cassandra_timeout_test.go index a02a433..87b1b8f 100644 --- a/http/cassandra_timeout_test.go +++ b/http/cassandra_timeout_test.go @@ -148,12 +148,12 @@ func TestIsDbTimeout(t *testing.T) { { name: "context.Canceled direct", err: context.Canceled, - want: true, + want: false, }, { name: "context.Canceled wrapped", err: fmt.Errorf("layer: %w", context.Canceled), - want: true, + want: false, }, { name: "ErrNotFound is not a timeout", diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 5ba1d9f..dcdaf30 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1312,8 +1312,9 @@ func (s *WebconfigServer) SpanMiddleware(next http.Handler) http.Handler { }) } -// dbErrToStatus maps a database error to its HTTP status code. -// Cassandra timeouts become 504; all other errors become 500. +// dbErrToStatus maps a dependency error to its HTTP status code. +// Timeout-like errors (Cassandra timeouts, connection closed, deadline exceeded) become 504; +// all other errors become 500. func (s *WebconfigServer) dbErrToStatus(err error) int { if s.IsDbTimeout(err) { return http.StatusGatewayTimeout From 3a709b5794b80c8bb9702431dd602bbae2d5cbbc Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 18 Aug 2026 00:35:16 -0700 Subject: [PATCH 07/14] Add token auth for /config api --- config/sample_webconfig.conf | 7 +++++++ http/router.go | 3 +++ http/webconfig_server.go | 12 ++++++++++++ http/webconfig_server_test.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index c336ac4..04eebcb 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -155,6 +155,13 @@ webconfig { server_api_token_auth { enabled = true } + + // Protect /config with API token authentication. + // Defaults to false to preserve existing endpoint access. + config_api_token_auth { + enabled = false + } + device_api_token_auth { enabled = true } diff --git a/http/router.go b/http/router.go index 103c77a..d3e621c 100644 --- a/http/router.go +++ b/http/router.go @@ -32,6 +32,9 @@ func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { r2.HandleFunc("", s.VersionHandler).Methods("GET") r3 := router.Path("/config").Subrouter() + if s.ConfigApiTokenAuthEnabled() { + r3.Use(s.ApiMiddleware) + } r3.HandleFunc("", s.ServerConfigHandler).Methods("GET") if s.TokenApiEnabled() { diff --git a/http/webconfig_server.go b/http/webconfig_server.go index dcdaf30..3ccd629 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -64,6 +64,7 @@ const ( MetricsEnabledDefault = true FactoryResetEnabledDefault = false serverApiTokenAuthEnabledDefault = true + configApiTokenAuthEnabledDefault = false deviceApiTokenAuthEnabledDefault = true tokenApiEnabledDefault = false activeDriverDefault = "cassandra" @@ -109,6 +110,7 @@ type WebconfigServer struct { metricsEnabled bool factoryResetEnabled bool serverApiTokenAuthEnabled bool + configApiTokenAuthEnabled bool deviceApiTokenAuthEnabled bool tokenApiEnabled bool kafkaEnabled bool @@ -270,6 +272,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer tlsConfig, _ := NewTlsConfig(conf) serverApiTokenAuthEnabled := conf.GetBoolean("webconfig.jwt.server_api_token_auth.enabled", serverApiTokenAuthEnabledDefault) + configApiTokenAuthEnabled := conf.GetBoolean("webconfig.jwt.config_api_token_auth.enabled", configApiTokenAuthEnabledDefault) if conf.GetNode("webconfig.jwt.server_api_token_auth.enabled") == nil { log.Warn("webconfig.jwt.server_api_token_auth.enabled is not set in config; defaulting to true (server API token auth enforced). See MIGRATION.md.") } @@ -399,6 +402,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer metricsEnabled: metricsEnabled, factoryResetEnabled: factoryResetEnabled, serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, + configApiTokenAuthEnabled: configApiTokenAuthEnabled, deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, tokenApiEnabled: tokenApiEnabled, kafkaEnabled: kafkaEnabled, @@ -642,6 +646,14 @@ func (s *WebconfigServer) SetServerApiTokenAuthEnabled(enabled bool) { s.serverApiTokenAuthEnabled = enabled } +func (s *WebconfigServer) ConfigApiTokenAuthEnabled() bool { + return s.configApiTokenAuthEnabled +} + +func (s *WebconfigServer) SetConfigApiTokenAuthEnabled(enabled bool) { + s.configApiTokenAuthEnabled = enabled +} + func (s *WebconfigServer) DeviceApiTokenAuthEnabled() bool { return s.deviceApiTokenAuthEnabled } diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index bc57955..52d093f 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -18,6 +18,7 @@ package http import ( + "net/http" "testing" "gotest.tools/assert" @@ -27,9 +28,33 @@ func TestApiTokenAuthSecureDefaults(t *testing.T) { // Admin write endpoints (document, rootdocument, poke, reference) MUST // be guarded by ApiMiddleware out of the box. Regression guard for f003. assert.Assert(t, serverApiTokenAuthEnabledDefault, "server API token auth must default to enabled") + assert.Assert(t, !configApiTokenAuthEnabledDefault, "config API token auth must default to disabled") assert.Assert(t, deviceApiTokenAuthEnabledDefault, "device API token auth must default to enabled") } +func TestConfigEndpointRemainsUnauthenticatedByDefault(t *testing.T) { + server := NewWebconfigServer(sc, true) + assert.Assert(t, !server.ConfigApiTokenAuthEnabled()) + router := server.GetRouter(false) + + req, err := http.NewRequest("GET", "/config", nil) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusOK) +} + +func TestConfigEndpointRequiresApiTokenWhenEnabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetConfigApiTokenAuthEnabled(true) + assert.Assert(t, server.ConfigApiTokenAuthEnabled()) + router := server.GetRouter(false) + + req, err := http.NewRequest("GET", "/config", nil) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) +} + func TestWebconfigServerSetterGetter(t *testing.T) { server := NewWebconfigServer(sc, true) @@ -49,6 +74,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetServerApiTokenAuthEnabled(enabled) assert.Equal(t, server.ServerApiTokenAuthEnabled(), enabled) + // config api token auth + enabled = true + server.SetConfigApiTokenAuthEnabled(enabled) + assert.Equal(t, server.ConfigApiTokenAuthEnabled(), enabled) + enabled = false + server.SetConfigApiTokenAuthEnabled(enabled) + assert.Equal(t, server.ConfigApiTokenAuthEnabled(), enabled) + // device api token auth enabled = true server.SetDeviceApiTokenAuthEnabled(enabled) From a8ebc616952456cd32c5a466c740ee86c6b09648 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 18 Aug 2026 12:10:36 -0700 Subject: [PATCH 08/14] skip logging for /config api --- http/webconfig_server.go | 12 ++++++++++-- http/webconfig_server_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 3ccd629..0da72bf 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -613,6 +613,10 @@ func (s *WebconfigServer) TestingCpeMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(fn) } +func isConfigEndpoint(r *http.Request) bool { + return r.URL.Path == "/config" +} + func (s *WebconfigServer) VerifyApiToken(tokenStr string) (bool, error) { if s.JwksEnabled() { if _, err := s.JwksManager.VerifyApiToken(tokenStr); err != nil { @@ -970,7 +974,9 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("Request started") + if !isConfigEndpoint(r) { + log.WithFields(tfields).Info("Request started") + } xwriter.LogDebug(r, "tracing", fmt.Sprintf("Trace final out_traceparent %s out_traceState %s", xpcTrace.OutTraceparent, xpcTrace.OutTracestate)) return xwriter @@ -1058,7 +1064,9 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { s.XpcTracer.SetSpan(fields, s.XpcTracer.MoracideTagPrefix()) tfields := common.FilterLogFields(fields) - log.WithFields(tfields).Info("Request finished") + if !isConfigEndpoint(r) { + log.WithFields(tfields).Info("Request finished") + } } func LogError(w http.ResponseWriter, err error) { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 52d093f..92d3e1a 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -19,8 +19,10 @@ package http import ( "net/http" + "strings" "testing" + log "github.com/sirupsen/logrus" "gotest.tools/assert" ) @@ -55,6 +57,39 @@ func TestConfigEndpointRequiresApiTokenWhenEnabled(t *testing.T) { assert.Equal(t, res.StatusCode, http.StatusForbidden) } +func TestApiMiddlewareSuppressesConfigRequestLogs(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetConfigApiTokenAuthEnabled(true) + router := server.GetRouter(false) + + var logs strings.Builder + previousOutput := log.StandardLogger().Out + previousLevel := log.StandardLogger().Level + log.SetOutput(&logs) + log.SetLevel(log.InfoLevel) + defer func() { + log.SetOutput(previousOutput) + log.SetLevel(previousLevel) + }() + + req, err := http.NewRequest("GET", "/config", nil) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + assert.Assert(t, !strings.Contains(logs.String(), "Request started")) + assert.Assert(t, !strings.Contains(logs.String(), "Request finished")) + + logs.Reset() + req, err = http.NewRequest("GET", "/other", nil) + assert.NilError(t, err) + res = ExecuteRequest(req, server.ApiMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + assert.Assert(t, strings.Contains(logs.String(), "Request started")) + assert.Assert(t, strings.Contains(logs.String(), "Request finished")) +} + func TestWebconfigServerSetterGetter(t *testing.T) { server := NewWebconfigServer(sc, true) From b2c12573c3ea8cb21c5f52be46bfdc8639fde0f6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 27 Aug 2026 00:34:53 -0700 Subject: [PATCH 09/14] return proper status code 401/403 for various token auth situations --- common/error.go | 1 + http/http_client.go | 2 +- http/multipart_test.go | 16 +- http/response.go | 2 +- http/webconfig_server.go | 31 ++- http/webconfig_server_test.go | 494 +++++++++++++++++++++++++++++++++- security/jwks_manager.go | 22 +- security/jwks_manager_test.go | 171 ++++++++++++ security/token.go | 101 +++---- security/token_test.go | 130 +++++++++ 10 files changed, 908 insertions(+), 62 deletions(-) create mode 100644 security/jwks_manager_test.go diff --git a/common/error.go b/common/error.go index 880ea08..959f2d6 100644 --- a/common/error.go +++ b/common/error.go @@ -36,6 +36,7 @@ var ( ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") ErrLowTrust = fmt.Errorf("token trust is lower than threshold") ErrPending = fmt.Errorf("application_status pending") + ErrNoCapabilities = NoCapabilitiesError{} ) type Http400Error struct { diff --git a/http/http_client.go b/http/http_client.go index 2761f9c..5d56575 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -349,7 +349,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] } switch res.StatusCode { - case http.StatusForbidden, http.StatusBadRequest, http.StatusNotFound: + case http.StatusUnauthorized, http.StatusForbidden, http.StatusBadRequest, http.StatusNotFound: return rbytes, nil, false, common.NewError(err) } return rbytes, nil, true, common.NewError(err) diff --git a/http/multipart_test.go b/http/multipart_test.go index 775c1e7..ad66c4e 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -256,7 +256,7 @@ func TestCpeMiddleware(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusForbidden) + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) // get a token token := server1.Generate(cpeMac, 86400) @@ -269,6 +269,20 @@ func TestCpeMiddleware(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) + // XPC-43727 a real signed token for one cpeMac presented against a + // different mac's /config URL must be rejected as unauthorized, not + // treated as an authorization (403) failure + otherCpeMac := util.GenerateRandomCpeMac() + otherConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", otherCpeMac) + req, err = http.NewRequest("GET", otherConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) + // change the min trust to 1000 server1.SetMinTrust(1000) assert.Equal(t, 1000, server1.MinTrust()) diff --git a/http/response.go b/http/response.go index baebe01..e4895ac 100644 --- a/http/response.go +++ b/http/response.go @@ -142,7 +142,7 @@ func Error(w http.ResponseWriter, status int, err error) { // calling WriteHeader() multiple times will cause errors in common.HeaderContentType // ==> errors like 'superfluous response.WriteHeader call' in stderr switch status { - case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: + case http.StatusNoContent, http.StatusNotModified, http.StatusUnauthorized, http.StatusForbidden: w.WriteHeader(status) case http.StatusAccepted: SetAuditValue(w, "response", err) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0da72bf..5277cd1 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -489,7 +489,8 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { params := mux.Vars(r) mac, ok := params["mac"] if !ok { - Error(xw, http.StatusForbidden, nil) + err := *common.NewHttp400Error("missing mac") + Error(xw, http.StatusBadRequest, common.NewError(err)) return } mac = strings.ToUpper(mac) @@ -534,7 +535,11 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { next.ServeHTTP(xw, r) } else { s.LogToken(xw, authorization, token, tokenErr) - Error(xw, http.StatusForbidden, nil) + if errors.Is(tokenErr, common.ErrNoCapabilities) || errors.Is(tokenErr, common.ErrLowTrust) { + Error(xw, http.StatusForbidden, nil) + } else { + Error(xw, http.StatusUnauthorized, nil) + } } } return http.HandlerFunc(fn) @@ -547,6 +552,7 @@ func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { defer s.logRequestEnds(xw, r) isValid := false + var verifyErr error token := xw.Token() if len(token) > 0 { var kid string @@ -562,6 +568,7 @@ func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { isValid = true log.WithFields(tfields).Debug("valid") } else { + verifyErr = err tfields["error"] = fmt.Sprintf("ApiMiddleware::VerifyApiToken() %v", err) log.WithFields(tfields).Debug("rejected") } @@ -572,7 +579,11 @@ func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { - Error(xw, http.StatusForbidden, nil) + if errors.Is(verifyErr, common.ErrNoCapabilities) { + Error(xw, http.StatusForbidden, nil) + } else { + Error(xw, http.StatusUnauthorized, nil) + } } } return http.HandlerFunc(fn) @@ -591,23 +602,31 @@ func (s *WebconfigServer) TestingCpeMiddleware(next http.Handler) http.Handler { } isValid := false + var verifyErr error if len(token) > 0 { params := mux.Vars(r) mac, ok := params["mac"] if !ok || len(mac) != 12 { - Error(xw, http.StatusForbidden, nil) + err := *common.NewHttp400Error("invalid mac") + Error(xw, http.StatusBadRequest, common.NewError(err)) return } - if ok, _, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, _, _, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true + } else { + verifyErr = err } } if isValid { next.ServeHTTP(xw, r) } else { - Error(xw, http.StatusForbidden, nil) + if errors.Is(verifyErr, common.ErrNoCapabilities) { + Error(xw, http.StatusForbidden, nil) + } else { + Error(xw, http.StatusUnauthorized, nil) + } } } return http.HandlerFunc(fn) diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 92d3e1a..6b611d4 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -18,10 +18,23 @@ package http import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" "net/http" + "net/http/httptest" "strings" "testing" + "time" + "github.com/MicahParks/keyfunc/v2" + "github.com/golang-jwt/jwt/v5" + "github.com/gorilla/mux" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/security" log "github.com/sirupsen/logrus" "gotest.tools/assert" ) @@ -54,7 +67,7 @@ func TestConfigEndpointRequiresApiTokenWhenEnabled(t *testing.T) { req, err := http.NewRequest("GET", "/config", nil) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - assert.Equal(t, res.StatusCode, http.StatusForbidden) + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) } func TestApiMiddlewareSuppressesConfigRequestLogs(t *testing.T) { @@ -75,7 +88,7 @@ func TestApiMiddlewareSuppressesConfigRequestLogs(t *testing.T) { req, err := http.NewRequest("GET", "/config", nil) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - assert.Equal(t, res.StatusCode, http.StatusForbidden) + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) assert.Assert(t, !strings.Contains(logs.String(), "Request started")) assert.Assert(t, !strings.Contains(logs.String(), "Request finished")) @@ -85,11 +98,486 @@ func TestApiMiddlewareSuppressesConfigRequestLogs(t *testing.T) { res = ExecuteRequest(req, server.ApiMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }))).Result() - assert.Equal(t, res.StatusCode, http.StatusForbidden) + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) assert.Assert(t, strings.Contains(logs.String(), "Request started")) assert.Assert(t, strings.Contains(logs.String(), "Request finished")) } +func TestApiMiddlewareReturnsUnauthorizedForAuthenticationFailures(t *testing.T) { + tests := []struct { + name string + auth string + err error + }{ + {name: "missing token"}, + {name: "malformed authorization", auth: "Basic credentials"}, + {name: "bearer with no token", auth: "Bearer"}, + {name: "bearer with multiple parts", auth: "Bearer a b"}, + {name: "malformed token", auth: "Bearer malformed", err: fmt.Errorf("malformed token")}, + {name: "invalid token", auth: "Bearer invalid", err: fmt.Errorf("invalid token")}, + {name: "expired token", auth: "Bearer expired", err: fmt.Errorf("token is expired")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return false, "", 0, tt.err + }) + + req, err := http.NewRequest("GET", "/", nil) + assert.NilError(t, err) + if tt.auth != "" { + req.Header.Set("Authorization", tt.auth) + } + res := ExecuteRequest(req, server.ApiMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authentication failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) + }) + } +} + +func TestApiMiddlewareReturnsForbiddenForInsufficientCapabilities(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return false, "", 0, common.ErrNoCapabilities + }) + + req, err := http.NewRequest("GET", "/", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer authenticated-but-not-capable") + res := ExecuteRequest(req, server.ApiMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authorization failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) +} + +// XPC-43727 an actually malformed JWT (not a mocked verifier error) must +// still be classified as 401 through the real TokenManager.VerifyToken path +func TestApiMiddlewareRejectsActualMalformedJwt(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + + req, err := http.NewRequest("GET", "/", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer this-is-not-a-jwt") + res := ExecuteRequest(req, server.ApiMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("malformed jwt reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + +func newJwksHttpTestFixture(t *testing.T) (*rsa.PrivateKey, *keyfunc.JWKS) { + t.Helper() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + assert.NilError(t, err) + + const kid = "jwks-http-test-key" + b64 := func(b []byte) string { + return base64.RawURLEncoding.EncodeToString(b) + } + eBytes := big.NewInt(int64(privateKey.PublicKey.E)).Bytes() + jwkJSON := fmt.Sprintf(`{"keys":[{"kty":"RSA","kid":%q,"alg":"RS256","n":%q,"e":%q}]}`, + kid, b64(privateKey.PublicKey.N.Bytes()), b64(eBytes)) + + jwks, err := keyfunc.NewJSON(json.RawMessage(jwkJSON)) + assert.NilError(t, err) + return privateKey, jwks +} + +func signJwksHttpTestToken(t *testing.T, privateKey *rsa.PrivateKey, claims jwt.MapClaims) string { + t.Helper() + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + token.Header["kid"] = "jwks-http-test-key" + tokenString, err := token.SignedString(privateKey) + assert.NilError(t, err) + return tokenString +} + +// XPC-43727 exercise the ApiMiddleware route actually used when JWKS is +// enabled (server.JwksManager / errors.Is classification via VerifyApiToken), +// not just the underlying JwksManager in isolation +func TestApiMiddlewareJwksEnabledRoute(t *testing.T) { + privateKey, jwks := newJwksHttpTestFixture(t) + utcnow := time.Now() + + tests := []struct { + name string + token string + status int + }{ + { + name: "valid capability", + token: signJwksHttpTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"webconfig:all"}, + "exp": utcnow.Add(time.Hour).Unix(), + }), + status: http.StatusOK, + }, + { + name: "missing capability", + token: signJwksHttpTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"some:other:capability"}, + "exp": utcnow.Add(time.Hour).Unix(), + }), + status: http.StatusForbidden, + }, + { + name: "expired token", + token: signJwksHttpTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"webconfig:all"}, + "exp": utcnow.Add(-time.Hour).Unix(), + }), + status: http.StatusUnauthorized, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.JwksManager = security.NewJwksManagerFromJWKS(jwks, []string{"webconfig:all"}) + server.SetJwksEnabled(true) + + req, err := http.NewRequest("GET", "/", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", tt.token)) + res := ExecuteRequest(req, server.ApiMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tt.status != http.StatusOK { + t.Fatal("authentication/authorization failure reached the next handler") + } + w.WriteHeader(http.StatusOK) + }))).Result() + assert.Equal(t, res.StatusCode, tt.status) + }) + } +} + +// XPC-43727 an invalid-signature token must return 401 through the JWKS +// route, never misclassified as 403 +func TestApiMiddlewareJwksEnabledRouteRejectsInvalidSignature(t *testing.T) { + privateKey, jwks := newJwksHttpTestFixture(t) + utcnow := time.Now() + token := signJwksHttpTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"webconfig:all"}, + "exp": utcnow.Add(time.Hour).Unix(), + }) + + parts := []rune(token) + lastDot := -1 + for i, r := range parts { + if r == '.' { + lastDot = i + } + } + assert.Assert(t, lastDot >= 0 && lastDot+2 < len(parts)) + mid := lastDot + 1 + (len(parts)-lastDot-1)/2 + if parts[mid] == 'A' { + parts[mid] = 'B' + } else { + parts[mid] = 'A' + } + tampered := string(parts) + + server := NewWebconfigServer(sc, true) + server.JwksManager = security.NewJwksManagerFromJWKS(jwks, []string{"webconfig:all"}) + server.SetJwksEnabled(true) + + req, err := http.NewRequest("GET", "/", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", tampered)) + res := ExecuteRequest(req, server.ApiMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("invalid signature reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + +func TestCpeMiddlewareReturnsUnauthorizedForInvalidToken(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return false, "", 0, fmt.Errorf("invalid token") + }) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer invalid") + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.CpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authentication failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + +func TestCpeMiddlewareReturnsForbiddenForInsufficientCapabilities(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return false, "", 0, common.ErrNoCapabilities + }) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer authenticated-but-not-capable") + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.CpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authorization failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) +} + +func TestCpeMiddlewareReturnsForbiddenForLowTrust(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetMinTrust(1000) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return true, "comcast", 0, nil + }) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer authenticated-but-low-trust") + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.CpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authorization failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) +} + +// XPC-43727 every class of authentication failure (malformed header, +// malformed/invalid/expired token, mac mismatch) must return 401, never 403. +func TestCpeMiddlewareReturnsUnauthorizedForAuthenticationFailures(t *testing.T) { + tests := []struct { + name string + auth string + err error + }{ + {name: "missing token"}, + {name: "malformed authorization", auth: "Basic credentials"}, + {name: "bearer with no token", auth: "Bearer"}, + {name: "bearer with multiple parts", auth: "Bearer a b"}, + {name: "malformed token", auth: "Bearer malformed", err: fmt.Errorf("malformed token")}, + {name: "invalid signature", auth: "Bearer invalid", err: fmt.Errorf("token signature is invalid")}, + {name: "expired token", auth: "Bearer expired", err: fmt.Errorf("token is expired")}, + {name: "mac mismatch", auth: "Bearer mismatched-mac", err: fmt.Errorf("mac in token does not match claims")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return false, "", 0, tt.err + }) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + if tt.auth != "" { + req.Header.Set("Authorization", tt.auth) + } + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.CpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authentication failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) + }) + } +} + +func TestCpeMiddlewareReturnsBadRequestForMissingMacRouteVar(t *testing.T) { + server := NewWebconfigServer(sc, true) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer irrelevant") + res := ExecuteRequest(req, server.CpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("missing route var reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) +} + +// XPC-43727 an actually malformed JWT (not a mocked verifier error) must +// still be classified as 401 through the real TokenManager.VerifyToken path +func TestCpeMiddlewareRejectsActualMalformedJwt(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer this-is-not-a-jwt") + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.CpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("malformed jwt reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + +// XPC-43727 a real signed token for one mac presented against a different +// mac must be rejected as 401, using an in-memory generated key pair so this +// assertion is isolated from external key files/db setup (unlike the +// equivalent case in multipart_test.go's TestCpeMiddleware) +func TestCpeMiddlewareRejectsRealSignedTokenWithWrongMac(t *testing.T) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + assert.NilError(t, err) + + const kid = "cpe-wrong-mac-test-key" + decodeKeys := map[string]*rsa.PublicKey{kid: &privateKey.PublicKey} + + sign := func(mac string) string { + token := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "mac": strings.ToLower(mac), + "exp": time.Now().Add(time.Hour).Unix(), + }) + token.Header["kid"] = kid + tokenString, err := token.SignedString(privateKey) + assert.NilError(t, err) + return tokenString + } + token := sign("AABBCCDDEEFF") + + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, int, error) { + return security.VerifyToken(decodeKeys, []string{kid}, requiredCapabilities, vargs...) + }) + + req, err := http.NewRequest("GET", "/api/v1/device/112233445566/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) + req = mux.SetURLVars(req, map[string]string{"mac": "112233445566"}) + res := ExecuteRequest(req, server.CpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("wrong-mac token reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + +func TestTestingCpeMiddlewareReturnsUnauthorizedForMissingToken(t *testing.T) { + server := NewWebconfigServer(sc, true) + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.TestingCpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authentication failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + +// XPC-43727 TestingCpeMiddleware independently parses the Authorization +// header and calls VerifyCpeToken directly; verify it classifies the same +// authentication failure classes as CpeMiddleware, despite the separate +// implementation. +func TestTestingCpeMiddlewareReturnsUnauthorizedForAuthenticationFailures(t *testing.T) { + tests := []struct { + name string + auth string + err error + }{ + {name: "malformed authorization", auth: "Basic credentials"}, + {name: "bearer with no token", auth: "Bearer"}, + {name: "bearer with multiple parts", auth: "Bearer a b"}, + {name: "malformed token", auth: "Bearer malformed", err: fmt.Errorf("malformed token")}, + {name: "invalid signature", auth: "Bearer invalid", err: fmt.Errorf("token signature is invalid")}, + {name: "expired token", auth: "Bearer expired", err: fmt.Errorf("token is expired")}, + {name: "mac mismatch", auth: "Bearer mismatched-mac", err: fmt.Errorf("mac in token does not match claims")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return false, "", 0, tt.err + }) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", tt.auth) + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.TestingCpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authentication failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) + }) + } +} + +// XPC-43727 an actually malformed JWT (not a mocked verifier error) must +// still be classified as 401 through the real TokenManager.VerifyToken path +func TestTestingCpeMiddlewareRejectsActualMalformedJwt(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer this-is-not-a-jwt") + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.TestingCpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("malformed jwt reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + +// XPC-43727 TestingCpeMiddleware also supports a genuine post-authentication +// authorization failure, distinct from an authentication failure: 403. +func TestTestingCpeMiddlewareReturnsForbiddenForInsufficientCapabilities(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.TokenManager = security.NewTokenManager(sc.Config) + server.TokenManager.SetVerifyFunc(func(_ map[string]*rsa.PublicKey, _ []string, _ []string, _ ...string) (bool, string, int, error) { + return false, "", 0, common.ErrNoCapabilities + }) + + req, err := http.NewRequest("GET", "/api/v1/device/AABBCCDDEEFF/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer authenticated-but-not-capable") + req = mux.SetURLVars(req, map[string]string{"mac": "AABBCCDDEEFF"}) + res := ExecuteRequest(req, server.TestingCpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("authorization failure reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) +} + +// XPC-43727 TestingCpeMiddleware independently validates the mac route var +// (exact length 12), unlike CpeMiddleware's ValidateMac; verify a malformed +// or missing mac is classified as 400, not 401, when a Bearer token is +// present. +func TestTestingCpeMiddlewareReturnsBadRequestForInvalidMac(t *testing.T) { + tests := []struct { + name string + vars map[string]string + }{ + {name: "mac too short", vars: map[string]string{"mac": "AABBCC"}}, + {name: "missing mac var", vars: map[string]string{}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := NewWebconfigServer(sc, true) + + req, err := http.NewRequest("GET", "/api/v1/device/x/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer irrelevant") + req = mux.SetURLVars(req, tt.vars) + res := ExecuteRequest(req, server.TestingCpeMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("invalid mac reached the next handler") + }))).Result() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + }) + } +} + +func TestErrorPreservesUnauthorizedResponse(t *testing.T) { + recorder := httptest.NewRecorder() + + Error(recorder, http.StatusUnauthorized, nil) + + assert.Equal(t, recorder.Code, http.StatusUnauthorized) + assert.Equal(t, recorder.Body.Len(), 0) +} + func TestWebconfigServerSetterGetter(t *testing.T) { server := NewWebconfigServer(sc, true) diff --git a/security/jwks_manager.go b/security/jwks_manager.go index 232fff3..60f8ffd 100644 --- a/security/jwks_manager.go +++ b/security/jwks_manager.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -23,10 +23,10 @@ import ( "time" "github.com/MicahParks/keyfunc/v2" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/golang-jwt/jwt/v5" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -80,7 +80,10 @@ func (m *JwksManager) VerifyApiToken(tokenStr string) (bool, error) { if itf, ok := mclaims["capabilities"]; ok { if iitfs, ok := itf.([]interface{}); ok { for _, iitf := range iitfs { - ss := iitf.(string) + ss, ok := iitf.(string) + if !ok { + continue + } if util.Contains(m.apiCapabilities, ss) { return true, nil } @@ -91,6 +94,17 @@ func (m *JwksManager) VerifyApiToken(tokenStr string) (bool, error) { return false, common.NoCapabilitiesError{} } +// NewJwksManagerFromJWKS builds a JwksManager from an already-constructed +// JWKS, bypassing the network fetch in NewJwksManager. Used by tests that +// need a self-contained, signature-verified JwksManager without a live JWKS +// endpoint. +func NewJwksManagerFromJWKS(jwks *keyfunc.JWKS, apiCapabilities []string) *JwksManager { + return &JwksManager{ + jwks: jwks, + apiCapabilities: apiCapabilities, + } +} + func LogRefreshError(err error) { fields := log.Fields{ "logger": "codebig", diff --git a/security/jwks_manager_test.go b/security/jwks_manager_test.go new file mode 100644 index 0000000..bc2ce2a --- /dev/null +++ b/security/jwks_manager_test.go @@ -0,0 +1,171 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package security + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/MicahParks/keyfunc/v2" + "github.com/golang-jwt/jwt/v5" + "github.com/rdkcentral/webconfig/common" + "gotest.tools/assert" +) + +// XPC-43727 real JWKS-backed fixtures: a self-contained RSA key pair and a +// keyfunc.JWKS built from it via keyfunc.NewJSON(), so VerifyApiToken can be +// exercised end-to-end (real signature verification) without any HTTP/JWKS +// server dependency. +const jwksTestKid = "jwks-test-key" + +func newJwksTestFixture(t *testing.T) (*rsa.PrivateKey, *keyfunc.JWKS) { + t.Helper() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + assert.NilError(t, err) + + b64 := func(b []byte) string { + return base64.RawURLEncoding.EncodeToString(b) + } + eBytes := big.NewInt(int64(privateKey.PublicKey.E)).Bytes() + + jwkJSON := fmt.Sprintf(`{"keys":[{"kty":"RSA","kid":%q,"alg":"RS256","n":%q,"e":%q}]}`, + jwksTestKid, b64(privateKey.PublicKey.N.Bytes()), b64(eBytes)) + + jwks, err := keyfunc.NewJSON(json.RawMessage(jwkJSON)) + assert.NilError(t, err) + + return privateKey, jwks +} + +func signJwksTestToken(t *testing.T, privateKey *rsa.PrivateKey, claims jwt.MapClaims) string { + t.Helper() + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + token.Header["kid"] = jwksTestKid + tokenString, err := token.SignedString(privateKey) + assert.NilError(t, err) + return tokenString +} + +func TestJwksManagerVerifyApiTokenValidCapability(t *testing.T) { + privateKey, jwks := newJwksTestFixture(t) + m := &JwksManager{jwks: jwks, apiCapabilities: []string{"webconfig:all"}} + + utcnow := time.Now() + token := signJwksTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"webconfig:all"}, + "exp": utcnow.Add(time.Hour).Unix(), + }) + + ok, err := m.VerifyApiToken(token) + assert.NilError(t, err) + assert.Assert(t, ok) +} + +func TestJwksManagerVerifyApiTokenMissingCapability(t *testing.T) { + privateKey, jwks := newJwksTestFixture(t) + m := &JwksManager{jwks: jwks, apiCapabilities: []string{"webconfig:all"}} + + utcnow := time.Now() + token := signJwksTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"some:other:capability"}, + "exp": utcnow.Add(time.Hour).Unix(), + }) + + ok, err := m.VerifyApiToken(token) + assert.Assert(t, !ok) + var noCapsErr common.NoCapabilitiesError + assert.Assert(t, errors.As(err, &noCapsErr)) +} + +// XPC-43727 a non-string entry in the capabilities claim must not panic; +// it must be treated as not-capable, same as an absent/mismatched capability. +func TestJwksManagerVerifyApiTokenMalformedCapabilityDoesNotPanic(t *testing.T) { + privateKey, jwks := newJwksTestFixture(t) + m := &JwksManager{jwks: jwks, apiCapabilities: []string{"webconfig:all"}} + + utcnow := time.Now() + token := signJwksTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{12345, "some:other:capability"}, + "exp": utcnow.Add(time.Hour).Unix(), + }) + + ok, err := m.VerifyApiToken(token) + assert.Assert(t, !ok) + var noCapsErr common.NoCapabilitiesError + assert.Assert(t, errors.As(err, &noCapsErr)) +} + +func TestJwksManagerVerifyApiTokenInvalidSignature(t *testing.T) { + privateKey, jwks := newJwksTestFixture(t) + m := &JwksManager{jwks: jwks, apiCapabilities: []string{"webconfig:all"}} + + utcnow := time.Now() + token := signJwksTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"webconfig:all"}, + "exp": utcnow.Add(time.Hour).Unix(), + }) + + // tamper a character in the middle of the signature segment; avoid the + // last char, whose low bits may be unused by base64url padding + parts := []rune(token) + // find the position of the last '.' to locate the signature segment + lastDot := -1 + for i, r := range parts { + if r == '.' { + lastDot = i + } + } + assert.Assert(t, lastDot >= 0 && lastDot+2 < len(parts)) + mid := lastDot + 1 + (len(parts)-lastDot-1)/2 + if parts[mid] == 'A' { + parts[mid] = 'B' + } else { + parts[mid] = 'A' + } + tampered := string(parts) + + ok, err := m.VerifyApiToken(tampered) + assert.Assert(t, !ok) + var noCapsErr common.NoCapabilitiesError + assert.Assert(t, !errors.As(err, &noCapsErr)) +} + +func TestJwksManagerVerifyApiTokenExpiredToken(t *testing.T) { + privateKey, jwks := newJwksTestFixture(t) + m := &JwksManager{jwks: jwks, apiCapabilities: []string{"webconfig:all"}} + + utcnow := time.Now() + token := signJwksTestToken(t, privateKey, jwt.MapClaims{ + "capabilities": []interface{}{"webconfig:all"}, + "exp": utcnow.Add(-time.Hour).Unix(), + }) + + ok, err := m.VerifyApiToken(token) + assert.Assert(t, !ok) + var noCapsErr common.NoCapabilitiesError + assert.Assert(t, !errors.As(err, &noCapsErr)) +} diff --git a/security/token.go b/security/token.go index 47270a4..7420a63 100644 --- a/security/token.go +++ b/security/token.go @@ -301,55 +301,33 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi parser := &jwt.Parser{} - // this is the claims before the token is verified by the public key + // this is only used to read the kid from the header; claims are not + // trusted until the token is verified against the public key below uclaims := jwt.MapClaims{} - if token, _, err := parser.ParseUnverified(tokenString, uclaims); err == nil { - // check kid - rawkid, ok := token.Header["kid"] - if !ok { - return false, "", trust, common.NewError(fmt.Errorf("missing kid in token")) - } - kid, ok = rawkid.(string) - if !ok { - return false, "", trust, common.NewError(fmt.Errorf("error in reading kid from header")) - } + token, _, err := parser.ParseUnverified(tokenString, uclaims) + if err != nil { + return false, "", trust, common.NewError(err) + } - ok = false - for _, k := range validKids { - if kid == k { - ok = true - break - } - } - if !ok { - return false, "", trust, common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) - } + // check kid + rawkid, ok := token.Header["kid"] + if !ok { + return false, "", trust, common.NewError(fmt.Errorf("missing kid in token")) + } + kid, ok = rawkid.(string) + if !ok { + return false, "", trust, common.NewError(fmt.Errorf("error in reading kid from header")) + } - // check capabilities, if requiredCapabilities is nonempty - if len(requiredCapabilities) > 0 { - isCapable := false - if capitfs, ok := uclaims["capabilities"]; ok { - capvalues, ok1 := capitfs.([]interface{}) - if ok1 { - for _, capvalue := range capvalues { - for _, rc := range requiredCapabilities { - if rc == capvalue { - isCapable = true - break - } - } - if isCapable { - break - } - } - } - } - if !isCapable { - return false, "", trust, common.NewError(fmt.Errorf("token without proper capabilities")) - } + ok = false + for _, k := range validKids { + if kid == k { + ok = true + break } - } else { - return false, "", trust, common.NewError(err) + } + if !ok { + return false, "", trust, common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) } decodeKey, ok := decodeKeys[kid] @@ -357,11 +335,38 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi return false, "", trust, common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) } + // verify signature and registered claims (exp, nbf, etc) before trusting + // anything in the token, including capabilities claims := jwt.MapClaims{} if _, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) { return decodeKey, nil }); err != nil { return false, "", trust, common.NewError(err) } + // check capabilities, if requiredCapabilities is nonempty + // this must run only after the token has been cryptographically verified + if len(requiredCapabilities) > 0 { + isCapable := false + if capitfs, ok := claims["capabilities"]; ok { + capvalues, ok1 := capitfs.([]interface{}) + if ok1 { + for _, capvalue := range capvalues { + for _, rc := range requiredCapabilities { + if rc == capvalue { + isCapable = true + break + } + } + if isCapable { + break + } + } + } + } + if !isCapable { + return false, "", trust, common.NewError(common.ErrNoCapabilities) + } + } + if len(vargs) > 1 { mac := vargs[1] // mac must match @@ -383,7 +388,11 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi // parse partner partner := "comcast" if itf, ok := claims["partner-id"]; ok { - partner = itf.(string) + partnerStr, ok := itf.(string) + if !ok { + return false, "", trust, common.NewError(fmt.Errorf("partner-id claim is not a string")) + } + partner = partnerStr } if itf, ok := claims["trust"]; ok { diff --git a/security/token_test.go b/security/token_test.go index 6f73aa6..ae7506b 100644 --- a/security/token_test.go +++ b/security/token_test.go @@ -22,7 +22,9 @@ import ( "os" "strings" "testing" + "time" + "github.com/golang-jwt/jwt/v5" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" @@ -86,3 +88,131 @@ func TestTokenValidation(t *testing.T) { assert.Equal(t, parsedPartner, partner1) assert.Equal(t, trust, 500) } + +// XPC-42727 a token with an invalid signature must fail as an authentication +// error, never misclassified as ErrNoCapabilities, even when its (untrusted) +// claims lack the required capability +func TestVerifyTokenRejectsInvalidSignatureBeforeCapabilities(t *testing.T) { + if tokenManager == nil { + t.Skip("webconfig.jwt.enabled = false") + } + + cpeMac := util.GenerateRandomCpeMac() + token := tokenManager.Generate(strings.ToLower(cpeMac), 86400) + + parts := strings.Split(token, ".") + assert.Equal(t, len(parts), 3) + // flip a character in the middle of the signature to invalidate it; + // avoid the last char, whose low bits may be unused by base64url padding + sig := []rune(parts[2]) + mid := len(sig) / 2 + if sig[mid] == 'A' { + sig[mid] = 'B' + } else { + sig[mid] = 'A' + } + tampered := parts[0] + "." + parts[1] + "." + string(sig) + + requiredCapabilities := []string{"capability-not-present-in-token"} + ok, _, _, verr := VerifyToken(tokenManager.decodeKeys, tokenManager.cpeKids, requiredCapabilities, tampered, cpeMac) + assert.Assert(t, !ok) + assert.Assert(t, !errors.Is(verr, common.ErrNoCapabilities)) +} + +// XPC-43727 a validly signed token whose partner-id claim is not a string +// must fail as a controlled authentication error, never panic +func TestVerifyTokenRejectsMalformedPartnerIdClaim(t *testing.T) { + if tokenManager == nil { + t.Skip("webconfig.jwt.enabled = false") + } + + cpeMac := util.GenerateRandomCpeMac() + utcnow := time.Now() + claims := jwt.MapClaims{ + "mac": strings.ToLower(cpeMac), + "partner-id": 12345, + "trust": 1000, + "exp": utcnow.Add(time.Hour).Unix(), + } + method := jwt.GetSigningMethod("RS256") + token := jwt.NewWithClaims(method, claims) + token.Header["kid"] = EncodingKeyId + tokenString, err := token.SignedString(tokenManager.encodeKey) + assert.NilError(t, err) + + ok, _, _, verr := VerifyToken(tokenManager.decodeKeys, tokenManager.cpeKids, nil, tokenString, cpeMac) + assert.Assert(t, !ok) + assert.ErrorContains(t, verr, "partner-id") +} + +// XPC-43727 if an operator configures webconfig.jwt.cpe_token.capabilities, +// VerifyToken enforces it correctly for a real signed CPE token: a matching +// capability succeeds, a non-matching capability fails as ErrNoCapabilities. +// The shipped sample config leaves cpe_token.capabilities empty by design +// (CPE authorization is governed by trust, not capabilities); this proves the +// capability-check code path itself works without changing that +// configuration. +func TestVerifyTokenEnforcesCpeCapabilitiesWhenConfigured(t *testing.T) { + if tokenManager == nil { + t.Skip("webconfig.jwt.enabled = false") + } + + cpeMac := util.GenerateRandomCpeMac() + sign := func(capabilities []string) string { + claims := jwt.MapClaims{ + "mac": strings.ToLower(cpeMac), + "capabilities": capabilities, + "exp": time.Now().Add(time.Hour).Unix(), + } + token := jwt.NewWithClaims(jwt.GetSigningMethod("RS256"), claims) + token.Header["kid"] = EncodingKeyId + tokenString, err := token.SignedString(tokenManager.encodeKey) + assert.NilError(t, err) + return tokenString + } + + requiredCapabilities := []string{"cpe:config:read"} + + matching := sign([]string{"cpe:config:read"}) + ok, _, _, err := VerifyToken(tokenManager.decodeKeys, tokenManager.cpeKids, requiredCapabilities, matching, cpeMac) + assert.NilError(t, err) + assert.Assert(t, ok) + + nonMatching := sign([]string{"cpe:other"}) + ok, _, _, err = VerifyToken(tokenManager.decodeKeys, tokenManager.cpeKids, requiredCapabilities, nonMatching, cpeMac) + assert.Assert(t, !ok) + assert.Assert(t, errors.Is(err, common.ErrNoCapabilities)) +} + +// XPC-42727 an expired token must fail as an authentication error, never +// misclassified as ErrNoCapabilities, even when its (untrusted) claims lack +// the required capability +func TestVerifyTokenRejectsExpiredTokenBeforeCapabilities(t *testing.T) { + if tokenManager == nil { + t.Skip("webconfig.jwt.enabled = false") + } + + cpeMac := util.GenerateRandomCpeMac() + utcnow := time.Now() + claims := ThemisClaims{ + KeyId: EncodingKeyId, + Mac: strings.ToLower(cpeMac), + PartnerId: "comcast", + Trust: 1000, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(utcnow.Add(-1 * time.Hour)), + IssuedAt: jwt.NewNumericDate(utcnow.Add(-2 * time.Hour)), + NotBefore: jwt.NewNumericDate(utcnow.Add(-2 * time.Hour)), + }, + } + method := jwt.GetSigningMethod("RS256") + token := jwt.NewWithClaims(method, claims) + token.Header["kid"] = EncodingKeyId + tokenString, err := token.SignedString(tokenManager.encodeKey) + assert.NilError(t, err) + + requiredCapabilities := []string{"capability-not-present-in-token"} + ok, _, _, verr := VerifyToken(tokenManager.decodeKeys, tokenManager.cpeKids, requiredCapabilities, tokenString, cpeMac) + assert.Assert(t, !ok) + assert.Assert(t, !errors.Is(verr, common.ErrNoCapabilities)) +} From e7059407bbdc45ea9c8bfe9370747582f165743e Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 14 Sep 2026 16:00:50 -0700 Subject: [PATCH 10/14] add support to combine logger kafka and kafkaproducer to one line to reduce logs --- README.md | 4 ++ config/sample_webconfig.conf | 4 ++ go.mod | 24 +++---- go.sum | 60 +++++++++--------- http/webconfig_server.go | 114 ++++++++++++++++++++++++++++------ http/webconfig_server_test.go | 99 +++++++++++++++++++++++++++++ kafka/consumer.go | 42 ++++++++++--- kafka/consumer_group_test.go | 75 +++++++++++++++++++++- 8 files changed, 353 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 7ef6617..4e46011 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,10 @@ Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumer ```shell kafka_producer { enabled = true + // one_line consolidates consumer and producer enqueue success under logger=kafka; + // two_line also emits the consumer processing success record and uses + // logger=kafkaproducer for producer records. + logging_mode = "one_line" brokers = "kafka-broker:9093" topic = "webconfig_downstream" diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 04eebcb..65acc9c 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -406,6 +406,10 @@ webconfig { // forward kafka messages if needed kafka_producer { enabled = false + // Kafka success logging: "one_line" consolidates consumer and producer enqueue success + // under logger=kafka; "two_line" also emits the consumer processing success record and + // uses logger=kafkaproducer for producer records. + logging_mode = "one_line" brokers = "localhost:9092" topic = "webconfig_downstream" diff --git a/go.mod b/go.mod index 1a7ca88..47b6ec3 100644 --- a/go.mod +++ b/go.mod @@ -16,14 +16,14 @@ require ( github.com/twmb/murmur3 v1.1.8 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 - go.opentelemetry.io/otel/sdk v1.43.0 - go.opentelemetry.io/otel/trace v1.43.0 + go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/automaxprocs v1.6.0 go.uber.org/ratelimit v0.3.1 - golang.org/x/sync v0.20.0 + golang.org/x/sync v0.22.0 gotest.tools v2.2.0+incompatible modernc.org/sqlite v1.50.0 ) @@ -63,17 +63,17 @@ require ( github.com/vmihailenco/tagparser v0.1.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/crypto v0.52.0 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 // indirect - google.golang.org/grpc v1.82.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/libc v1.72.2 // indirect diff --git a/go.sum b/go.sum index 6af8102..5d4c236 100644 --- a/go.sum +++ b/go.sum @@ -138,22 +138,22 @@ github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= @@ -169,11 +169,11 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -182,20 +182,20 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -205,13 +205,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -219,12 +219,12 @@ gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348 h1:U8orV30l6KpDsi9dxU0CoJZGbjS8EEpw+6ba+XwGPQA= -google.golang.org/genproto/googleapis/api v0.0.0-20260504160031-60b97b32f348/go.mod h1:Yzdzr5OOZFgSsEV2D/Xi9NL3bszpXFAg0hFJiRohcD8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348 h1:pfIbyB44sWzHiCpRqIen67ZQnVXSfIxWrqUMk1qwODE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260504160031-60b97b32f348/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 5277cd1..f403e63 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -74,6 +74,10 @@ const ( defaultSupplementaryAppendingEnabled = true authPrefixLength = 60 defaultMaxRequestBodyBytes = 1048576 + maxKafkaProducerLogPayloadBytes = 4096 + KafkaLoggingModeOneLine = "one_line" + KafkaLoggingModeTwoLine = "two_line" + defaultKafkaLoggingMode = KafkaLoggingModeOneLine ) var ( @@ -124,6 +128,7 @@ type WebconfigServer struct { supplementaryAppendingEnabled bool kafkaProducerEnabled bool kafkaProducerTopic string + kafkaLoggingMode string upstreamProfilesEnabled bool queryParamsValidationEnabled bool minTrust int @@ -306,6 +311,13 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer var kafkaProducer sarama.AsyncProducer kafkaProducerEnabled := conf.GetBoolean("webconfig.kafka_producer.enabled") var kafkaProducerTopic string + kafkaLoggingMode := conf.GetString("webconfig.kafka_producer.logging_mode", defaultKafkaLoggingMode) + if kafkaLoggingMode != KafkaLoggingModeOneLine && kafkaLoggingMode != KafkaLoggingModeTwoLine { + if kafkaProducerEnabled { + panic(fmt.Errorf("webconfig.kafka_producer.logging_mode must be %q or %q", KafkaLoggingModeOneLine, KafkaLoggingModeTwoLine)) + } + kafkaLoggingMode = defaultKafkaLoggingMode + } if kafkaProducerEnabled { brokersStr := conf.GetString("webconfig.kafka_producer.brokers") if len(brokersStr) == 0 { @@ -414,6 +426,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, + kafkaLoggingMode: kafkaLoggingMode, upstreamProfilesEnabled: upstreamProfilesEnabled, queryParamsValidationEnabled: queryParamsValidationEnabled, minTrust: minTrust, @@ -796,6 +809,17 @@ func (s *WebconfigServer) SetKafkaProducerTopic(x string) { s.kafkaProducerTopic = x } +func (s *WebconfigServer) KafkaLoggingMode() string { + return s.kafkaLoggingMode +} + +func (s *WebconfigServer) SetKafkaLoggingMode(mode string) { + if mode != KafkaLoggingModeOneLine && mode != KafkaLoggingModeTwoLine { + return + } + s.kafkaLoggingMode = mode +} + func (s *WebconfigServer) UpstreamProfilesEnabled() bool { return s.upstreamProfilesEnabled } @@ -973,6 +997,11 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques mac = strings.ToUpper(mac) fields["cpe_mac"] = mac } + if r.Method == "POST" { + if subdocID, ok := params["subdoc_id"]; ok { + fields["subdoc_id"] = subdocID + } + } xwriter := NewXResponseWriter(w, time.Now(), token, fields) @@ -1145,8 +1174,11 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { - tfields := common.CopyCoreLogFields(fields) +func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields, logMessage string) { + tfields := kafkaProducerLogFields(fields) + if _, ok := tfields["kafka_operation"]; !ok { + tfields["kafka_operation"] = "producer_send" + } bbytes, err := json.Marshal(m) if err != nil { @@ -1168,7 +1200,7 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess if m := s.Metrics(); m != nil { m.ObserveKafkaProducerErr(s.KafkaProducerTopic(), -1) } - tfields["logger"] = "kafkaproducer" + tfields["logger"] = s.KafkaProducerLogger() tfields["error"] = r log.WithFields(tfields).Warn("dropped: producer closed during shutdown") } @@ -1176,16 +1208,32 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess s.Input() <- outMessage - tfields["logger"] = "kafkaproducer" + tfields["logger"] = s.KafkaProducerLogger() tfields["output_topic"] = outMessage.Topic tfields["output_key"] = string(kbytes) - tfields["output_body"] = m - log.WithFields(tfields).Info("send") + log.WithFields(tfields).Info(logMessage + "; send") +} + +func kafkaProducerLogFields(fields log.Fields) log.Fields { + const producerFieldLimit = 15 + allowedFields := [...]string{ + "app_name", "audit_id", "body", "cpe_mac", "subdoc_id", + "kafka_key", "topic", "cluster_name", "kafka_partition", "kafka_offset", + "message_length", "duration", "event_name", "rpt", "kafka_operation", + } + producerFields := make(log.Fields, producerFieldLimit) + for _, field := range allowedFields { + if value, ok := fields[field]; ok { + producerFields[field] = common.FilterLogFields(log.Fields{field: value})[field] + } + } + return producerFields } func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMessage, fields log.Fields) { - tfields := common.CopyCoreLogFields(fields) - tfields["logger"] = "kafkaproducer" + tfields := kafkaProducerLogFields(fields) + tfields["logger"] = s.KafkaProducerLogger() + tfields["kafka_operation"] = "state_correction_send" tfields["output_topic"] = s.KafkaProducerTopic() for _, m := range messages { @@ -1228,7 +1276,6 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } tfields["output_key"] = mac - tfields["output_body"] = m log.WithFields(tfields).Info("send") } } @@ -1288,7 +1335,8 @@ func (s *WebconfigServer) HandleKafkaProducerResults(ctx context.Context) { continue } fields := make(log.Fields) - fields["logger"] = "kafkaproducer" + fields["logger"] = s.KafkaProducerLogger() + fields["kafka_operation"] = "producer_result" fields["output_topic"] = success.Topic fields["output_partition"] = success.Partition fields["output_offset"] = success.Offset @@ -1304,7 +1352,8 @@ func (s *WebconfigServer) HandleKafkaProducerResults(ctx context.Context) { m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) } fields := make(log.Fields) - fields["logger"] = "kafkaproducer" + fields["logger"] = s.KafkaProducerLogger() + fields["kafka_operation"] = "producer_result" fields["output_topic"] = pErr.Msg.Topic fields["output_partition"] = pErr.Msg.Partition kbytes, err := pErr.Msg.Key.Encode() @@ -1318,14 +1367,7 @@ func (s *WebconfigServer) HandleKafkaProducerResults(ctx context.Context) { if err != nil { log.WithFields(fields).Error(common.NewError(err)) } else { - var itf interface{} - err1 := json.Unmarshal(vbytes, &itf) - if err1 != nil { - log.WithFields(fields).Error(common.NewError(err1)) - fields["output_body_text"] = base64.StdEncoding.EncodeToString(vbytes) - } else { - fields["output_body"] = itf - } + common.UpdateLogFields(fields, producerErrorPayloadFields(vbytes)) } log.WithFields(fields).Error(pErr.Err) @@ -1333,6 +1375,40 @@ func (s *WebconfigServer) HandleKafkaProducerResults(ctx context.Context) { } } +func (s *WebconfigServer) KafkaProducerLogger() string { + if s.KafkaLoggingMode() == KafkaLoggingModeTwoLine { + return "kafkaproducer" + } + return "kafka" +} + +func producerErrorPayloadFields(payload []byte) log.Fields { + fields := log.Fields{ + "output_body_bytes": len(payload), + "output_body_truncated": false, + } + maxRawBytes := (maxKafkaProducerLogPayloadBytes / 4) * 3 + if len(payload) <= maxRawBytes { + var body interface{} + if err := json.Unmarshal(payload, &body); err == nil { + fields["output_body"] = body + return fields + } + } + + fields["output_body_text"], fields["output_body_truncated"] = boundedKafkaPayload(payload) + return fields +} + +func boundedKafkaPayload(payload []byte) (string, bool) { + maxRawBytes := (maxKafkaProducerLogPayloadBytes / 4) * 3 + truncated := len(payload) > maxRawBytes + if truncated { + payload = payload[:maxRawBytes] + } + return base64.StdEncoding.EncodeToString(payload), truncated +} + func (s *WebconfigServer) StopXpcTracer() { sdkTraceProvider, ok := s.XpcTracer.OtelTracerProvider().(*sdktrace.TracerProvider) if ok && sdkTraceProvider != nil { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 6b611d4..832b8a0 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -47,6 +47,105 @@ func TestApiTokenAuthSecureDefaults(t *testing.T) { assert.Assert(t, deviceApiTokenAuthEnabledDefault, "device API token auth must default to enabled") } +func TestPostSubDocumentRequestIncludesSubdocID(t *testing.T) { + server := NewWebconfigServer(sc, true) + + req, err := http.NewRequest("POST", "/api/v1/device/001122334455/document/lan", nil) + assert.NilError(t, err) + req = mux.SetURLVars(req, map[string]string{ + "mac": "001122334455", + "subdoc_id": "lan", + }) + xw := server.logRequestStarts(httptest.NewRecorder(), req) + assert.Equal(t, xw.Audit()["subdoc_id"], "lan") + + getReq, err := http.NewRequest("GET", "/api/v1/device/001122334455/document/lan", nil) + assert.NilError(t, err) + getReq = mux.SetURLVars(getReq, map[string]string{ + "mac": "001122334455", + "subdoc_id": "lan", + }) + getXw := server.logRequestStarts(httptest.NewRecorder(), getReq) + _, ok := getXw.Audit()["subdoc_id"] + assert.Assert(t, !ok) +} + +func TestKafkaLoggingModeDefaultsToOneLine(t *testing.T) { + server := NewWebconfigServer(sc, true) + assert.Equal(t, server.KafkaLoggingMode(), KafkaLoggingModeOneLine) + + server.SetKafkaLoggingMode(KafkaLoggingModeTwoLine) + assert.Equal(t, server.KafkaLoggingMode(), KafkaLoggingModeTwoLine) + server.SetKafkaLoggingMode("invalid") + assert.Equal(t, server.KafkaLoggingMode(), KafkaLoggingModeTwoLine) +} + +func TestKafkaProducerLoggerFollowsLoggingMode(t *testing.T) { + server := NewWebconfigServer(sc, true) + assert.Equal(t, server.KafkaProducerLogger(), "kafka") + + server.SetKafkaLoggingMode(KafkaLoggingModeTwoLine) + assert.Equal(t, server.KafkaProducerLogger(), "kafkaproducer") +} + +func TestKafkaProducerLogFieldsAllowlist(t *testing.T) { + fields := kafkaProducerLogFields(log.Fields{ + "app_name": "webconfig", + "audit_id": "audit", + "cpe_mac": "001122334455", + "subdoc_id": "lan", + "event_name": "mqtt-get", + "header": map[string]string{"Authorization": "secret", "X-Test": "value"}, + "authorization": "secret", + }) + + _, hasHeader := fields["header"] + _, hasAuthorization := fields["authorization"] + assert.Assert(t, !hasHeader) + assert.Assert(t, !hasAuthorization) + assert.Equal(t, fields["subdoc_id"], "lan") + assert.Equal(t, fields["event_name"], "mqtt-get") + assert.Equal(t, fields["kafka_operation"], nil) +} + +func TestBoundedKafkaPayload(t *testing.T) { + payload := make([]byte, maxKafkaProducerLogPayloadBytes) + bounded, truncated := boundedKafkaPayload(payload) + assert.Equal(t, len(bounded), maxKafkaProducerLogPayloadBytes) + assert.Assert(t, truncated) + _, err := base64.StdEncoding.DecodeString(bounded) + assert.NilError(t, err) + + payload = append(payload, 'x') + bounded, truncated = boundedKafkaPayload(payload) + assert.Equal(t, len(bounded), maxKafkaProducerLogPayloadBytes) + assert.Assert(t, truncated) + _, err = base64.StdEncoding.DecodeString(bounded) + assert.NilError(t, err) + + payload = payload[:(maxKafkaProducerLogPayloadBytes/4)*3] + bounded, truncated = boundedKafkaPayload(payload) + assert.Assert(t, !truncated) + assert.Equal(t, len(bounded), maxKafkaProducerLogPayloadBytes) +} + +func TestProducerErrorPayloadFields(t *testing.T) { + fields := producerErrorPayloadFields([]byte(`{"status":"failed"}`)) + assert.Equal(t, fields["output_body"].(map[string]interface{})["status"], "failed") + _, hasText := fields["output_body_text"] + assert.Assert(t, !hasText) + assert.Assert(t, !fields["output_body_truncated"].(bool)) + + fields = producerErrorPayloadFields([]byte("not-json")) + _, hasBody := fields["output_body"] + assert.Assert(t, !hasBody) + _, hasText = fields["output_body_text"] + assert.Assert(t, hasText) + + fields = producerErrorPayloadFields(make([]byte, maxKafkaProducerLogPayloadBytes)) + assert.Assert(t, fields["output_body_truncated"].(bool)) +} + func TestConfigEndpointRemainsUnauthenticatedByDefault(t *testing.T) { server := NewWebconfigServer(sc, true) assert.Assert(t, !server.ConfigApiTokenAuthEnabled()) diff --git a/kafka/consumer.go b/kafka/consumer.go index adb95e6..90eea28 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -46,6 +46,30 @@ type Consumer struct { topicPartitionsMap map[string][]int32 } +func addKafkaEventLogFields(fields log.Fields, eventName, rptHeaderValue string) { + if eventName != "webpa-state" { + fields["event_name"] = eventName + } + if rptHeaderValue != "" { + fields["rpt"] = rptHeaderValue + } +} + +const maxKafkaMessageLogPayloadBytes = 4096 + +func boundedKafkaMessage(payload []byte) (string, bool) { + maxRawBytes := (maxKafkaMessageLogPayloadBytes / 4) * 3 + truncated := len(payload) > maxRawBytes + if truncated { + payload = payload[:maxRawBytes] + } + return base64.StdEncoding.EncodeToString(payload), truncated +} + +func shouldLogConsumerSuccess(producerEnabled bool, message *common.EventMessage, loggingMode string) bool { + return !producerEnabled || message == nil || loggingMode == wchttp.KafkaLoggingModeTwoLine +} + func NewConsumer(s *wchttp.WebconfigServer, ratelimitMessagesPerSecond int, m *common.AppMetrics, clusterName string, offsetEnum int64, topicPartitionsMap map[string][]int32) *Consumer { return &Consumer{ WebconfigServer: s, @@ -210,6 +234,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram messageLength := len(message.Value) fields := log.Fields{ "logger": "kafka", + "kafka_operation": "consumer_process", "app_name": c.AppName(), "kafka_lag": lag, "kafka_key": kafkaKey, @@ -244,8 +269,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram session.MarkMessage(message, "") duration := int(time.Since(start).Nanoseconds() / 1000000) fields["duration"] = duration - fields["event_name"] = eventName - fields["rpt"] = rptHeaderValue + addKafkaEventLogFields(fields, eventName, rptHeaderValue) forwardMessage := false if err != nil { @@ -255,12 +279,15 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram log.WithFields(fields).Trace("pending") } else { fields["error"] = err.Error() - fields["kafka_message"] = base64.StdEncoding.EncodeToString(message.Value) + fields["kafka_message"], fields["kafka_message_truncated"] = boundedKafkaMessage(message.Value) + fields["kafka_message_bytes"] = len(message.Value) log.WithFields(fields).Error("errors") } } else { forwardMessage = true - log.WithFields(fields).Info(logMessage) + if shouldLogConsumerSuccess(c.KafkaProducerEnabled(), m, c.KafkaLoggingMode()) { + log.WithFields(fields).Info(logMessage) + } } // build metrics dimensions and update metrics @@ -281,7 +308,8 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() && m != nil && forwardMessage { - c.ForwardKafkaMessage(message.Key, m, fields) + fields["kafka_operation"] = "producer_send" + c.ForwardKafkaMessage(message.Key, m, fields, logMessage) if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { // build a root/success message @@ -294,7 +322,8 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram TransactionUuid: m.TransactionUuid, Version: m.Version, } - c.ForwardKafkaMessage(message.Key, em, fields) + fields["kafka_operation"] = "state_correction_send" + c.ForwardKafkaMessage(message.Key, em, fields, logMessage) } } } @@ -303,7 +332,6 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram return nil } } - return nil } func (c *Consumer) AppName() string { diff --git a/kafka/consumer_group_test.go b/kafka/consumer_group_test.go index d720eb2..c1fcd78 100644 --- a/kafka/consumer_group_test.go +++ b/kafka/consumer_group_test.go @@ -14,17 +14,90 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( + "encoding/base64" "testing" "time" "github.com/IBM/sarama" + "github.com/rdkcentral/webconfig/common" + wchttp "github.com/rdkcentral/webconfig/http" + log "github.com/sirupsen/logrus" "gotest.tools/assert" ) +func TestKafkaSuccessLoggingMode(t *testing.T) { + message := &common.EventMessage{} + tests := []struct { + name string + producerEnabled bool + message *common.EventMessage + loggingMode string + want bool + }{ + {name: "producer disabled", want: true}, + {name: "one line", producerEnabled: true, message: message, loggingMode: wchttp.KafkaLoggingModeOneLine, want: false}, + {name: "two line", producerEnabled: true, message: message, loggingMode: wchttp.KafkaLoggingModeTwoLine, want: true}, + {name: "no message", producerEnabled: true, loggingMode: wchttp.KafkaLoggingModeOneLine, want: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, shouldLogConsumerSuccess(test.producerEnabled, test.message, test.loggingMode), test.want) + }) + } +} + +func TestBoundedKafkaMessage(t *testing.T) { + payload := make([]byte, maxKafkaMessageLogPayloadBytes+1) + encoded, truncated := boundedKafkaMessage(payload) + assert.Assert(t, truncated) + assert.Equal(t, len(encoded), maxKafkaMessageLogPayloadBytes) + _, err := base64.StdEncoding.DecodeString(encoded) + assert.NilError(t, err) + + payload = payload[:(maxKafkaMessageLogPayloadBytes/4)*3] + encoded, truncated = boundedKafkaMessage(payload) + assert.Assert(t, !truncated) + assert.Equal(t, len(encoded), maxKafkaMessageLogPayloadBytes) +} + +func TestKafkaEventLogFields(t *testing.T) { + tests := []struct { + name string + eventName string + rpt string + wantEvent string + wantRPT string + hasEvent bool + hasRPT bool + }{ + {name: "webpa state default", eventName: "webpa-state"}, + {name: "mqtt get", eventName: "mqtt-get", rpt: "x/fr/webconfig/get", wantEvent: "mqtt-get", wantRPT: "x/fr/webconfig/get", hasEvent: true, hasRPT: true}, + {name: "mqtt state", eventName: "mqtt-state", rpt: "x/fr/webconfig/poke", wantEvent: "mqtt-state", wantRPT: "x/fr/webconfig/poke", hasEvent: true, hasRPT: true}, + {name: "unknown event without rpt", eventName: "unknown-no-rpt", wantEvent: "unknown-no-rpt", hasEvent: true}, + {name: "unknown event with rpt", eventName: "unknown-rpt", rpt: "indigo", wantEvent: "unknown-rpt", wantRPT: "indigo", hasEvent: true, hasRPT: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fields := log.Fields{} + addKafkaEventLogFields(fields, test.eventName, test.rpt) + value, hasEvent := fields["event_name"] + assert.Equal(t, hasEvent, test.hasEvent) + if test.hasEvent { + assert.Equal(t, value, test.wantEvent) + } + value, hasRPT := fields["rpt"] + assert.Equal(t, hasRPT, test.hasRPT) + if test.hasRPT { + assert.Equal(t, value, test.wantRPT) + } + }) + } +} + func TestGetEventName(t *testing.T) { // ==== mqtt-get ==== rheader := &sarama.RecordHeader{ From af801acb0b8575ae7d307bf966e726d7c28ba24a Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 22 Sep 2026 22:45:27 -0700 Subject: [PATCH 11/14] add server_name for kafka tls connection cert verification --- README.md | 58 +++++++------- common/kafka_tls.go | 98 +++++++++++++----------- common/kafka_tls_test.go | 34 +++++++++ config/sample_webconfig.conf | 9 ++- db/cassandra/cassandra_client.go | 127 +------------------------------ http/router.go | 6 +- http/webconfig_server_test.go | 12 +++ security/token.go | 51 +++++++------ security/token_test.go | 44 +++++++++++ 9 files changed, 212 insertions(+), 227 deletions(-) diff --git a/README.md b/README.md index 4e46011..a0a2089 100644 --- a/README.md +++ b/README.md @@ -84,11 +84,14 @@ Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumer **TLS Configuration Options:** -- `tls.enabled` - Enable/disable TLS for Kafka connections (default: false) -- `tls.cert_file` - Path to client certificate file for mTLS authentication (optional) -- `tls.key_file` - Path to client private key file for mTLS authentication (optional) -- `tls.ca_cert_file` - Path to CA certificate file for broker verification (optional) -- `tls.insecure_skip_verify` - Skip certificate verification (insecure, for testing only, default: false) +- `tls_enabled` - Enable/disable TLS for Kafka connections (default: false) +- `tls_cert_file` - Path to client certificate file for mTLS authentication (optional) +- `tls_key_file` - Path to client private key file for mTLS authentication (optional) +- `tls_ca_cert_file` - Path to CA certificate file for broker verification (optional) +- `tls_insecure_skip_verify` - Skip certificate verification (insecure, for testing only, default: false) +- `tls_server_name` - Optional SNI and certificate name override (default: empty) + +Kafka keeps its existing `tls_enabled` flag for each consumer cluster and producer. Cassandra uses its existing `is_ssl_enabled` flag; after TLS is enabled, both integrations use the same flat `tls_*` fields, including identical `tls_insecure_skip_verify` and `tls_server_name` behavior. **Consumer TLS Configuration Example:** @@ -99,13 +102,12 @@ Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumer topics = "config-version-report" consumer_group = "webconfig" - tls { - enabled = true - cert_file = "/etc/webconfig/kafka/client.crt" - key_file = "/etc/webconfig/kafka/client.key" - ca_cert_file = "/etc/webconfig/kafka/ca.crt" - insecure_skip_verify = false - } + tls_enabled = true + tls_cert_file = "/etc/webconfig/kafka/client.crt" + tls_key_file = "/etc/webconfig/kafka/client.key" + tls_ca_cert_file = "/etc/webconfig/kafka/ca.crt" + tls_insecure_skip_verify = false + tls_server_name = "kafka.example.com" # Per-cluster TLS configuration clusters { @@ -114,12 +116,12 @@ Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumer brokers = "kafka-mesh:9093" topics = "staging-chi-onewifi-from-device" - tls { - enabled = true - cert_file = "/etc/webconfig/kafka/mesh-client.crt" - key_file = "/etc/webconfig/kafka/mesh-client.key" - ca_cert_file = "/etc/webconfig/kafka/mesh-ca.crt" - } + tls_enabled = true + tls_cert_file = "/etc/webconfig/kafka/mesh-client.crt" + tls_key_file = "/etc/webconfig/kafka/mesh-client.key" + tls_ca_cert_file = "/etc/webconfig/kafka/mesh-ca.crt" + tls_insecure_skip_verify = false + tls_server_name = "kafka-mesh.example.com" } } } @@ -137,20 +139,20 @@ Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumer brokers = "kafka-broker:9093" topic = "webconfig_downstream" - tls { - enabled = true - cert_file = "/etc/webconfig/kafka/producer-client.crt" - key_file = "/etc/webconfig/kafka/producer-client.key" - ca_cert_file = "/etc/webconfig/kafka/ca.crt" - } + tls_enabled = true + tls_cert_file = "/etc/webconfig/kafka/producer-client.crt" + tls_key_file = "/etc/webconfig/kafka/producer-client.key" + tls_ca_cert_file = "/etc/webconfig/kafka/ca.crt" + tls_insecure_skip_verify = false + tls_server_name = "kafka.example.com" } ``` **Certificate Requirements:** -1. **Client Certificate (mTLS)**: If `cert_file` and `key_file` are provided, mutual TLS authentication is enabled. The certificate and key must be in PEM format. +1. **Client Certificate (mTLS)**: If `tls_cert_file` and `tls_key_file` are provided, mutual TLS authentication is enabled. The certificate and key must be in PEM format. -2. **CA Certificate**: If `ca_cert_file` is provided, it will be used to verify the Kafka broker's certificate. This is useful when using self-signed certificates or internal CAs. +2. **CA Certificate**: If `tls_ca_cert_file` is provided, it will be used to verify the Kafka broker's certificate. This is useful when using self-signed certificates or internal CAs. 3. **Certificate Validation**: All certificate files are validated at startup. The application will fail to start with clear error messages if: - Certificate files are missing or unreadable @@ -160,8 +162,8 @@ Webconfig supports secure TLS/SSL connections to Kafka brokers for both consumer **TLS Security Best Practices:** 1. **Use TLS in Production**: Always enable TLS for production Kafka connections to encrypt data in transit -2. **Use mTLS**: Provide client certificates (`cert_file` and `key_file`) for mutual authentication -3. **Verify Certificates**: Never use `insecure_skip_verify = true` in production - it disables certificate verification and is insecure +2. **Use mTLS**: Provide client certificates (`tls_cert_file` and `tls_key_file`) for mutual authentication +3. **Verify Certificates**: Never use `tls_insecure_skip_verify = true` in production - it disables certificate verification and is insecure 4. **Protect Certificate Files**: Set appropriate file permissions (0600) on certificate and key files 5. **Use Secure Ports**: Configure Kafka brokers to listen on secure ports (typically 9093 for TLS) 6. **Certificate Rotation**: Plan for certificate rotation - the service must be restarted to pick up new certificates diff --git a/common/kafka_tls.go b/common/kafka_tls.go index bcc182a..9c76a31 100644 --- a/common/kafka_tls.go +++ b/common/kafka_tls.go @@ -27,109 +27,115 @@ import ( log "github.com/sirupsen/logrus" ) -// LoadKafkaTLSConfig loads TLS configuration from HOCON config at the specified prefix. -// Prefix should be like "webconfig.kafka" or "webconfig.kafka.clusters.mesh" or "webconfig.kafka_producer". -// Returns nil if TLS is not enabled. -// Returns error if TLS is enabled but configuration is invalid. -func LoadKafkaTLSConfig(conf *configuration.Config, prefix string) (*tls.Config, error) { - tlsEnabled := conf.GetBoolean(prefix + ".tls_enabled") - if !tlsEnabled { - return nil, nil +// LoadTLSConfig loads the shared flat tls_* configuration used by Kafka and Cassandra. +// Prefix should be empty for a driver sub-config, or a full HOCON path such as +// "webconfig.kafka" or "webconfig.kafka_producer". +func LoadTLSConfig(conf *configuration.Config, prefix, component string) (*tls.Config, error) { + key := func(name string) string { + if prefix == "" { + return name + } + return prefix + "." + name } - tlsConfig := &tls.Config{} - - // Check insecure_skip_verify flag first - insecureSkipVerify := conf.GetBoolean(prefix + ".tls_insecure_skip_verify") + insecureSkipVerify := conf.GetBoolean(key("tls_insecure_skip_verify")) + certFile := conf.GetString(key("tls_cert_file")) + keyFile := conf.GetString(key("tls_key_file")) + caCertFile := conf.GetString(key("tls_ca_cert_file")) + serverName := conf.GetString(key("tls_server_name")) + + cipherSuites := []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + } + if component == "Cassandra" { + cipherSuites = append(cipherSuites, tls.TLS_RSA_WITH_AES_128_CBC_SHA) + } - // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) - certFile := conf.GetString(prefix + ".tls_cert_file") - keyFile := conf.GetString(prefix + ".tls_key_file") + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: serverName, + CipherSuites: cipherSuites, + InsecureSkipVerify: insecureSkipVerify, + } - // When insecure_skip_verify is true and no cert files configured, skip loading certificates - // This allows TLS without client authentication (server-only TLS) if insecureSkipVerify && (len(certFile) == 0 || len(keyFile) == 0) { - // Insecure mode without client certificates - skip cert loading + log.WithField("component", component).Warn("TLS enabled in insecure mode without client certificates") } else if len(certFile) > 0 && len(keyFile) > 0 { - // Only validate cert files exist when verification is enabled if !insecureSkipVerify { - // Validate certificate file exists if _, err := os.Stat(certFile); os.IsNotExist(err) { return nil, NewError(fmt.Errorf("TLS certificate file does not exist: %s", certFile)) } - - // Validate key file exists if _, err := os.Stat(keyFile); os.IsNotExist(err) { return nil, NewError(fmt.Errorf("TLS key file does not exist: %s", keyFile)) } } - // Load and parse the certificate and key cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { return nil, NewError(fmt.Errorf("failed to load TLS certificate and key from %s and %s: %v", certFile, keyFile, err)) } - tlsConfig.Certificates = []tls.Certificate{cert} log.WithFields(log.Fields{ - "prefix": prefix, + "component": component, "cert_file": certFile, "key_file": keyFile, }).Info("Loaded TLS client certificate for mTLS") } else if len(certFile) > 0 || len(keyFile) > 0 { - // Partial cert configuration detected - require both cert and key when verification is enabled if !insecureSkipVerify { return nil, NewError(fmt.Errorf("TLS enabled with verification but incomplete certificate configuration (cert: %s, key: %s)", certFile, keyFile)) } } - // Load CA certificate if provided (optional when insecure_skip_verify is true) - caCertFile := conf.GetString(prefix + ".tls_ca_cert_file") - // When insecure_skip_verify is true and no CA file configured, skip loading CA cert - // This allows TLS without broker verification (insecure mode) if insecureSkipVerify && len(caCertFile) == 0 { - // Insecure mode without CA cert - skip CA loading + log.WithField("component", component).Warn("TLS enabled in insecure mode without CA certificate") } else if len(caCertFile) > 0 { - // Only validate CA cert file exists when verification is enabled if !insecureSkipVerify { - // Validate CA certificate file exists if _, err := os.Stat(caCertFile); os.IsNotExist(err) { return nil, NewError(fmt.Errorf("TLS CA certificate file does not exist: %s", caCertFile)) } } - // Load CA certificate caCert, err := os.ReadFile(caCertFile) if err != nil { return nil, NewError(fmt.Errorf("failed to read TLS CA certificate from %s: %v", caCertFile, err)) } - - // Parse CA certificate caCertPool := x509.NewCertPool() if !caCertPool.AppendCertsFromPEM(caCert) { return nil, NewError(fmt.Errorf("failed to parse TLS CA certificate from %s", caCertFile)) } - tlsConfig.RootCAs = caCertPool log.WithFields(log.Fields{ - "prefix": prefix, + "component": component, "ca_cert_file": caCertFile, - }).Info("Loaded TLS CA certificate for broker verification") + }).Info("Loaded TLS CA certificate for server verification") } if insecureSkipVerify { - tlsConfig.InsecureSkipVerify = true - log.WithFields(log.Fields{ - "prefix": prefix, - }).Warn("TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") + log.WithField("component", component).Warn("TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") } - log.WithFields(log.Fields{ - "prefix": prefix, + "component": component, "has_client_cert": len(tlsConfig.Certificates) > 0, "has_ca_cert": tlsConfig.RootCAs != nil, "insecure_skip_verify": insecureSkipVerify, - }).Info("TLS configuration loaded for Kafka connection") + "server_name": serverName, + }).Info("TLS configuration loaded") return tlsConfig, nil } + +// LoadKafkaTLSConfig loads enabled Kafka TLS configuration from HOCON. +// Prefix should be like "webconfig.kafka", "webconfig.kafka.clusters.mesh", +// or "webconfig.kafka_producer". +func LoadKafkaTLSConfig(conf *configuration.Config, prefix string) (*tls.Config, error) { + tlsEnabled := conf.GetBoolean(prefix + ".tls_enabled") + if !tlsEnabled { + return nil, nil + } + return LoadTLSConfig(conf, prefix, "Kafka") +} diff --git a/common/kafka_tls_test.go b/common/kafka_tls_test.go index 688d07d..46cd2bb 100644 --- a/common/kafka_tls_test.go +++ b/common/kafka_tls_test.go @@ -20,6 +20,7 @@ package common import ( "crypto/rand" "crypto/rsa" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -325,6 +326,7 @@ webconfig { tls_cert_file = "` + certFile + `" tls_key_file = "` + keyFile + `" tls_ca_cert_file = "` + caCertFile + `" + tls_server_name = "kafka.example.com" } } ` @@ -336,6 +338,38 @@ webconfig { assert.Assert(t, len(tlsConfig.Certificates) == 1, "Should have one client certificate") assert.Assert(t, tlsConfig.RootCAs != nil, "Should have custom CA") assert.Assert(t, !tlsConfig.InsecureSkipVerify, "Should not skip verification") + assert.Equal(t, tlsConfig.ServerName, "kafka.example.com") + assert.Equal(t, tlsConfig.MinVersion, uint16(0x0303)) + assert.Assert(t, len(tlsConfig.CipherSuites) > 0, "Should configure TLS cipher suites") + assert.Assert(t, !containsCipherSuite(tlsConfig.CipherSuites, tls.TLS_RSA_WITH_AES_128_CBC_SHA), "Kafka should not enable the Cassandra legacy cipher suite") +} + +func TestLoadTLSConfig_CassandraIncludesLegacyCipher(t *testing.T) { + conf := configuration.ParseString(` +webconfig { + database { + cassandra { + tls_server_name = "cassandra.example.com" + tls_insecure_skip_verify = false + } + } +} +`) + + tlsConfig, err := LoadTLSConfig(conf.GetConfig("webconfig.database.cassandra"), "", "Cassandra") + + assert.NilError(t, err) + assert.Equal(t, tlsConfig.ServerName, "cassandra.example.com") + assert.Assert(t, containsCipherSuite(tlsConfig.CipherSuites, tls.TLS_RSA_WITH_AES_128_CBC_SHA), "Cassandra should retain its legacy 3.11.x cipher suite") +} + +func containsCipherSuite(cipherSuites []uint16, wanted uint16) bool { + for _, cipherSuite := range cipherSuites { + if cipherSuite == wanted { + return true + } + } + return false } func TestLoadKafkaTLSConfig_DifferentPrefixes(t *testing.T) { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 65acc9c..4ec2bad 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -286,14 +286,16 @@ webconfig { // TLS configuration for secure Kafka connections // NOTE: These are flat keys read by LoadKafkaTLSConfig at prefix "webconfig.kafka" - // Certificate files (tls_cert_file, tls_key_file, tls_ca_cert_file) are REQUIRED when - // tls_insecure_skip_verify=false (default). They are OPTIONAL when tls_insecure_skip_verify=true. + // Client certificate/key are optional unless the Kafka broker requires mTLS. + // tls_ca_cert_file is optional; when unset, the system CA pool is used. tls_enabled = false tls_cert_file = "/etc/webconfig/kafka/client-cert.pem" tls_key_file = "/etc/webconfig/kafka/client-key.pem" tls_ca_cert_file = "/etc/webconfig/kafka/ca-cert.pem" // WARNING: set true only in dev/test environments, NEVER in production tls_insecure_skip_verify = false + // Optional SNI/server certificate name. Leave empty to use the broker address. + tls_server_name = "" // Reconnect resilience — tuned for broker restart recovery. @@ -359,6 +361,7 @@ webconfig { tls_key_file = "/etc/webconfig/kafka/mesh-client-key.pem" tls_ca_cert_file = "/etc/webconfig/kafka/mesh-ca-cert.pem" tls_insecure_skip_verify = false + tls_server_name = "" } east { enabled = false @@ -379,6 +382,7 @@ webconfig { tls_key_file = "/etc/webconfig/kafka/east-client-key.pem" tls_ca_cert_file = "/etc/webconfig/kafka/east-ca-cert.pem" tls_insecure_skip_verify = false + tls_server_name = "" } } } @@ -419,6 +423,7 @@ webconfig { tls_key_file = "/etc/webconfig/kafka/producer-client-key.pem" tls_ca_cert_file = "/etc/webconfig/kafka/producer-ca-cert.pem" tls_insecure_skip_verify = false + tls_server_name = "" // Reconnect resilience — tuned for broker restart recovery. diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 8ceed76..a78bdbd 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -19,8 +19,6 @@ package cassandra import ( "context" - "crypto/tls" - "crypto/x509" "errors" "fmt" "os" @@ -33,7 +31,6 @@ import ( "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - log "github.com/sirupsen/logrus" ) const ( @@ -165,7 +162,7 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl if isSslEnabled { insecureSkipVerify := dbconf.GetBoolean("tls_insecure_skip_verify") - tlsConfig, err := loadCassandraTLSConfig(dbconf, dbdriver, insecureSkipVerify) + tlsConfig, err := common.LoadTLSConfig(dbconf, "", "Cassandra") if err != nil { return nil, common.NewError(err) } @@ -214,128 +211,6 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl }, nil } -// loadCassandraTLSConfig loads TLS configuration for Cassandra connection. -// Returns a tls.Config with certificates loaded from the configuration. -// Reads flat tls_* keys directly under the database driver config (cassandra or yugabyte). -func loadCassandraTLSConfig(dbconf *configuration.Config, dbdriver string, insecureSkipVerify bool) (*tls.Config, error) { - // Load client certificates for mTLS if provided (optional when insecure_skip_verify is true) - certFile := dbconf.GetString("tls_cert_file") - keyFile := dbconf.GetString("tls_key_file") - caCertFile := dbconf.GetString("tls_ca_cert_file") - serverName := dbconf.GetString("tls_server_name") - - // Create TLS config for Cassandra connection. - // Prefer modern ECDHE+AEAD suites for forward secrecy; keep TLS_RSA_WITH_AES_128_CBC_SHA - // last as a fallback for legacy Cassandra 3.11.x nodes that only negotiate that suite. - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - ServerName: serverName, - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - // Legacy fallback for Cassandra 3.11.x — do not remove - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - }, - InsecureSkipVerify: insecureSkipVerify, - } - - // When insecure_skip_verify is true and no cert files configured, skip loading certificates - // This allows TLS without client authentication (server-only TLS) - if insecureSkipVerify && (len(certFile) == 0 || len(keyFile) == 0) { - // Insecure mode without client certificates - skip cert loading - log.WithFields(log.Fields{ - "driver": dbdriver, - }).Warn("Cassandra TLS enabled in insecure mode without client certificates") - } else if len(certFile) > 0 && len(keyFile) > 0 { - // Only validate cert files exist when verification is enabled - if !insecureSkipVerify { - // Validate certificate file exists - if _, err := os.Stat(certFile); os.IsNotExist(err) { - return nil, fmt.Errorf("Cassandra TLS certificate file does not exist: %s", certFile) - } - - // Validate key file exists - if _, err := os.Stat(keyFile); os.IsNotExist(err) { - return nil, fmt.Errorf("Cassandra TLS key file does not exist: %s", keyFile) - } - } - - // Load and parse the certificate and key - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return nil, fmt.Errorf("failed to load Cassandra TLS certificate and key from %s and %s: %v", certFile, keyFile, err) - } - - tlsConfig.Certificates = []tls.Certificate{cert} - log.WithFields(log.Fields{ - "driver": dbdriver, - "cert_file": certFile, - "key_file": keyFile, - }).Info("Loaded Cassandra TLS client certificate for mTLS") - } else if len(certFile) > 0 || len(keyFile) > 0 { - // Partial cert configuration detected - require both cert and key when verification is enabled - if !insecureSkipVerify { - return nil, fmt.Errorf("Cassandra TLS enabled with verification but incomplete certificate configuration (cert: %s, key: %s)", certFile, keyFile) - } - } - - // Load CA certificate if provided (optional when insecure_skip_verify is true) - // When insecure_skip_verify is true and no CA file configured, skip loading CA cert - // This allows TLS without server verification (insecure mode) - if insecureSkipVerify && len(caCertFile) == 0 { - // Insecure mode without CA cert - skip CA loading - log.WithFields(log.Fields{ - "driver": dbdriver, - }).Warn("Cassandra TLS enabled in insecure mode without CA certificate") - } else if len(caCertFile) > 0 { - // Only validate CA cert file exists when verification is enabled - if !insecureSkipVerify { - // Validate CA certificate file exists - if _, err := os.Stat(caCertFile); os.IsNotExist(err) { - return nil, fmt.Errorf("Cassandra TLS CA certificate file does not exist: %s", caCertFile) - } - } - - // Load CA certificate - caCert, err := os.ReadFile(caCertFile) - if err != nil { - return nil, fmt.Errorf("failed to read Cassandra TLS CA certificate from %s: %v", caCertFile, err) - } - - // Parse CA certificate - caCertPool := x509.NewCertPool() - if !caCertPool.AppendCertsFromPEM(caCert) { - return nil, fmt.Errorf("failed to parse Cassandra TLS CA certificate from %s", caCertFile) - } - - tlsConfig.RootCAs = caCertPool - log.WithFields(log.Fields{ - "driver": dbdriver, - "ca_cert_file": caCertFile, - }).Info("Loaded Cassandra TLS CA certificate for server verification") - } - - if insecureSkipVerify { - log.WithFields(log.Fields{ - "driver": dbdriver, - }).Warn("Cassandra TLS certificate verification is disabled (insecure_skip_verify=true). This is insecure and should only be used for testing.") - } - - log.WithFields(log.Fields{ - "driver": dbdriver, - "has_client_cert": len(tlsConfig.Certificates) > 0, - "has_ca_cert": tlsConfig.RootCAs != nil, - "insecure_skip_verify": insecureSkipVerify, - "cipher_suites": len(tlsConfig.CipherSuites), - }).Info("TLS configuration loaded for Cassandra connection") - - return tlsConfig, nil -} - func (c *CassandraClient) Codec() *security.AesCodec { return c.AesCodec } diff --git a/http/router.go b/http/router.go index d3e621c..ae4f8f8 100644 --- a/http/router.go +++ b/http/router.go @@ -32,8 +32,12 @@ func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { r2.HandleFunc("", s.VersionHandler).Methods("GET") r3 := router.Path("/config").Subrouter() - if s.ConfigApiTokenAuthEnabled() { + if testOnly { + r3.Use(s.TestingMiddleware) + } else if s.ConfigApiTokenAuthEnabled() { r3.Use(s.ApiMiddleware) + } else { + r3.Use(s.NoAuthMiddleware) } r3.HandleFunc("", s.ServerConfigHandler).Methods("GET") diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 832b8a0..d7b23f8 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -169,6 +169,18 @@ func TestConfigEndpointRequiresApiTokenWhenEnabled(t *testing.T) { assert.Equal(t, res.StatusCode, http.StatusUnauthorized) } +func TestConfigEndpointUsesTestingMiddlewareInTestRouter(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetConfigApiTokenAuthEnabled(true) + router := server.GetRouter(true) + + req, err := http.NewRequest("GET", "/config", nil) + assert.NilError(t, err) + req.Header.Set("Authorization", "Bearer test-token") + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusOK) +} + func TestApiMiddlewareSuppressesConfigRequestLogs(t *testing.T) { server := NewWebconfigServer(sc, true) server.SetConfigApiTokenAuthEnabled(true) diff --git a/security/token.go b/security/token.go index 7420a63..42b160c 100644 --- a/security/token.go +++ b/security/token.go @@ -342,16 +342,37 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi return false, "", trust, common.NewError(err) } - // check capabilities, if requiredCapabilities is nonempty - // this must run only after the token has been cryptographically verified + if len(vargs) > 1 { + mac := vargs[1] + // mac must match + isMatched := false + if macitf, ok := claims["mac"]; ok { + if macstr, ok := macitf.(string); ok { + if strings.ToLower(mac) == strings.ToLower(macstr) { + isMatched = true + } else { + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) + } + } + } + if !isMatched { + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) + } + } + + // Check capabilities after signature and route binding validation. Entries + // are untrusted claim values and must be strings before comparison. if len(requiredCapabilities) > 0 { isCapable := false if capitfs, ok := claims["capabilities"]; ok { - capvalues, ok1 := capitfs.([]interface{}) - if ok1 { + if capvalues, ok := capitfs.([]interface{}); ok { for _, capvalue := range capvalues { - for _, rc := range requiredCapabilities { - if rc == capvalue { + capability, ok := capvalue.(string) + if !ok { + continue + } + for _, requiredCapability := range requiredCapabilities { + if requiredCapability == capability { isCapable = true break } @@ -367,24 +388,6 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } - if len(vargs) > 1 { - mac := vargs[1] - // mac must match - isMatched := false - if macitf, ok := claims["mac"]; ok { - if macstr, ok := macitf.(string); ok { - if strings.ToLower(mac) == strings.ToLower(macstr) { - isMatched = true - } else { - return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) - } - } - } - if !isMatched { - return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) - } - } - // parse partner partner := "comcast" if itf, ok := claims["partner-id"]; ok { diff --git a/security/token_test.go b/security/token_test.go index ae7506b..728c8c6 100644 --- a/security/token_test.go +++ b/security/token_test.go @@ -184,6 +184,50 @@ func TestVerifyTokenEnforcesCpeCapabilitiesWhenConfigured(t *testing.T) { assert.Assert(t, errors.Is(err, common.ErrNoCapabilities)) } +func TestVerifyTokenRejectsNonStringCapabilityWithoutPanic(t *testing.T) { + if tokenManager == nil { + t.Skip("webconfig.jwt.enabled = false") + } + + cpeMac := util.GenerateRandomCpeMac() + claims := jwt.MapClaims{ + "mac": strings.ToLower(cpeMac), + "capabilities": []interface{}{map[string]interface{}{"nested": true}}, + "exp": time.Now().Add(time.Hour).Unix(), + } + token := jwt.NewWithClaims(jwt.GetSigningMethod("RS256"), claims) + token.Header["kid"] = EncodingKeyId + tokenString, err := token.SignedString(tokenManager.encodeKey) + assert.NilError(t, err) + + ok, _, _, verifyErr := VerifyToken(tokenManager.decodeKeys, tokenManager.cpeKids, []string{"cpe:config:read"}, tokenString, cpeMac) + assert.Assert(t, !ok) + assert.Assert(t, errors.Is(verifyErr, common.ErrNoCapabilities)) +} + +func TestVerifyTokenChecksMacBeforeCapabilities(t *testing.T) { + if tokenManager == nil { + t.Skip("webconfig.jwt.enabled = false") + } + + tokenMac := util.GenerateRandomCpeMac() + requestMac := util.GenerateRandomCpeMac() + claims := jwt.MapClaims{ + "mac": strings.ToLower(tokenMac), + "capabilities": []string{"cpe:other"}, + "exp": time.Now().Add(time.Hour).Unix(), + } + token := jwt.NewWithClaims(jwt.GetSigningMethod("RS256"), claims) + token.Header["kid"] = EncodingKeyId + tokenString, err := token.SignedString(tokenManager.encodeKey) + assert.NilError(t, err) + + ok, _, _, verifyErr := VerifyToken(tokenManager.decodeKeys, tokenManager.cpeKids, []string{"cpe:config:read"}, tokenString, requestMac) + assert.Assert(t, !ok) + assert.Assert(t, !errors.Is(verifyErr, common.ErrNoCapabilities)) + assert.ErrorContains(t, verifyErr, "does not match") +} + // XPC-42727 an expired token must fail as an authentication error, never // misclassified as ErrNoCapabilities, even when its (untrusted) claims lack // the required capability From 1fa5d648293964c5c7ef4375e97f35f11b2a39e6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 25 Sep 2026 23:40:32 -0700 Subject: [PATCH 12/14] Add auth for create device token api --- config/sample_webconfig.conf | 6 ++++++ http/router.go | 6 +++++- http/webconfig_server.go | 12 ++++++++++++ http/webconfig_server_test.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 4ec2bad..48fcfeb 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -162,6 +162,12 @@ webconfig { enabled = false } + // Protect /api/v1/token with API token authentication. + // Defaults to false to preserve existing endpoint access. + token_api_token_auth { + enabled = false + } + device_api_token_auth { enabled = true } diff --git a/http/router.go b/http/router.go index ae4f8f8..24b6c62 100644 --- a/http/router.go +++ b/http/router.go @@ -43,7 +43,11 @@ func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { if s.TokenApiEnabled() { r4 := router.Path("/api/v1/token").Subrouter() - r4.Use(s.NoAuthMiddleware) + if s.TokenApiTokenAuthEnabled() { + r4.Use(s.ApiMiddleware) + } else { + r4.Use(s.NoAuthMiddleware) + } r4.HandleFunc("", s.CreateTokenHandler).Methods("POST") } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index f403e63..2755954 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -65,6 +65,7 @@ const ( FactoryResetEnabledDefault = false serverApiTokenAuthEnabledDefault = true configApiTokenAuthEnabledDefault = false + tokenApiTokenAuthEnabledDefault = false deviceApiTokenAuthEnabledDefault = true tokenApiEnabledDefault = false activeDriverDefault = "cassandra" @@ -115,6 +116,7 @@ type WebconfigServer struct { factoryResetEnabled bool serverApiTokenAuthEnabled bool configApiTokenAuthEnabled bool + tokenApiTokenAuthEnabled bool deviceApiTokenAuthEnabled bool tokenApiEnabled bool kafkaEnabled bool @@ -278,6 +280,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer serverApiTokenAuthEnabled := conf.GetBoolean("webconfig.jwt.server_api_token_auth.enabled", serverApiTokenAuthEnabledDefault) configApiTokenAuthEnabled := conf.GetBoolean("webconfig.jwt.config_api_token_auth.enabled", configApiTokenAuthEnabledDefault) + tokenApiTokenAuthEnabled := conf.GetBoolean("webconfig.jwt.token_api_token_auth.enabled", tokenApiTokenAuthEnabledDefault) if conf.GetNode("webconfig.jwt.server_api_token_auth.enabled") == nil { log.Warn("webconfig.jwt.server_api_token_auth.enabled is not set in config; defaulting to true (server API token auth enforced). See MIGRATION.md.") } @@ -415,6 +418,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer factoryResetEnabled: factoryResetEnabled, serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, configApiTokenAuthEnabled: configApiTokenAuthEnabled, + tokenApiTokenAuthEnabled: tokenApiTokenAuthEnabled, deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, tokenApiEnabled: tokenApiEnabled, kafkaEnabled: kafkaEnabled, @@ -690,6 +694,14 @@ func (s *WebconfigServer) SetConfigApiTokenAuthEnabled(enabled bool) { s.configApiTokenAuthEnabled = enabled } +func (s *WebconfigServer) TokenApiTokenAuthEnabled() bool { + return s.tokenApiTokenAuthEnabled +} + +func (s *WebconfigServer) SetTokenApiTokenAuthEnabled(enabled bool) { + s.tokenApiTokenAuthEnabled = enabled +} + func (s *WebconfigServer) DeviceApiTokenAuthEnabled() bool { return s.deviceApiTokenAuthEnabled } diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index d7b23f8..ae8d35e 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -44,6 +44,7 @@ func TestApiTokenAuthSecureDefaults(t *testing.T) { // be guarded by ApiMiddleware out of the box. Regression guard for f003. assert.Assert(t, serverApiTokenAuthEnabledDefault, "server API token auth must default to enabled") assert.Assert(t, !configApiTokenAuthEnabledDefault, "config API token auth must default to disabled") + assert.Assert(t, !tokenApiTokenAuthEnabledDefault, "token API token auth must default to disabled") assert.Assert(t, deviceApiTokenAuthEnabledDefault, "device API token auth must default to enabled") } @@ -169,6 +170,30 @@ func TestConfigEndpointRequiresApiTokenWhenEnabled(t *testing.T) { assert.Equal(t, res.StatusCode, http.StatusUnauthorized) } +func TestTokenEndpointRemainsUnauthenticatedWhenDisabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetTokenApiEnabled(true) + assert.Assert(t, !server.TokenApiTokenAuthEnabled()) + router := server.GetRouter(false) + + req, err := http.NewRequest("POST", "/api/v1/token", strings.NewReader("not-json")) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusInternalServerError) +} + +func TestTokenEndpointRequiresApiTokenWhenEnabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetTokenApiEnabled(true) + server.SetTokenApiTokenAuthEnabled(true) + router := server.GetRouter(false) + + req, err := http.NewRequest("POST", "/api/v1/token", strings.NewReader("not-json")) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusUnauthorized) +} + func TestConfigEndpointUsesTestingMiddlewareInTestRouter(t *testing.T) { server := NewWebconfigServer(sc, true) server.SetConfigApiTokenAuthEnabled(true) @@ -716,6 +741,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetConfigApiTokenAuthEnabled(enabled) assert.Equal(t, server.ConfigApiTokenAuthEnabled(), enabled) + // token api token auth + enabled = true + server.SetTokenApiTokenAuthEnabled(enabled) + assert.Equal(t, server.TokenApiTokenAuthEnabled(), enabled) + enabled = false + server.SetTokenApiTokenAuthEnabled(enabled) + assert.Equal(t, server.TokenApiTokenAuthEnabled(), enabled) + // device api token auth enabled = true server.SetDeviceApiTokenAuthEnabled(enabled) From 14705bd67027cc06dc8363ec29f06e32626ec509 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 25 Sep 2026 23:52:11 -0700 Subject: [PATCH 13/14] fix a bug at router setup --- http/router.go | 4 +++- http/webconfig_server_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/http/router.go b/http/router.go index 24b6c62..ac8f7f9 100644 --- a/http/router.go +++ b/http/router.go @@ -43,7 +43,9 @@ func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { if s.TokenApiEnabled() { r4 := router.Path("/api/v1/token").Subrouter() - if s.TokenApiTokenAuthEnabled() { + if testOnly { + r4.Use(s.TestingMiddleware) + } else if s.TokenApiTokenAuthEnabled() { r4.Use(s.ApiMiddleware) } else { r4.Use(s.NoAuthMiddleware) diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index ae8d35e..a60a68f 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -194,6 +194,18 @@ func TestTokenEndpointRequiresApiTokenWhenEnabled(t *testing.T) { assert.Equal(t, res.StatusCode, http.StatusUnauthorized) } +func TestTokenEndpointUsesTestingMiddlewareInTestRouter(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetTokenApiEnabled(true) + server.SetTokenApiTokenAuthEnabled(true) + router := server.GetRouter(true) + + req, err := http.NewRequest("POST", "/api/v1/token", strings.NewReader("not-json")) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusInternalServerError) +} + func TestConfigEndpointUsesTestingMiddlewareInTestRouter(t *testing.T) { server := NewWebconfigServer(sc, true) server.SetConfigApiTokenAuthEnabled(true) From 324c4b8e6e72556c8317b4b0e9919aa341b6570f Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 1 Oct 2026 15:39:23 -0700 Subject: [PATCH 14/14] modify .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5125374..8bd6dbd 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ openspec/ .opencode/ AGENTS.md CLAUDE.md +opencode.json