diff --git a/CHANGELOG.md b/CHANGELOG.md index f72e17c6..b050de73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,17 @@ the compatibility and migration notes before upgrading. egress proxy recorded `unavailable` for every package while the GitHub client worked. Registry requests now honour the standard proxy variables; the private-address policy is applied to the route host rather than the - proxy address. + proxy address. ([#117]) +- A registry route configured after inventories existed was never consulted + for them: enrichment queued only on publication, and a repeat collection of + an unchanged export publishes nothing. The enrichment worker now queues + every stream's current snapshot at start-up (idempotent: fresh evidence and + active jobs are skipped). + +### Added + +- Helm: `server.extraEnv` renders plain variables into the server ConfigMap, + for `HTTPS_PROXY`/`NO_PROXY` on clusters whose only egress is a proxy. ## [0.6.0] - 2026-09-23 @@ -352,3 +362,4 @@ MCP client sign-in, and an expanded experimental graph-analysis foundation. [#111]: https://github.com/balcsida/graphnest/pull/111 [#112]: https://github.com/balcsida/graphnest/pull/112 [#113]: https://github.com/balcsida/graphnest/pull/113 +[#117]: https://github.com/balcsida/graphnest/pull/117 diff --git a/cmd/graphnest-server/main.go b/cmd/graphnest-server/main.go index 6803a7b9..5231cb98 100644 --- a/cmd/graphnest-server/main.go +++ b/cmd/graphnest-server/main.go @@ -555,6 +555,11 @@ func startSupplyChain(ctx context.Context, settings config.SupplyChain, store *p done = append(done, enrichDone) go func() { defer close(enrichDone) + if created, err := enricher.Backfill(ctx); err != nil && ctx.Err() == nil { + logger.Error("supply chain enrichment backfill failed", "error", err) + } else if created > 0 { + logger.Info("supply chain enrichment backfill queued", "jobs", created) + } if err := enricher.Run(ctx); err != nil && ctx.Err() == nil { logger.Error("supply chain enrichment worker stopped", "error", err) } diff --git a/deploy/helm/graphnest/README.md b/deploy/helm/graphnest/README.md index 8b2cc271..de4a199c 100644 --- a/deploy/helm/graphnest/README.md +++ b/deploy/helm/graphnest/README.md @@ -177,7 +177,10 @@ key of the existing Secret named by `secrets.supplyChainRegistries` at `/var/run/secrets/graphnest/registries/` (npm and NuGet: bearer token; Maven: `user:password`); `registries.ca: true` mounts its `caKey` as the route CA. Credentials never render into a ConfigMap. A private route is never bypassed -toward a public registry. +toward a public registry. On a cluster whose only egress is an HTTP proxy, +set `server.extraEnv` (plain variables rendered into the server ConfigMap), +e.g. `{HTTPS_PROXY: "http://proxy:3128", NO_PROXY: ".svc,.cluster.local"}`; +registry and GitHub requests honour them. `breakGlass.enabled=true` exposes only the disabled-by-default local recovery routes. It provisions no user name, password, hash, salt, or Secret and never diff --git a/deploy/helm/graphnest/ci/optional-values.yaml b/deploy/helm/graphnest/ci/optional-values.yaml index b969bda6..b777babd 100644 --- a/deploy/helm/graphnest/ci/optional-values.yaml +++ b/deploy/helm/graphnest/ci/optional-values.yaml @@ -22,6 +22,9 @@ secrets: supplyChainRegistries: {name: graphnest-registries, npmTokenKey: npm-token, nugetTokenKey: nuget-token, mavenBasicKey: maven-basic, caKey: ca.crt} server: scim: {enabled: true} + extraEnv: + HTTPS_PROXY: "http://proxy.example.invalid:3128" + NO_PROXY: ".svc,.cluster.local,github.example.invalid" supplyChain: enabled: true interval: 12h diff --git a/deploy/helm/graphnest/templates/configmaps.yaml b/deploy/helm/graphnest/templates/configmaps.yaml index 6123ea7d..b81f4ab3 100644 --- a/deploy/helm/graphnest/templates/configmaps.yaml +++ b/deploy/helm/graphnest/templates/configmaps.yaml @@ -26,6 +26,9 @@ data: GRAPHNEST_SCIP_MAX_UPLOAD_BYTES: {{ printf "%d" (int64 .Values.server.config.scipMaxUploadBytes) | quote }} GRAPHNEST_GITHUB_PRIVATE_KEY_FILE: /var/run/secrets/graphnest/github/private-key.pem GRAPHNEST_GITHUB_WEBHOOK_SECRET_FILE: /var/run/secrets/graphnest/github/webhook-secret + {{- range $name, $value := .Values.server.extraEnv }} + {{ $name }}: {{ $value | quote }} + {{- end }} {{- if .Values.breakGlass.enabled }} GRAPHNEST_BREAK_GLASS_ENABLED: "true" {{- end }} diff --git a/deploy/helm/graphnest/tests/render.sh b/deploy/helm/graphnest/tests/render.sh index 0c5b35f4..a09e58c0 100644 --- a/deploy/helm/graphnest/tests/render.sh +++ b/deploy/helm/graphnest/tests/render.sh @@ -396,6 +396,9 @@ reject 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NUGET|GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NP require 'mountPath: /var/run/secrets/graphnest/registries' "$tmp/optional.yaml" require 'secretName: graphnest-registries' "$tmp/optional.yaml" reject 'supply-chain-registries|GRAPHNEST_SUPPLY_CHAIN_REGISTRY' "$tmp/minimal.yaml" +require 'HTTPS_PROXY: "http://proxy.example.invalid:3128"' "$tmp/optional.yaml" +require 'NO_PROXY: ".svc,.cluster.local,github.example.invalid"' "$tmp/optional.yaml" +reject 'HTTPS_PROXY|NO_PROXY' "$tmp/minimal.yaml" reject '^kind: Secret$|GRAPHNEST_SCIM_TOKEN: ' "$tmp/optional.yaml" require 'GRAPHNEST_PUBLIC_URL: "https://graphnest.example.invalid"' "$tmp/scim.yaml" require 'GRAPHNEST_SCIM_TOKEN_FILE: /var/run/secrets/graphnest/scim/token' "$tmp/scim.yaml" diff --git a/deploy/helm/graphnest/values.schema.json b/deploy/helm/graphnest/values.schema.json index 1f4fcfdc..f499c9d2 100644 --- a/deploy/helm/graphnest/values.schema.json +++ b/deploy/helm/graphnest/values.schema.json @@ -125,7 +125,8 @@ "topologySpreadConstraints", "podSecurityContext", "podAnnotations", - "pdb" + "pdb", + "extraEnv" ], "properties": { "replicas": { @@ -321,6 +322,9 @@ "podAnnotations": { "$ref": "#/definitions/stringMap" }, + "extraEnv": { + "$ref": "#/definitions/stringMap" + }, "pdb": { "type": "object", "additionalProperties": false, diff --git a/deploy/helm/graphnest/values.yaml b/deploy/helm/graphnest/values.yaml index fe4ec6e2..ebea9ee9 100644 --- a/deploy/helm/graphnest/values.yaml +++ b/deploy/helm/graphnest/values.yaml @@ -76,6 +76,9 @@ server: topologySpreadConstraints: [] podSecurityContext: {} podAnnotations: {} + # Plain (non-secret) variables added to the server ConfigMap, e.g. + # HTTPS_PROXY/NO_PROXY on a cluster whose only egress is a proxy. + extraEnv: {} pdb: {enabled: true, minAvailable: 1} node: replicaCount: 1 diff --git a/docs/operations.md b/docs/operations.md index bdbfb5b2..eb0d7edb 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -233,6 +233,12 @@ different expressions, or an expression against `UNLICENSED`/`NONE`), An assessment is evidence, not approval; the review workflow records conclusions and decisions separately. +Lookups are queued when a snapshot is published and, at every server start, +for every stream's current snapshot, so a route configured after inventories +exist is consulted for them without waiting for the exports to change. +Coordinates with a queued job or fresh evidence are skipped, so the start-up +pass is idempotent. + ### Standards-based imports `POST /v1/supply-chain/imports?repository_id=&subject=&label=` diff --git a/internal/postgres/supply_chain_license.go b/internal/postgres/supply_chain_license.go index 2adfc697..9d0f5aed 100644 --- a/internal/postgres/supply_chain_license.go +++ b/internal/postgres/supply_chain_license.go @@ -313,6 +313,24 @@ func (s *Store) UpsertAssessment(ctx context.Context, assessment license.Assessm return err } +// LatestSnapshotIDs lists every stream's current snapshot. +func (s *Store) LatestSnapshotIDs(ctx context.Context) ([]int64, error) { + rows, err := s.pool.Query(ctx, `select latest_snapshot_id from supply_chain_streams where latest_snapshot_id is not null order by latest_snapshot_id`) + if err != nil { + return nil, err + } + defer rows.Close() + var result []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + result = append(result, id) + } + return result, rows.Err() +} + // SnapshotCoordinates lists distinct exact coordinates of a snapshot's // components that have a purl name and a version. func (s *Store) SnapshotCoordinates(ctx context.Context, snapshotID int64) ([]license.Coordinates, error) { diff --git a/internal/supplychain/license/store.go b/internal/supplychain/license/store.go index 5b218aad..86a11794 100644 --- a/internal/supplychain/license/store.go +++ b/internal/supplychain/license/store.go @@ -84,4 +84,6 @@ type Store interface { UpsertAssessment(ctx context.Context, assessment Assessment) error // SnapshotCoordinates lists distinct resolvable coordinates in a snapshot. SnapshotCoordinates(ctx context.Context, snapshotID int64) ([]Coordinates, error) + // LatestSnapshotIDs lists every stream's current snapshot. + LatestSnapshotIDs(ctx context.Context) ([]int64, error) } diff --git a/internal/supplychain/license/worker.go b/internal/supplychain/license/worker.go index 5d8074de..d5d7c48a 100644 --- a/internal/supplychain/license/worker.go +++ b/internal/supplychain/license/worker.go @@ -122,6 +122,31 @@ func (worker *Worker) EnqueueSnapshot(ctx context.Context, snapshotID int64) (in return created, nil } +// Backfill queues lookups for every stream's current snapshot. Publication +// queues enrichment only for the snapshot it publishes and a repeat +// collection of an unchanged export publishes nothing, so a route configured +// after inventories exist would otherwise never be consulted for them. +// EnqueueEnrichment skips coordinates with fresh evidence or an active job, +// so running this at every start is idempotent. +func (worker *Worker) Backfill(ctx context.Context) (int, error) { + if worker.Registry == nil || len(worker.Registry.Ecosystems()) == 0 { + return 0, nil + } + snapshots, err := worker.Store.LatestSnapshotIDs(ctx) + if err != nil { + return 0, err + } + created := 0 + for _, snapshotID := range snapshots { + count, err := worker.EnqueueSnapshot(ctx, snapshotID) + created += count + if err != nil { + return created, err + } + } + return created, nil +} + // Run processes enrichment jobs until the context ends. func (worker *Worker) Run(ctx context.Context) error { poll := worker.Poll diff --git a/internal/supplychain/license/worker_test.go b/internal/supplychain/license/worker_test.go index 22c47671..68b0e4b1 100644 --- a/internal/supplychain/license/worker_test.go +++ b/internal/supplychain/license/worker_test.go @@ -18,6 +18,7 @@ type memoryStore struct { occurrences map[Coordinates][][2]int64 declared map[int64]*string coordinates []Coordinates + snapshots []int64 } func newMemoryStore() *memoryStore { @@ -121,6 +122,10 @@ func (store *memoryStore) SnapshotCoordinates(context.Context, int64) ([]Coordin return store.coordinates, nil } +func (store *memoryStore) LatestSnapshotIDs(context.Context) ([]int64, error) { + return store.snapshots, nil +} + func (store *memoryStore) ComponentDeclarations(_ context.Context, componentID int64) (*string, *string, error) { store.mu.Lock() defer store.mu.Unlock() @@ -169,6 +174,32 @@ func TestWorkerEnqueuesOnlyRoutedEcosystemsAndAssesses(t *testing.T) { } } +// TestWorkerBackfillQueuesLatestSnapshots covers a route configured after +// inventories exist: every stream's current snapshot is queued once, and a +// second start queues nothing more. +func TestWorkerBackfillQueuesLatestSnapshots(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { fmt.Fprint(writer, npmLeftPad) }) + registry, err := NewRegistry([]Route{r.route(t, "npm", "/")}) + if err != nil { + t.Fatal(err) + } + store := newMemoryStore() + store.snapshots = []int64{3, 4} + store.coordinates = []Coordinates{{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.3.0"}} + worker := &Worker{Store: store, Registry: registry, Owner: "w"} + if created, err := worker.Backfill(t.Context()); err != nil || created != 1 || len(store.jobs) != 1 { + t.Fatalf("backfill created=%d jobs=%d err=%v (same coordinate in two snapshots is one job)", created, len(store.jobs), err) + } + if created, err := worker.Backfill(t.Context()); err != nil || created != 0 || len(store.jobs) != 1 { + t.Fatalf("second backfill created=%d jobs=%d err=%v", created, len(store.jobs), err) + } + unrouted := &Worker{Store: store, Registry: &Registry{resolvers: map[string]Resolver{}, routes: map[string]string{}}, Owner: "w"} + store.snapshots = nil // a nil store answer must not matter: no routes means no lookups at all + if created, err := unrouted.Backfill(t.Context()); err != nil || created != 0 { + t.Fatalf("unrouted backfill created=%d err=%v", created, err) + } +} + func TestWorkerWithoutRoutesProducesNoTraffic(t *testing.T) { r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { t.Error("registry was called") }) registry, err := NewRegistry(nil)