diff --git a/CHANGELOG.md b/CHANGELOG.md index a1e01792..2c5ccd30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,16 @@ the compatibility and migration notes before upgrading. (`subject_assurance: unknown`) and license fields are preserved verbatim. Migration 033 adds the `supply_chain_*` tables; with the module disabled nothing else changes. See ADR-0017 and `docs/execplans/supply-chain.md`. +- Exact-version license evidence for npm, NuGet, and Maven components from + explicitly configured registry routes (`GRAPHNEST_SUPPLY_CHAIN_REGISTRY_*`), + parsed with a bounded SPDX 2.3 expression parser against the pinned SPDX + License List 3.27.0. Evidence rows are immutable and carry raw values, + parse status, resolver and list versions, content hashes, and outcomes; + per-occurrence assessments report resolved, declared, conflict, unlicensed, + or unknown and are shown in the component table and a new evidence detail + view (`GET /v1/supply-chain/repositories/{id}/component`). No route means no + outbound license traffic. Migration 034 adds the evidence, enrichment-job, + and assessment tables. ## [0.5.0] - 2026-09-18 diff --git a/README.md b/README.md index 832bf892..6b505aa4 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ With `GRAPHNEST_SUPPLY_CHAIN=true` in durable mode, `graphnest-server` collects What the inventory is and is not: - A GitHub dependency-graph export is a **timestamped observation of the default branch**. The endpoint has no ref selector, so snapshots report `subject_assurance: unknown`; GraphNest never copies the indexed or current HEAD into a snapshot. -- GitHub Enterprise Server does not populate dependency license fields; `license_declared_raw`/`license_concluded_raw` are preserved verbatim (typically `NOASSERTION`) and are never mapped to a license. Exact-version license evidence is a separate, later enrichment layer. +- GitHub Enterprise Server does not populate dependency license fields; `license_declared_raw`/`license_concluded_raw` are preserved verbatim (typically `NOASSERTION`) and are never mapped to a license. Exact-version license evidence comes only from registry routes you configure (`GRAPHNEST_SUPPLY_CHAIN_REGISTRY_{NPM,NUGET,MAVEN}_URL` and companion secret-file settings); without a route no license traffic is produced, and a private route never falls back to a public registry. SPDX expressions are parsed against the pinned SPDX License List 3.27.0 with AND/OR/WITH structure preserved; `NOASSERTION`, `NONE`, `UNLICENSED`, unknown identifiers, license files, and URLs stay what they are. - Components without a purl or version stay visible. Dependency scope (`root`/`direct`/`transitive`) is derived only from resolved `DEPENDS_ON` edges leaving a described root; a flattened list yields `unknown`, never `direct`. - A failed refresh (403, 404, rate limit, malformed or oversized document, outage) records a collection attempt and leaves the last successful snapshot in place; the status reports `collection: failed` alongside the retained inventory. - Inventory eligibility is repository authorization alone. It works for repositories with no Zoekt index, no SCIP upload, and no graph enrichment, and inventory work never blocks lexical indexing. diff --git a/cmd/graphnest-server/main.go b/cmd/graphnest-server/main.go index 62038c1f..ba5bae9c 100644 --- a/cmd/graphnest-server/main.go +++ b/cmd/graphnest-server/main.go @@ -43,6 +43,7 @@ import ( "github.com/balcsida/graphnest/internal/sso/githuboauth" "github.com/balcsida/graphnest/internal/sso/oidc" "github.com/balcsida/graphnest/internal/supplychain" + "github.com/balcsida/graphnest/internal/supplychain/license" "github.com/balcsida/graphnest/internal/webhook" "github.com/balcsida/graphnest/internal/webui" "github.com/balcsida/graphnest/internal/zoekt" @@ -451,8 +452,23 @@ func newDurableRuntime(ctx context.Context, settings config.Config, logger *slog var extras []func(*http.ServeMux) var supplyChainDone []<-chan struct{} if settings.SupplyChain.Enabled { - supplyChainService := &supplychain.Service{Store: store, Authorizer: authz.NewPostgres(store), Interval: settings.SupplyChain.Interval, MaxResults: settings.Limits.MaxResults} - supplyChainDone = startSupplyChain(loopCtx, settings.SupplyChain, store, githubClient, metrics, logger) + routes, err := license.RoutesFromEnv(os.Getenv, license.ReadSecretFile) + if err != nil { + cancel() + <-done + <-reconcileDone + return fail(fmt.Errorf("supply chain registry routes: %w", err)) + } + registry, err := license.NewRegistry(routes) + if err != nil { + cancel() + <-done + <-reconcileDone + return fail(fmt.Errorf("supply chain registry routes: %w", err)) + } + supplyChainService := &supplychain.Service{Store: store, Authorizer: authz.NewPostgres(store), Interval: settings.SupplyChain.Interval, MaxResults: settings.Limits.MaxResults, + License: store, EnrichmentEcosystems: registry.Ecosystems()} + supplyChainDone = startSupplyChain(loopCtx, settings.SupplyChain, store, githubClient, registry, metrics, logger) extras = append(extras, func(mux *http.ServeMux) { httpapi.RegisterSupplyChain(mux, auth.requestAuth, supplyChainService, settings.Limits.MaxResults, settings.Limits.MaxResponseBytes) }) @@ -478,8 +494,21 @@ func newDurableRuntime(ctx context.Context, settings config.Config, logger *slog // startSupplyChain runs the inventory scheduler and collection workers inside // the server process. They share nothing with the indexer, so inventory work // can neither block nor be blocked by lexical indexing (ADR-0017). -func startSupplyChain(ctx context.Context, settings config.SupplyChain, store *postgres.Store, client *githubapp.Client, metrics *observability.Metrics, logger *slog.Logger) []<-chan struct{} { +func startSupplyChain(ctx context.Context, settings config.SupplyChain, store *postgres.Store, client *githubapp.Client, registry *license.Registry, metrics *observability.Metrics, logger *slog.Logger) []<-chan struct{} { var done []<-chan struct{} + hostname, _ := os.Hostname() + var enricher *license.Worker + if len(registry.Ecosystems()) > 0 { + enricher = &license.Worker{Store: store, Registry: registry, Owner: fmt.Sprintf("%s-%d-enrich", hostname, os.Getpid()), Logger: logger, Observer: metrics} + enrichDone := make(chan struct{}) + done = append(done, enrichDone) + go func() { + defer close(enrichDone) + if err := enricher.Run(ctx); err != nil && ctx.Err() == nil { + logger.Error("supply chain enrichment worker stopped", "error", err) + } + }() + } scheduler := &supplychain.Scheduler{Store: store, Interval: settings.Interval} schedulerDone := make(chan struct{}) done = append(done, schedulerDone) @@ -502,12 +531,14 @@ func startSupplyChain(ctx context.Context, settings config.SupplyChain, store *p } } }() - hostname, _ := os.Hostname() for worker := range settings.Workers { collector := &supplychain.Collector{ Store: store, GitHub: client, Owner: fmt.Sprintf("%s-%d-%d", hostname, os.Getpid(), worker), MaxDocumentBytes: settings.MaxDocumentBytes, Limits: supplychain.Limits{MaxComponents: settings.MaxComponents}, Logger: logger, Observer: metrics, } + if enricher != nil { + collector.Enricher = enricher + } workerDone := make(chan struct{}) done = append(done, workerDone) go func() { diff --git a/deploy/helm/graphnest/README.md b/deploy/helm/graphnest/README.md index 15a1daa5..8b2cc271 100644 --- a/deploy/helm/graphnest/README.md +++ b/deploy/helm/graphnest/README.md @@ -169,6 +169,16 @@ the only outbound call is the configured GitHub API endpoint. Snapshots live in PostgreSQL (`supply_chain_*` tables, created by the normal migration Job); see the repository operations guide for lifecycle, recovery, and metrics. +License enrichment is off until a registry route is set under +`server.supplyChain.registries.{npm,nuget,maven}.url` (HTTPS). Optional +`namespaces` restrict what the route may answer for, `allowPrivate` permits an +internal mirror on a private address, and `token`/`basic` mount the matching +key of the existing Secret named by `secrets.supplyChainRegistries` at +`/var/run/secrets/graphnest/registries/` (npm and NuGet: bearer token; Maven: +`user:password`); `registries.ca: true` mounts its `caKey` as the route CA. +Credentials never render into a ConfigMap. A private route is never bypassed +toward a public registry. + `breakGlass.enabled=true` exposes only the disabled-by-default local recovery routes. It provisions no user name, password, hash, salt, or Secret and never activates because OIDC is unavailable. Provision and rotate the operator diff --git a/deploy/helm/graphnest/ci/optional-values.yaml b/deploy/helm/graphnest/ci/optional-values.yaml index d6cc1254..1f898269 100644 --- a/deploy/helm/graphnest/ci/optional-values.yaml +++ b/deploy/helm/graphnest/ci/optional-values.yaml @@ -19,9 +19,20 @@ secrets: githubOAuth: {name: graphnest-github-oauth, clientSecretKey: client-secret} oidcCA: {name: graphnest-oidc-ca, key: ca.crt} scim: {name: graphnest-scim, tokenKey: token} + supplyChainRegistries: {name: graphnest-registries, npmTokenKey: npm-token, nugetTokenKey: nuget-token, mavenBasicKey: maven-basic, caKey: ca.crt} server: scim: {enabled: true} - supplyChain: {enabled: true, interval: 12h, workers: 2, maxDocumentBytes: 33554432, maxComponents: 60000} + supplyChain: + enabled: true + interval: 12h + workers: 2 + maxDocumentBytes: 33554432 + maxComponents: 60000 + registries: + npm: {url: https://npm.example.invalid/, namespaces: ["@acme", "@internal"], allowPrivate: false, token: true} + nuget: {url: "", namespaces: [], allowPrivate: false} + maven: {url: https://maven.example.invalid/repository/public/, namespaces: [com.acme], allowPrivate: true, basic: true} + ca: true sso: publicURL: https://graphnest.example.invalid oidc: {enabled: true, issuerURL: https://id.example.invalid, clientID: graphnest, scopes: [openid, profile, email], linkClaim: sub, displayNameClaim: name} diff --git a/deploy/helm/graphnest/templates/configmaps.yaml b/deploy/helm/graphnest/templates/configmaps.yaml index 3b769ab3..56fe9b0f 100644 --- a/deploy/helm/graphnest/templates/configmaps.yaml +++ b/deploy/helm/graphnest/templates/configmaps.yaml @@ -73,6 +73,27 @@ data: GRAPHNEST_SUPPLY_CHAIN_WORKERS: {{ .Values.server.supplyChain.workers | quote }} GRAPHNEST_SUPPLY_CHAIN_MAX_DOCUMENT_BYTES: {{ printf "%d" (int64 .Values.server.supplyChain.maxDocumentBytes) | quote }} GRAPHNEST_SUPPLY_CHAIN_MAX_COMPONENTS: {{ .Values.server.supplyChain.maxComponents | quote }} + {{- range $ecosystem, $route := (pick .Values.server.supplyChain.registries "npm" "nuget" "maven") }} + {{- if $route.url }} + {{- $prefix := printf "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_%s_" (upper $ecosystem) }} + {{ $prefix }}URL: {{ $route.url | quote }} + {{- if $route.namespaces }} + {{ $prefix }}NAMESPACES: {{ join "," $route.namespaces | quote }} + {{- end }} + {{- if $route.allowPrivate }} + {{ $prefix }}ALLOW_PRIVATE: "true" + {{- end }} + {{- if $route.token }} + {{ $prefix }}TOKEN_FILE: /var/run/secrets/graphnest/registries/{{ $ecosystem }}-token + {{- end }} + {{- if $route.basic }} + {{ $prefix }}BASIC_FILE: /var/run/secrets/graphnest/registries/{{ $ecosystem }}-basic + {{- end }} + {{- if $.Values.server.supplyChain.registries.ca }} + {{ $prefix }}CA_FILE: /var/run/secrets/graphnest/registries/ca.crt + {{- end }} + {{- end }} + {{- end }} {{- end }} --- apiVersion: v1 diff --git a/deploy/helm/graphnest/templates/server.yaml b/deploy/helm/graphnest/templates/server.yaml index 9fae58f0..2d7d723a 100644 --- a/deploy/helm/graphnest/templates/server.yaml +++ b/deploy/helm/graphnest/templates/server.yaml @@ -92,6 +92,9 @@ spec: {{- if .Values.server.scim.enabled }} - {name: scim-token, mountPath: /var/run/secrets/graphnest/scim/token, subPath: token, readOnly: true} {{- end }} + {{- if and .Values.server.supplyChain.enabled .Values.secrets.supplyChainRegistries.name }} + - {name: supply-chain-registries, mountPath: /var/run/secrets/graphnest/registries, readOnly: true} + {{- end }} {{- if .Values.server.sso.oidc.enabled }} - {name: oidc-client-secret, mountPath: /var/run/secrets/graphnest/oidc/client-secret, subPath: client-secret, readOnly: true} {{- if .Values.secrets.oidcCA.name }} @@ -151,6 +154,25 @@ spec: items: - {key: {{ .Values.secrets.scim.tokenKey }}, path: token} {{- end }} + {{- if and .Values.server.supplyChain.enabled .Values.secrets.supplyChainRegistries.name }} + - name: supply-chain-registries + secret: + secretName: {{ .Values.secrets.supplyChainRegistries.name }} + optional: false + items: + {{- if .Values.server.supplyChain.registries.npm.token }} + - {key: {{ .Values.secrets.supplyChainRegistries.npmTokenKey }}, path: npm-token} + {{- end }} + {{- if .Values.server.supplyChain.registries.nuget.token }} + - {key: {{ .Values.secrets.supplyChainRegistries.nugetTokenKey }}, path: nuget-token} + {{- end }} + {{- if .Values.server.supplyChain.registries.maven.basic }} + - {key: {{ .Values.secrets.supplyChainRegistries.mavenBasicKey }}, path: maven-basic} + {{- end }} + {{- if .Values.server.supplyChain.registries.ca }} + - {key: {{ .Values.secrets.supplyChainRegistries.caKey }}, path: ca.crt} + {{- end }} + {{- end }} --- apiVersion: v1 kind: Service diff --git a/deploy/helm/graphnest/tests/render.sh b/deploy/helm/graphnest/tests/render.sh index 0ed03831..d1fe3dbd 100644 --- a/deploy/helm/graphnest/tests/render.sh +++ b/deploy/helm/graphnest/tests/render.sh @@ -384,6 +384,17 @@ require 'GRAPHNEST_SUPPLY_CHAIN_INTERVAL: "12h"' "$tmp/optional.yaml" require 'GRAPHNEST_SUPPLY_CHAIN_WORKERS: "2"' "$tmp/optional.yaml" require 'GRAPHNEST_SUPPLY_CHAIN_MAX_DOCUMENT_BYTES: "33554432"' "$tmp/optional.yaml" require 'GRAPHNEST_SUPPLY_CHAIN_MAX_COMPONENTS: "60000"' "$tmp/optional.yaml" +require 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL: "https://npm.example.invalid/"' "$tmp/optional.yaml" +require 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_NAMESPACES: "@acme,@internal"' "$tmp/optional.yaml" +require 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_TOKEN_FILE: /var/run/secrets/graphnest/registries/npm-token' "$tmp/optional.yaml" +require 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_CA_FILE: /var/run/secrets/graphnest/registries/ca.crt' "$tmp/optional.yaml" +require 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_MAVEN_URL: "https://maven.example.invalid/repository/public/"' "$tmp/optional.yaml" +require 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_MAVEN_ALLOW_PRIVATE: "true"' "$tmp/optional.yaml" +require 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_MAVEN_BASIC_FILE: /var/run/secrets/graphnest/registries/maven-basic' "$tmp/optional.yaml" +reject 'GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NUGET|GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_ALLOW_PRIVATE|GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_BASIC|nuget-token' "$tmp/optional.yaml" +require 'mountPath: /var/run/secrets/graphnest/registries' "$tmp/optional.yaml" +require 'secretName: graphnest-registries' "$tmp/optional.yaml" +reject 'supply-chain-registries|GRAPHNEST_SUPPLY_CHAIN_REGISTRY' "$tmp/minimal.yaml" reject '^kind: Secret$|GRAPHNEST_SCIM_TOKEN: ' "$tmp/optional.yaml" require 'GRAPHNEST_PUBLIC_URL: "https://graphnest.example.invalid"' "$tmp/scim.yaml" require 'GRAPHNEST_SCIM_TOKEN_FILE: /var/run/secrets/graphnest/scim/token' "$tmp/scim.yaml" diff --git a/deploy/helm/graphnest/values.schema.json b/deploy/helm/graphnest/values.schema.json index 922bcee9..5097edc3 100644 --- a/deploy/helm/graphnest/values.schema.json +++ b/deploy/helm/graphnest/values.schema.json @@ -55,7 +55,8 @@ "githubOAuth", "mcpOAuth", "oidcCA", - "scim" + "scim", + "supplyChainRegistries" ], "properties": { "runtime": { @@ -81,6 +82,18 @@ }, "scim": { "$ref": "#/definitions/scimSecret" + }, + "supplyChainRegistries": { + "type": "object", + "additionalProperties": false, + "required": ["name", "npmTokenKey", "nugetTokenKey", "mavenBasicKey", "caKey"], + "properties": { + "name": {"$ref": "#/definitions/optionalKubernetesObjectName"}, + "npmTokenKey": {"$ref": "#/definitions/secretKey"}, + "nugetTokenKey": {"$ref": "#/definitions/secretKey"}, + "mavenBasicKey": {"$ref": "#/definitions/secretKey"}, + "caKey": {"$ref": "#/definitions/secretKey"} + } } } }, @@ -266,13 +279,24 @@ "supplyChain": { "type": "object", "additionalProperties": false, - "required": ["enabled", "interval", "workers", "maxDocumentBytes", "maxComponents"], + "required": ["enabled", "interval", "workers", "maxDocumentBytes", "maxComponents", "registries"], "properties": { "enabled": {"type": "boolean"}, "interval": {"$ref": "#/definitions/duration"}, "workers": {"type": "integer", "minimum": 1, "maximum": 8}, "maxDocumentBytes": {"type": "integer", "minimum": 1, "maximum": 268435456}, - "maxComponents": {"type": "integer", "minimum": 1, "maximum": 500000} + "maxComponents": {"type": "integer", "minimum": 1, "maximum": 500000}, + "registries": { + "type": "object", + "additionalProperties": false, + "required": ["npm", "nuget", "maven", "ca"], + "properties": { + "npm": {"$ref": "#/definitions/registryRoute"}, + "nuget": {"$ref": "#/definitions/registryRoute"}, + "maven": {"$ref": "#/definitions/registryRoute"}, + "ca": {"type": "boolean", "description": "Mount secrets.supplyChainRegistries[caKey] as the route CA for every configured registry"} + } + } } }, "resources": { @@ -1030,6 +1054,18 @@ "type": "string" } }, + "registryRoute": { + "type": "object", + "additionalProperties": false, + "required": ["url", "namespaces", "allowPrivate"], + "properties": { + "url": {"type": "string", "pattern": "^(|https://.+)$"}, + "namespaces": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "allowPrivate": {"type": "boolean"}, + "token": {"type": "boolean", "description": "Mount the ecosystem token key from secrets.supplyChainRegistries as a bearer token"}, + "basic": {"type": "boolean", "description": "Mount the ecosystem basic key from secrets.supplyChainRegistries as user:password"} + } + }, "scimSecret": { "type": "object", "additionalProperties": false, diff --git a/deploy/helm/graphnest/values.yaml b/deploy/helm/graphnest/values.yaml index add63654..e16cd2dc 100644 --- a/deploy/helm/graphnest/values.yaml +++ b/deploy/helm/graphnest/values.yaml @@ -14,6 +14,8 @@ secrets: mcpOAuth: {name: "", keyKey: sealing-key} oidcCA: {name: "", key: ca.crt} scim: {name: "", tokenKey: token} + # Optional per-ecosystem registry credentials for license enrichment. Keys are mounted read-only; a missing key is simply not mounted. + supplyChainRegistries: {name: "", npmTokenKey: npm-token, nugetTokenKey: nuget-token, mavenBasicKey: maven-basic, caKey: ca.crt} breakGlass: {enabled: false} server: replicas: 2 @@ -53,7 +55,18 @@ server: mcpOAuth: {enabled: false} scim: {enabled: false} # Dependencies & Licenses inventory (ADR-0017). Disabled by default; enabling it needs no repository changes. - supplyChain: {enabled: false, interval: 24h, workers: 1, maxDocumentBytes: 16777216, maxComponents: 50000} + supplyChain: + enabled: false + interval: 24h + workers: 1 + maxDocumentBytes: 16777216 + maxComponents: 50000 + # License enrichment routes; an ecosystem without a url produces no registry traffic. + registries: + npm: {url: "", namespaces: [], allowPrivate: false, token: false} + nuget: {url: "", namespaces: [], allowPrivate: false, token: false} + maven: {url: "", namespaces: [], allowPrivate: false, basic: false} + ca: false resources: {requests: {cpu: 250m, memory: 256Mi}, limits: {cpu: "1", memory: 1Gi}} nodeSelector: {} affinity: {} diff --git a/docs/images/supply-chain-dark.png b/docs/images/supply-chain-dark.png index 77c04cc7..ebc30f7e 100644 Binary files a/docs/images/supply-chain-dark.png and b/docs/images/supply-chain-dark.png differ diff --git a/docs/images/supply-chain-light.png b/docs/images/supply-chain-light.png index 56741d40..c147ddf1 100644 Binary files a/docs/images/supply-chain-light.png and b/docs/images/supply-chain-light.png differ diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 8e946d21..3adc58fc 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -667,6 +667,23 @@ paths: '405': {$ref: '#/components/responses/InvalidRequest'} '500': {description: Response exceeded the configured byte limit} '503': {$ref: '#/components/responses/Unavailable'} + /v1/supply-chain/repositories/{id}/component: + get: + description: Evidence detail for one occurrence identified by its document element ID (SPDXID). Returns the producer's declarations as evidence rows, the immutable registry evidence history for the exact coordinates (only reachable through an authorized occurrence), bounded relationships, and the derived assessment. Evidence is not approval. + security: [{bearerAuth: []}, {sessionCookie: []}] + parameters: + - {name: id, in: path, required: true, schema: {type: integer, format: int64, minimum: 1}} + - {name: element, in: query, required: true, schema: {type: string, minLength: 1, maxLength: 512}} + - {name: stream, in: query, required: false, schema: {type: string, enum: ['github:source']}} + - {name: snapshot_id, in: query, required: false, schema: {type: integer, format: int64, minimum: 1}} + responses: + '200': {description: Component evidence detail, content: {application/json: {schema: {$ref: '#/components/schemas/SupplyChainComponentDetail'}}}} + '400': {$ref: '#/components/responses/InvalidRequest'} + '401': {$ref: '#/components/responses/Unauthenticated'} + '404': {$ref: '#/components/responses/SupplyChainNotFound'} + '405': {$ref: '#/components/responses/InvalidRequest'} + '500': {description: Response exceeded the configured byte limit} + '503': {$ref: '#/components/responses/Unavailable'} /v1/supply-chain/repositories/{id}/snapshots: get: security: [{bearerAuth: []}, {sessionCookie: []}] @@ -2125,7 +2142,7 @@ components: SupplyChainRepositoryStatus: type: object additionalProperties: false - required: [repository_id, repository, stream, producer, subject, collection, freshness_seconds, latest_snapshot, last_collection, active_job, enrichment, opt_out, notes, documents] + required: [repository_id, repository, stream, producer, subject, collection, freshness_seconds, latest_snapshot, last_collection, active_job, enrichment, enrichment_ecosystems, license_summary, opt_out, notes, documents] properties: repository_id: {type: integer, format: int64} repository: {type: string} @@ -2137,7 +2154,9 @@ components: latest_snapshot: {oneOf: [{$ref: '#/components/schemas/SupplyChainSnapshot'}, {type: 'null'}]} last_collection: {oneOf: [{$ref: '#/components/schemas/SupplyChainCollection'}, {type: 'null'}]} active_job: {oneOf: [{$ref: '#/components/schemas/SupplyChainJob'}, {type: 'null'}]} - enrichment: {type: string, enum: [not_configured], description: License enrichment state; a later milestone adds resolver states} + enrichment: {type: string, enum: [not_configured, configured], description: Whether any registry route is configured for license enrichment} + enrichment_ecosystems: {type: array, items: {type: string, enum: [npm, nuget, maven]}} + license_summary: {type: object, additionalProperties: {type: integer, minimum: 0}, description: Latest snapshot's assessment counts by status (unknown, declared, resolved, conflict, unlicensed, not_applicable, pending)} opt_out: {type: boolean} notes: {type: array, items: {type: string}} documents: {type: array, items: {$ref: '#/components/schemas/SupplyChainDocumentRef'}} @@ -2209,6 +2228,69 @@ components: checksums: {type: array, items: {type: object, additionalProperties: false, required: [algorithm, value], properties: {algorithm: {type: string}, value: {type: string}}}} is_root: {type: boolean} scope: {type: string, enum: [root, direct, transitive, unknown]} + license: {oneOf: [{$ref: '#/components/schemas/SupplyChainLicenseAssessment'}, {type: 'null'}]} + SupplyChainLicenseAssessment: + type: object + additionalProperties: false + required: [status, evidence_count, assessed_at, evidence_fingerprint] + description: Derived view over producer declarations and registry evidence for one occurrence. Not an approval. + properties: + status: {type: string, enum: [unknown, declared, resolved, conflict, unlicensed, not_applicable, pending]} + expression: {type: string, description: Normalized SPDX expression when status is declared or resolved} + conflict_detail: {type: string} + evidence_count: {type: integer, minimum: 0} + assessed_at: {type: string, format: date-time} + evidence_fingerprint: {type: string, pattern: '^[0-9a-f]{64}$'} + SupplyChainLicenseEvidence: + type: object + additionalProperties: false + required: [id, source, ecosystem, name, version, raw_value, raw_kind, parse_status, resolver_version, license_list_version, fetched_at, outcome] + properties: + id: {type: integer, format: int64, minimum: 0, description: 0 for producer declarations synthesized from the snapshot} + source: {type: string, enum: [producer_declared, producer_concluded, registry_npm, registry_nuget, registry_maven, import, human]} + route: {type: string} + ecosystem: {type: string} + namespace: {type: string} + name: {type: string} + version: {type: string} + artifact_sha256: {type: string} + raw_value: {type: string, description: Verbatim value as observed} + raw_kind: {type: string, enum: [expression, expression_or_file, license_file, license_url, license_name, legacy_object, missing, sentinel]} + parse_status: {type: string, enum: [parsed, unknown_terms, no_assertion, none, unlicensed, invalid, not_applicable]} + expression: {type: string} + unknown_terms: {type: array, items: {type: string}} + license_url: {type: string} + license_file_name: {type: string} + detail: {type: object} + resolver_version: {type: integer, minimum: 1} + license_list_version: {type: string} + content_sha256: {type: string, pattern: '^[0-9a-f]{64}$'} + fetched_at: {type: string, format: date-time} + expires_at: {type: string, format: date-time, description: Present on negative outcomes; the lookup is retried after this time} + outcome: {type: string, enum: [resolved, not_found, no_license_metadata, unavailable, rejected, too_large, malformed]} + http_status: {type: integer} + message: {type: string} + SupplyChainComponentDetail: + type: object + additionalProperties: false + required: [component, snapshot, declarations, evidence, relationships, notes, truncated] + properties: + component: {$ref: '#/components/schemas/SupplyChainComponent'} + snapshot: {$ref: '#/components/schemas/SupplyChainSnapshot'} + declarations: {type: array, items: {$ref: '#/components/schemas/SupplyChainLicenseEvidence'}} + evidence: {type: array, maxItems: 100, items: {$ref: '#/components/schemas/SupplyChainLicenseEvidence'}} + relationships: {type: array, maxItems: 100, items: {$ref: '#/components/schemas/SupplyChainRelationship'}} + notes: {type: array, items: {type: string}} + truncated: {type: boolean} + SupplyChainRelationship: + type: object + additionalProperties: false + required: [from, type, to, resolved] + properties: + from: {type: string} + type: {type: string} + to: {type: string} + resolved: {type: boolean} SupplyChainComponentList: type: object additionalProperties: false diff --git a/docs/operations.md b/docs/operations.md index efe3e89c..00c716e1 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -156,9 +156,73 @@ distinct SBOM exports retained. Retention pruning is not implemented in this milestone. Metrics: `graphnest_supply_chain_collections_total{outcome}`, -`graphnest_supply_chain_collection_duration_seconds{outcome}`, and -`graphnest_supply_chain_queue_depth{state}`. Labels use fixed vocabularies; -no repository or component identity is exported. +`graphnest_supply_chain_collection_duration_seconds{outcome}`, +`graphnest_supply_chain_queue_depth{state}`, +`graphnest_supply_chain_enrichment_total{outcome}`, and +`graphnest_supply_chain_enrichment_duration_seconds{outcome}`. Labels use +fixed vocabularies; no repository or component identity is exported. + +### License enrichment routes + +GitHub exports carry no license data. Exact-version license evidence comes +only from registry routes you configure; with none configured, GraphNest +produces no license traffic at all and components show only the producer's +(usually `NOASSERTION`) declaration. + +| Variable (per ecosystem `NPM`, `NUGET`, `MAVEN`) | Meaning | +| --- | --- | +| `GRAPHNEST_SUPPLY_CHAIN_REGISTRY__URL` | HTTPS registry root, without credentials, query, or fragment. npm: the registry root (`https://npm.example/`); NuGet: the V3 flat container (`https://nuget.example/v3-flatcontainer/`); Maven: the repository root (`https://maven.example/repository/public/`). | +| `..._TOKEN_FILE` | Optional bearer token secret file (regular file, at most 64 KiB). | +| `..._BASIC_FILE` | Optional `user:password` secret file; mutually exclusive with the token file. | +| `..._CA_FILE` | Optional PEM bundle appended to the system roots for this route. | +| `..._ALLOW_PRIVATE` | `true` to permit a registry that resolves to a private, loopback, or link-local address (internal mirrors). Default `false`; cloud metadata ranges stay blocked regardless. | +| `..._NAMESPACES` | Optional comma-separated npm scopes / Maven groupId prefixes / NuGet id prefixes this route may answer for; anything else is rejected without a request. | + +One route per ecosystem. A package the route does not know is recorded as +`not_found` at that route; GraphNest never retries it against a public +registry, so a private-registry deployment cannot leak package names. Requests +are pinned to the route's origin and base path (redirects elsewhere are +rejected), bodies are bounded after decompression (4 MiB), and credentials are +attached only to the route's own origin. + +What each resolver reads and how it records it: + +- **npm**: `GET {root}/{name}/{version}` for the exact version only, never + dist-tags or the packument's `latest`. A string `license` is parsed as an + SPDX expression; `SEE LICENSE IN ` is recorded as a license-file + reference; legacy `{type,url}` objects and `licenses` arrays are kept as + legacy metadata (an array of names has no SPDX AND/OR meaning and stays + unparsed); `UNLICENSED` stays `unlicensed`. A document naming a different + version is rejected. +- **NuGet**: the exact-version `.nuspec` from the flat container; the + `.nupkg` is never downloaded. `` is parsed, + `` is a file reference, and a legacy `` + alone is recorded as a URL, not a concluded license. +- **Maven**: the exact-version POM. `` are names and URLs, not SPDX + expressions; only unambiguous names (Apache 2.0, MIT, BSD, EPL, LGPL, MPL, + ISC, CDDL, Unlicense, CC0, GPL-2.0 with Classpath) are normalized, and a + name that is already a valid SPDX expression parses as such. Several + `` elements are kept as a list without invented structure. Missing + `` are inherited through `` on the same route only, at + most eight levels, with cycle detection and bounded `${property}` + expansion; anything unresolved stays `no_license_metadata`. Repository + declarations inside POMs are never followed. + +Evidence rows are immutable and carry the raw value, parse status, normalized +expression, unknown terms, resolver version, SPDX License List version +(3.27.0), content hash, fetch time, and outcome. A re-fetch that yields the +same facts is a new observation flagged as a duplicate; a change is a new +row. Negative results (`not_found`, `no_license_metadata`, `unavailable`) +expire after 24 hours and are retried; an outage keeps the earlier resolved +evidence visible with its age rather than replacing it with "no license". + +Assessments are derived per occurrence from the producer's declaration and +the latest evidence per route: `resolved` (registry expression, consistent), +`declared` (only the producer's expression parsed), `conflict` (structurally +different expressions, or an expression against `UNLICENSED`/`NONE`), +`unlicensed`, `unknown` (nothing parseable), `pending`, or `not_applicable`. +An assessment is evidence, not approval; the review workflow records +conclusions and decisions separately. ## Break-glass administrator recovery diff --git a/docs/threat-model.md b/docs/threat-model.md index 6cb22026..c9ddc01e 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -141,6 +141,17 @@ all clients appear as one source. last successful snapshot. - Telemetry labels use fixed outcome and state vocabularies; no repository or component identity is exported. +- License enrichment produces no outbound traffic unless a registry route is + configured. A route is pinned to one HTTPS origin and base path: redirects + elsewhere are rejected, private/loopback/link-local/metadata destinations + are refused unless the route explicitly allows private hosts, hostile path + segments are rejected before any request, bodies are bounded after + decompression, and credentials from secret files are attached only to that + origin. A package unknown to a private route is never retried against a + public registry, so private package names do not leak. Registry metadata is + untrusted content: SPDX expressions are parsed with a bounded grammar + (input size, token count, nesting depth), XML is decoded without external + entities, and free text or unknown identifiers never become a license. ## Known limits diff --git a/internal/httpapi/supply_chain.go b/internal/httpapi/supply_chain.go index 70dbe2a3..93e57f60 100644 --- a/internal/httpapi/supply_chain.go +++ b/internal/httpapi/supply_chain.go @@ -75,6 +75,27 @@ func RegisterSupplyChain(mux *http.ServeMux, authenticator authn.RequestAuthenti } writeBoundedJSON(writer, response, maxResponseBytes) }).ServeHTTP(writer, request) + case "component": + authenticated(http.MethodGet, func(writer http.ResponseWriter, request *http.Request) { + var snapshotID int64 + ok := true + if value := query.Get("snapshot_id"); value != "" { + var err error + snapshotID, err = strconv.ParseInt(value, 10, 64) + ok = err == nil && snapshotID > 0 + } + element := query.Get("element") + if !ok || element == "" || len(query["element"]) != 1 { + writeError(writer, http.StatusBadRequest, "invalid_request", "request is invalid", false) + return + } + response, err := service.ComponentDetail(request.Context(), PrincipalFromContext(request.Context()), githubID, stream, snapshotID, element) + if err != nil { + writeSupplyChainError(writer, err) + return + } + writeBoundedJSON(writer, response, maxResponseBytes) + }).ServeHTTP(writer, request) case "snapshots": authenticated(http.MethodGet, func(writer http.ResponseWriter, request *http.Request) { limit, ok := limitFrom(query) diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go index e40b328d..cef7a7c0 100644 --- a/internal/observability/metrics.go +++ b/internal/observability/metrics.go @@ -34,6 +34,8 @@ type Metrics struct { supplyChainRuns *prometheus.CounterVec supplyChainTime *prometheus.HistogramVec supplyChainDepth *prometheus.GaugeVec + enrichmentRuns *prometheus.CounterVec + enrichmentTime *prometheus.HistogramVec } func New() *Metrics { @@ -59,8 +61,10 @@ func New() *Metrics { metrics.authEvents = prometheus.NewCounterVec(prometheus.CounterOpts{Name: "graphnest_auth_events_total", Help: "Authentication events."}, []string{"provider", "event", "result"}) metrics.supplyChainRuns = prometheus.NewCounterVec(prometheus.CounterOpts{Name: "graphnest_supply_chain_collections_total", Help: "Supply chain collection attempts by outcome."}, []string{"outcome"}) metrics.supplyChainTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: "graphnest_supply_chain_collection_duration_seconds", Help: "Supply chain collection duration by outcome."}, []string{"outcome"}) - metrics.supplyChainDepth = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "graphnest_supply_chain_queue_depth", Help: "Supply chain refresh jobs by state."}, []string{"state"}) - metrics.registry.MustRegister(metrics.archiveOperations, metrics.archiveDuration, metrics.activeRequests, metrics.httpRequests, metrics.httpDuration, metrics.httpResponseSize, metrics.backendCalls, metrics.backendDuration, metrics.githubRequests, metrics.webhookDeliveries, metrics.indexQueueDepth, metrics.indexPhases, metrics.indexDuration, metrics.graphQueueDepth, metrics.graphPhases, metrics.graphDuration, metrics.graphQueries, metrics.graphQueryDuration, metrics.authEvents, metrics.supplyChainRuns, metrics.supplyChainTime, metrics.supplyChainDepth) + metrics.supplyChainDepth = prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "graphnest_supply_chain_queue_depth", Help: "Supply chain refresh and enrichment jobs by state."}, []string{"state"}) + metrics.enrichmentRuns = prometheus.NewCounterVec(prometheus.CounterOpts{Name: "graphnest_supply_chain_enrichment_total", Help: "License enrichment lookups by outcome."}, []string{"outcome"}) + metrics.enrichmentTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: "graphnest_supply_chain_enrichment_duration_seconds", Help: "License enrichment lookup duration by outcome."}, []string{"outcome"}) + metrics.registry.MustRegister(metrics.enrichmentRuns, metrics.enrichmentTime, metrics.archiveOperations, metrics.archiveDuration, metrics.activeRequests, metrics.httpRequests, metrics.httpDuration, metrics.httpResponseSize, metrics.backendCalls, metrics.backendDuration, metrics.githubRequests, metrics.webhookDeliveries, metrics.indexQueueDepth, metrics.indexPhases, metrics.indexDuration, metrics.graphQueueDepth, metrics.graphPhases, metrics.graphDuration, metrics.graphQueries, metrics.graphQueryDuration, metrics.authEvents, metrics.supplyChainRuns, metrics.supplyChainTime, metrics.supplyChainDepth) return metrics } @@ -73,7 +77,14 @@ func (metrics *Metrics) ObserveSupplyChainCollection(outcome string, duration ti } func (metrics *Metrics) SetSupplyChainQueueDepth(state string, depth int64) { - metrics.supplyChainDepth.WithLabelValues(fixed(state, "queued", "running")).Set(float64(depth)) + metrics.supplyChainDepth.WithLabelValues(fixed(state, "queued", "running", "enrichment_queued", "enrichment_running")).Set(float64(depth)) +} + +// ObserveSupplyChainEnrichment records one registry lookup by outcome class. +func (metrics *Metrics) ObserveSupplyChainEnrichment(outcome string, duration time.Duration) { + label := fixed(outcome, "resolved", "not_found", "no_license_metadata", "unavailable", "rejected", "too_large", "malformed") + metrics.enrichmentRuns.WithLabelValues(label).Inc() + metrics.enrichmentTime.WithLabelValues(label).Observe(duration.Seconds()) } func (metrics *Metrics) ObserveGraphQuery(operation, result string, duration time.Duration) { diff --git a/internal/postgres/migrations/034_supply_chain_license.sql b/internal/postgres/migrations/034_supply_chain_license.sql new file mode 100644 index 00000000..0eace8a4 --- /dev/null +++ b/internal/postgres/migrations/034_supply_chain_license.sql @@ -0,0 +1,78 @@ +-- Exact-version license evidence (ADR-0017, Milestone 2). Evidence rows are +-- immutable: a metadata change becomes a new row, never an edit. Evidence is +-- keyed by coordinates and the registry route that produced it, so a private +-- registry's answer never mixes with a public one. +create table supply_chain_license_evidence ( + id bigint generated always as identity primary key, + -- Evidence origin. + source varchar(32) not null check (source in ('producer_declared', 'producer_concluded', 'registry_npm', 'registry_nuget', 'registry_maven', 'import', 'human')), + -- Registry route name from configuration ('' for producer/import/human evidence). + route varchar(128) not null default '', + -- Exact coordinates the evidence applies to. + ecosystem varchar(64) not null, + namespace text not null default '', + name text not null, + version text not null, + -- Artifact identity when the evidence is tied to verified bytes ('' for coordinate-level evidence). + artifact_sha256 varchar(64) not null default '' check (artifact_sha256 = '' or artifact_sha256 ~ '^[0-9a-f]{64}$'), + -- What was observed. + raw_value text not null, + raw_kind varchar(32) not null check (raw_kind in ('expression', 'expression_or_file', 'license_file', 'license_url', 'license_name', 'legacy_object', 'missing', 'sentinel')), + parse_status varchar(32) not null check (parse_status in ('parsed', 'unknown_terms', 'no_assertion', 'none', 'unlicensed', 'invalid', 'not_applicable')), + normalized_expression text not null default '', + expression_tree jsonb, + unknown_terms text[] not null default '{}', + license_url text not null default '', + license_file_name text not null default '', + -- Raw metadata snippet the resolver used (bounded, never the whole artifact). + detail jsonb not null default '{}', + -- Provenance. + resolver_version integer not null check (resolver_version > 0), + license_list_version varchar(32) not null, + content_sha256 bytea check (content_sha256 is null or octet_length(content_sha256) = 32), + fetched_at timestamptz not null default now(), + -- Negative results (not found, no license metadata) expire and are retried. + expires_at timestamptz, + outcome varchar(32) not null check (outcome in ('resolved', 'not_found', 'no_license_metadata', 'unavailable', 'rejected', 'too_large', 'malformed')), + http_status integer, + message text not null default '' +); +create index supply_chain_license_evidence_coordinates on supply_chain_license_evidence (ecosystem, namespace, name, version, route, id desc); +create index supply_chain_license_evidence_expiry on supply_chain_license_evidence (expires_at) where expires_at is not null; + +-- Enrichment work: which coordinates need a lookup, leased like other jobs. +create table supply_chain_enrichment_jobs ( + id bigint generated always as identity primary key, + ecosystem varchar(64) not null, + namespace text not null default '', + name text not null, + version text not null, + route varchar(128) not null, + state varchar(16) not null check (state in ('queued', 'running', 'succeeded', 'failed', 'skipped')), + attempt integer not null default 0 check (attempt >= 0), + max_attempts integer not null default 3 check (max_attempts between 1 and 10), + run_after timestamptz not null default now(), + lease_owner varchar(128), + lease_expires_at timestamptz, + fence bigint not null default 0, + error_code varchar(64) not null default '', + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); +create unique index supply_chain_enrichment_jobs_one_active on supply_chain_enrichment_jobs (ecosystem, namespace, name, version, route) where state in ('queued', 'running'); +create index supply_chain_enrichment_jobs_claim on supply_chain_enrichment_jobs (state, run_after, id); + +-- Derived per-occurrence assessment: which evidence rows apply to a snapshot +-- component and whether they agree. Rebuilt whenever evidence changes; the +-- underlying evidence is never edited. +create table supply_chain_component_assessments ( + component_id bigint primary key references supply_chain_components(id) on delete cascade, + snapshot_id bigint not null references supply_chain_snapshots(id) on delete cascade, + status varchar(32) not null check (status in ('unknown', 'declared', 'resolved', 'conflict', 'unlicensed', 'not_applicable', 'pending')), + normalized_expression text not null default '', + evidence_ids bigint[] not null default '{}', + conflict_detail text not null default '', + assessed_at timestamptz not null default now(), + evidence_fingerprint bytea check (evidence_fingerprint is null or octet_length(evidence_fingerprint) = 32) +); +create index supply_chain_component_assessments_snapshot on supply_chain_component_assessments (snapshot_id, status); diff --git a/internal/postgres/supply_chain_license.go b/internal/postgres/supply_chain_license.go new file mode 100644 index 00000000..2adfc697 --- /dev/null +++ b/internal/postgres/supply_chain_license.go @@ -0,0 +1,397 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/balcsida/graphnest/internal/supplychain" + "github.com/balcsida/graphnest/internal/supplychain/license" + "github.com/balcsida/graphnest/internal/supplychain/spdxexpr" + "github.com/jackc/pgx/v5" +) + +const enrichmentLease = 2 * time.Minute + +// InsertLicenseEvidence appends an immutable evidence row. duplicate reports +// that the newest existing row for the same coordinates, source, and route +// has the same material fingerprint. +func (s *Store) InsertLicenseEvidence(ctx context.Context, evidence license.Evidence) (int64, bool, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return 0, false, err + } + defer tx.Rollback(ctx) + duplicate := false + previous, err := s.latestEvidenceRow(ctx, tx, evidence.Coordinates, string(evidence.Source), evidence.Route) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return 0, false, err + } + if err == nil && string(previous.Fingerprint()) == string(evidence.Fingerprint()) { + duplicate = true + } + var tree []byte + if evidence.ExpressionTree != nil { + if tree, err = json.Marshal(evidence.ExpressionTree); err != nil { + return 0, false, err + } + } + detail, err := json.Marshal(nonNilAny(evidence.Detail)) + if err != nil { + return 0, false, err + } + unknown := evidence.UnknownTerms + if unknown == nil { + unknown = []string{} + } + var id int64 + err = tx.QueryRow(ctx, `insert into supply_chain_license_evidence (source, route, ecosystem, namespace, name, version, artifact_sha256, raw_value, raw_kind, parse_status, + normalized_expression, expression_tree, unknown_terms, license_url, license_file_name, detail, resolver_version, license_list_version, content_sha256, fetched_at, expires_at, outcome, http_status, message) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) returning id`, + string(evidence.Source), evidence.Route, evidence.Coordinates.Ecosystem, evidence.Coordinates.Namespace, evidence.Coordinates.Name, evidence.Coordinates.Version, evidence.ArtifactSHA256, + evidence.RawValue, string(evidence.RawKind), string(evidence.ParseStatus), evidence.NormalizedExpression, nullableJSON(tree), unknown, evidence.LicenseURL, evidence.LicenseFileName, detail, + evidence.ResolverVersion, evidence.LicenseListVersion, evidence.ContentSHA256, evidence.FetchedAt.UTC(), evidence.ExpiresAt, string(evidence.Outcome), evidence.HTTPStatus, evidence.Message).Scan(&id) + if err != nil { + return 0, false, err + } + return id, duplicate, tx.Commit(ctx) +} + +func nullableJSON(data []byte) any { + if len(data) == 0 { + return nil + } + return data +} + +func nonNilAny(values map[string]any) map[string]any { + if values == nil { + return map[string]any{} + } + return values +} + +const licenseEvidenceColumns = `id, source, route, ecosystem, namespace, name, version, artifact_sha256, raw_value, raw_kind, parse_status, normalized_expression, expression_tree, unknown_terms, + license_url, license_file_name, detail, resolver_version, license_list_version, content_sha256, fetched_at, expires_at, outcome, http_status, message` + +func scanLicenseEvidence(row interface{ Scan(...any) error }) (license.Evidence, error) { + var evidence license.Evidence + var source, rawKind, parseStatus, outcome string + var tree, detail []byte + if err := row.Scan(&evidence.ID, &source, &evidence.Route, &evidence.Coordinates.Ecosystem, &evidence.Coordinates.Namespace, &evidence.Coordinates.Name, &evidence.Coordinates.Version, &evidence.ArtifactSHA256, + &evidence.RawValue, &rawKind, &parseStatus, &evidence.NormalizedExpression, &tree, &evidence.UnknownTerms, &evidence.LicenseURL, &evidence.LicenseFileName, &detail, &evidence.ResolverVersion, + &evidence.LicenseListVersion, &evidence.ContentSHA256, &evidence.FetchedAt, &evidence.ExpiresAt, &outcome, &evidence.HTTPStatus, &evidence.Message); err != nil { + return license.Evidence{}, err + } + evidence.Source, evidence.RawKind, evidence.ParseStatus, evidence.Outcome = license.Source(source), license.RawKind(rawKind), spdxexpr.Status(parseStatus), license.Outcome(outcome) + if len(tree) > 0 { + evidence.ExpressionTree = &spdxexpr.Node{} + if err := json.Unmarshal(tree, evidence.ExpressionTree); err != nil { + return license.Evidence{}, err + } + } + if err := json.Unmarshal(detail, &evidence.Detail); err != nil { + return license.Evidence{}, err + } + return evidence, nil +} + +func (s *Store) latestEvidenceRow(ctx context.Context, tx pgx.Tx, coordinates license.Coordinates, source, route string) (license.Evidence, error) { + return scanLicenseEvidence(tx.QueryRow(ctx, `select `+licenseEvidenceColumns+` from supply_chain_license_evidence + where ecosystem=$1 and namespace=$2 and name=$3 and version=$4 and source=$5 and route=$6 order by id desc limit 1`, + coordinates.Ecosystem, coordinates.Namespace, coordinates.Name, coordinates.Version, source, route)) +} + +// LatestLicenseEvidence returns, per (source, route), the newest row and, +// when that row is a negative outcome, also the newest resolved row, so an +// outage keeps earlier evidence visible with its age. Ordered by ID. +func (s *Store) LatestLicenseEvidence(ctx context.Context, coordinates license.Coordinates) ([]license.Evidence, error) { + rows, err := s.pool.Query(ctx, `with scoped as ( + select `+licenseEvidenceColumns+` from supply_chain_license_evidence where ecosystem=$1 and namespace=$2 and name=$3 and version=$4 + ), newest as ( + select distinct on (source, route) * from scoped order by source, route, id desc + ), newest_resolved as ( + select distinct on (source, route) * from scoped where outcome='resolved' order by source, route, id desc + ) + select * from newest + union + select r.* from newest_resolved r join newest n on n.source=r.source and n.route=r.route and n.outcome<>'resolved' + order by id`, coordinates.Ecosystem, coordinates.Namespace, coordinates.Name, coordinates.Version) + if err != nil { + return nil, err + } + defer rows.Close() + var result []license.Evidence + for rows.Next() { + evidence, err := scanLicenseEvidence(rows) + if err != nil { + return nil, err + } + result = append(result, evidence) + } + return result, rows.Err() +} + +// LicenseEvidenceHistory lists every evidence row for the coordinates, newest +// first, bounded. Callers authorize the coordinates through an authorized +// occurrence first. +func (s *Store) LicenseEvidenceHistory(ctx context.Context, coordinates license.Coordinates, limit int) ([]license.Evidence, error) { + rows, err := s.pool.Query(ctx, `select `+licenseEvidenceColumns+` from supply_chain_license_evidence + where ecosystem=$1 and namespace=$2 and name=$3 and version=$4 order by id desc limit $5`, coordinates.Ecosystem, coordinates.Namespace, coordinates.Name, coordinates.Version, limit) + if err != nil { + return nil, err + } + defer rows.Close() + result := []license.Evidence{} + for rows.Next() { + evidence, err := scanLicenseEvidence(rows) + if err != nil { + return nil, err + } + result = append(result, evidence) + } + return result, rows.Err() +} + +// EnqueueEnrichment queues a lookup unless one is active or fresh evidence +// exists for the route: a resolved row (any age; metadata changes are picked +// up by explicit re-enrichment), or an unexpired negative row. +func (s *Store) EnqueueEnrichment(ctx context.Context, coordinates license.Coordinates, route string) (bool, error) { + var fresh bool + if err := s.pool.QueryRow(ctx, `select exists(select 1 from supply_chain_license_evidence where ecosystem=$1 and namespace=$2 and name=$3 and version=$4 and route=$5 + and (outcome='resolved' or expires_at is null or expires_at>now()) order by id desc limit 1)`, coordinates.Ecosystem, coordinates.Namespace, coordinates.Name, coordinates.Version, route).Scan(&fresh); err != nil { + return false, err + } + if fresh { + // Only the newest row decides; check it explicitly. + var outcome string + var expires *time.Time + err := s.pool.QueryRow(ctx, `select outcome, expires_at from supply_chain_license_evidence where ecosystem=$1 and namespace=$2 and name=$3 and version=$4 and route=$5 order by id desc limit 1`, + coordinates.Ecosystem, coordinates.Namespace, coordinates.Name, coordinates.Version, route).Scan(&outcome, &expires) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, err + } + if err == nil && (outcome == "resolved" || expires == nil || expires.After(time.Now())) { + return false, nil + } + } + result, err := s.pool.Exec(ctx, `insert into supply_chain_enrichment_jobs (ecosystem, namespace, name, version, route, state) values ($1, $2, $3, $4, $5, 'queued') + on conflict do nothing`, coordinates.Ecosystem, coordinates.Namespace, coordinates.Name, coordinates.Version, route) + if err != nil { + return false, err + } + return result.RowsAffected() == 1, nil +} + +const enrichmentJobColumns = `id, ecosystem, namespace, name, version, route, state, attempt, max_attempts, run_after, coalesce(lease_owner, ''), lease_expires_at, fence, error_code` + +func scanEnrichmentJob(row interface{ Scan(...any) error }) (license.Job, error) { + var job license.Job + err := row.Scan(&job.ID, &job.Coordinates.Ecosystem, &job.Coordinates.Namespace, &job.Coordinates.Name, &job.Coordinates.Version, &job.Route, &job.State, &job.Attempt, &job.MaxAttempts, &job.RunAfter, &job.LeaseOwner, &job.LeaseExpiresAt, &job.Fence, &job.ErrorCode) + return job, err +} + +func (s *Store) ClaimEnrichment(ctx context.Context, owner string) (license.Job, error) { + job, err := scanEnrichmentJob(s.pool.QueryRow(ctx, `with next as ( + select id from supply_chain_enrichment_jobs where state='queued' and run_after<=now() order by run_after, id for update skip locked limit 1 + ) + update supply_chain_enrichment_jobs set state='running', attempt=attempt+1, lease_owner=$1, fence=fence+1, lease_expires_at=now()+$2::interval, updated_at=now() + where id=(select id from next) returning `+enrichmentJobColumns, owner, enrichmentLease)) + if errors.Is(err, pgx.ErrNoRows) { + return license.Job{}, license.ErrNoJob + } + return job, err +} + +func (s *Store) RenewEnrichment(ctx context.Context, id int64, owner string, fence int64) error { + result, err := s.pool.Exec(ctx, `update supply_chain_enrichment_jobs set lease_expires_at=now()+$4::interval, updated_at=now() + where id=$1 and state='running' and lease_owner=$2 and fence=$3 and lease_expires_at>now()`, id, owner, fence, enrichmentLease) + if err != nil { + return err + } + if result.RowsAffected() != 1 { + return license.ErrFenced + } + return nil +} + +// CompleteEnrichment finishes a leased job. Unavailable outcomes retry with +// backoff up to max_attempts; everything else is terminal (the evidence row +// records what happened). +func (s *Store) CompleteEnrichment(ctx context.Context, id int64, owner string, fence int64, outcome license.Outcome, errorCode string) error { + state := "succeeded" + switch outcome { + case license.OutcomeUnavailable: + state = "retry" + case license.OutcomeRejected: + state = "skipped" + } + result, err := s.pool.Exec(ctx, `update supply_chain_enrichment_jobs set + state=case when $4='retry' and attempt'' order by 1, 2, 3, 4`, snapshotID) + if err != nil { + return nil, err + } + defer rows.Close() + var result []license.Coordinates + for rows.Next() { + var item license.Coordinates + if err := rows.Scan(&item.Ecosystem, &item.Namespace, &item.Name, &item.Version); err != nil { + return nil, err + } + result = append(result, item) + } + return result, rows.Err() +} + +// SupplyChainAssessments returns assessments for a snapshot's components, +// keyed by component ID. Callers authorize the snapshot first. +func (s *Store) SupplyChainAssessments(ctx context.Context, snapshotID int64, componentIDs []int64) (map[int64]license.Assessment, error) { + rows, err := s.pool.Query(ctx, `select component_id, snapshot_id, status, normalized_expression, evidence_ids, conflict_detail, assessed_at, evidence_fingerprint + from supply_chain_component_assessments where snapshot_id=$1 and component_id=any($2)`, snapshotID, componentIDs) + if err != nil { + return nil, err + } + defer rows.Close() + result := map[int64]license.Assessment{} + for rows.Next() { + var assessment license.Assessment + var status string + if err := rows.Scan(&assessment.ComponentID, &assessment.SnapshotID, &status, &assessment.NormalizedExpression, &assessment.EvidenceIDs, &assessment.ConflictDetail, &assessment.AssessedAt, &assessment.EvidenceFingerprint); err != nil { + return nil, err + } + assessment.Status = license.AssessmentStatus(status) + result[assessment.ComponentID] = assessment + } + return result, rows.Err() +} + +// SupplyChainAssessmentCounts summarizes assessment statuses for a snapshot. +func (s *Store) SupplyChainAssessmentCounts(ctx context.Context, snapshotID int64) (map[string]int, error) { + rows, err := s.pool.Query(ctx, `select status, count(*) from supply_chain_component_assessments where snapshot_id=$1 group by status`, snapshotID) + if err != nil { + return nil, err + } + defer rows.Close() + counts := map[string]int{} + for rows.Next() { + var status string + var count int + if err := rows.Scan(&status, &count); err != nil { + return nil, err + } + counts[status] = count + } + return counts, rows.Err() +} + +// SupplyChainComponentByElement resolves one occurrence in an authorized +// snapshot by its document element ID. +func (s *Store) SupplyChainComponentByElement(ctx context.Context, snapshotID int64, elementID string) (supplychain.Component, error) { + var component supplychain.Component + var qualifiers, checksums []byte + err := s.pool.QueryRow(ctx, `select id, snapshot_id, ordinal, element_id, name, version, purl, coalesce(ecosystem, ''), coalesce(purl_namespace, ''), coalesce(purl_name, ''), + coalesce(purl_version, ''), qualifiers, license_declared_raw, license_concluded_raw, download_location, supplier, checksums, is_root + from supply_chain_components where snapshot_id=$1 and element_id=$2`, snapshotID, elementID).Scan(&component.ID, &component.SnapshotID, &component.Ordinal, &component.ElementID, &component.Name, + &component.Version, &component.PURL, &component.Ecosystem, &component.PURLNamespace, &component.PURLName, &component.PURLVersion, &qualifiers, &component.LicenseDeclaredRaw, &component.LicenseConcludedRaw, + &component.DownloadLocation, &component.Supplier, &checksums, &component.IsRoot) + if err != nil { + return supplychain.Component{}, err + } + if err := json.Unmarshal(qualifiers, &component.Qualifiers); err != nil { + return supplychain.Component{}, err + } + if err := json.Unmarshal(checksums, &component.Checksums); err != nil { + return supplychain.Component{}, err + } + return component, nil +} diff --git a/internal/postgres/supply_chain_license_test.go b/internal/postgres/supply_chain_license_test.go new file mode 100644 index 00000000..7f3e69ca --- /dev/null +++ b/internal/postgres/supply_chain_license_test.go @@ -0,0 +1,238 @@ +//go:build integration + +package postgres + +import ( + "errors" + "testing" + "time" + + "github.com/balcsida/graphnest/internal/supplychain" + "github.com/balcsida/graphnest/internal/supplychain/license" + "github.com/balcsida/graphnest/internal/supplychain/spdxexpr" +) + +func resolvedEvidence(coordinates license.Coordinates, route, raw string) license.Evidence { + evidence := license.Evidence{Source: license.SourceRegistryNPM, Route: route, Coordinates: coordinates, Outcome: license.OutcomeResolved, FetchedAt: time.Now().UTC(), ContentSHA256: make([]byte, 32), + RawValue: raw, RawKind: license.RawExpression, ResolverVersion: license.ResolverVersion, LicenseListVersion: spdxexpr.ListVersion, Detail: map[string]any{"integrity": "sha512-x"}} + parsed := spdxexpr.Parse(raw) + evidence.ParseStatus, evidence.NormalizedExpression, evidence.ExpressionTree, evidence.UnknownTerms = parsed.Status, parsed.Normalized, parsed.Expression, parsed.UnknownTerms + return evidence +} + +func TestLicenseEvidenceIsImmutableAndRouteScoped(t *testing.T) { + store := migratedStore(t) + coordinates := license.Coordinates{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.3.0"} + first, duplicate, err := store.InsertLicenseEvidence(t.Context(), resolvedEvidence(coordinates, "npm:public", "MIT")) + if err != nil || duplicate || first == 0 { + t.Fatalf("first insert = %d %v %v", first, duplicate, err) + } + second, duplicate, err := store.InsertLicenseEvidence(t.Context(), resolvedEvidence(coordinates, "npm:public", "MIT")) + if err != nil || !duplicate || second == first { + t.Fatalf("identical re-fetch must be a new row flagged duplicate: %d %v %v", second, duplicate, err) + } + changed, duplicate, err := store.InsertLicenseEvidence(t.Context(), resolvedEvidence(coordinates, "npm:public", "ISC")) + if err != nil || duplicate { + t.Fatalf("changed metadata = %d %v %v", changed, duplicate, err) + } + private, duplicate, err := store.InsertLicenseEvidence(t.Context(), resolvedEvidence(coordinates, "npm:private", "LicenseRef-Acme-Internal")) + if err != nil || duplicate { + t.Fatalf("private route = %d %v %v", private, duplicate, err) + } + latest, err := store.LatestLicenseEvidence(t.Context(), coordinates) + if err != nil || len(latest) != 2 { + t.Fatalf("latest = %+v, %v", latest, err) + } + byRoute := map[string]license.Evidence{} + for _, evidence := range latest { + byRoute[evidence.Route] = evidence + } + if byRoute["npm:public"].NormalizedExpression != "ISC" || byRoute["npm:public"].ID != changed || byRoute["npm:private"].NormalizedExpression != "LicenseRef-Acme-Internal" { + t.Fatalf("latest per route = %+v", byRoute) + } + if byRoute["npm:public"].ExpressionTree == nil || byRoute["npm:public"].ExpressionTree.ID != "ISC" || byRoute["npm:public"].Detail["integrity"] != "sha512-x" { + t.Fatalf("round trip lost tree or detail: %+v", byRoute["npm:public"]) + } + // An outage after resolved evidence keeps the resolved row alongside the negative one. + outage := license.Evidence{Source: license.SourceRegistryNPM, Route: "npm:public", Coordinates: coordinates, Outcome: license.OutcomeUnavailable, FetchedAt: time.Now().UTC(), RawKind: license.RawMissing, + ParseStatus: license.NotApplicable, ResolverVersion: 1, LicenseListVersion: spdxexpr.ListVersion, Message: "registry unavailable"} + if _, _, err := store.InsertLicenseEvidence(t.Context(), outage); err != nil { + t.Fatal(err) + } + latest, err = store.LatestLicenseEvidence(t.Context(), coordinates) + if err != nil || len(latest) != 3 { + t.Fatalf("latest with outage = %d rows, %v", len(latest), err) + } + outcomes := map[license.Outcome]int{} + for _, evidence := range latest { + outcomes[evidence.Outcome]++ + if evidence.Route == "npm:public" && evidence.Outcome == license.OutcomeResolved && evidence.NormalizedExpression != "ISC" { + t.Fatalf("earlier resolved evidence must be the newest resolved row: %+v", evidence) + } + } + if outcomes[license.OutcomeResolved] != 2 || outcomes[license.OutcomeUnavailable] != 1 { + t.Fatalf("outcomes = %v", outcomes) + } + history, err := store.LicenseEvidenceHistory(t.Context(), coordinates, 10) + if err != nil || len(history) != 5 || history[1].ID != private || history[4].ID != first { + t.Fatalf("history = %d rows, %v", len(history), err) + } + // A same-name package at a different version shares nothing. + other, err := store.LatestLicenseEvidence(t.Context(), license.Coordinates{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.4.0"}) + if err != nil || len(other) != 0 { + t.Fatalf("other version = %+v, %v", other, err) + } + // Evidence rows cannot be edited through the store API; verify the DB has no update path used here by checking counts are additive. + var count int + if err := store.pool.QueryRow(t.Context(), `select count(*) from supply_chain_license_evidence`).Scan(&count); err != nil || count != 5 { + t.Fatalf("rows = %d, %v", count, err) + } +} + +func TestEnrichmentQueueFreshnessAndLeases(t *testing.T) { + store := migratedStore(t) + coordinates := license.Coordinates{Ecosystem: "maven", Namespace: "org.example", Name: "core", Version: "2.1.0"} + created, err := store.EnqueueEnrichment(t.Context(), coordinates, "maven:test") + if err != nil || !created { + t.Fatalf("enqueue = %v %v", created, err) + } + if created, err := store.EnqueueEnrichment(t.Context(), coordinates, "maven:test"); err != nil || created { + t.Fatalf("duplicate enqueue = %v %v", created, err) + } + job, err := store.ClaimEnrichment(t.Context(), "worker-a") + if err != nil || job.Coordinates != coordinates || job.Route != "maven:test" || job.Fence != 1 || job.Attempt != 1 { + t.Fatalf("claim = %+v %v", job, err) + } + if _, err := store.ClaimEnrichment(t.Context(), "worker-b"); !errors.Is(err, license.ErrNoJob) { + t.Fatalf("second claim = %v", err) + } + // Unavailable retries with backoff; the stale fence cannot complete. + if err := store.CompleteEnrichment(t.Context(), job.ID, "worker-a", job.Fence+1, license.OutcomeResolved, ""); !errors.Is(err, license.ErrFenced) { + t.Fatalf("stale fence completed: %v", err) + } + if err := store.CompleteEnrichment(t.Context(), job.ID, job.LeaseOwner, job.Fence, license.OutcomeUnavailable, "registry_down"); err != nil { + t.Fatal(err) + } + var state string + var runAfter time.Time + if err := store.pool.QueryRow(t.Context(), `select state, run_after from supply_chain_enrichment_jobs where id=$1`, job.ID).Scan(&state, &runAfter); err != nil || state != "queued" || !runAfter.After(time.Now()) { + t.Fatalf("after unavailable: state=%s run_after=%v err=%v", state, runAfter, err) + } + // Negative evidence with an unexpired TTL suppresses re-enqueue; expired does not. + negative := license.Evidence{Source: license.SourceRegistryMaven, Route: "maven:test", Coordinates: coordinates, Outcome: license.OutcomeNotFound, FetchedAt: time.Now().UTC(), RawKind: license.RawMissing, + ParseStatus: license.NotApplicable, ResolverVersion: 1, LicenseListVersion: spdxexpr.ListVersion} + expires := time.Now().Add(time.Hour) + negative.ExpiresAt = &expires + if _, err := store.pool.Exec(t.Context(), `delete from supply_chain_enrichment_jobs`); err != nil { + t.Fatal(err) + } + if _, _, err := store.InsertLicenseEvidence(t.Context(), negative); err != nil { + t.Fatal(err) + } + if created, err := store.EnqueueEnrichment(t.Context(), coordinates, "maven:test"); err != nil || created { + t.Fatalf("fresh negative evidence must suppress lookups: %v %v", created, err) + } + expired := time.Now().Add(-time.Minute) + negative.ExpiresAt = &expired + if _, _, err := store.InsertLicenseEvidence(t.Context(), negative); err != nil { + t.Fatal(err) + } + if created, err := store.EnqueueEnrichment(t.Context(), coordinates, "maven:test"); err != nil || !created { + t.Fatalf("expired negative evidence must allow a lookup: %v %v", created, err) + } + // Another route is independent. + if created, err := store.EnqueueEnrichment(t.Context(), coordinates, "maven:other"); err != nil || !created { + t.Fatalf("other route = %v %v", created, err) + } + // Reaping requeues an expired running lease. + job, err = store.ClaimEnrichment(t.Context(), "worker-a") + if err != nil { + t.Fatal(err) + } + if _, err := store.pool.Exec(t.Context(), `update supply_chain_enrichment_jobs set lease_expires_at=now()-interval '1 second' where id=$1`, job.ID); err != nil { + t.Fatal(err) + } + if reaped, err := store.ReapExpiredEnrichment(t.Context(), 10); err != nil || reaped != 1 { + t.Fatalf("reaped = %d %v", reaped, err) + } + if err := store.RenewEnrichment(t.Context(), job.ID, job.LeaseOwner, job.Fence); !errors.Is(err, license.ErrFenced) { + t.Fatalf("renew after reap = %v", err) + } + depths, err := store.EnrichmentQueueDepths(t.Context()) + if err != nil || depths["queued"] != 2 || depths["running"] != 0 { + t.Fatalf("depths = %v %v", depths, err) + } +} + +func TestSnapshotCoordinatesAndAssessments(t *testing.T) { + store := migratedStore(t) + repositoryID := supplyChainRepository(t, store, 101, "widgets") + collection, err := store.PublishSupplyChainSnapshot(t.Context(), publication(t, repositoryID, claimSupplyChain(t, store, repositoryID, "worker"), supplyChainFixture(t))) + if err != nil { + t.Fatal(err) + } + coordinates, err := store.SnapshotCoordinates(t.Context(), *collection.SnapshotID) + if err != nil { + t.Fatal(err) + } + // github root (no version), vendored (no purl) are excluded; npm, maven, nuget, githubactions, golang remain. + if len(coordinates) != 5 { + t.Fatalf("coordinates = %+v", coordinates) + } + npm := license.Coordinates{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.3.0"} + occurrences, err := store.ComponentsForCoordinates(t.Context(), npm, 10) + if err != nil || len(occurrences) != 1 || occurrences[0][1] != *collection.SnapshotID { + t.Fatalf("occurrences = %v %v", occurrences, err) + } + declared, concluded, err := store.ComponentDeclarations(t.Context(), occurrences[0][0]) + if err != nil || declared == nil || *declared != "NOASSERTION" || concluded == nil || *concluded != "NOASSERTION" { + t.Fatalf("declarations = %v %v %v", declared, concluded, err) + } + id, _, err := store.InsertLicenseEvidence(t.Context(), resolvedEvidence(npm, "npm:test", "MIT")) + if err != nil { + t.Fatal(err) + } + evidence, err := store.LatestLicenseEvidence(t.Context(), npm) + if err != nil { + t.Fatal(err) + } + assessment := license.Assess(occurrences[0][0], occurrences[0][1], declared, concluded, evidence, time.Now()) + if assessment.Status != license.AssessmentResolved || assessment.NormalizedExpression != "MIT" || len(assessment.EvidenceIDs) != 1 || assessment.EvidenceIDs[0] != id { + t.Fatalf("assessment = %+v", assessment) + } + if err := store.UpsertAssessment(t.Context(), assessment); err != nil { + t.Fatal(err) + } + stored, err := store.SupplyChainAssessments(t.Context(), *collection.SnapshotID, []int64{occurrences[0][0]}) + if err != nil || len(stored) != 1 || stored[occurrences[0][0]].Status != license.AssessmentResolved || string(stored[occurrences[0][0]].EvidenceFingerprint) != string(assessment.EvidenceFingerprint) { + t.Fatalf("stored = %+v %v", stored, err) + } + counts, err := store.SupplyChainAssessmentCounts(t.Context(), *collection.SnapshotID) + if err != nil || counts["resolved"] != 1 { + t.Fatalf("counts = %v %v", counts, err) + } + // The assessment is rebuilt in place (one row per component) and cascades with the snapshot. + assessment.Status = license.AssessmentConflict + if err := store.UpsertAssessment(t.Context(), assessment); err != nil { + t.Fatal(err) + } + if counts, _ := store.SupplyChainAssessmentCounts(t.Context(), *collection.SnapshotID); counts["conflict"] != 1 || counts["resolved"] != 0 { + t.Fatalf("counts after upsert = %v", counts) + } + component, err := store.SupplyChainComponentByElement(t.Context(), *collection.SnapshotID, "SPDXRef-npm-scope-left-pad-1.3.0") + if err != nil || component.PURLName != "left-pad" || component.ID != occurrences[0][0] { + t.Fatalf("component by element = %+v %v", component, err) + } + if _, err := store.pool.Exec(t.Context(), `delete from repositories where id=$1`, repositoryID); err != nil { + t.Fatal(err) + } + var remaining int + if err := store.pool.QueryRow(t.Context(), `select count(*) from supply_chain_component_assessments`).Scan(&remaining); err != nil || remaining != 0 { + t.Fatalf("assessments after repository removal = %d %v", remaining, err) + } + // Evidence is shared registry knowledge, not repository data: it survives. + if history, err := store.LicenseEvidenceHistory(t.Context(), npm, 10); err != nil || len(history) != 1 { + t.Fatalf("evidence after repository removal = %d %v", len(history), err) + } + _ = supplychain.StreamGitHubSource +} diff --git a/internal/supplychain/collector.go b/internal/supplychain/collector.go index b2c6ba58..2784a9e3 100644 --- a/internal/supplychain/collector.go +++ b/internal/supplychain/collector.go @@ -40,6 +40,12 @@ type Observer interface { SetSupplyChainQueueDepth(state string, depth int64) } +// Enricher receives every newly published snapshot so license lookups can +// be queued. It must not block publication on registry availability. +type Enricher interface { + EnqueueSnapshot(ctx context.Context, snapshotID int64) (int, error) +} + // Collector leases refresh jobs and turns GHES SBOM exports into published // snapshots. It runs in the server process, independent of the indexer. type Collector struct { @@ -53,6 +59,8 @@ type Collector struct { Now func() time.Time // Poll is how long the worker waits when no job is available. Poll time.Duration + // Enricher is optional; nil means no license enrichment is configured. + Enricher Enricher } func (collector *Collector) now() time.Time { @@ -176,6 +184,13 @@ func (collector *Collector) process(ctx context.Context, job Job, started time.T return collection.Outcome, err } } + if collector.Enricher != nil { + if _, err := collector.Enricher.EnqueueSnapshot(ctx, *collection.SnapshotID); err != nil { + // Enrichment is best-effort background work; the snapshot is + // published regardless and the next publication re-queues. + collector.logger().Warn("supply chain enrichment enqueue failed", "repository_id", repo.ID, "error", err) + } + } } return collection.Outcome, nil } diff --git a/internal/supplychain/license/assess.go b/internal/supplychain/license/assess.go new file mode 100644 index 00000000..8ddc5ae4 --- /dev/null +++ b/internal/supplychain/license/assess.go @@ -0,0 +1,133 @@ +package license + +import ( + "crypto/sha256" + "sort" + "strings" + "time" + + "github.com/balcsida/graphnest/internal/supplychain/spdxexpr" +) + +// Assess derives one component's assessment from the producer's raw +// declaration/conclusion and the latest registry evidence. Rules: +// +// - Producer NOASSERTION/empty contributes nothing; NONE/UNLICENSED asserts +// "no license granted" and is kept as such, never mapped. +// - A registry expression that parsed is the resolved basis. A parsed +// producer expression that is structurally different from a parsed +// registry expression is a conflict, with both shown. +// - License files, URLs, and unparseable names never become an expression; +// they leave the status unknown (visible in the evidence detail). +// - Two registry routes disagreeing is also a conflict. +// +// The result carries the IDs of every evidence row considered and a +// fingerprint of their material fields so a review can detect later change. +func Assess(componentID, snapshotID int64, declaredRaw, concludedRaw *string, registry []Evidence, now time.Time) Assessment { + assessment := Assessment{ComponentID: componentID, SnapshotID: snapshotID, AssessedAt: now.UTC(), Status: AssessmentUnknown} + var candidates []candidate + hash := sha256.New() + consider := func(label, raw string) { + parsed := spdxexpr.Parse(raw) + hash.Write([]byte(label + "\x00" + raw + "\x00" + string(parsed.Status) + "\x00")) + switch parsed.Status { + case spdxexpr.StatusParsed, spdxexpr.StatusUnknownTerms: + candidates = append(candidates, candidate{label: label, expression: parsed.Expression, normalized: parsed.Normalized, status: parsed.Status}) + case spdxexpr.StatusNone, spdxexpr.StatusUnlicensed: + candidates = append(candidates, candidate{label: label, status: parsed.Status}) + } + } + if concludedRaw != nil { + consider("producer_concluded", *concludedRaw) + } + if declaredRaw != nil { + consider("producer_declared", *declaredRaw) + } + pending := false + sort.SliceStable(registry, func(i, j int) bool { return registry[i].ID < registry[j].ID }) + for _, evidence := range registry { + assessment.EvidenceIDs = append(assessment.EvidenceIDs, evidence.ID) + hash.Write(evidence.Fingerprint()) + switch evidence.Outcome { + case OutcomeResolved: + switch evidence.ParseStatus { + case spdxexpr.StatusParsed, spdxexpr.StatusUnknownTerms: + candidates = append(candidates, candidate{label: string(evidence.Source) + "@" + evidence.Route, expression: evidence.ExpressionTree, normalized: evidence.NormalizedExpression, status: evidence.ParseStatus}) + case spdxexpr.StatusNone, spdxexpr.StatusUnlicensed: + candidates = append(candidates, candidate{label: string(evidence.Source) + "@" + evidence.Route, status: evidence.ParseStatus}) + } + case OutcomeUnavailable: + // An outage retains earlier evidence with its age; it does not + // make the component "unlicensed". Nothing to add. + } + } + if len(assessment.EvidenceIDs) == 0 { + assessment.EvidenceIDs = []int64{} + } + assessment.EvidenceFingerprint = hash.Sum(nil) + if len(candidates) == 0 { + if pending { + assessment.Status = AssessmentPending + } + return assessment + } + var expressions []candidate + var refusals []candidate + for _, item := range candidates { + if item.expression != nil { + expressions = append(expressions, item) + } else { + refusals = append(refusals, item) + } + } + // Structural disagreement among parsed expressions is a conflict. + for index := 1; index < len(expressions); index++ { + if !spdxexpr.Equal(expressions[0].expression, expressions[index].expression) { + assessment.Status = AssessmentConflict + assessment.ConflictDetail = describe(expressions) + return assessment + } + } + if len(expressions) > 0 && len(refusals) > 0 { + assessment.Status = AssessmentConflict + assessment.ConflictDetail = describe(append(expressions, refusals...)) + return assessment + } + if len(expressions) == 0 { + assessment.Status = AssessmentUnlicensed + assessment.ConflictDetail = "" + return assessment + } + assessment.NormalizedExpression = expressions[0].normalized + registryBacked := false + for _, item := range expressions { + if strings.HasPrefix(item.label, "registry_") { + registryBacked = true + } + } + if registryBacked { + assessment.Status = AssessmentResolved + } else { + assessment.Status = AssessmentDeclared + } + return assessment +} + +type candidate struct { + label string + expression *spdxexpr.Node + normalized string + status spdxexpr.Status +} + +func describe(items []candidate) string { + parts := make([]string, 0, len(items)) + for _, item := range items { + value := item.normalized + if value == "" { + value = strings.ToUpper(string(item.status)) + } + parts = append(parts, item.label+": "+value) + } + return strings.Join(parts, " | ") +} diff --git a/internal/supplychain/license/assess_test.go b/internal/supplychain/license/assess_test.go new file mode 100644 index 00000000..9665108c --- /dev/null +++ b/internal/supplychain/license/assess_test.go @@ -0,0 +1,84 @@ +package license + +import ( + "strings" + "testing" + "time" + + "github.com/balcsida/graphnest/internal/supplychain/spdxexpr" +) + +func ptr(value string) *string { return &value } + +func registryEvidence(id int64, route, raw string, outcome Outcome) Evidence { + evidence := Evidence{ID: id, Source: SourceRegistryNPM, Route: route, Coordinates: Coordinates{Ecosystem: "npm", Name: "a", Version: "1"}, Outcome: outcome, FetchedAt: time.Now()} + if outcome == OutcomeResolved { + classify(&evidence, raw, RawExpression) + } else { + evidence = negative(evidence, outcome, nil, "x") + } + return evidence +} + +func TestAssessCombinesDeclarationsAndRegistryEvidence(t *testing.T) { + now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC) + cases := map[string]struct { + declared, concluded *string + registry []Evidence + status AssessmentStatus + normalized string + conflictIn string + evidence int + }{ + "nothing": {declared: ptr("NOASSERTION"), concluded: ptr("NOASSERTION"), status: AssessmentUnknown}, + "declared only": {declared: ptr("MIT"), concluded: ptr("NOASSERTION"), status: AssessmentDeclared, normalized: "MIT"}, + "registry only": {declared: ptr("NOASSERTION"), registry: []Evidence{registryEvidence(1, "npm:a", "MIT", OutcomeResolved)}, status: AssessmentResolved, normalized: "MIT", evidence: 1}, + "agree": {declared: ptr("mit"), registry: []Evidence{registryEvidence(1, "npm:a", "MIT", OutcomeResolved)}, status: AssessmentResolved, normalized: "MIT", evidence: 1}, + "agree structurally": {declared: ptr("(MIT OR Apache-2.0)"), registry: []Evidence{registryEvidence(1, "npm:a", "MIT OR Apache-2.0", OutcomeResolved)}, status: AssessmentResolved, normalized: "MIT OR Apache-2.0", evidence: 1}, + "disagree": {declared: ptr("MIT"), registry: []Evidence{registryEvidence(1, "npm:a", "ISC", OutcomeResolved)}, status: AssessmentConflict, conflictIn: "producer_declared: MIT | registry_npm@npm:a: ISC", evidence: 1}, + "grouping disagrees": {declared: ptr("MIT AND (ISC OR Apache-2.0)"), registry: []Evidence{registryEvidence(1, "npm:a", "(MIT AND ISC) OR Apache-2.0", OutcomeResolved)}, status: AssessmentConflict, conflictIn: "|", evidence: 1}, + "or branch vs choice": {declared: ptr("MIT OR Apache-2.0"), concluded: ptr("MIT"), status: AssessmentConflict, conflictIn: "producer_concluded: MIT | producer_declared: MIT OR Apache-2.0"}, + "routes disagree": {registry: []Evidence{registryEvidence(1, "npm:public", "MIT", OutcomeResolved), registryEvidence(2, "npm:private", "LicenseRef-Acme", OutcomeResolved)}, status: AssessmentConflict, conflictIn: "npm:private: LicenseRef-Acme", evidence: 2}, + "unlicensed registry": {declared: ptr("NOASSERTION"), registry: []Evidence{registryEvidence(1, "npm:a", "UNLICENSED", OutcomeResolved)}, status: AssessmentUnlicensed, evidence: 1}, + "unlicensed vs declared": {declared: ptr("MIT"), registry: []Evidence{registryEvidence(1, "npm:a", "UNLICENSED", OutcomeResolved)}, status: AssessmentConflict, conflictIn: "UNLICENSED", evidence: 1}, + "none declared": {declared: ptr("NONE"), status: AssessmentUnlicensed}, + "outage keeps unknown": {declared: ptr("NOASSERTION"), registry: []Evidence{registryEvidence(1, "npm:a", "", OutcomeUnavailable)}, status: AssessmentUnknown, evidence: 1}, + "outage keeps earlier evidence": {registry: []Evidence{registryEvidence(1, "npm:a", "MIT", OutcomeResolved), registryEvidence(2, "npm:a", "", OutcomeUnavailable)}, status: AssessmentResolved, normalized: "MIT", evidence: 2}, + "not found keeps declared": {declared: ptr("Apache-2.0"), registry: []Evidence{registryEvidence(1, "npm:a", "", OutcomeNotFound)}, status: AssessmentDeclared, normalized: "Apache-2.0", evidence: 1}, + "unknown terms still count": {registry: []Evidence{registryEvidence(1, "npm:a", "Custom-1.0", OutcomeResolved)}, status: AssessmentResolved, normalized: "Custom-1.0", evidence: 1}, + "free text is not evidence": {declared: ptr("see LICENSE"), registry: []Evidence{registryEvidence(1, "npm:a", "Copyright Acme", OutcomeResolved)}, status: AssessmentUnknown, evidence: 1}, + } + for name, test := range cases { + t.Run(name, func(t *testing.T) { + got := Assess(7, 3, test.declared, test.concluded, test.registry, now) + if got.Status != test.status || got.NormalizedExpression != test.normalized || len(got.EvidenceIDs) != test.evidence || got.ComponentID != 7 || got.SnapshotID != 3 { + t.Fatalf("assessment = %+v", got) + } + if test.conflictIn != "" && !strings.Contains(got.ConflictDetail, test.conflictIn) { + t.Fatalf("conflict detail = %q, want it to contain %q", got.ConflictDetail, test.conflictIn) + } + if test.status != AssessmentConflict && got.ConflictDetail != "" { + t.Fatalf("unexpected conflict detail %q", got.ConflictDetail) + } + if len(got.EvidenceFingerprint) != 32 { + t.Fatalf("fingerprint = %x", got.EvidenceFingerprint) + } + }) + } +} + +func TestAssessFingerprintTracksMaterialChange(t *testing.T) { + now := time.Now() + base := Assess(1, 1, ptr("MIT"), nil, []Evidence{registryEvidence(1, "npm:a", "MIT", OutcomeResolved)}, now) + same := Assess(1, 1, ptr("MIT"), nil, []Evidence{registryEvidence(1, "npm:a", "MIT", OutcomeResolved)}, now.Add(time.Hour)) + if string(base.EvidenceFingerprint) != string(same.EvidenceFingerprint) { + t.Fatal("assessment time must not change the evidence fingerprint") + } + changed := Assess(1, 1, ptr("MIT"), nil, []Evidence{registryEvidence(1, "npm:a", "ISC", OutcomeResolved)}, now) + if string(base.EvidenceFingerprint) == string(changed.EvidenceFingerprint) { + t.Fatal("new registry evidence must change the fingerprint so reviews can detect it") + } + if spdxexpr.Parse("MIT").Status != spdxexpr.StatusParsed { + t.Fatal("sanity") + } +} diff --git a/internal/supplychain/license/evidence.go b/internal/supplychain/license/evidence.go new file mode 100644 index 00000000..2324c454 --- /dev/null +++ b/internal/supplychain/license/evidence.go @@ -0,0 +1,217 @@ +package license + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + "github.com/balcsida/graphnest/internal/supplychain/spdxexpr" +) + +// ResolverVersion is recorded on every registry evidence row. Bump it when +// resolver behavior changes so old rows are distinguishable. +const ResolverVersion = 1 + +// NegativeTTL bounds how long a not-found / no-metadata / unavailable result +// is trusted before the coordinates are looked up again. +const NegativeTTL = 24 * time.Hour + +type Source string + +const ( + SourceProducerDeclared Source = "producer_declared" + SourceProducerConcluded Source = "producer_concluded" + SourceRegistryNPM Source = "registry_npm" + SourceRegistryNuGet Source = "registry_nuget" + SourceRegistryMaven Source = "registry_maven" + SourceImport Source = "import" + SourceHuman Source = "human" +) + +type RawKind string + +const ( + RawExpression RawKind = "expression" + RawExpressionOrFile RawKind = "expression_or_file" + RawLicenseFile RawKind = "license_file" + RawLicenseURL RawKind = "license_url" + RawLicenseName RawKind = "license_name" + RawLegacyObject RawKind = "legacy_object" + RawMissing RawKind = "missing" + RawSentinel RawKind = "sentinel" +) + +type Outcome string + +const ( + OutcomeResolved Outcome = "resolved" + OutcomeNotFound Outcome = "not_found" + OutcomeNoLicenseMetadata Outcome = "no_license_metadata" + OutcomeUnavailable Outcome = "unavailable" + OutcomeRejected Outcome = "rejected" + OutcomeTooLarge Outcome = "too_large" + OutcomeMalformed Outcome = "malformed" +) + +// Coordinates identify the exact package version evidence applies to. +type Coordinates struct { + Ecosystem, Namespace, Name, Version string +} + +// Evidence is one immutable observation about a package version's license. +// Parsed fields come from spdxexpr; the raw value is always kept. +type Evidence struct { + ID int64 + Source Source + Route string + Coordinates Coordinates + ArtifactSHA256 string + RawValue string + RawKind RawKind + ParseStatus spdxexpr.Status + NormalizedExpression string + ExpressionTree *spdxexpr.Node + UnknownTerms []string + LicenseURL string + LicenseFileName string + Detail map[string]any + ResolverVersion int + LicenseListVersion string + ContentSHA256 []byte + FetchedAt time.Time + ExpiresAt *time.Time + Outcome Outcome + HTTPStatus *int + Message string +} + +// Fingerprint hashes the fields that make evidence materially different, so +// a re-fetch that produced the same facts is recognizable without comparing +// rows field by field. +func (evidence Evidence) Fingerprint() []byte { + hash := sha256.New() + for _, part := range []string{string(evidence.Source), evidence.Route, evidence.Coordinates.Ecosystem, evidence.Coordinates.Namespace, evidence.Coordinates.Name, evidence.Coordinates.Version, + evidence.ArtifactSHA256, evidence.RawValue, string(evidence.RawKind), string(evidence.ParseStatus), evidence.NormalizedExpression, evidence.LicenseURL, evidence.LicenseFileName, string(evidence.Outcome)} { + hash.Write([]byte(part)) + hash.Write([]byte{0}) + } + return hash.Sum(nil) +} + +// Resolver resolves one exact version through one configured route. +type Resolver interface { + Ecosystem() string + Resolve(ctx context.Context, coordinates Coordinates) (Evidence, error) +} + +// NotApplicable is the parse status of evidence that carries no expression +// (a license file, URL, or a missing field). +const NotApplicable spdxexpr.Status = "not_applicable" + +// Classify records a producer declaration as evidence: sentinels stay +// sentinels, expressions are parsed, and nothing is mapped. +func Classify(evidence *Evidence, raw string) { + trimmed := strings.TrimSpace(raw) + switch strings.ToUpper(trimmed) { + case "", "NOASSERTION", "NONE", "UNLICENSED": + classify(evidence, raw, RawExpression) + evidence.RawKind = RawSentinel + if trimmed == "" { + evidence.RawKind = RawMissing + } + return + } + classify(evidence, raw, RawExpression) +} + +// classify parses a raw expression candidate into evidence fields. +func classify(evidence *Evidence, raw string, kind RawKind) { + evidence.RawValue, evidence.RawKind = raw, kind + evidence.LicenseListVersion = spdxexpr.ListVersion + evidence.ResolverVersion = ResolverVersion + if kind != RawExpression && kind != RawExpressionOrFile && kind != RawLicenseName { + evidence.ParseStatus = NotApplicable + return + } + parsed := spdxexpr.Parse(raw) + evidence.ParseStatus = parsed.Status + evidence.NormalizedExpression = parsed.Normalized + evidence.ExpressionTree = parsed.Expression + evidence.UnknownTerms = parsed.UnknownTerms + if parsed.Status == spdxexpr.StatusInvalid { + evidence.Message = parsed.Problem + } +} + +func negative(evidence Evidence, outcome Outcome, status *int, message string) Evidence { + evidence.Outcome = outcome + evidence.HTTPStatus = status + evidence.Message = message + evidence.RawKind = RawMissing + evidence.ParseStatus = NotApplicable + evidence.ResolverVersion = ResolverVersion + evidence.LicenseListVersion = spdxexpr.ListVersion + expires := evidence.FetchedAt.Add(NegativeTTL) + evidence.ExpiresAt = &expires + return evidence +} + +// fromFetchError maps route errors to negative evidence outcomes. +func fromFetchError(evidence Evidence, err error) Evidence { + switch { + case errors.Is(err, ErrRouteRejected), errors.Is(err, ErrNamespaceDeny): + return negative(evidence, OutcomeRejected, nil, "request rejected by the registry route policy") + case errors.Is(err, ErrResponseLarge): + return negative(evidence, OutcomeTooLarge, nil, "registry response exceeds the configured limit") + default: + return negative(evidence, OutcomeUnavailable, nil, "registry unavailable") + } +} + +func statusOutcome(evidence Evidence, response Response) (Evidence, bool) { + status := response.Status + switch { + case status == http.StatusOK: + return evidence, true + case status == http.StatusNotFound || status == http.StatusGone: + return negative(evidence, OutcomeNotFound, &status, "package version not found at the configured route"), false + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return negative(evidence, OutcomeUnavailable, &status, "registry refused the configured credentials"), false + default: + return negative(evidence, OutcomeUnavailable, &status, "registry returned an unexpected status"), false + } +} + +func contentHash(body []byte) []byte { + sum := sha256.Sum256(body) + return sum[:] +} + +// boundedDetail keeps a small JSON-serializable excerpt for the evidence view. +func boundedDetail(values map[string]any) map[string]any { + data, err := json.Marshal(values) + if err != nil || len(data) > 4096 { + return map[string]any{"truncated": true} + } + return values +} + +func truncate(value string, max int) string { + value = strings.TrimSpace(value) + if len(value) > max { + return value[:max] + } + return value +} + +// ListVersion exposes the pinned license list release for evidence rows. +func ListVersion() string { return spdxexpr.ListVersion } + +// Now is replaceable in tests. +var Now = time.Now + +func now() time.Time { return Now().UTC() } diff --git a/internal/supplychain/license/maven.go b/internal/supplychain/license/maven.go new file mode 100644 index 00000000..3cda3025 --- /dev/null +++ b/internal/supplychain/license/maven.go @@ -0,0 +1,293 @@ +package license + +import ( + "context" + "encoding/xml" + "fmt" + "regexp" + "strings" + + "github.com/balcsida/graphnest/internal/supplychain/spdxexpr" +) + +// MavenResolver reads the exact-version POM from the configured repository: +// GET {base}/{group/as/path}/{artifact}/{version}/{artifact}-{version}.pom. +// Licenses are inherited from parents; resolution follows through the +// same route only, at most maxParentDepth levels, detects cycles, and bounds +// ${property} expansion. Unresolved inheritance stays unknown rather than +// guessed. Repository declarations inside POMs are never followed. +type MavenResolver struct { + Route Route + Fetcher *Fetcher +} + +const maxParentDepth = 8 + +func NewMavenResolver(route Route) (*MavenResolver, error) { + fetcher, err := NewFetcher(route) + if err != nil { + return nil, err + } + return &MavenResolver{Route: route, Fetcher: fetcher}, nil +} + +func (resolver *MavenResolver) Ecosystem() string { return "maven" } + +type pom struct { + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Parent struct { + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + } `xml:"parent"` + Licenses []struct { + Name string `xml:"name"` + URL string `xml:"url"` + } `xml:"licenses>license"` + Properties struct { + Entries []xmlEntry `xml:",any"` + } `xml:"properties"` +} + +type xmlEntry struct { + XMLName xml.Name + Value string `xml:",chardata"` +} + +var mavenCoordinate = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +func (resolver *MavenResolver) Resolve(ctx context.Context, coordinates Coordinates) (Evidence, error) { + evidence := Evidence{Source: SourceRegistryMaven, Route: resolver.Route.Name, Coordinates: coordinates, FetchedAt: now()} + if coordinates.Version == "" || coordinates.Name == "" || coordinates.Namespace == "" { + return negative(evidence, OutcomeRejected, nil, "exact groupId, artifactId, and version are required"), nil + } + if !resolver.Route.ServesNamespace(coordinates.Namespace) { + return negative(evidence, OutcomeRejected, nil, "groupId is not served by the configured route"), nil + } + if !mavenCoordinate.MatchString(coordinates.Namespace) || !mavenCoordinate.MatchString(coordinates.Name) || !mavenCoordinate.MatchString(coordinates.Version) { + return negative(evidence, OutcomeRejected, nil, "coordinates contain characters outside the Maven identifier alphabet"), nil + } + document, response, err := resolver.fetchPOM(ctx, coordinates.Namespace, coordinates.Name, coordinates.Version) + if err != nil { + return fromFetchError(evidence, err), nil + } + evidence.FetchedAt = response.FetchedAt + evidence, ok := statusOutcome(evidence, response) + if !ok { + return evidence, nil + } + if document == nil { + return negative(evidence, OutcomeMalformed, &response.Status, "POM is not well-formed XML"), nil + } + status := response.Status + evidence.HTTPStatus = &status + evidence.ContentSHA256 = contentHash(response.Body) + evidence.Outcome = OutcomeResolved + evidence.Detail = boundedDetail(map[string]any{"groupId": truncate(document.GroupID, 200), "artifactId": truncate(document.ArtifactID, 200), "version": truncate(document.Version, 100)}) + + // Walk the parent chain until a element is found. + properties := map[string]string{} + chain := []string{} + current := document + depth := 0 + for { + collectProperties(current, properties) + key := fmt.Sprintf("%s:%s:%s", current.GroupID, current.ArtifactID, current.Version) + for _, seen := range chain { + if seen == key && key != "::" { + evidence.Message = "parent chain is cyclic; license inheritance is unresolved" + evidence.Detail["parent_chain"] = chain + return withoutLicense(evidence), nil + } + } + chain = append(chain, key) + if len(current.Licenses) > 0 { + break + } + if current.Parent.ArtifactID == "" { + evidence.Message = "no element in the POM or its parents" + evidence.Detail["parent_chain"] = chain + return withoutLicense(evidence), nil + } + depth++ + if depth > maxParentDepth { + evidence.Message = "parent chain exceeds the resolution depth; license inheritance is unresolved" + evidence.Detail["parent_chain"] = chain + return withoutLicense(evidence), nil + } + parentGroup := expand(current.Parent.GroupID, properties) + parentArtifact := expand(current.Parent.ArtifactID, properties) + parentVersion := expand(current.Parent.Version, properties) + if strings.Contains(parentGroup+parentArtifact+parentVersion, "${") || !mavenCoordinate.MatchString(parentGroup) || !mavenCoordinate.MatchString(parentArtifact) || !mavenCoordinate.MatchString(parentVersion) { + evidence.Message = "parent coordinates could not be resolved; license inheritance is unresolved" + evidence.Detail["parent_chain"] = chain + return withoutLicense(evidence), nil + } + if !resolver.Route.ServesNamespace(parentGroup) { + evidence.Message = "parent groupId is outside the configured route; license inheritance is unresolved" + evidence.Detail["parent_chain"] = chain + return withoutLicense(evidence), nil + } + parent, parentResponse, err := resolver.fetchPOM(ctx, parentGroup, parentArtifact, parentVersion) + if err != nil || parent == nil || parentResponse.Status != 200 { + evidence.Message = "parent POM could not be fetched from the configured route; license inheritance is unresolved" + evidence.Detail["parent_chain"] = chain + return withoutLicense(evidence), nil + } + current = parent + } + evidence.Detail["parent_chain"] = chain + + // Maven licenses are names plus URLs, not SPDX expressions. Normalize only + // unambiguous single names; several licenses have no defined AND/OR + // semantics and stay a name list with unknown structure. + names := make([]string, 0, len(current.Licenses)) + urls := make([]string, 0, len(current.Licenses)) + for _, item := range current.Licenses { + name := truncate(expand(item.Name, properties), 200) + if name != "" { + names = append(names, name) + } + if url := truncate(expand(item.URL, properties), 500); url != "" { + urls = append(urls, url) + } + } + if len(urls) > 0 { + evidence.LicenseURL = urls[0] + } + switch { + case len(names) == 0 && len(urls) > 0: + classify(&evidence, urls[0], RawLicenseURL) + evidence.Message = "license is declared by URL only; a URL is not a concluded SPDX license" + case len(names) == 1: + raw := names[0] + if canonical, ok := mavenNameToSPDX(raw); ok { + classify(&evidence, canonical, RawLicenseName) + evidence.RawValue = raw + evidence.Detail["normalized_from_name"] = true + } else { + classify(&evidence, raw, RawLicenseName) + } + default: + classify(&evidence, strings.Join(names, "; "), RawLicenseName) + evidence.Message = "multiple elements have no defined AND/OR semantics; the list is kept without invented structure" + evidence.ParseStatus = spdxexpr.StatusInvalid + evidence.NormalizedExpression, evidence.ExpressionTree, evidence.UnknownTerms = "", nil, nil + evidence.Detail["license_names"] = names + } + return evidence, nil +} + +func withoutLicense(evidence Evidence) Evidence { + evidence.Outcome = OutcomeNoLicenseMetadata + evidence.RawKind = RawMissing + evidence.ParseStatus = NotApplicable + evidence.ResolverVersion = ResolverVersion + evidence.LicenseListVersion = ListVersion() + expires := evidence.FetchedAt.Add(NegativeTTL) + evidence.ExpiresAt = &expires + return evidence +} + +func (resolver *MavenResolver) fetchPOM(ctx context.Context, group, artifact, version string) (*pom, Response, error) { + segments := strings.Split(group, ".") + segments = append(segments, artifact, version, artifact+"-"+version+".pom") + response, err := resolver.Fetcher.Get(ctx, "application/xml", segments...) + if err != nil { + return nil, Response{}, err + } + if response.Status != 200 { + return nil, response, nil + } + var document pom + decoder := xml.NewDecoder(strings.NewReader(string(response.Body))) + decoder.Strict = true + if err := decoder.Decode(&document); err != nil { + return nil, response, nil + } + // Inherit missing groupId/version from the parent declaration, as Maven does. + if document.GroupID == "" { + document.GroupID = document.Parent.GroupID + } + if document.Version == "" { + document.Version = document.Parent.Version + } + return &document, response, nil +} + +func collectProperties(document *pom, properties map[string]string) { + // Child values win over parent values; only set keys not yet present. + if _, ok := properties["project.groupId"]; !ok && document.GroupID != "" { + properties["project.groupId"] = document.GroupID + } + if _, ok := properties["project.version"]; !ok && document.Version != "" { + properties["project.version"] = document.Version + } + for _, entry := range document.Properties.Entries { + if _, ok := properties[entry.XMLName.Local]; !ok && len(properties) < 512 { + properties[entry.XMLName.Local] = strings.TrimSpace(entry.Value) + } + } +} + +var propertyPattern = regexp.MustCompile(`\$\{([A-Za-z0-9._-]+)\}`) + +// expand substitutes ${property} references, bounded to a few passes so +// self-referential properties cannot loop. Unknown properties are left as +// written, which callers treat as unresolved. +func expand(value string, properties map[string]string) string { + value = strings.TrimSpace(value) + for pass := 0; pass < 4 && strings.Contains(value, "${"); pass++ { + value = propertyPattern.ReplaceAllStringFunc(value, func(match string) string { + key := match[2 : len(match)-1] + if replacement, ok := properties[key]; ok { + return replacement + } + return match + }) + } + return value +} + +// mavenNameToSPDX maps only unambiguous, widely used Maven license names to +// SPDX identifiers. Anything else stays a name. +func mavenNameToSPDX(name string) (string, bool) { + normalized := strings.ToLower(strings.Join(strings.Fields(strings.ReplaceAll(strings.ReplaceAll(name, ",", " "), "-", " ")), " ")) + switch normalized { + case "apache license version 2.0", "the apache software license version 2.0", "apache 2.0", "apache license 2.0", "the apache license version 2.0", "apache software license version 2.0", "apache 2", "asl 2.0": + return "Apache-2.0", true + case "mit license", "the mit license", "mit": + return "MIT", true + case "bsd 3 clause", "bsd 3 clause license", "the bsd 3 clause license", "bsd 3 clause \"new\" or \"revised\" license", "new bsd license", "the new bsd license", "bsd new": + return "BSD-3-Clause", true + case "bsd 2 clause", "bsd 2 clause license", "the bsd 2 clause license", "simplified bsd license": + return "BSD-2-Clause", true + case "eclipse public license version 2.0", "eclipse public license v2.0", "epl 2.0", "eclipse public license 2.0": + return "EPL-2.0", true + case "eclipse public license version 1.0", "eclipse public license v1.0", "epl 1.0", "eclipse public license 1.0": + return "EPL-1.0", true + case "gnu lesser general public license version 2.1", "lgpl 2.1", "gnu lesser general public license v2.1": + return "LGPL-2.1-only", true + case "gnu lesser general public license version 3", "lgpl 3.0", "gnu lesser general public license v3", "gnu lesser general public license version 3.0": + return "LGPL-3.0-only", true + case "mozilla public license version 2.0", "mpl 2.0", "mozilla public license 2.0": + return "MPL-2.0", true + case "isc license", "isc": + return "ISC", true + case "cddl 1.0", "common development and distribution license (cddl) v1.0": + return "CDDL-1.0", true + case "the unlicense", "unlicense": + return "Unlicense", true + case "cc0 1.0 universal", "cc0 1.0", "public domain via cc0": + return "CC0-1.0", true + case "gnu general public license version 2 with the classpath exception", "gpl2 w/ cpe", "gplv2 with classpath exception", "gpl 2.0 with classpath exception": + return "GPL-2.0-only WITH Classpath-exception-2.0", true + } + // A name that already is a valid SPDX identifier maps to itself. + if parsed := spdxexpr.Parse(name); parsed.Status == spdxexpr.StatusParsed { + return parsed.Normalized, true + } + return "", false +} diff --git a/internal/supplychain/license/npm.go b/internal/supplychain/license/npm.go new file mode 100644 index 00000000..acdfa9e8 --- /dev/null +++ b/internal/supplychain/license/npm.go @@ -0,0 +1,168 @@ +package license + +import ( + "context" + "encoding/json" + "strings" +) + +// NPMResolver reads the exact version document +// GET {base}/{name}/{version} (scoped names are one percent-encoded segment). +// It never reads dist-tags or the "latest" document, and it treats legacy +// {"type":..,"url":..} objects, arrays, "UNLICENSED", and "SEE LICENSE IN" +// conservatively: they are recorded as what they are, not as a license. +type NPMResolver struct { + Route Route + Fetcher *Fetcher +} + +func NewNPMResolver(route Route) (*NPMResolver, error) { + fetcher, err := NewFetcher(route) + if err != nil { + return nil, err + } + return &NPMResolver{Route: route, Fetcher: fetcher}, nil +} + +func (resolver *NPMResolver) Ecosystem() string { return "npm" } + +func (resolver *NPMResolver) Resolve(ctx context.Context, coordinates Coordinates) (Evidence, error) { + evidence := Evidence{Source: SourceRegistryNPM, Route: resolver.Route.Name, Coordinates: coordinates, FetchedAt: now()} + if coordinates.Version == "" || coordinates.Name == "" { + return negative(evidence, OutcomeRejected, nil, "exact version and name are required"), nil + } + if !resolver.Route.ServesNamespace(coordinates.Namespace) { + return negative(evidence, OutcomeRejected, nil, "package scope is not served by the configured route"), nil + } + name := coordinates.Name + if coordinates.Namespace != "" { + // The registry expects "@scope/name" as one segment with the slash + // percent-encoded; the fetcher escapes "%" again unless told the + // segment is pre-encoded, so mark it. + name = coordinates.Namespace + "%2F" + coordinates.Name + } + response, err := resolver.Fetcher.Get(ctx, "application/json", name, coordinates.Version) + if err != nil { + return fromFetchError(evidence, err), nil + } + evidence.FetchedAt = response.FetchedAt + evidence, ok := statusOutcome(evidence, response) + if !ok { + return evidence, nil + } + var document struct { + Name string `json:"name"` + Version string `json:"version"` + License json.RawMessage `json:"license"` + Licenses json.RawMessage `json:"licenses"` + Dist struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + } `json:"dist"` + } + if err := json.Unmarshal(response.Body, &document); err != nil { + return negative(evidence, OutcomeMalformed, &response.Status, "registry document is not valid JSON"), nil + } + if document.Version != "" && document.Version != coordinates.Version { + // A registry that answers a different version (dist-tag resolution, + // redirect to latest) must not become evidence for the requested one. + return negative(evidence, OutcomeRejected, &response.Status, "registry answered a different version than requested"), nil + } + evidence.ContentSHA256 = contentHash(response.Body) + evidence.Outcome = OutcomeResolved + evidence.Detail = boundedDetail(map[string]any{"name": truncate(document.Name, 200), "version": truncate(document.Version, 100), "integrity": truncate(document.Dist.Integrity, 200)}) + status := response.Status + evidence.HTTPStatus = &status + + raw, kind := npmLicenseValue(document.License, document.Licenses) + switch kind { + case RawMissing: + evidence.Outcome = OutcomeNoLicenseMetadata + evidence.RawKind = RawMissing + evidence.ParseStatus = NotApplicable + evidence.ResolverVersion = ResolverVersion + evidence.LicenseListVersion = ListVersion() + evidence.Message = "package.json declares no license" + expires := evidence.FetchedAt.Add(NegativeTTL) + evidence.ExpiresAt = &expires + return evidence, nil + case RawLicenseFile: + // "SEE LICENSE IN ": a pointer to a file, not an expression. + classify(&evidence, raw, RawLicenseFile) + evidence.LicenseFileName = strings.TrimSpace(strings.TrimPrefix(raw, "SEE LICENSE IN")) + evidence.Message = "license is declared by reference to a file in the package; the expression is unknown until the file is reviewed" + return evidence, nil + case RawLegacyObject: + // Legacy {type,url} / [{type,url}] metadata: keep the type as a + // candidate name and the URL, but do not treat the URL as a license. + classify(&evidence, raw, RawLicenseName) + evidence.RawKind = RawLegacyObject + return evidence, nil + default: + classify(&evidence, raw, RawExpression) + return evidence, nil + } +} + +// npmLicenseValue reduces the license/licenses members to a raw value and +// kind. Only a plain string license is an expression candidate. +func npmLicenseValue(license, licenses json.RawMessage) (string, RawKind) { + if len(license) > 0 && string(license) != "null" { + var text string + if err := json.Unmarshal(license, &text); err == nil { + text = strings.TrimSpace(text) + switch { + case text == "": + return "", RawMissing + case strings.HasPrefix(strings.ToUpper(text), "SEE LICENSE IN"): + return text, RawLicenseFile + default: + return text, RawExpression + } + } + var object struct { + Type string `json:"type"` + URL string `json:"url"` + } + if err := json.Unmarshal(license, &object); err == nil && (object.Type != "" || object.URL != "") { + return legacyValue([]struct{ Type, URL string }{{object.Type, object.URL}}), RawLegacyObject + } + } + if len(licenses) > 0 && string(licenses) != "null" { + var objects []struct { + Type string `json:"type"` + URL string `json:"url"` + } + if err := json.Unmarshal(licenses, &objects); err == nil && len(objects) > 0 { + items := make([]struct{ Type, URL string }, 0, len(objects)) + for _, object := range objects { + items = append(items, struct{ Type, URL string }{object.Type, object.URL}) + } + return legacyValue(items), RawLegacyObject + } + var names []string + if err := json.Unmarshal(licenses, &names); err == nil && len(names) > 0 { + // An array of names is ambiguous: SPDX gives it no AND/OR meaning. + // Keep the list verbatim; it will parse as invalid and stay visible. + return strings.Join(names, ", "), RawLegacyObject + } + } + return "", RawMissing +} + +func legacyValue(items []struct{ Type, URL string }) string { + parts := make([]string, 0, len(items)) + for _, item := range items { + part := strings.TrimSpace(item.Type) + if url := strings.TrimSpace(item.URL); url != "" { + if part != "" { + part += " " + } + part += "(" + url + ")" + } + if part != "" { + parts = append(parts, part) + } + } + return strings.Join(parts, ", ") +} diff --git a/internal/supplychain/license/nuget.go b/internal/supplychain/license/nuget.go new file mode 100644 index 00000000..d132ee59 --- /dev/null +++ b/internal/supplychain/license/nuget.go @@ -0,0 +1,106 @@ +package license + +import ( + "context" + "encoding/xml" + "strings" +) + +// NuGetResolver reads the exact-version nuspec through the V3 flat container: +// GET {base}/{id-lower}/{version-lower}/{id-lower}.nuspec. It preserves the +// versus distinction and +// treats a legacy alone as a URL, not a concluded license. It +// never downloads or unpacks the .nupkg. +type NuGetResolver struct { + Route Route + Fetcher *Fetcher +} + +func NewNuGetResolver(route Route) (*NuGetResolver, error) { + fetcher, err := NewFetcher(route) + if err != nil { + return nil, err + } + return &NuGetResolver{Route: route, Fetcher: fetcher}, nil +} + +func (resolver *NuGetResolver) Ecosystem() string { return "nuget" } + +type nuspec struct { + Metadata struct { + ID string `xml:"id"` + Version string `xml:"version"` + License struct { + Type string `xml:"type,attr"` + Version string `xml:"version,attr"` + Value string `xml:",chardata"` + } `xml:"license"` + LicenseURL string `xml:"licenseUrl"` + } `xml:"metadata"` +} + +func (resolver *NuGetResolver) Resolve(ctx context.Context, coordinates Coordinates) (Evidence, error) { + evidence := Evidence{Source: SourceRegistryNuGet, Route: resolver.Route.Name, Coordinates: coordinates, FetchedAt: now()} + if coordinates.Version == "" || coordinates.Name == "" { + return negative(evidence, OutcomeRejected, nil, "exact version and package id are required"), nil + } + if !resolver.Route.ServesNamespace(coordinates.Name) { + return negative(evidence, OutcomeRejected, nil, "package id is not served by the configured route"), nil + } + id, version := strings.ToLower(coordinates.Name), strings.ToLower(coordinates.Version) + response, err := resolver.Fetcher.Get(ctx, "application/xml", id, version, id+".nuspec") + if err != nil { + return fromFetchError(evidence, err), nil + } + evidence.FetchedAt = response.FetchedAt + evidence, ok := statusOutcome(evidence, response) + if !ok { + return evidence, nil + } + var document nuspec + decoder := xml.NewDecoder(strings.NewReader(string(response.Body))) + decoder.Strict = true + // External entities and DTDs are not resolved by encoding/xml; CharsetReader is nil so only UTF-8 is accepted. + if err := decoder.Decode(&document); err != nil { + return negative(evidence, OutcomeMalformed, &response.Status, "nuspec is not well-formed XML"), nil + } + if document.Metadata.ID != "" && !strings.EqualFold(document.Metadata.ID, coordinates.Name) || document.Metadata.Version != "" && !strings.EqualFold(strings.TrimSpace(document.Metadata.Version), coordinates.Version) { + return negative(evidence, OutcomeRejected, &response.Status, "nuspec identifies a different package or version than requested"), nil + } + status := response.Status + evidence.HTTPStatus = &status + evidence.ContentSHA256 = contentHash(response.Body) + evidence.Outcome = OutcomeResolved + evidence.Detail = boundedDetail(map[string]any{"id": truncate(document.Metadata.ID, 200), "version": truncate(document.Metadata.Version, 100), "license_type": truncate(document.Metadata.License.Type, 32)}) + licenseValue := strings.TrimSpace(document.Metadata.License.Value) + switch strings.ToLower(document.Metadata.License.Type) { + case "expression": + classify(&evidence, licenseValue, RawExpression) + if document.Metadata.License.Version != "" { + evidence.Detail["license_expression_version"] = truncate(document.Metadata.License.Version, 16) + } + case "file": + classify(&evidence, licenseValue, RawLicenseFile) + evidence.LicenseFileName = licenseValue + evidence.Message = "license is embedded as a file in the package; the expression is unknown until the file is reviewed" + default: + if url := strings.TrimSpace(document.Metadata.LicenseURL); url != "" { + classify(&evidence, url, RawLicenseURL) + evidence.LicenseURL = url + evidence.Message = "only a legacy licenseUrl is declared; a URL is not a concluded SPDX license" + return evidence, nil + } + evidence.Outcome = OutcomeNoLicenseMetadata + evidence.RawKind = RawMissing + evidence.ParseStatus = NotApplicable + evidence.ResolverVersion = ResolverVersion + evidence.LicenseListVersion = ListVersion() + evidence.Message = "nuspec declares no license" + expires := evidence.FetchedAt.Add(NegativeTTL) + evidence.ExpiresAt = &expires + } + if url := strings.TrimSpace(document.Metadata.LicenseURL); url != "" && evidence.LicenseURL == "" { + evidence.LicenseURL = url + } + return evidence, nil +} diff --git a/internal/supplychain/license/resolvers_test.go b/internal/supplychain/license/resolvers_test.go new file mode 100644 index 00000000..34436fd8 --- /dev/null +++ b/internal/supplychain/license/resolvers_test.go @@ -0,0 +1,562 @@ +package license + +import ( + "compress/gzip" + "context" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/balcsida/graphnest/internal/supplychain/spdxexpr" +) + +// registry is a TLS fake registry whose handler decides per path. Requests +// are counted so "no outbound traffic" assertions are exact. +type registry struct { + server *httptest.Server + handler func(http.ResponseWriter, *http.Request) + calls atomic.Int32 + last atomic.Pointer[http.Request] +} + +func newRegistry(t *testing.T, handler func(http.ResponseWriter, *http.Request)) *registry { + t.Helper() + r := ®istry{handler: handler} + r.server = httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + r.calls.Add(1) + r.last.Store(request.Clone(context.Background())) + r.handler(writer, request) + })) + t.Cleanup(r.server.Close) + return r +} + +func (r *registry) route(t *testing.T, ecosystem, basePath string) Route { + t.Helper() + base, err := url.Parse(r.server.URL + basePath) + if err != nil { + t.Fatal(err) + } + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: r.server.Certificate().Raw}) + // httptest listens on 127.0.0.1, so tests opt into private hosts; the + // private-host denial is tested separately. + return Route{Name: ecosystem + ":test", Ecosystem: ecosystem, BaseURL: base, CAPEM: certificate, AllowPrivateHosts: true, Timeout: 5 * time.Second, MaxResponseBytes: 64 << 10} +} + +const npmLeftPad = `{"name":"@scope/left-pad","version":"1.3.0","license":"MIT","dist":{"integrity":"sha512-abc","shasum":"deadbeef"}}` + +func TestNPMResolverExactVersion(t *testing.T) { + cases := map[string]struct { + body string + status int + outcome Outcome + kind RawKind + parse spdxexpr.Status + normal string + raw string + wantURL string + wantFile string + wantPath string + wantMsgIn string + }{ + "plain expression": {body: npmLeftPad, status: 200, outcome: OutcomeResolved, kind: RawExpression, parse: spdxexpr.StatusParsed, normal: "MIT", raw: "MIT", wantPath: "/registry/@scope%2Fleft-pad/1.3.0"}, + "compound expression": {body: `{"version":"1.3.0","license":"(MIT OR Apache-2.0)"}`, status: 200, outcome: OutcomeResolved, kind: RawExpression, parse: spdxexpr.StatusParsed, normal: "MIT OR Apache-2.0", raw: "(MIT OR Apache-2.0)"}, + "unlicensed": {body: `{"version":"1.3.0","license":"UNLICENSED"}`, status: 200, outcome: OutcomeResolved, kind: RawExpression, parse: spdxexpr.StatusUnlicensed, raw: "UNLICENSED"}, + "see license in": {body: `{"version":"1.3.0","license":"SEE LICENSE IN LICENSE.md"}`, status: 200, outcome: OutcomeResolved, kind: RawLicenseFile, parse: NotApplicable, raw: "SEE LICENSE IN LICENSE.md", wantFile: "LICENSE.md", wantMsgIn: "reference to a file"}, + "legacy object": {body: `{"version":"1.3.0","license":{"type":"BSD-3-Clause","url":"https://example.invalid/LICENSE"}}`, status: 200, outcome: OutcomeResolved, kind: RawLegacyObject, parse: spdxexpr.StatusInvalid, raw: "BSD-3-Clause (https://example.invalid/LICENSE)"}, + "legacy array": {body: `{"version":"1.3.0","licenses":[{"type":"MIT","url":"https://a"},{"type":"Apache-2.0","url":"https://b"}]}`, status: 200, outcome: OutcomeResolved, kind: RawLegacyObject, parse: spdxexpr.StatusInvalid, raw: "MIT (https://a), Apache-2.0 (https://b)"}, + "array of names": {body: `{"version":"1.3.0","licenses":["MIT","Apache-2.0"]}`, status: 200, outcome: OutcomeResolved, kind: RawLegacyObject, parse: spdxexpr.StatusInvalid, raw: "MIT, Apache-2.0"}, + "unknown identifier": {body: `{"version":"1.3.0","license":"Custom-Corp-1.0"}`, status: 200, outcome: OutcomeResolved, kind: RawExpression, parse: spdxexpr.StatusUnknownTerms, normal: "Custom-Corp-1.0", raw: "Custom-Corp-1.0"}, + "free text": {body: `{"version":"1.3.0","license":"Copyright Acme, all rights reserved"}`, status: 200, outcome: OutcomeResolved, kind: RawExpression, parse: spdxexpr.StatusInvalid, raw: "Copyright Acme, all rights reserved"}, + "missing license": {body: `{"version":"1.3.0","name":"x"}`, status: 200, outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable}, + "empty license": {body: `{"version":"1.3.0","license":""}`, status: 200, outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable}, + "version mismatch": {body: `{"version":"1.4.0","license":"MIT"}`, status: 200, outcome: OutcomeRejected, kind: RawMissing, parse: NotApplicable, wantMsgIn: "different version"}, + "not found": {body: `{"error":"Not found"}`, status: 404, outcome: OutcomeNotFound, kind: RawMissing, parse: NotApplicable}, + "forbidden": {body: `{}`, status: 403, outcome: OutcomeUnavailable, kind: RawMissing, parse: NotApplicable}, + "server error": {body: `{}`, status: 503, outcome: OutcomeUnavailable, kind: RawMissing, parse: NotApplicable}, + "malformed": {body: ``, status: 200, outcome: OutcomeMalformed, kind: RawMissing, parse: NotApplicable}, + } + for name, test := range cases { + t.Run(name, func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(test.status) + fmt.Fprint(writer, test.body) + }) + resolver, err := NewNPMResolver(r.route(t, "npm", "/registry/")) + if err != nil { + t.Fatal(err) + } + evidence, err := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.3.0"}) + if err != nil { + t.Fatal(err) + } + if evidence.Outcome != test.outcome || evidence.RawKind != test.kind || evidence.ParseStatus != test.parse || evidence.NormalizedExpression != test.normal || evidence.RawValue != test.raw { + t.Fatalf("evidence = outcome %s kind %s parse %s normalized %q raw %q message %q", evidence.Outcome, evidence.RawKind, evidence.ParseStatus, evidence.NormalizedExpression, evidence.RawValue, evidence.Message) + } + if evidence.Source != SourceRegistryNPM || evidence.Route != "npm:test" || evidence.ResolverVersion != ResolverVersion || evidence.LicenseListVersion != spdxexpr.ListVersion || evidence.Coordinates.Version != "1.3.0" { + t.Fatalf("provenance = %+v", evidence) + } + if test.wantFile != "" && evidence.LicenseFileName != test.wantFile { + t.Fatalf("license file = %q", evidence.LicenseFileName) + } + if test.wantMsgIn != "" && !strings.Contains(evidence.Message, test.wantMsgIn) { + t.Fatalf("message = %q", evidence.Message) + } + if test.wantPath != "" && r.last.Load().URL.EscapedPath() != test.wantPath { + t.Fatalf("path = %q, want %q", r.last.Load().URL.EscapedPath(), test.wantPath) + } + negativeOutcome := evidence.Outcome != OutcomeResolved + if negativeOutcome != (evidence.ExpiresAt != nil) { + t.Fatalf("negative results must expire, positive must not: outcome %s expires %v", evidence.Outcome, evidence.ExpiresAt) + } + if evidence.Outcome == OutcomeResolved && len(evidence.ContentSHA256) != 32 { + t.Fatalf("resolved evidence must hash its content: %+v", evidence.ContentSHA256) + } + if r.last.Load().Header.Get("Accept") != "application/json" || r.last.Load().Header.Get("User-Agent") != "GraphNest" { + t.Fatalf("headers = %v", r.last.Load().Header) + } + }) + } +} + +func TestNPMResolverNeverUsesDistTags(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + if strings.Contains(request.URL.Path, "latest") || strings.Count(strings.Trim(request.URL.Path, "/"), "/") < 1 { + t.Errorf("resolver asked for a dist-tag or packument: %s", request.URL.Path) + } + fmt.Fprint(writer, npmLeftPad) + }) + resolver, _ := NewNPMResolver(r.route(t, "npm", "/")) + if _, err := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.3.0"}); err != nil { + t.Fatal(err) + } + if got := r.last.Load().URL.EscapedPath(); got != "/@scope%2Fleft-pad/1.3.0" { + t.Fatalf("path = %q", got) + } + // A missing version is rejected before any request is made. + before := r.calls.Load() + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad"}) + if evidence.Outcome != OutcomeRejected || r.calls.Load() != before { + t.Fatalf("versionless lookup made a request: %+v calls=%d", evidence, r.calls.Load()-before) + } +} + +func TestRouteNamespaceRestrictionBlocksRequests(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprint(writer, `{"version":"1.0.0","license":"MIT"}`) + }) + route := r.route(t, "npm", "/") + route.AllowedNamespaces = []string{"@acme"} + resolver, _ := NewNPMResolver(route) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeRejected || r.calls.Load() != 0 { + t.Fatalf("out-of-scope package reached the registry: %+v calls=%d", evidence, r.calls.Load()) + } + evidence, _ = resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Namespace: "@acme", Name: "widget", Version: "1.0.0"}) + if evidence.Outcome != OutcomeResolved || r.calls.Load() != 1 { + t.Fatalf("in-scope package = %+v calls=%d", evidence, r.calls.Load()) + } +} + +func TestFetcherRejectsHostileRegistries(t *testing.T) { + t.Run("cross-origin redirect", func(t *testing.T) { + other := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { fmt.Fprint(writer, npmLeftPad) }) + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + http.Redirect(writer, request, other.server.URL+request.URL.Path, http.StatusFound) + }) + resolver, _ := NewNPMResolver(r.route(t, "npm", "/")) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeRejected || other.calls.Load() != 0 { + t.Fatalf("redirect followed off-route: %+v other calls=%d", evidence, other.calls.Load()) + } + }) + t.Run("redirect outside base path", func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + if strings.HasPrefix(request.URL.Path, "/registry/") { + http.Redirect(writer, request, "/admin/secret", http.StatusFound) + return + } + fmt.Fprint(writer, npmLeftPad) + }) + resolver, _ := NewNPMResolver(r.route(t, "npm", "/registry/")) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeRejected || r.calls.Load() != 1 { + t.Fatalf("redirect escaped the base path: %+v calls=%d", evidence, r.calls.Load()) + } + }) + t.Run("same-origin redirect within base is followed", func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + if strings.HasSuffix(request.URL.Path, "/1.3.0") { + http.Redirect(writer, request, "/registry/left-pad/1.3.0/", http.StatusMovedPermanently) + return + } + fmt.Fprint(writer, `{"version":"1.3.0","license":"ISC"}`) + }) + resolver, _ := NewNPMResolver(r.route(t, "npm", "/registry/")) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeResolved || evidence.NormalizedExpression != "ISC" || r.calls.Load() != 2 { + t.Fatalf("same-origin redirect = %+v calls=%d", evidence, r.calls.Load()) + } + }) + t.Run("oversized body", func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprint(writer, `{"version":"1.3.0","license":"`+strings.Repeat("M", 70<<10)+`"}`) + }) + resolver, _ := NewNPMResolver(r.route(t, "npm", "/")) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeTooLarge { + t.Fatalf("oversized = %+v", evidence) + } + }) + t.Run("decompression bomb is bounded", func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Encoding", "gzip") + compressor := gzip.NewWriter(writer) + fmt.Fprint(compressor, `{"version":"1.3.0","license":"`+strings.Repeat("A", 1<<20)+`"}`) + compressor.Close() + }) + resolver, _ := NewNPMResolver(r.route(t, "npm", "/")) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeTooLarge { + t.Fatalf("bomb = %+v", evidence) + } + }) + t.Run("private address denied by default", func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { fmt.Fprint(writer, npmLeftPad) }) + route := r.route(t, "npm", "/") + route.AllowPrivateHosts = false + resolver, _ := NewNPMResolver(route) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeRejected || r.calls.Load() != 0 { + t.Fatalf("loopback registry reached without AllowPrivateHosts: %+v calls=%d", evidence, r.calls.Load()) + } + }) + t.Run("untrusted certificate", func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { fmt.Fprint(writer, npmLeftPad) }) + route := r.route(t, "npm", "/") + route.CAPEM = nil + resolver, _ := NewNPMResolver(route) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeUnavailable { + t.Fatalf("untrusted TLS = %+v", evidence) + } + }) + t.Run("hostile names cannot traverse", func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { fmt.Fprint(writer, npmLeftPad) }) + resolver, _ := NewNPMResolver(r.route(t, "npm", "/registry/")) + for _, name := range []string{"..", "a/b", "a?b", "a#b", ".", "a\\b"} { + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: name, Version: "1.0.0"}) + if evidence.Outcome != OutcomeRejected { + t.Fatalf("name %q was requested: %+v", name, evidence) + } + } + if r.calls.Load() != 0 { + t.Fatalf("hostile names produced %d requests", r.calls.Load()) + } + }) +} + +func TestCredentialsStayOnTheRoute(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + if request.Header.Get("Authorization") != "Bearer secret-token" { + writer.WriteHeader(http.StatusUnauthorized) + return + } + fmt.Fprint(writer, npmLeftPad) + }) + route := r.route(t, "npm", "/") + route.BearerToken = "secret-token" + resolver, _ := NewNPMResolver(route) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeResolved { + t.Fatalf("bearer route = %+v", evidence) + } + if strings.Contains(fmt.Sprintf("%+v", evidence), "secret-token") { + t.Fatal("evidence leaks the route credential") + } + basic := r.route(t, "npm", "/") + basic.BasicUser, basic.BasicPassword = "user", "pw" + resolver, _ = NewNPMResolver(basic) + evidence, _ = resolver.Resolve(t.Context(), Coordinates{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}) + if evidence.Outcome != OutcomeUnavailable || evidence.HTTPStatus == nil || *evidence.HTTPStatus != 401 { + t.Fatalf("basic auth refused = %+v", evidence) + } +} + +const nuspecExpression = `Newtonsoft.Json13.0.3MIThttps://licenses.nuget.org/MIT` + +func TestNuGetResolverPreservesLicenseKinds(t *testing.T) { + cases := map[string]struct { + body string + status int + outcome Outcome + kind RawKind + parse spdxexpr.Status + normal string + file string + url string + }{ + "expression": {body: nuspecExpression, status: 200, outcome: OutcomeResolved, kind: RawExpression, parse: spdxexpr.StatusParsed, normal: "MIT", url: "https://licenses.nuget.org/MIT"}, + "compound": {body: strings.Replace(nuspecExpression, `MIT`, `Apache-2.0 OR MIT`, 1), status: 200, outcome: OutcomeResolved, kind: RawExpression, parse: spdxexpr.StatusParsed, normal: "Apache-2.0 OR MIT", url: "https://licenses.nuget.org/MIT"}, + "file": {body: strings.Replace(nuspecExpression, `MIT`, `LICENSE.txt`, 1), status: 200, outcome: OutcomeResolved, kind: RawLicenseFile, parse: NotApplicable, file: "LICENSE.txt", url: "https://licenses.nuget.org/MIT"}, + "legacy url only": {body: strings.Replace(nuspecExpression, `MIT`, ``, 1), status: 200, outcome: OutcomeResolved, kind: RawLicenseURL, parse: NotApplicable, url: "https://licenses.nuget.org/MIT"}, + "nothing declared": {body: `Newtonsoft.Json13.0.3`, status: 200, outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable}, + "wrong version": {body: strings.Replace(nuspecExpression, "13.0.3", "13.0.4", 1), status: 200, outcome: OutcomeRejected, kind: RawMissing, parse: NotApplicable}, + "wrong id": {body: strings.Replace(nuspecExpression, "Newtonsoft.Json", "Evil.Json", 1), status: 200, outcome: OutcomeRejected, kind: RawMissing, parse: NotApplicable}, + "not found": {body: ``, status: 404, outcome: OutcomeNotFound, kind: RawMissing, parse: NotApplicable}, + "malformed": {body: ``, status: 200, outcome: OutcomeMalformed, kind: RawMissing, parse: NotApplicable}, + "external entity": {body: `]>Newtonsoft.Json13.0.3&x;`, status: 200, outcome: OutcomeMalformed, kind: RawMissing, parse: NotApplicable}, + } + for name, test := range cases { + t.Run(name, func(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(test.status) + fmt.Fprint(writer, test.body) + }) + resolver, err := NewNuGetResolver(r.route(t, "nuget", "/v3-flatcontainer/")) + if err != nil { + t.Fatal(err) + } + evidence, err := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "nuget", Name: "Newtonsoft.Json", Version: "13.0.3"}) + if err != nil { + t.Fatal(err) + } + if evidence.Outcome != test.outcome || evidence.RawKind != test.kind || evidence.ParseStatus != test.parse || evidence.NormalizedExpression != test.normal || evidence.LicenseFileName != test.file || evidence.LicenseURL != test.url { + t.Fatalf("evidence = outcome %s kind %s parse %s normalized %q file %q url %q message %q", evidence.Outcome, evidence.RawKind, evidence.ParseStatus, evidence.NormalizedExpression, evidence.LicenseFileName, evidence.LicenseURL, evidence.Message) + } + if got := r.last.Load().URL.EscapedPath(); got != "/v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec" { + t.Fatalf("path = %q", got) + } + if evidence.Source != SourceRegistryNuGet || evidence.Route != "nuget:test" { + t.Fatalf("provenance = %+v", evidence) + } + }) + } +} + +func TestNuGetResolverRejectsWithoutVersion(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { fmt.Fprint(writer, nuspecExpression) }) + resolver, _ := NewNuGetResolver(r.route(t, "nuget", "/")) + evidence, _ := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "nuget", Name: "Newtonsoft.Json"}) + if evidence.Outcome != OutcomeRejected || r.calls.Load() != 0 { + t.Fatalf("versionless = %+v calls=%d", evidence, r.calls.Load()) + } +} + +func mavenPOM(group, artifact, version, parent, licenses, properties string) string { + return `4.0.0` + parent + + `` + group + `` + artifact + `` + version + `` + properties + licenses + `` +} + +func mavenParent(group, artifact, version string) string { + return `` + group + `` + artifact + `` + version + `` +} + +func TestMavenResolverDeclarationsAndInheritance(t *testing.T) { + poms := map[string]string{ + "/maven2/org/example/core/2.1.0/core-2.1.0.pom": mavenPOM("org.example", "core", "2.1.0", "", `The Apache Software License, Version 2.0https://www.apache.org/licenses/LICENSE-2.0.txt`, ""), + "/maven2/org/example/child/1.0.0/child-1.0.0.pom": mavenPOM("", "child", "", mavenParent("org.example", "parent", "3.0.0"), "", ""), + "/maven2/org/example/parent/3.0.0/parent-3.0.0.pom": mavenPOM("org.example", "parent", "3.0.0", mavenParent("org.example", "grandparent", "1.0.0"), "", ""), + "/maven2/org/example/grandparent/1.0.0/grandparent-1.0.0.pom": mavenPOM("org.example", "grandparent", "1.0.0", "", `${license.name}${license.url}`, `MIT Licensehttps://opensource.org/licenses/MIT`), + "/maven2/org/example/cyclic-a/1.0.0/cyclic-a-1.0.0.pom": mavenPOM("org.example", "cyclic-a", "1.0.0", mavenParent("org.example", "cyclic-b", "1.0.0"), "", ""), + "/maven2/org/example/cyclic-b/1.0.0/cyclic-b-1.0.0.pom": mavenPOM("org.example", "cyclic-b", "1.0.0", mavenParent("org.example", "cyclic-a", "1.0.0"), "", ""), + "/maven2/org/example/orphan/1.0.0/orphan-1.0.0.pom": mavenPOM("org.example", "orphan", "1.0.0", mavenParent("org.example", "missing-parent", "9.9.9"), "", ""), + "/maven2/org/example/dual/1.0.0/dual-1.0.0.pom": mavenPOM("org.example", "dual", "1.0.0", "", `MIT LicenseApache License, Version 2.0`, ""), + "/maven2/org/example/urlonly/1.0.0/urlonly-1.0.0.pom": mavenPOM("org.example", "urlonly", "1.0.0", "", `https://example.invalid/LICENSE`, ""), + "/maven2/org/example/nolicense/1.0.0/nolicense-1.0.0.pom": mavenPOM("org.example", "nolicense", "1.0.0", "", "", ""), + "/maven2/org/example/customname/1.0.0/customname-1.0.0.pom": mavenPOM("org.example", "customname", "1.0.0", "", `Acme Internal License`, ""), + "/maven2/org/example/spdxname/1.0.0/spdxname-1.0.0.pom": mavenPOM("org.example", "spdxname", "1.0.0", "", `EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0`, ""), + "/maven2/org/example/unresolved/1.0.0/unresolved-1.0.0.pom": mavenPOM("org.example", "unresolved", "1.0.0", mavenParent("${parent.group}", "parent", "3.0.0"), "", ""), + "/maven2/org/example/foreign/1.0.0/foreign-1.0.0.pom": mavenPOM("org.example", "foreign", "1.0.0", mavenParent("com.other", "parent", "1.0.0"), "", ""), + "/maven2/org/example/selfref/1.0.0/selfref-1.0.0.pom": mavenPOM("org.example", "selfref", "1.0.0", "", `${a}`, `${b}${a}`), + } + var deep strings.Builder + for level := 0; level <= maxParentDepth+1; level++ { + _ = deep + parent := "" + if level < maxParentDepth+1 { + parent = mavenParent("org.example", fmt.Sprintf("deep-%d", level+1), "1.0.0") + } + poms[fmt.Sprintf("/maven2/org/example/deep-%d/1.0.0/deep-%d-1.0.0.pom", level, level)] = mavenPOM("org.example", fmt.Sprintf("deep-%d", level), "1.0.0", parent, "", "") + } + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + body, ok := poms[request.URL.EscapedPath()] + if !ok { + writer.WriteHeader(http.StatusNotFound) + return + } + fmt.Fprint(writer, body) + }) + route := r.route(t, "maven", "/maven2/") + route.AllowedNamespaces = []string{"org.example"} + resolver, err := NewMavenResolver(route) + if err != nil { + t.Fatal(err) + } + cases := map[string]struct { + artifact string + outcome Outcome + kind RawKind + parse spdxexpr.Status + normal string + raw string + url string + msgIn string + calls int32 + }{ + "apache name normalized": {artifact: "core", outcome: OutcomeResolved, kind: RawLicenseName, parse: spdxexpr.StatusParsed, normal: "Apache-2.0", raw: "The Apache Software License, Version 2.0", url: "https://www.apache.org/licenses/LICENSE-2.0.txt", calls: 1}, + "inherited through parents": {artifact: "child", outcome: OutcomeResolved, kind: RawLicenseName, parse: spdxexpr.StatusParsed, normal: "MIT", raw: "MIT License", url: "https://opensource.org/licenses/MIT", calls: 3}, + "cycle detected": {artifact: "cyclic-a", outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable, msgIn: "cyclic", calls: 3}, + "missing parent": {artifact: "orphan", outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable, msgIn: "could not be fetched", calls: 2}, + "depth bounded": {artifact: "deep-0", outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable, msgIn: "depth", calls: maxParentDepth + 1}, + "multiple licenses": {artifact: "dual", outcome: OutcomeResolved, kind: RawLicenseName, parse: spdxexpr.StatusInvalid, normal: "", raw: "MIT License; Apache License, Version 2.0", msgIn: "no defined AND/OR", calls: 1}, + "url only": {artifact: "urlonly", outcome: OutcomeResolved, kind: RawLicenseURL, parse: NotApplicable, raw: "https://example.invalid/LICENSE", url: "https://example.invalid/LICENSE", msgIn: "not a concluded", calls: 1}, + "no license": {artifact: "nolicense", outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable, msgIn: "no ", calls: 1}, + "custom name kept": {artifact: "customname", outcome: OutcomeResolved, kind: RawLicenseName, parse: spdxexpr.StatusInvalid, raw: "Acme Internal License", calls: 1}, + "spdx expression as name": {artifact: "spdxname", outcome: OutcomeResolved, kind: RawLicenseName, parse: spdxexpr.StatusParsed, normal: "EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0", raw: "EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0", calls: 1}, + "unresolved property": {artifact: "unresolved", outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable, msgIn: "could not be resolved", calls: 1}, + "foreign parent group": {artifact: "foreign", outcome: OutcomeNoLicenseMetadata, kind: RawMissing, parse: NotApplicable, msgIn: "outside the configured route", calls: 1}, + "self-referential property": {artifact: "selfref", outcome: OutcomeResolved, kind: RawLicenseName, parse: spdxexpr.StatusInvalid, raw: "${a}", calls: 1}, + "not found": {artifact: "absent", outcome: OutcomeNotFound, kind: RawMissing, parse: NotApplicable, calls: 1}, + } + for name, test := range cases { + t.Run(name, func(t *testing.T) { + r.calls.Store(0) + evidence, err := resolver.Resolve(t.Context(), Coordinates{Ecosystem: "maven", Namespace: "org.example", Name: test.artifact, Version: "1.0.0"}) + if test.artifact == "core" { + evidence, err = resolver.Resolve(t.Context(), Coordinates{Ecosystem: "maven", Namespace: "org.example", Name: "core", Version: "2.1.0"}) + r.calls.Store(1) + } + if err != nil { + t.Fatal(err) + } + if evidence.Outcome != test.outcome || evidence.RawKind != test.kind || evidence.ParseStatus != test.parse || evidence.NormalizedExpression != test.normal || evidence.RawValue != test.raw || evidence.LicenseURL != test.url { + t.Fatalf("evidence = outcome %s kind %s parse %s normalized %q raw %q url %q message %q", evidence.Outcome, evidence.RawKind, evidence.ParseStatus, evidence.NormalizedExpression, evidence.RawValue, evidence.LicenseURL, evidence.Message) + } + if test.msgIn != "" && !strings.Contains(evidence.Message, test.msgIn) { + t.Fatalf("message = %q, want it to contain %q", evidence.Message, test.msgIn) + } + if r.calls.Load() != test.calls { + t.Fatalf("registry calls = %d, want %d", r.calls.Load(), test.calls) + } + if evidence.Source != SourceRegistryMaven || evidence.Route != "maven:test" { + t.Fatalf("provenance = %+v", evidence) + } + }) + } +} + +func TestMavenResolverRejectsBeforeRequesting(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprint(writer, mavenPOM("g", "a", "1", "", "", "")) + }) + route := r.route(t, "maven", "/maven2/") + route.AllowedNamespaces = []string{"org.example"} + resolver, _ := NewMavenResolver(route) + for _, coordinates := range []Coordinates{ + {Ecosystem: "maven", Namespace: "org.example", Name: "core"}, + {Ecosystem: "maven", Name: "core", Version: "1.0"}, + {Ecosystem: "maven", Namespace: "com.other", Name: "core", Version: "1.0"}, + {Ecosystem: "maven", Namespace: "org.example", Name: "../etc", Version: "1.0"}, + {Ecosystem: "maven", Namespace: "org.example", Name: "core", Version: "1.0/../../x"}, + } { + evidence, _ := resolver.Resolve(t.Context(), coordinates) + if evidence.Outcome != OutcomeRejected { + t.Fatalf("%+v = %+v", coordinates, evidence) + } + } + if r.calls.Load() != 0 { + t.Fatalf("rejected coordinates produced %d requests", r.calls.Load()) + } +} + +func TestRoutesFromEnvDefaultsToNoRoutes(t *testing.T) { + routes, err := RoutesFromEnv(func(string) string { return "" }, func(string) ([]byte, error) { return nil, errors.New("unexpected read") }) + if err != nil || len(routes) != 0 { + t.Fatalf("routes = %v, %v", routes, err) + } + env := map[string]string{ + "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL": "https://npm.example/registry", + "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_TOKEN_FILE": "/run/secrets/npm", + "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_NAMESPACES": "@acme, @Internal", + "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_MAVEN_URL": "https://maven.example/repository/public/", + "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_MAVEN_ALLOW_PRIVATE": "true", + "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_MAVEN_BASIC_FILE": "/run/secrets/maven", + } + files := map[string]string{"/run/secrets/npm": "tok\n", "/run/secrets/maven": "deploy:pw"} + routes, err = RoutesFromEnv(func(key string) string { return env[key] }, func(path string) ([]byte, error) { return []byte(files[path]), nil }) + if err != nil || len(routes) != 2 { + t.Fatalf("routes = %+v, %v", routes, err) + } + npm, maven := routes[0], routes[1] + if npm.Ecosystem != "npm" || npm.BaseURL.String() != "https://npm.example/registry/" || npm.BearerToken != "tok" || npm.AllowPrivateHosts || strings.Join(npm.AllowedNamespaces, ",") != "@acme,@internal" || npm.Name != "npm:npm.example" { + t.Fatalf("npm route = %+v", npm) + } + if maven.Ecosystem != "maven" || !maven.AllowPrivateHosts || maven.BasicUser != "deploy" || maven.BasicPassword != "pw" || maven.BearerToken != "" { + t.Fatalf("maven route = %+v", maven) + } + for name, bad := range map[string]map[string]string{ + "http": {"GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL": "http://npm.example/"}, + "credentials": {"GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL": "https://user:pw@npm.example/"}, + "query": {"GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL": "https://npm.example/?x=1"}, + "orphan token": {"GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NUGET_TOKEN_FILE": "/x"}, + "bad bool": {"GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL": "https://npm.example/", "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_ALLOW_PRIVATE": "yes"}, + "both credentials": {"GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL": "https://npm.example/", "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_TOKEN_FILE": "/run/secrets/npm", "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_BASIC_FILE": "/run/secrets/maven"}, + "empty token": {"GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL": "https://npm.example/", "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_TOKEN_FILE": "/run/secrets/empty"}, + } { + if _, err := RoutesFromEnv(func(key string) string { return bad[key] }, func(path string) ([]byte, error) { return []byte(files[path]), nil }); err == nil { + t.Fatalf("%s accepted", name) + } + } +} + +func TestPublicAddressPolicy(t *testing.T) { + for address, public := range map[string]bool{ + "93.184.216.34": true, "2606:2800:220:1:248:1893:25c8:1946": true, + "127.0.0.1": false, "10.1.2.3": false, "172.16.0.1": false, "192.168.1.1": false, "169.254.169.254": false, "100.64.0.1": false, + "0.0.0.0": false, "::1": false, "fe80::1": false, "fd00::1": false, "::ffff:10.0.0.1": false, "224.0.0.1": false, "198.18.0.1": false, + } { + parsed, err := parseAddress(address) + if err != nil { + t.Fatal(err) + } + if publicAddress(parsed) != public { + t.Fatalf("publicAddress(%s) = %v, want %v", address, !public, public) + } + } +} + +func TestEvidenceFingerprintChangesWithMaterialFields(t *testing.T) { + base := Evidence{Source: SourceRegistryNPM, Route: "npm:test", Coordinates: Coordinates{Ecosystem: "npm", Name: "a", Version: "1"}, RawValue: "MIT", RawKind: RawExpression, ParseStatus: spdxexpr.StatusParsed, NormalizedExpression: "MIT", Outcome: OutcomeResolved} + same := base + same.FetchedAt = time.Now() + same.Detail = map[string]any{"integrity": "x"} + if string(base.Fingerprint()) != string(same.Fingerprint()) { + t.Fatal("fetch time and detail must not change the fingerprint") + } + changed := base + changed.RawValue, changed.NormalizedExpression = "ISC", "ISC" + if string(base.Fingerprint()) == string(changed.Fingerprint()) { + t.Fatal("a different license must change the fingerprint") + } + otherRoute := base + otherRoute.Route = "npm:private" + if string(base.Fingerprint()) == string(otherRoute.Fingerprint()) { + t.Fatal("evidence from another route must not collide") + } +} + +func TestCertificateHelper(t *testing.T) { + r := newRegistry(t, func(http.ResponseWriter, *http.Request) {}) + if _, err := x509.ParseCertificate(r.server.Certificate().Raw); err != nil { + t.Fatal(err) + } +} diff --git a/internal/supplychain/license/route.go b/internal/supplychain/license/route.go new file mode 100644 index 00000000..8f0cc219 --- /dev/null +++ b/internal/supplychain/license/route.go @@ -0,0 +1,331 @@ +// Package license resolves exact-version license evidence from explicitly +// configured registry routes. Nothing here produces outbound traffic unless a +// route is configured, and a route never falls back from a private registry +// to a public one. +package license + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "strings" + "time" +) + +// Route is one configured registry destination for one ecosystem. All +// requests for the ecosystem go to this route (and only this route); a +// package that the route does not know is "not found" here, never retried +// against a public registry. +type Route struct { + // Name is the operator label recorded on evidence rows. + Name string + // Ecosystem is the PURL type this route serves: npm, nuget, or maven. + Ecosystem string + // BaseURL is the registry API root, e.g. https://registry.npmjs.org/, + // https://nuget.example/v3-flatcontainer/, https://repo1.maven.org/maven2/. + BaseURL *url.URL + // Bearer/Basic credentials loaded from secret files; never logged. + BearerToken string + BasicUser string + BasicPassword string + AllowPrivateHosts bool + CAPEM []byte + Timeout time.Duration + MaxResponseBytes int64 + // AllowedNamespaces restricts npm scopes / maven groupIds this route may + // answer for (prefix match, case-insensitive). Empty means any. + AllowedNamespaces []string +} + +var ( + ErrRouteMissing = errors.New("no registry route is configured for this ecosystem") + ErrRouteRejected = errors.New("registry request rejected by route policy") + ErrResponseLarge = errors.New("registry response exceeds the configured limit") + ErrUnavailable = errors.New("registry unavailable") + ErrNotFound = errors.New("package version not found at the configured route") + ErrMalformed = errors.New("registry response is malformed") + ErrNamespaceDeny = errors.New("package namespace is not served by this route") + errRedirectDenied = errors.New("registry redirect rejected") +) + +// RoutesFromEnv loads routes from GRAPHNEST_SUPPLY_CHAIN_REGISTRY__URL +// and companion variables. With no variables set, no routes exist and +// enrichment produces no traffic. +// +// GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_URL=https://npm.example/ +// GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_TOKEN_FILE=/run/secrets/npm-token (optional bearer) +// GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_BASIC_FILE=/run/secrets/npm-basic (optional "user:password") +// GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_CA_FILE=/run/secrets/npm-ca.pem (optional) +// GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_ALLOW_PRIVATE=true (optional; permit private/loopback addresses) +// GRAPHNEST_SUPPLY_CHAIN_REGISTRY_NPM_NAMESPACES=@acme,@internal (optional restriction) +func RoutesFromEnv(getenv func(string) string, readFile func(string) ([]byte, error)) ([]Route, error) { + var routes []Route + for _, ecosystem := range []string{"npm", "nuget", "maven"} { + prefix := "GRAPHNEST_SUPPLY_CHAIN_REGISTRY_" + strings.ToUpper(ecosystem) + "_" + raw := getenv(prefix + "URL") + if raw == "" { + for _, suffix := range []string{"TOKEN_FILE", "BASIC_FILE", "CA_FILE", "ALLOW_PRIVATE", "NAMESPACES"} { + if getenv(prefix+suffix) != "" { + return nil, fmt.Errorf("%s%s requires %sURL", prefix, suffix, prefix) + } + } + continue + } + base, err := url.Parse(raw) + if err != nil || base.Scheme != "https" || base.Host == "" || base.User != nil || base.RawQuery != "" || base.Fragment != "" { + return nil, fmt.Errorf("%sURL must be an HTTPS URL without credentials, query, or fragment", prefix) + } + if !strings.HasSuffix(base.Path, "/") { + base.Path += "/" + } + route := Route{Name: ecosystem + ":" + base.Host, Ecosystem: ecosystem, BaseURL: base, Timeout: 15 * time.Second, MaxResponseBytes: 4 << 20} + switch getenv(prefix + "ALLOW_PRIVATE") { + case "", "false": + case "true": + route.AllowPrivateHosts = true + default: + return nil, fmt.Errorf("%sALLOW_PRIVATE must be true or false", prefix) + } + if file := getenv(prefix + "TOKEN_FILE"); file != "" { + token, err := readFile(file) + if err != nil { + return nil, fmt.Errorf("%sTOKEN_FILE: %w", prefix, err) + } + route.BearerToken = strings.TrimSpace(string(token)) + if route.BearerToken == "" { + return nil, fmt.Errorf("%sTOKEN_FILE is empty", prefix) + } + } + if file := getenv(prefix + "BASIC_FILE"); file != "" { + if route.BearerToken != "" { + return nil, fmt.Errorf("%s: configure either TOKEN_FILE or BASIC_FILE", prefix) + } + credential, err := readFile(file) + if err != nil { + return nil, fmt.Errorf("%sBASIC_FILE: %w", prefix, err) + } + user, password, ok := strings.Cut(strings.TrimSpace(string(credential)), ":") + if !ok || user == "" { + return nil, fmt.Errorf("%sBASIC_FILE must contain user:password", prefix) + } + route.BasicUser, route.BasicPassword = user, password + } + if file := getenv(prefix + "CA_FILE"); file != "" { + pemBytes, err := readFile(file) + if err != nil { + return nil, fmt.Errorf("%sCA_FILE: %w", prefix, err) + } + route.CAPEM = pemBytes + } + if namespaces := getenv(prefix + "NAMESPACES"); namespaces != "" { + for _, namespace := range strings.Split(namespaces, ",") { + if namespace = strings.TrimSpace(namespace); namespace != "" { + route.AllowedNamespaces = append(route.AllowedNamespaces, strings.ToLower(namespace)) + } + } + } + routes = append(routes, route) + } + return routes, nil +} + +// ReadSecretFile reads a bounded regular file for RoutesFromEnv. +func ReadSecretFile(path string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() || info.Size() > 64<<10 { + return nil, errors.New("secret file must be a regular file under 64 KiB") + } + return os.ReadFile(path) +} + +// ServesNamespace reports whether the route may answer for a namespace. +func (route Route) ServesNamespace(namespace string) bool { + if len(route.AllowedNamespaces) == 0 { + return true + } + lower := strings.ToLower(namespace) + for _, allowed := range route.AllowedNamespaces { + if lower == allowed || strings.HasPrefix(lower, allowed+".") || strings.HasPrefix(lower, allowed+"/") { + return true + } + } + return false +} + +// Fetcher performs route-bound GETs. It pins the origin, rejects redirects +// that leave the route, blocks private and link-local destinations unless +// the route allows them, bounds the body (after decompression), and isolates +// credentials to the route's origin. +type Fetcher struct { + route Route + client *http.Client +} + +func NewFetcher(route Route) (*Fetcher, error) { + roots, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + if roots == nil { + roots = x509.NewCertPool() + } + if len(route.CAPEM) > 0 && !roots.AppendCertsFromPEM(route.CAPEM) { + return nil, errors.New("invalid registry CA certificate") + } + dialer := &net.Dialer{Timeout: 5 * time.Second} + transport := &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + ForceAttemptHTTP2: true, + MaxIdleConns: 4, + IdleConnTimeout: 30 * time.Second, + ResponseHeaderTimeout: route.Timeout, + DisableCompression: false, + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + addresses, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil { + return nil, err + } + for _, candidate := range addresses { + if !route.AllowPrivateHosts && !publicAddress(candidate) { + return nil, fmt.Errorf("%w: %s resolves to a non-public address", ErrRouteRejected, host) + } + } + var lastErr error + for _, candidate := range addresses { + conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(candidate.Unmap().String(), port)) + if err == nil { + return conn, nil + } + lastErr = err + } + if lastErr == nil { + lastErr = errors.New("no addresses") + } + return nil, lastErr + }, + } + timeout := route.Timeout + if timeout <= 0 { + timeout = 15 * time.Second + } + client := &http.Client{Transport: transport, Timeout: timeout, CheckRedirect: func(request *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return errRedirectDenied + } + if !sameOrigin(request.URL, route.BaseURL) || !strings.HasPrefix(request.URL.Path, route.BaseURL.Path) { + return errRedirectDenied + } + // Go strips Authorization on cross-host redirects; same-origin keeps it, which is what we want. + return nil + }} + return &Fetcher{route: route, client: client}, nil +} + +// publicAddress rejects loopback, private, link-local, multicast, +// unspecified, and cloud-metadata ranges. +func publicAddress(address netip.Addr) bool { + address = address.Unmap() + if address.IsLoopback() || address.IsPrivate() || address.IsLinkLocalUnicast() || address.IsLinkLocalMulticast() || address.IsMulticast() || address.IsUnspecified() || address.IsInterfaceLocalMulticast() { + return false + } + for _, blocked := range []string{"169.254.0.0/16", "100.64.0.0/10", "192.0.0.0/24", "198.18.0.0/15", "240.0.0.0/4", "fc00::/7", "fe80::/10", "::ffff:0:0/96"} { + if prefix, err := netip.ParsePrefix(blocked); err == nil && prefix.Contains(address) { + return false + } + } + return true +} + +func sameOrigin(left, right *url.URL) bool { + return left.Scheme == right.Scheme && strings.EqualFold(left.Host, right.Host) +} + +// Response is a bounded, fully read registry response. +type Response struct { + Status int + Body []byte + ContentType string + FetchedAt time.Time +} + +// Get fetches a path relative to the route base. The path segments are +// escaped individually so a hostile package name cannot traverse. A literal +// "%2F" inside a segment (npm scoped names) is kept as the encoded slash. +func (fetcher *Fetcher) Get(ctx context.Context, accept string, segments ...string) (Response, error) { + target := *fetcher.route.BaseURL + for _, segment := range segments { + if segment == "" || segment == "." || segment == ".." || strings.ContainsAny(segment, "/\\?#") { + return Response{}, fmt.Errorf("%w: invalid path segment", ErrRouteRejected) + } + } + rawPath := strings.TrimSuffix(target.EscapedPath(), "/") + for _, segment := range segments { + rawPath += "/" + strings.ReplaceAll(url.PathEscape(strings.ReplaceAll(segment, "%2F", "\x00")), "%00", "%2F") + } + path, err := url.PathUnescape(rawPath) + if err != nil { + return Response{}, fmt.Errorf("%w: invalid path", ErrRouteRejected) + } + target.Path, target.RawPath = path, rawPath + request, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) + if err != nil { + return Response{}, err + } + request.Header.Set("User-Agent", "GraphNest") + if accept != "" { + request.Header.Set("Accept", accept) + } + if fetcher.route.BearerToken != "" { + request.Header.Set("Authorization", "Bearer "+fetcher.route.BearerToken) + } else if fetcher.route.BasicUser != "" { + request.SetBasicAuth(fetcher.route.BasicUser, fetcher.route.BasicPassword) + } + response, err := fetcher.client.Do(request) + if err != nil { + if errors.Is(err, errRedirectDenied) || errors.Is(err, ErrRouteRejected) { + return Response{}, fmt.Errorf("%w: %v", ErrRouteRejected, unwrapURLError(err)) + } + return Response{}, fmt.Errorf("%w: request failed", ErrUnavailable) + } + defer response.Body.Close() + limit := fetcher.route.MaxResponseBytes + if limit <= 0 { + limit = 4 << 20 + } + // The transport transparently decompresses gzip; the limit applies to the + // decompressed bytes we actually read, so a compression bomb is cut off. + body, err := io.ReadAll(io.LimitReader(response.Body, limit+1)) + if err != nil { + return Response{}, fmt.Errorf("%w: read failed", ErrUnavailable) + } + if int64(len(body)) > limit { + return Response{}, ErrResponseLarge + } + return Response{Status: response.StatusCode, Body: body, ContentType: response.Header.Get("Content-Type"), FetchedAt: time.Now().UTC()}, nil +} + +func unwrapURLError(err error) error { + var urlErr *url.Error + if errors.As(err, &urlErr) { + return urlErr.Err + } + return err +} + +// parseAddress is a small test seam around netip.ParseAddr. +func parseAddress(value string) (netip.Addr, error) { return netip.ParseAddr(value) } diff --git a/internal/supplychain/license/store.go b/internal/supplychain/license/store.go new file mode 100644 index 00000000..5b218aad --- /dev/null +++ b/internal/supplychain/license/store.go @@ -0,0 +1,87 @@ +package license + +import ( + "context" + "errors" + "time" +) + +// ErrNoJob: no runnable enrichment job. +var ErrNoJob = errors.New("no enrichment job available") + +// ErrFenced: the job's lease was lost; nothing was written. +var ErrFenced = errors.New("enrichment job lease lost") + +// Job is one exact-coordinates lookup through one route. +type Job struct { + ID int64 + Coordinates Coordinates + Route string + State string + Attempt int + MaxAttempts int + RunAfter time.Time + LeaseOwner string + LeaseExpiresAt *time.Time + Fence int64 + ErrorCode string +} + +// Assessment is the derived, per-occurrence view of the evidence that applies +// to one snapshot component. It is rebuilt when evidence changes; evidence +// rows themselves are never edited. +type Assessment struct { + ComponentID int64 + SnapshotID int64 + Status AssessmentStatus + NormalizedExpression string + EvidenceIDs []int64 + ConflictDetail string + AssessedAt time.Time + EvidenceFingerprint []byte +} + +type AssessmentStatus string + +const ( + // AssessmentUnknown: no usable expression from any source. + AssessmentUnknown AssessmentStatus = "unknown" + // AssessmentDeclared: only the producer's declaration parsed (no registry evidence). + AssessmentDeclared AssessmentStatus = "declared" + // AssessmentResolved: registry evidence parsed and agrees with any declaration. + AssessmentResolved AssessmentStatus = "resolved" + // AssessmentConflict: two parsed expressions disagree structurally. + AssessmentConflict AssessmentStatus = "conflict" + // AssessmentUnlicensed: a source asserts UNLICENSED/NONE. + AssessmentUnlicensed AssessmentStatus = "unlicensed" + // AssessmentNotApplicable: the component has no exact coordinates to resolve. + AssessmentNotApplicable AssessmentStatus = "not_applicable" + // AssessmentPending: a lookup is queued or running and nothing parsed yet. + AssessmentPending AssessmentStatus = "pending" +) + +// Store is the persistence the enrichment worker and assessor need. +type Store interface { + // InsertLicenseEvidence appends an immutable evidence row and returns its ID. + // When an identical fingerprint already exists as the newest row for the + // same coordinates/route/source, it still inserts (a new observation) but + // reports duplicate=true so callers can avoid re-assessing. + InsertLicenseEvidence(ctx context.Context, evidence Evidence) (id int64, duplicate bool, err error) + // LatestLicenseEvidence returns, per (source, route), the newest row and, + // when that row is a negative outcome, also the newest resolved row, so an + // outage does not erase earlier evidence (it is shown with its age). + LatestLicenseEvidence(ctx context.Context, coordinates Coordinates) ([]Evidence, error) + // EnqueueEnrichment queues a lookup unless one is queued/running or fresh + // evidence (resolved, or unexpired negative) exists for the route. + EnqueueEnrichment(ctx context.Context, coordinates Coordinates, route string) (created bool, err error) + ClaimEnrichment(ctx context.Context, owner string) (Job, error) + RenewEnrichment(ctx context.Context, id int64, owner string, fence int64) error + CompleteEnrichment(ctx context.Context, id int64, owner string, fence int64, outcome Outcome, errorCode string) error + ReapExpiredEnrichment(ctx context.Context, limit int) (int64, error) + // ComponentsForCoordinates returns (component_id, snapshot_id) pairs of + // latest-stream occurrences matching the coordinates, bounded. + ComponentsForCoordinates(ctx context.Context, coordinates Coordinates, limit int) ([][2]int64, error) + UpsertAssessment(ctx context.Context, assessment Assessment) error + // SnapshotCoordinates lists distinct resolvable coordinates in a snapshot. + SnapshotCoordinates(ctx context.Context, snapshotID int64) ([]Coordinates, error) +} diff --git a/internal/supplychain/license/worker.go b/internal/supplychain/license/worker.go new file mode 100644 index 00000000..5d8074de --- /dev/null +++ b/internal/supplychain/license/worker.go @@ -0,0 +1,231 @@ +package license + +import ( + "context" + "errors" + "log/slog" + "math/rand/v2" + "time" +) + +// Registry maps ecosystems to configured resolvers. An ecosystem without a +// route has no resolver and its components stay at their producer-declared +// assessment; nothing is fetched for them. +type Registry struct { + resolvers map[string]Resolver + routes map[string]string +} + +func NewRegistry(routes []Route) (*Registry, error) { + registry := &Registry{resolvers: map[string]Resolver{}, routes: map[string]string{}} + for _, route := range routes { + var resolver Resolver + var err error + switch route.Ecosystem { + case "npm": + resolver, err = NewNPMResolver(route) + case "nuget": + resolver, err = NewNuGetResolver(route) + case "maven": + resolver, err = NewMavenResolver(route) + default: + return nil, errors.New("unsupported registry ecosystem " + route.Ecosystem) + } + if err != nil { + return nil, err + } + if _, duplicate := registry.resolvers[route.Ecosystem]; duplicate { + return nil, errors.New("duplicate registry route for " + route.Ecosystem) + } + registry.resolvers[route.Ecosystem] = resolver + registry.routes[route.Ecosystem] = route.Name + } + return registry, nil +} + +// Route returns the configured route name for an ecosystem. +func (registry *Registry) Route(ecosystem string) (string, bool) { + name, ok := registry.routes[ecosystem] + return name, ok +} + +func (registry *Registry) Resolver(ecosystem string) (Resolver, bool) { + resolver, ok := registry.resolvers[ecosystem] + return resolver, ok +} + +// Ecosystems lists configured ecosystems in a stable order. +func (registry *Registry) Ecosystems() []string { + var names []string + for _, ecosystem := range []string{"npm", "nuget", "maven"} { + if _, ok := registry.resolvers[ecosystem]; ok { + names = append(names, ecosystem) + } + } + return names +} + +// Worker turns published snapshots into enrichment jobs and jobs into +// immutable evidence and rebuilt assessments. +type Worker struct { + Store Store + Registry *Registry + Owner string + Logger *slog.Logger + Observer interface { + ObserveSupplyChainEnrichment(outcome string, duration time.Duration) + } + Poll time.Duration + Now func() time.Time +} + +func (worker *Worker) now() time.Time { + if worker.Now != nil { + return worker.Now().UTC() + } + return time.Now().UTC() +} + +func (worker *Worker) logger() *slog.Logger { + if worker.Logger == nil { + return slog.Default() + } + return worker.Logger +} + +// EnqueueSnapshot queues lookups for every resolvable coordinate of a +// snapshot whose ecosystem has a route, and writes an initial assessment for +// every component from its producer declaration alone so the inventory shows +// declared/unknown immediately rather than waiting for registries. +func (worker *Worker) EnqueueSnapshot(ctx context.Context, snapshotID int64) (int, error) { + if worker.Registry == nil { + return 0, nil + } + coordinates, err := worker.Store.SnapshotCoordinates(ctx, snapshotID) + if err != nil { + return 0, err + } + created := 0 + for _, item := range coordinates { + route, ok := worker.Registry.Route(item.Ecosystem) + if !ok || item.Version == "" || item.Name == "" { + continue + } + ok, err := worker.Store.EnqueueEnrichment(ctx, item, route) + if err != nil { + return created, err + } + if ok { + created++ + } + } + return created, nil +} + +// Run processes enrichment jobs until the context ends. +func (worker *Worker) Run(ctx context.Context) error { + poll := worker.Poll + if poll <= 0 { + poll = 10 * time.Second + } + for { + if _, err := worker.Store.ReapExpiredEnrichment(ctx, 100); err != nil && ctx.Err() == nil { + worker.logger().Error("enrichment reap failed", "error", err) + } + processed, err := worker.RunOnce(ctx) + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil { + worker.logger().Error("enrichment failed", "error", err) + } + if processed && err == nil { + continue + } + timer := time.NewTimer(poll + time.Duration(rand.Int64N(int64(poll)/4+1))) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +// RunOnce claims one job, resolves it, stores evidence, and rebuilds the +// assessments of every latest-stream occurrence with those coordinates. +func (worker *Worker) RunOnce(ctx context.Context) (bool, error) { + job, err := worker.Store.ClaimEnrichment(ctx, worker.Owner) + if errors.Is(err, ErrNoJob) { + return false, nil + } + if err != nil { + return false, err + } + started := worker.now() + resolver, ok := worker.Registry.Resolver(job.Coordinates.Ecosystem) + if !ok { + return true, worker.Store.CompleteEnrichment(ctx, job.ID, job.LeaseOwner, job.Fence, OutcomeRejected, "no_route") + } + evidence, err := resolver.Resolve(ctx, job.Coordinates) + if err != nil { + if ctx.Err() != nil { + return true, ctx.Err() + } + return true, worker.Store.CompleteEnrichment(ctx, job.ID, job.LeaseOwner, job.Fence, OutcomeUnavailable, "resolver_error") + } + evidence.Route = job.Route + id, _, err := worker.Store.InsertLicenseEvidence(ctx, evidence) + if err != nil { + return true, err + } + evidence.ID = id + if err := worker.Store.CompleteEnrichment(ctx, job.ID, job.LeaseOwner, job.Fence, evidence.Outcome, ""); err != nil { + if errors.Is(err, ErrFenced) { + // The evidence row is still valid (it is what the registry said); + // only the job bookkeeping belongs to another lease. + return true, nil + } + return true, err + } + if worker.Observer != nil { + worker.Observer.ObserveSupplyChainEnrichment(string(evidence.Outcome), worker.now().Sub(started)) + } + return true, worker.Reassess(ctx, job.Coordinates) +} + +// Reassess rebuilds assessments for every latest-stream occurrence of the +// coordinates from the newest evidence per source/route. +func (worker *Worker) Reassess(ctx context.Context, coordinates Coordinates) error { + evidence, err := worker.Store.LatestLicenseEvidence(ctx, coordinates) + if err != nil { + return err + } + occurrences, err := worker.Store.ComponentsForCoordinates(ctx, coordinates, 10000) + if err != nil { + return err + } + for _, occurrence := range occurrences { + declared, concluded, err := worker.declarations(ctx, occurrence[0]) + if err != nil { + return err + } + if err := worker.Store.UpsertAssessment(ctx, Assess(occurrence[0], occurrence[1], declared, concluded, evidence, worker.now())); err != nil { + return err + } + } + return nil +} + +// Declarations is optional: stores that can return the producer's raw values +// per component implement it; otherwise assessments use registry evidence only. +type Declarations interface { + ComponentDeclarations(ctx context.Context, componentID int64) (declared, concluded *string, err error) +} + +func (worker *Worker) declarations(ctx context.Context, componentID int64) (*string, *string, error) { + if store, ok := worker.Store.(Declarations); ok { + return store.ComponentDeclarations(ctx, componentID) + } + return nil, nil, nil +} diff --git a/internal/supplychain/license/worker_test.go b/internal/supplychain/license/worker_test.go new file mode 100644 index 00000000..22c47671 --- /dev/null +++ b/internal/supplychain/license/worker_test.go @@ -0,0 +1,246 @@ +package license + +import ( + "context" + "fmt" + "net/http" + "sync" + "testing" + "time" +) + +// memoryStore is an in-memory Store for worker tests. +type memoryStore struct { + mu sync.Mutex + evidence []Evidence + jobs []*Job + assessments map[int64]Assessment + occurrences map[Coordinates][][2]int64 + declared map[int64]*string + coordinates []Coordinates +} + +func newMemoryStore() *memoryStore { + return &memoryStore{assessments: map[int64]Assessment{}, occurrences: map[Coordinates][][2]int64{}, declared: map[int64]*string{}} +} + +func (store *memoryStore) InsertLicenseEvidence(_ context.Context, evidence Evidence) (int64, bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + evidence.ID = int64(len(store.evidence) + 1) + store.evidence = append(store.evidence, evidence) + return evidence.ID, false, nil +} + +func (store *memoryStore) LatestLicenseEvidence(_ context.Context, coordinates Coordinates) ([]Evidence, error) { + store.mu.Lock() + defer store.mu.Unlock() + latest := map[string]Evidence{} + resolved := map[string]Evidence{} + for _, evidence := range store.evidence { + if evidence.Coordinates == coordinates { + key := string(evidence.Source) + "|" + evidence.Route + latest[key] = evidence + if evidence.Outcome == OutcomeResolved { + resolved[key] = evidence + } + } + } + var result []Evidence + for key, evidence := range latest { + result = append(result, evidence) + if evidence.Outcome != OutcomeResolved { + if earlier, ok := resolved[key]; ok { + result = append(result, earlier) + } + } + } + return result, nil +} + +func (store *memoryStore) EnqueueEnrichment(_ context.Context, coordinates Coordinates, route string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + for _, job := range store.jobs { + if job.Coordinates == coordinates && job.Route == route && (job.State == "queued" || job.State == "running") { + return false, nil + } + } + store.jobs = append(store.jobs, &Job{ID: int64(len(store.jobs) + 1), Coordinates: coordinates, Route: route, State: "queued", MaxAttempts: 3}) + return true, nil +} + +func (store *memoryStore) ClaimEnrichment(_ context.Context, owner string) (Job, error) { + store.mu.Lock() + defer store.mu.Unlock() + for _, job := range store.jobs { + if job.State == "queued" { + job.State, job.LeaseOwner, job.Fence, job.Attempt = "running", owner, job.Fence+1, job.Attempt+1 + return *job, nil + } + } + return Job{}, ErrNoJob +} + +func (store *memoryStore) RenewEnrichment(context.Context, int64, string, int64) error { return nil } + +func (store *memoryStore) CompleteEnrichment(_ context.Context, id int64, owner string, fence int64, outcome Outcome, errorCode string) error { + store.mu.Lock() + defer store.mu.Unlock() + for _, job := range store.jobs { + if job.ID == id { + if job.LeaseOwner != owner || job.Fence != fence { + return ErrFenced + } + job.State, job.ErrorCode = "succeeded", errorCode + if outcome == OutcomeUnavailable { + job.State = "queued" + } + return nil + } + } + return ErrFenced +} + +func (store *memoryStore) ReapExpiredEnrichment(context.Context, int) (int64, error) { return 0, nil } + +func (store *memoryStore) ComponentsForCoordinates(_ context.Context, coordinates Coordinates, _ int) ([][2]int64, error) { + store.mu.Lock() + defer store.mu.Unlock() + return store.occurrences[coordinates], nil +} + +func (store *memoryStore) UpsertAssessment(_ context.Context, assessment Assessment) error { + store.mu.Lock() + defer store.mu.Unlock() + store.assessments[assessment.ComponentID] = assessment + return nil +} + +func (store *memoryStore) SnapshotCoordinates(context.Context, int64) ([]Coordinates, error) { + return store.coordinates, nil +} + +func (store *memoryStore) ComponentDeclarations(_ context.Context, componentID int64) (*string, *string, error) { + store.mu.Lock() + defer store.mu.Unlock() + return store.declared[componentID], nil, nil +} + +func TestWorkerEnqueuesOnlyRoutedEcosystemsAndAssesses(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + fmt.Fprint(writer, `{"version":"1.3.0","license":"MIT"}`) + }) + registry, err := NewRegistry([]Route{r.route(t, "npm", "/")}) + if err != nil { + t.Fatal(err) + } + store := newMemoryStore() + npm := Coordinates{Ecosystem: "npm", Namespace: "@scope", Name: "left-pad", Version: "1.3.0"} + store.coordinates = []Coordinates{npm, {Ecosystem: "maven", Namespace: "org.example", Name: "core", Version: "2.1.0"}, {Ecosystem: "golang", Namespace: "golang.org/x", Name: "text", Version: "0.14.0"}, {Ecosystem: "npm", Name: "versionless"}} + store.occurrences[npm] = [][2]int64{{7, 3}, {8, 4}} + mit := "MIT" + store.declared[7] = &mit + isc := "ISC" + store.declared[8] = &isc + worker := &Worker{Store: store, Registry: registry, Owner: "w"} + created, err := worker.EnqueueSnapshot(t.Context(), 3) + if err != nil || created != 1 { + t.Fatalf("created = %d err = %v (only the npm coordinate has a route and a version)", created, err) + } + if created, err := worker.EnqueueSnapshot(t.Context(), 3); err != nil || created != 0 { + t.Fatalf("second enqueue created %d", created) + } + processed, err := worker.RunOnce(t.Context()) + if err != nil || !processed { + t.Fatalf("processed=%v err=%v", processed, err) + } + if r.calls.Load() != 1 || len(store.evidence) != 1 || store.evidence[0].NormalizedExpression != "MIT" || store.evidence[0].Route != "npm:test" { + t.Fatalf("calls=%d evidence=%+v", r.calls.Load(), store.evidence) + } + if store.assessments[7].Status != AssessmentResolved || store.assessments[8].Status != AssessmentConflict || len(store.assessments) != 2 { + t.Fatalf("assessments = %+v", store.assessments) + } + if processed, err := worker.RunOnce(t.Context()); err != nil || processed { + t.Fatalf("queue drained: processed=%v err=%v", processed, err) + } + if store.jobs[0].State != "succeeded" { + t.Fatalf("job = %+v", store.jobs[0]) + } +} + +func TestWorkerWithoutRoutesProducesNoTraffic(t *testing.T) { + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { t.Error("registry was called") }) + registry, err := NewRegistry(nil) + if err != nil { + t.Fatal(err) + } + store := newMemoryStore() + store.coordinates = []Coordinates{{Ecosystem: "npm", Name: "left-pad", Version: "1.3.0"}} + worker := &Worker{Store: store, Registry: registry, Owner: "w"} + if created, err := worker.EnqueueSnapshot(t.Context(), 1); err != nil || created != 0 || len(store.jobs) != 0 { + t.Fatalf("created=%d jobs=%d err=%v", created, len(store.jobs), err) + } + if processed, err := worker.RunOnce(t.Context()); err != nil || processed { + t.Fatalf("processed=%v err=%v", processed, err) + } + if r.calls.Load() != 0 || len(registry.Ecosystems()) != 0 { + t.Fatalf("calls=%d ecosystems=%v", r.calls.Load(), registry.Ecosystems()) + } +} + +func TestWorkerOutageRetainsEarlierEvidence(t *testing.T) { + var down bool + r := newRegistry(t, func(writer http.ResponseWriter, request *http.Request) { + if down { + writer.WriteHeader(http.StatusBadGateway) + return + } + fmt.Fprint(writer, `{"version":"1.0.0","license":"Apache-2.0"}`) + }) + registry, _ := NewRegistry([]Route{r.route(t, "npm", "/")}) + store := newMemoryStore() + coordinates := Coordinates{Ecosystem: "npm", Name: "pkg", Version: "1.0.0"} + store.occurrences[coordinates] = [][2]int64{{1, 1}} + worker := &Worker{Store: store, Registry: registry, Owner: "w", Now: func() time.Time { return time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC) }} + if _, err := store.EnqueueEnrichment(t.Context(), coordinates, "npm:test"); err != nil { + t.Fatal(err) + } + if _, err := worker.RunOnce(t.Context()); err != nil { + t.Fatal(err) + } + if store.assessments[1].Status != AssessmentResolved || store.assessments[1].NormalizedExpression != "Apache-2.0" { + t.Fatalf("initial = %+v", store.assessments[1]) + } + down = true + store.jobs = nil + if _, err := store.EnqueueEnrichment(t.Context(), coordinates, "npm:test"); err != nil { + t.Fatal(err) + } + if _, err := worker.RunOnce(t.Context()); err != nil { + t.Fatal(err) + } + if len(store.evidence) != 2 || store.evidence[1].Outcome != OutcomeUnavailable || store.evidence[1].ExpiresAt == nil { + t.Fatalf("outage evidence = %+v", store.evidence) + } + if store.assessments[1].Status != AssessmentResolved || store.assessments[1].NormalizedExpression != "Apache-2.0" { + t.Fatalf("outage replaced earlier evidence: %+v", store.assessments[1]) + } + if store.jobs[0].State != "queued" { + t.Fatalf("unavailable outcome must retry: %+v", store.jobs[0]) + } +} + +func TestNewRegistryRejectsDuplicatesAndUnknownEcosystems(t *testing.T) { + r := newRegistry(t, func(http.ResponseWriter, *http.Request) {}) + if _, err := NewRegistry([]Route{r.route(t, "npm", "/"), r.route(t, "npm", "/other/")}); err == nil { + t.Fatal("duplicate route accepted") + } + if _, err := NewRegistry([]Route{r.route(t, "cargo", "/")}); err == nil { + t.Fatal("unsupported ecosystem accepted") + } + registry, err := NewRegistry([]Route{r.route(t, "maven", "/"), r.route(t, "nuget", "/")}) + if err != nil || len(registry.Ecosystems()) != 2 || registry.Ecosystems()[0] != "nuget" { + t.Fatalf("registry = %v %v", registry, err) + } +} diff --git a/internal/supplychain/service.go b/internal/supplychain/service.go index a07f68c5..3f8b8ec8 100644 --- a/internal/supplychain/service.go +++ b/internal/supplychain/service.go @@ -13,6 +13,7 @@ import ( "github.com/balcsida/graphnest/internal/authn" "github.com/balcsida/graphnest/internal/repository" + "github.com/balcsida/graphnest/internal/supplychain/license" "github.com/balcsida/graphnest/pkg/api" "github.com/jackc/pgx/v5" ) @@ -40,6 +41,16 @@ type Store interface { EnqueueSupplyChainJob(context.Context, int64, string, string, string, int, time.Time) (Job, bool, error) } +// LicenseStore is the optional evidence read side. When nil, component pages +// carry no assessments and the detail view reports enrichment as not +// configured. +type LicenseStore interface { + SupplyChainAssessments(context.Context, int64, []int64) (map[int64]license.Assessment, error) + SupplyChainAssessmentCounts(context.Context, int64) (map[string]int, error) + SupplyChainComponentByElement(context.Context, int64, string) (Component, error) + LicenseEvidenceHistory(context.Context, license.Coordinates, int) ([]license.Evidence, error) +} + // Authorizer resolves the live principal's repository scope. *authz.Postgres // satisfies it (postgres imports this package for its models, so the authz // package cannot be imported here). @@ -57,6 +68,10 @@ type Service struct { Interval time.Duration MaxResults int Now func() time.Time + // License is optional evidence storage; EnrichmentEcosystems lists the + // ecosystems with configured registry routes. + License LicenseStore + EnrichmentEcosystems []string } func (service *Service) now() time.Time { @@ -137,7 +152,11 @@ func (service *Service) Status(ctx context.Context, principal authn.Principal, g return api.SupplyChainRepositoryStatus{}, err } status := api.SupplyChainRepositoryStatus{RepositoryID: repo.GitHubID, Repository: repo.Name, Stream: streamKey, Producer: string(ProducerGitHub), Subject: string(SubjectSource), - Collection: "never", Enrichment: "not_configured", Notes: []string{}, Documents: []api.SupplyChainDocumentRef{}} + Collection: "never", Enrichment: "not_configured", EnrichmentEcosystems: []string{}, LicenseSummary: map[string]int{}, Notes: []string{}, Documents: []api.SupplyChainDocumentRef{}} + if len(service.EnrichmentEcosystems) > 0 { + status.Enrichment = "configured" + status.EnrichmentEcosystems = append(status.EnrichmentEcosystems, service.EnrichmentEcosystems...) + } stream, err := service.Store.SupplyChainStream(ctx, repo.ID, streamKey) if err != nil && !errors.Is(err, pgx.ErrNoRows) { return api.SupplyChainRepositoryStatus{}, err @@ -158,6 +177,13 @@ func (service *Service) Status(ctx context.Context, principal authn.Principal, g if service.now().Sub(snapshot.CollectedAt) > 2*service.interval() { status.Collection = "stale" } + if service.License != nil { + counts, err := service.License.SupplyChainAssessmentCounts(ctx, snapshot.ID) + if err != nil { + return api.SupplyChainRepositoryStatus{}, err + } + status.LicenseSummary = counts + } } collections, err := service.Store.SupplyChainCollections(ctx, repo.ID, streamKey, 0, 1) if err != nil { @@ -285,6 +311,10 @@ func (service *Service) Components(ctx context.Context, principal authn.Principa if err != nil { return api.SupplyChainComponentList{}, err } + assessments, err := service.assessments(ctx, snapshot.ID, components) + if err != nil { + return api.SupplyChainComponentList{}, err + } result := api.SupplyChainComponentList{SnapshotID: snapshot.ID, Components: make([]api.SupplyChainComponent, 0, len(components))} for index, component := range components { if index == limit { @@ -292,11 +322,139 @@ func (service *Service) Components(ctx context.Context, principal authn.Principa result.NextCursor = encodeComponentCursor(componentCursor{SnapshotID: snapshot.ID, Ordinal: components[index-1].Ordinal, Search: search}) break } - result.Components = append(result.Components, componentSummary(component, scopes)) + summary := componentSummary(component, scopes) + if assessment, ok := assessments[component.ID]; ok { + summary.License = assessmentSummary(assessment) + } + result.Components = append(result.Components, summary) } return result, nil } +func (service *Service) assessments(ctx context.Context, snapshotID int64, components []Component) (map[int64]license.Assessment, error) { + if service.License == nil || len(components) == 0 { + return nil, nil + } + ids := make([]int64, 0, len(components)) + for _, component := range components { + ids = append(ids, component.ID) + } + return service.License.SupplyChainAssessments(ctx, snapshotID, ids) +} + +// ComponentDetail returns one occurrence with its producer declarations, +// registry evidence history, relationships, and assessment. Evidence is keyed +// by coordinates shared across repositories, so it is only reachable through +// an occurrence in an authorized snapshot. +func (service *Service) ComponentDetail(ctx context.Context, principal authn.Principal, githubID int64, streamKey string, snapshotID int64, elementID string) (api.SupplyChainComponentDetail, error) { + streamKey, err := normalizeStream(streamKey) + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + if elementID == "" || len(elementID) > 512 { + return api.SupplyChainComponentDetail{}, ErrInvalidRequest + } + repo, err := service.authorizedRepository(ctx, principal, githubID) + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + if snapshotID == 0 { + stream, err := service.Store.SupplyChainStream(ctx, repo.ID, streamKey) + if errors.Is(err, pgx.ErrNoRows) || err == nil && stream.LatestSnapshotID == nil { + return api.SupplyChainComponentDetail{}, ErrNoInventory + } + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + snapshotID = *stream.LatestSnapshotID + } + snapshot, err := service.Store.SupplyChainSnapshot(ctx, snapshotID, []int64{repo.ID}) + if errors.Is(err, pgx.ErrNoRows) { + return api.SupplyChainComponentDetail{}, ErrNotFound + } + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + if service.License == nil { + return api.SupplyChainComponentDetail{}, ErrNotFound + } + component, err := service.License.SupplyChainComponentByElement(ctx, snapshot.ID, elementID) + if errors.Is(err, pgx.ErrNoRows) { + return api.SupplyChainComponentDetail{}, ErrNotFound + } + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + scopes, err := service.scopes(ctx, snapshot) + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + detail := api.SupplyChainComponentDetail{Component: componentSummary(component, scopes), Snapshot: snapshotSummary(snapshot), Declarations: []api.SupplyChainLicenseEvidence{}, + Evidence: []api.SupplyChainLicenseEvidence{}, Relationships: []api.SupplyChainRelationship{}, Notes: []string{}} + assessments, err := service.License.SupplyChainAssessments(ctx, snapshot.ID, []int64{component.ID}) + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + if assessment, ok := assessments[component.ID]; ok { + detail.Component.License = assessmentSummary(assessment) + } + for _, declaration := range []struct { + source string + value *string + }{{"producer_declared", component.LicenseDeclaredRaw}, {"producer_concluded", component.LicenseConcludedRaw}} { + if declaration.value == nil { + continue + } + evidence := license.Evidence{Source: license.Source(declaration.source), Coordinates: license.Coordinates{Ecosystem: component.Ecosystem, Namespace: component.PURLNamespace, Name: component.PURLName, Version: component.PURLVersion}, + FetchedAt: snapshot.CollectedAt, Outcome: license.OutcomeResolved} + license.Classify(&evidence, *declaration.value) + detail.Declarations = append(detail.Declarations, evidenceSummary(evidence)) + } + if component.Ecosystem != "" && component.PURLName != "" && component.PURLVersion != "" { + history, err := service.License.LicenseEvidenceHistory(ctx, license.Coordinates{Ecosystem: component.Ecosystem, Namespace: component.PURLNamespace, Name: component.PURLName, Version: component.PURLVersion}, service.maxResults()+1) + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + if len(history) > service.maxResults() { + history, detail.Truncated = history[:service.maxResults()], true + } + for _, evidence := range history { + detail.Evidence = append(detail.Evidence, evidenceSummary(evidence)) + } + } else { + detail.Notes = append(detail.Notes, "This occurrence has no exact package coordinates (purl name and version), so no registry lookup applies.") + } + edges, err := service.Store.SupplyChainRelationships(ctx, snapshot.ID, component.ElementID, service.maxResults()) + if err != nil { + return api.SupplyChainComponentDetail{}, err + } + for _, edge := range edges { + detail.Relationships = append(detail.Relationships, api.SupplyChainRelationship{From: edge.FromElement, Type: edge.Type, To: edge.ToElement, Resolved: edge.Resolved}) + } + detail.Notes = append(detail.Notes, "Publisher declarations and registry metadata are evidence, not approval; a human conclusion or policy decision is recorded separately.") + if len(service.EnrichmentEcosystems) == 0 { + detail.Notes = append(detail.Notes, "No registry routes are configured; only producer declarations are shown.") + } + return detail, nil +} + +func assessmentSummary(assessment license.Assessment) *api.SupplyChainLicenseAssessment { + return &api.SupplyChainLicenseAssessment{Status: string(assessment.Status), Expression: assessment.NormalizedExpression, ConflictDetail: assessment.ConflictDetail, + EvidenceCount: len(assessment.EvidenceIDs), AssessedAt: assessment.AssessedAt, EvidenceFingerprint: hex.EncodeToString(assessment.EvidenceFingerprint)} +} + +func evidenceSummary(evidence license.Evidence) api.SupplyChainLicenseEvidence { + summary := api.SupplyChainLicenseEvidence{ID: evidence.ID, Source: string(evidence.Source), Route: evidence.Route, Ecosystem: evidence.Coordinates.Ecosystem, Namespace: evidence.Coordinates.Namespace, + Name: evidence.Coordinates.Name, Version: evidence.Coordinates.Version, ArtifactSHA256: evidence.ArtifactSHA256, RawValue: evidence.RawValue, RawKind: string(evidence.RawKind), + ParseStatus: string(evidence.ParseStatus), Expression: evidence.NormalizedExpression, UnknownTerms: evidence.UnknownTerms, LicenseURL: evidence.LicenseURL, LicenseFileName: evidence.LicenseFileName, + Detail: evidence.Detail, ResolverVersion: evidence.ResolverVersion, LicenseListVersion: evidence.LicenseListVersion, FetchedAt: evidence.FetchedAt, ExpiresAt: evidence.ExpiresAt, + Outcome: string(evidence.Outcome), HTTPStatus: evidence.HTTPStatus, Message: evidence.Message} + if len(evidence.ContentSHA256) > 0 { + summary.ContentSHA256 = hex.EncodeToString(evidence.ContentSHA256) + } + return summary +} + // scopes derives root/direct/transitive/unknown per element from resolved // DEPENDS_ON edges. It is bounded by the snapshot's own edge count. func (service *Service) scopes(ctx context.Context, snapshot Snapshot) (map[string]string, error) { diff --git a/internal/supplychain/spdxexpr/exceptions.tsv b/internal/supplychain/spdxexpr/exceptions.tsv new file mode 100644 index 00000000..7a90dc23 --- /dev/null +++ b/internal/supplychain/spdxexpr/exceptions.tsv @@ -0,0 +1,81 @@ +# SPDX License List 3.27.0 exceptions +# idflags: D=deprecated +389-exception +Asterisk-exception +Asterisk-linking-protocols-exception +Autoconf-exception-2.0 +Autoconf-exception-3.0 +Autoconf-exception-generic +Autoconf-exception-generic-3.0 +Autoconf-exception-macro +Bison-exception-1.24 +Bison-exception-2.2 +Bootloader-exception +CGAL-linking-exception +Classpath-exception-2.0 +CLISP-exception-2.0 +cryptsetup-OpenSSL-exception +Digia-Qt-LGPL-exception-1.1 +DigiRule-FOSS-exception +eCos-exception-2.0 +erlang-otp-linking-exception +Fawkes-Runtime-exception +FLTK-exception +fmt-exception +Font-exception-2.0 +freertos-exception-2.0 +GCC-exception-2.0 +GCC-exception-2.0-note +GCC-exception-3.1 +Gmsh-exception +GNAT-exception +GNOME-examples-exception +GNU-compiler-exception +gnu-javamail-exception +GPL-3.0-389-ds-base-exception +GPL-3.0-interface-exception +GPL-3.0-linking-exception +GPL-3.0-linking-source-exception +GPL-CC-1.0 +GStreamer-exception-2005 +GStreamer-exception-2008 +harbour-exception +i2p-gpl-java-exception +Independent-modules-exception +KiCad-libraries-exception +LGPL-3.0-linking-exception +libpri-OpenH323-exception +Libtool-exception +Linux-syscall-note +LLGPL +LLVM-exception +LZMA-exception +mif-exception +mxml-exception +Nokia-Qt-exception-1.1 D +OCaml-LGPL-linking-exception +OCCT-exception-1.0 +OpenJDK-assembly-exception-1.0 +openvpn-openssl-exception +PCRE2-exception +polyparse-exception +PS-or-PDF-font-exception-20170817 +QPL-1.0-INRIA-2004-exception +Qt-GPL-exception-1.0 +Qt-LGPL-exception-1.1 +Qwt-exception-1.0 +romic-exception +RRDtool-FLOSS-exception-2.0 +SANE-exception +SHL-2.0 +SHL-2.1 +stunnel-exception +SWI-exception +Swift-exception +Texinfo-exception +u-boot-exception-2.0 +UBDL-exception +Universal-FOSS-exception-1.0 +vsftpd-openssl-exception +WxWindows-exception-3.1 +x11vnc-openssl-exception diff --git a/internal/supplychain/spdxexpr/licenses.tsv b/internal/supplychain/spdxexpr/licenses.tsv new file mode 100644 index 00000000..d39299b5 --- /dev/null +++ b/internal/supplychain/spdxexpr/licenses.tsv @@ -0,0 +1,701 @@ +# SPDX License List 3.27.0 (https://github.com/spdx/license-list-data/tree/v3.27.0) +# idflags: D=deprecated O=OSI approved F=FSF libre +0BSD O +3D-Slicer-1.0 +AAL O +Abstyles +AdaCore-doc +Adobe-2006 +Adobe-Display-PostScript +Adobe-Glyph +Adobe-Utopia +ADSL +AFL-1.1 OF +AFL-1.2 OF +AFL-2.0 OF +AFL-2.1 OF +AFL-3.0 OF +Afmparse +AGPL-1.0 DF +AGPL-1.0-only +AGPL-1.0-or-later +AGPL-3.0 DOF +AGPL-3.0-only OF +AGPL-3.0-or-later OF +Aladdin +AMD-newlib +AMDPLPA +AML +AML-glslang +AMPAS +ANTLR-PD +ANTLR-PD-fallback +any-OSI +any-OSI-perl-modules +Apache-1.0 F +Apache-1.1 OF +Apache-2.0 OF +APAFML +APL-1.0 O +App-s2p +APSL-1.0 O +APSL-1.1 O +APSL-1.2 O +APSL-2.0 OF +Arphic-1999 +Artistic-1.0 O +Artistic-1.0-cl8 O +Artistic-1.0-Perl O +Artistic-2.0 OF +Artistic-dist +Aspell-RU +ASWF-Digital-Assets-1.0 +ASWF-Digital-Assets-1.1 +Baekmuk +Bahyph +Barr +bcrypt-Solar-Designer +Beerware +Bitstream-Charter +Bitstream-Vera +BitTorrent-1.0 +BitTorrent-1.1 F +blessing +BlueOak-1.0.0 O +Boehm-GC +Boehm-GC-without-fee +Borceux +Brian-Gladman-2-Clause +Brian-Gladman-3-Clause +BSD-1-Clause O +BSD-2-Clause OF +BSD-2-Clause-Darwin +BSD-2-Clause-first-lines +BSD-2-Clause-FreeBSD DF +BSD-2-Clause-NetBSD DF +BSD-2-Clause-Patent O +BSD-2-Clause-pkgconf-disclaimer +BSD-2-Clause-Views +BSD-3-Clause OF +BSD-3-Clause-acpica +BSD-3-Clause-Attribution +BSD-3-Clause-Clear F +BSD-3-Clause-flex +BSD-3-Clause-HP +BSD-3-Clause-LBNL O +BSD-3-Clause-Modification +BSD-3-Clause-No-Military-License +BSD-3-Clause-No-Nuclear-License +BSD-3-Clause-No-Nuclear-License-2014 +BSD-3-Clause-No-Nuclear-Warranty +BSD-3-Clause-Open-MPI +BSD-3-Clause-Sun +BSD-4-Clause F +BSD-4-Clause-Shortened +BSD-4-Clause-UC +BSD-4.3RENO +BSD-4.3TAHOE +BSD-Advertising-Acknowledgement +BSD-Attribution-HPND-disclaimer +BSD-Inferno-Nettverk +BSD-Protection +BSD-Source-beginning-file +BSD-Source-Code +BSD-Systemics +BSD-Systemics-W3Works +BSL-1.0 OF +BUSL-1.1 +bzip2-1.0.5 D +bzip2-1.0.6 +C-UDA-1.0 +CAL-1.0 O +CAL-1.0-Combined-Work-Exception O +Caldera +Caldera-no-preamble +Catharon +CATOSL-1.1 O +CC-BY-1.0 +CC-BY-2.0 +CC-BY-2.5 +CC-BY-2.5-AU +CC-BY-3.0 +CC-BY-3.0-AT +CC-BY-3.0-AU +CC-BY-3.0-DE +CC-BY-3.0-IGO +CC-BY-3.0-NL +CC-BY-3.0-US +CC-BY-4.0 F +CC-BY-NC-1.0 +CC-BY-NC-2.0 +CC-BY-NC-2.5 +CC-BY-NC-3.0 +CC-BY-NC-3.0-DE +CC-BY-NC-4.0 +CC-BY-NC-ND-1.0 +CC-BY-NC-ND-2.0 +CC-BY-NC-ND-2.5 +CC-BY-NC-ND-3.0 +CC-BY-NC-ND-3.0-DE +CC-BY-NC-ND-3.0-IGO +CC-BY-NC-ND-4.0 +CC-BY-NC-SA-1.0 +CC-BY-NC-SA-2.0 +CC-BY-NC-SA-2.0-DE +CC-BY-NC-SA-2.0-FR +CC-BY-NC-SA-2.0-UK +CC-BY-NC-SA-2.5 +CC-BY-NC-SA-3.0 +CC-BY-NC-SA-3.0-DE +CC-BY-NC-SA-3.0-IGO +CC-BY-NC-SA-4.0 +CC-BY-ND-1.0 +CC-BY-ND-2.0 +CC-BY-ND-2.5 +CC-BY-ND-3.0 +CC-BY-ND-3.0-DE +CC-BY-ND-4.0 +CC-BY-SA-1.0 +CC-BY-SA-2.0 +CC-BY-SA-2.0-UK +CC-BY-SA-2.1-JP +CC-BY-SA-2.5 +CC-BY-SA-3.0 +CC-BY-SA-3.0-AT +CC-BY-SA-3.0-DE +CC-BY-SA-3.0-IGO +CC-BY-SA-4.0 F +CC-PDDC +CC-PDM-1.0 +CC-SA-1.0 +CC0-1.0 F +CDDL-1.0 OF +CDDL-1.1 +CDL-1.0 +CDLA-Permissive-1.0 +CDLA-Permissive-2.0 +CDLA-Sharing-1.0 +CECILL-1.0 +CECILL-1.1 +CECILL-2.0 F +CECILL-2.1 O +CECILL-B F +CECILL-C F +CERN-OHL-1.1 +CERN-OHL-1.2 +CERN-OHL-P-2.0 O +CERN-OHL-S-2.0 O +CERN-OHL-W-2.0 O +CFITSIO +check-cvs +checkmk +ClArtistic F +Clips +CMU-Mach +CMU-Mach-nodoc +CNRI-Jython +CNRI-Python O +CNRI-Python-GPL-Compatible +COIL-1.0 +Community-Spec-1.0 +Condor-1.1 F +copyleft-next-0.3.0 +copyleft-next-0.3.1 +Cornell-Lossless-JPEG +CPAL-1.0 OF +CPL-1.0 OF +CPOL-1.02 +Cronyx +Crossword +CryptoSwift +CrystalStacker +CUA-OPL-1.0 O +Cube +curl +cve-tou +D-FSL-1.0 +DEC-3-Clause +diffmark +DL-DE-BY-2.0 +DL-DE-ZERO-2.0 +DOC +DocBook-DTD +DocBook-Schema +DocBook-Stylesheet +DocBook-XML +Dotseqn +DRL-1.0 +DRL-1.1 +DSDP +dtoa +dvipdfm +ECL-1.0 O +ECL-2.0 OF +eCos-2.0 DF +EFL-1.0 O +EFL-2.0 OF +eGenix +Elastic-2.0 +Entessa O +EPICS +EPL-1.0 OF +EPL-2.0 OF +ErlPL-1.1 +etalab-2.0 +EUDatagrid OF +EUPL-1.0 +EUPL-1.1 OF +EUPL-1.2 OF +Eurosym +Fair O +FBM +FDK-AAC +Ferguson-Twofish +Frameworx-1.0 O +FreeBSD-DOC +FreeImage +FSFAP F +FSFAP-no-warranty-disclaimer +FSFUL +FSFULLR +FSFULLRSD +FSFULLRWD +FSL-1.1-ALv2 +FSL-1.1-MIT +FTL F +Furuseth +fwlw +Game-Programming-Gems +GCR-docs +GD +generic-xts +GFDL-1.1 DF +GFDL-1.1-invariants-only +GFDL-1.1-invariants-or-later +GFDL-1.1-no-invariants-only +GFDL-1.1-no-invariants-or-later +GFDL-1.1-only F +GFDL-1.1-or-later F +GFDL-1.2 DF +GFDL-1.2-invariants-only +GFDL-1.2-invariants-or-later +GFDL-1.2-no-invariants-only +GFDL-1.2-no-invariants-or-later +GFDL-1.2-only F +GFDL-1.2-or-later F +GFDL-1.3 DF +GFDL-1.3-invariants-only +GFDL-1.3-invariants-or-later +GFDL-1.3-no-invariants-only +GFDL-1.3-no-invariants-or-later +GFDL-1.3-only F +GFDL-1.3-or-later F +Giftware +GL2PS +Glide +Glulxe +GLWTPL +gnuplot F +GPL-1.0 D +GPL-1.0+ D +GPL-1.0-only +GPL-1.0-or-later +GPL-2.0 DOF +GPL-2.0+ DOF +GPL-2.0-only OF +GPL-2.0-or-later OF +GPL-2.0-with-autoconf-exception D +GPL-2.0-with-bison-exception D +GPL-2.0-with-classpath-exception D +GPL-2.0-with-font-exception D +GPL-2.0-with-GCC-exception D +GPL-3.0 DOF +GPL-3.0+ DOF +GPL-3.0-only OF +GPL-3.0-or-later OF +GPL-3.0-with-autoconf-exception D +GPL-3.0-with-GCC-exception DO +Graphics-Gems +gSOAP-1.3b +gtkbook +Gutmann +HaskellReport +HDF5 +hdparm +HIDAPI +Hippocratic-2.1 +HP-1986 +HP-1989 +HPND OF +HPND-DEC +HPND-doc +HPND-doc-sell +HPND-export-US +HPND-export-US-acknowledgement +HPND-export-US-modify +HPND-export2-US +HPND-Fenneberg-Livingston +HPND-INRIA-IMAG +HPND-Intel +HPND-Kevlin-Henney +HPND-Markus-Kuhn +HPND-merchantability-variant +HPND-MIT-disclaimer +HPND-Netrek +HPND-Pbmplus +HPND-sell-MIT-disclaimer-xserver +HPND-sell-regexpr +HPND-sell-variant +HPND-sell-variant-MIT-disclaimer +HPND-sell-variant-MIT-disclaimer-rev +HPND-UC +HPND-UC-export-US +HTMLTIDY +IBM-pibs +ICU O +IEC-Code-Components-EULA +IJG F +IJG-short +ImageMagick +iMatix F +Imlib2 F +Info-ZIP +Inner-Net-2.0 +InnoSetup +Intel OF +Intel-ACPI +Interbase-1.0 +IPA OF +IPL-1.0 OF +ISC OF +ISC-Veillard +Jam O +JasPer-2.0 +jove +JPL-image +JPNIC +JSON +Kastrup +Kazlib +Knuth-CTAN +LAL-1.2 +LAL-1.3 +Latex2e +Latex2e-translated-notice +Leptonica +LGPL-2.0 DO +LGPL-2.0+ DO +LGPL-2.0-only O +LGPL-2.0-or-later O +LGPL-2.1 DOF +LGPL-2.1+ DOF +LGPL-2.1-only OF +LGPL-2.1-or-later OF +LGPL-3.0 DOF +LGPL-3.0+ DOF +LGPL-3.0-only OF +LGPL-3.0-or-later OF +LGPLLR +Libpng +libpng-1.6.35 +libpng-2.0 +libselinux-1.0 +libtiff +libutil-David-Nugent +LiLiQ-P-1.1 O +LiLiQ-R-1.1 O +LiLiQ-Rplus-1.1 O +Linux-man-pages-1-para +Linux-man-pages-copyleft +Linux-man-pages-copyleft-2-para +Linux-man-pages-copyleft-var +Linux-OpenIB +LOOP +LPD-document +LPL-1.0 O +LPL-1.02 OF +LPPL-1.0 +LPPL-1.1 +LPPL-1.2 F +LPPL-1.3a F +LPPL-1.3c O +lsof +Lucida-Bitmap-Fonts +LZMA-SDK-9.11-to-9.20 +LZMA-SDK-9.22 +Mackerras-3-Clause +Mackerras-3-Clause-acknowledgment +magaz +mailprio +MakeIndex +man2html +Martin-Birgmeier +McPhee-slideshow +metamail +Minpack +MIPS +MirOS O +MIT OF +MIT-0 O +MIT-advertising +MIT-Click +MIT-CMU +MIT-enna +MIT-feh +MIT-Festival +MIT-Khronos-old +MIT-Modern-Variant O +MIT-open-group +MIT-testregex +MIT-Wu +MITNFA +MMIXware +Motosoto O +MPEG-SSG +mpi-permissive +mpich2 +MPL-1.0 O +MPL-1.1 OF +MPL-2.0 OF +MPL-2.0-no-copyleft-exception O +mplus +MS-LPL +MS-PL OF +MS-RL OF +MTLL +MulanPSL-1.0 +MulanPSL-2.0 O +Multics O +Mup +NAIST-2003 +NASA-1.3 O +Naumen O +NBPL-1.0 +NCBI-PD +NCGL-UK-2.0 +NCL +NCSA OF +Net-SNMP D +NetCDF +Newsletr +NGPL O +ngrep +NICTA-1.0 +NIST-PD +NIST-PD-fallback +NIST-Software +NLOD-1.0 +NLOD-2.0 +NLPL +Nokia OF +NOSL F +Noweb +NPL-1.0 F +NPL-1.1 F +NPOSL-3.0 O +NRL +NTIA-PD +NTP O +NTP-0 +Nunit DF +O-UDA-1.0 +OAR +OCCT-PL +OCLC-2.0 O +ODbL-1.0 F +ODC-By-1.0 +OFFIS +OFL-1.0 F +OFL-1.0-no-RFN +OFL-1.0-RFN +OFL-1.1 OF +OFL-1.1-no-RFN O +OFL-1.1-RFN O +OGC-1.0 +OGDL-Taiwan-1.0 +OGL-Canada-2.0 +OGL-UK-1.0 +OGL-UK-2.0 +OGL-UK-3.0 +OGTSL O +OLDAP-1.1 +OLDAP-1.2 +OLDAP-1.3 +OLDAP-1.4 +OLDAP-2.0 +OLDAP-2.0.1 +OLDAP-2.1 +OLDAP-2.2 +OLDAP-2.2.1 +OLDAP-2.2.2 +OLDAP-2.3 F +OLDAP-2.4 +OLDAP-2.5 +OLDAP-2.6 +OLDAP-2.7 F +OLDAP-2.8 O +OLFL-1.3 O +OML +OpenPBS-2.3 +OpenSSL F +OpenSSL-standalone +OpenVision +OPL-1.0 +OPL-UK-3.0 +OPUBL-1.0 +OSET-PL-2.1 O +OSL-1.0 OF +OSL-1.1 F +OSL-2.0 OF +OSL-2.1 OF +OSL-3.0 OF +PADL +Parity-6.0.0 +Parity-7.0.0 +PDDL-1.0 +PHP-3.0 O +PHP-3.01 OF +Pixar +pkgconf +Plexus +pnmstitch +PolyForm-Noncommercial-1.0.0 +PolyForm-Small-Business-1.0.0 +PostgreSQL O +PPL +PSF-2.0 +psfrag +psutils +Python-2.0 OF +Python-2.0.1 +python-ldap +Qhull +QPL-1.0 OF +QPL-1.0-INRIA-2004 +radvd +Rdisc +RHeCos-1.1 +RPL-1.1 O +RPL-1.5 O +RPSL-1.0 OF +RSA-MD +RSCPL O +Ruby F +Ruby-pty +SAX-PD +SAX-PD-2.0 +Saxpath +SCEA +SchemeReport +Sendmail +Sendmail-8.23 +Sendmail-Open-Source-1.1 +SGI-B-1.0 +SGI-B-1.1 +SGI-B-2.0 F +SGI-OpenGL +SGP4 +SHL-0.5 +SHL-0.51 +SimPL-2.0 O +SISSL OF +SISSL-1.2 +SL +Sleepycat OF +SMAIL-GPL +SMLNJ F +SMPPL +SNIA +snprintf +SOFA +softSurfer +Soundex +Spencer-86 +Spencer-94 +Spencer-99 +SPL-1.0 OF +ssh-keyscan +SSH-OpenSSH +SSH-short +SSLeay-standalone +SSPL-1.0 +StandardML-NJ DF +SugarCRM-1.1.3 +SUL-1.0 +Sun-PPP +Sun-PPP-2000 +SunPro +SWL +swrule +Symlinks +TAPR-OHL-1.0 +TCL +TCP-wrappers +TermReadKey +TGPPL-1.0 +ThirdEye +threeparttable +TMate +TORQUE-1.1 +TOSL +TPDL +TPL-1.0 +TrustedQSL +TTWL +TTYP0 +TU-Berlin-1.0 +TU-Berlin-2.0 +Ubuntu-font-1.0 +UCAR +UCL-1.0 O +ulem +UMich-Merit +Unicode-3.0 O +Unicode-DFS-2015 +Unicode-DFS-2016 O +Unicode-TOU +UnixCrypt +Unlicense OF +Unlicense-libtelnet +Unlicense-libwhirlpool +UPL-1.0 OF +URT-RLE +Vim F +VOSTROM +VSL-1.0 O +W3C OF +W3C-19980720 +W3C-20150513 O +w3m +Watcom-1.0 O +Widget-Workshop +Wsuipa +WTFPL F +wwl +wxWindows DO +X11 F +X11-distribute-modifications-variant +X11-swapped +Xdebug-1.03 +Xerox +Xfig +XFree86-1.1 F +xinetd F +xkeyboard-config-Zinoviev +xlock +Xnet O +xpp +XSkat +xzoom +YPL-1.0 +YPL-1.1 F +Zed +Zeeff +Zend-2.0 F +Zimbra-1.3 F +Zimbra-1.4 +Zlib OF +zlib-acknowledgement +ZPL-1.1 +ZPL-2.0 OF +ZPL-2.1 OF diff --git a/internal/supplychain/spdxexpr/spdxexpr.go b/internal/supplychain/spdxexpr/spdxexpr.go new file mode 100644 index 00000000..81446f33 --- /dev/null +++ b/internal/supplychain/spdxexpr/spdxexpr.go @@ -0,0 +1,518 @@ +// Package spdxexpr parses SPDX 2.3 license expressions with a bounded +// grammar and a pinned license list. It preserves AND, OR, parentheses, and +// WITH exceptions as a tree; it never flattens an expression into a bag of +// names, never maps NOASSERTION/NONE/UNLICENSED or unknown identifiers to a +// license, and reports what it could not recognize. +// +// Grammar (SPDX 2.3 Annex D): +// +// expression := disjunction +// disjunction := conjunction ( "OR" conjunction )* +// conjunction := primary ( "AND" primary )* +// primary := simple [ "WITH" exception ] | "(" expression ")" +// simple := license-id [ "+" ] | "LicenseRef-" idstring | "DocumentRef-" idstring ":" "LicenseRef-" idstring +package spdxexpr + +import ( + "bufio" + "embed" + "errors" + "fmt" + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +// ListVersion is the pinned SPDX License List release embedded in this +// package. Recorded on every parsed evidence row so a list upgrade is a new +// resolver version, not a silent change. +const ListVersion = "3.27.0" + +//go:embed licenses.tsv exceptions.tsv +var lists embed.FS + +// Status classifies a raw license value before or after parsing. +type Status string + +const ( + // StatusParsed: a syntactically valid expression whose every identifier + // is on the pinned list (LicenseRef/DocumentRef identifiers are allowed + // and reported separately). + StatusParsed Status = "parsed" + // StatusUnknownTerms: valid syntax with at least one identifier that is + // not on the pinned list and not a LicenseRef. + StatusUnknownTerms Status = "unknown_terms" + // StatusNoAssertion: NOASSERTION (or empty): no claim was made. + StatusNoAssertion Status = "no_assertion" + // StatusNone: NONE: the producer asserts there is no license information. + StatusNone Status = "none" + // StatusUnlicensed: npm's UNLICENSED sentinel: use is not granted. + StatusUnlicensed Status = "unlicensed" + // StatusInvalid: the value is not an SPDX expression (free text, "SEE + // LICENSE IN ...", unbalanced parentheses, bad operators). + StatusInvalid Status = "invalid" +) + +// Kind is the node type of a parsed expression. +type Kind string + +const ( + KindLicense Kind = "license" + KindWith Kind = "with" + KindAnd Kind = "and" + KindOr Kind = "or" + KindReference Kind = "license_ref" +) + +// Node is one node of the expression tree. Children is ordered as written. +type Node struct { + Kind Kind `json:"kind"` + // ID is the license identifier for KindLicense (canonical list casing), + // or the full LicenseRef-/DocumentRef- identifier for KindReference. + ID string `json:"id,omitempty"` + // OrLater is the "+" suffix on a license identifier. + OrLater bool `json:"or_later,omitempty"` + // Exception is the exception identifier for KindWith; Children[0] is the + // licensed term the exception applies to. + Exception string `json:"exception,omitempty"` + Children []*Node `json:"children,omitempty"` + // Known is false for KindLicense/KindWith identifiers that are not on the + // pinned list. It is always true for KindReference nodes. + Known bool `json:"known"` + // Deprecated marks identifiers the list marks deprecated (kept, flagged). + Deprecated bool `json:"deprecated,omitempty"` +} + +// Result is the outcome of Parse. +type Result struct { + Raw string + Status Status + Expression *Node + // Normalized is the canonical rendering (list casing, single spaces, + // minimal parentheses preserved as written). Empty unless Status is + // parsed or unknown_terms. + Normalized string + // UnknownTerms lists identifiers not on the pinned list, in order. + UnknownTerms []string + // References lists LicenseRef-/DocumentRef- identifiers, in order. + References []string + // Deprecated lists deprecated identifiers used, in order. + Deprecated []string + // Problem explains StatusInvalid without echoing more than a bounded + // prefix of the input. + Problem string +} + +// Licenses returns whether an identifier is on the pinned list, its canonical +// casing, and whether it is deprecated. +type Licenses interface { + License(id string) (canonical string, deprecated, ok bool) + Exception(id string) (canonical string, deprecated, ok bool) +} + +type list struct { + licenses map[string]entry + exceptions map[string]entry +} + +type entry struct { + id string + deprecated bool +} + +var pinned = mustLoad() + +func mustLoad() *list { + result := &list{licenses: map[string]entry{}, exceptions: map[string]entry{}} + load := func(name string, target map[string]entry) { + file, err := lists.Open(name) + if err != nil { + panic(err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if line == "" || strings.HasPrefix(line, "#") { + continue + } + id, flags, _ := strings.Cut(line, "\t") + target[strings.ToLower(id)] = entry{id: id, deprecated: strings.Contains(flags, "D")} + } + if err := scanner.Err(); err != nil { + panic(err) + } + } + load("licenses.tsv", result.licenses) + load("exceptions.tsv", result.exceptions) + return result +} + +func (l *list) License(id string) (string, bool, bool) { + item, ok := l.licenses[strings.ToLower(id)] + return item.id, item.deprecated, ok +} + +func (l *list) Exception(id string) (string, bool, bool) { + item, ok := l.exceptions[strings.ToLower(id)] + return item.id, item.deprecated, ok +} + +// Pinned returns the embedded license list. +func Pinned() Licenses { return pinned } + +// Count reports the embedded list sizes for tests and diagnostics. +func Count() (licenses, exceptions int) { return len(pinned.licenses), len(pinned.exceptions) } + +const ( + maxInputBytes = 4096 + maxTokens = 512 + maxDepth = 32 +) + +var errTooLong = errors.New("expression exceeds the size limit") + +// Parse classifies and parses a raw license value using the pinned list. +func Parse(raw string) Result { + return ParseWith(raw, pinned) +} + +// ParseWith parses against a caller-supplied list (tests, future versions). +func ParseWith(raw string, licenses Licenses) Result { + result := Result{Raw: raw} + trimmed := strings.TrimSpace(raw) + switch { + case trimmed == "" || strings.EqualFold(trimmed, "NOASSERTION"): + result.Status = StatusNoAssertion + return result + case strings.EqualFold(trimmed, "NONE"): + result.Status = StatusNone + return result + case strings.EqualFold(trimmed, "UNLICENSED"): + result.Status = StatusUnlicensed + return result + } + if len(trimmed) > maxInputBytes { + result.Status, result.Problem = StatusInvalid, errTooLong.Error() + return result + } + tokens, err := tokenize(trimmed) + if err != nil { + result.Status, result.Problem = StatusInvalid, err.Error() + return result + } + parser := &parser{tokens: tokens, licenses: licenses} + node, err := parser.expression(0) + if err == nil && parser.pos != len(parser.tokens) { + err = fmt.Errorf("unexpected %q after the expression", parser.tokens[parser.pos].text) + } + if err != nil { + result.Status, result.Problem = StatusInvalid, err.Error() + return result + } + result.Expression = node + result.Normalized = node.String() + result.UnknownTerms, result.References, result.Deprecated = parser.unknown, parser.references, parser.deprecated + result.Status = StatusParsed + if len(result.UnknownTerms) > 0 { + result.Status = StatusUnknownTerms + } + return result +} + +type token struct { + kind string // "id", "(", ")", "AND", "OR", "WITH", "+" + text string +} + +func tokenize(input string) ([]token, error) { + var tokens []token + index := 0 + for index < len(input) { + character := input[index] + switch { + case character == ' ' || character == '\t' || character == '\n' || character == '\r': + index++ + case character == '(' || character == ')': + tokens = append(tokens, token{kind: string(character), text: string(character)}) + index++ + case character == '+': + tokens = append(tokens, token{kind: "+", text: "+"}) + index++ + case isIDByte(character): + start := index + for index < len(input) && isIDByte(input[index]) { + index++ + } + text := input[start:index] + switch strings.ToUpper(text) { + case "AND", "OR", "WITH": + tokens = append(tokens, token{kind: strings.ToUpper(text), text: text}) + default: + tokens = append(tokens, token{kind: "id", text: text}) + } + default: + return nil, fmt.Errorf("unexpected character %q", sanitizeRune(input[index:])) + } + if len(tokens) > maxTokens { + return nil, errTooLong + } + } + if len(tokens) == 0 { + return nil, errors.New("empty expression") + } + return tokens, nil +} + +// isIDByte accepts the SPDX idstring alphabet (letters, digits, "-", ".") +// plus ":" so DocumentRef-x:LicenseRef-y stays one token. +func isIDByte(character byte) bool { + return character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || character == '-' || character == '.' || character == ':' +} + +func sanitizeRune(rest string) string { + r, size := utf8.DecodeRuneInString(rest) + if size == 0 { + return "" + } + if unicode.IsPrint(r) { + return string(r) + } + return fmt.Sprintf("U+%04X", r) +} + +type parser struct { + tokens []token + pos int + licenses Licenses + unknown []string + references []string + deprecated []string +} + +func (p *parser) peek() (token, bool) { + if p.pos >= len(p.tokens) { + return token{}, false + } + return p.tokens[p.pos], true +} + +func (p *parser) expression(depth int) (*Node, error) { + if depth > maxDepth { + return nil, errors.New("expression nesting exceeds the depth limit") + } + left, err := p.conjunction(depth) + if err != nil { + return nil, err + } + var children []*Node + for { + next, ok := p.peek() + if !ok || next.kind != "OR" { + break + } + p.pos++ + right, err := p.conjunction(depth) + if err != nil { + return nil, err + } + if children == nil { + children = []*Node{left} + } + children = append(children, right) + } + if children == nil { + return left, nil + } + return &Node{Kind: KindOr, Children: flatten(KindOr, children), Known: true}, nil +} + +// flatten merges children that are the same associative operator so +// "(A OR B) OR C" and "A OR B OR C" produce one tree. Order is preserved; +// mixed operators keep their grouping. +func flatten(kind Kind, children []*Node) []*Node { + result := make([]*Node, 0, len(children)) + for _, child := range children { + if child.Kind == kind { + result = append(result, child.Children...) + continue + } + result = append(result, child) + } + return result +} + +func (p *parser) conjunction(depth int) (*Node, error) { + left, err := p.primary(depth) + if err != nil { + return nil, err + } + var children []*Node + for { + next, ok := p.peek() + if !ok || next.kind != "AND" { + break + } + p.pos++ + right, err := p.primary(depth) + if err != nil { + return nil, err + } + if children == nil { + children = []*Node{left} + } + children = append(children, right) + } + if children == nil { + return left, nil + } + return &Node{Kind: KindAnd, Children: flatten(KindAnd, children), Known: true}, nil +} + +func (p *parser) primary(depth int) (*Node, error) { + next, ok := p.peek() + if !ok { + return nil, errors.New("expression ends where a license was expected") + } + switch next.kind { + case "(": + p.pos++ + inner, err := p.expression(depth + 1) + if err != nil { + return nil, err + } + closing, ok := p.peek() + if !ok || closing.kind != ")" { + return nil, errors.New("missing closing parenthesis") + } + p.pos++ + // A parenthesized group is kept as-is; parentheses are re-rendered + // around compound children when they change grouping. + return inner, nil + case "id": + p.pos++ + node := p.simple(next.text) + if plus, ok := p.peek(); ok && plus.kind == "+" { + p.pos++ + if node.Kind != KindLicense { + return nil, errors.New("\"+\" is only valid after a license identifier") + } + node.OrLater = true + } + if with, ok := p.peek(); ok && with.kind == "WITH" { + p.pos++ + exception, ok := p.peek() + if !ok || exception.kind != "id" { + return nil, errors.New("WITH must be followed by an exception identifier") + } + p.pos++ + canonical, deprecated, known := p.licenses.Exception(exception.text) + if !known { + canonical = exception.text + p.unknown = append(p.unknown, exception.text) + } + if deprecated { + p.deprecated = append(p.deprecated, canonical) + } + return &Node{Kind: KindWith, Exception: canonical, Children: []*Node{node}, Known: known, Deprecated: deprecated}, nil + } + return node, nil + case ")": + return nil, errors.New("unexpected closing parenthesis") + default: + return nil, fmt.Errorf("unexpected %q where a license was expected", next.text) + } +} + +func (p *parser) simple(text string) *Node { + lower := strings.ToLower(text) + if strings.HasPrefix(lower, "licenseref-") || strings.HasPrefix(lower, "documentref-") { + p.references = append(p.references, text) + return &Node{Kind: KindReference, ID: text, Known: true} + } + canonical, deprecated, known := p.licenses.License(text) + if !known { + p.unknown = append(p.unknown, text) + return &Node{Kind: KindLicense, ID: text, Known: false} + } + if deprecated { + p.deprecated = append(p.deprecated, canonical) + } + return &Node{Kind: KindLicense, ID: canonical, Known: true, Deprecated: deprecated} +} + +// String renders the canonical expression. Compound children of a different +// operator are parenthesized so structure is never lost. +func (node *Node) String() string { + if node == nil { + return "" + } + switch node.Kind { + case KindLicense: + if node.OrLater { + return node.ID + "+" + } + return node.ID + case KindReference: + return node.ID + case KindWith: + return node.Children[0].String() + " WITH " + node.Exception + case KindAnd, KindOr: + operator := " AND " + if node.Kind == KindOr { + operator = " OR " + } + parts := make([]string, 0, len(node.Children)) + for _, child := range node.Children { + text := child.String() + if (child.Kind == KindAnd || child.Kind == KindOr) && child.Kind != node.Kind { + text = "(" + text + ")" + } + parts = append(parts, text) + } + return strings.Join(parts, operator) + } + return "" +} + +// Terms returns the distinct leaf identifiers (licenses with their "+" and +// WITH exception rendered, and references) in sorted order. It is a display +// aid; policy evaluation must walk the tree. +func (node *Node) Terms() []string { + seen := map[string]bool{} + var walk func(*Node) + walk = func(current *Node) { + switch current.Kind { + case KindAnd, KindOr: + for _, child := range current.Children { + walk(child) + } + default: + seen[current.String()] = true + } + } + walk(node) + terms := make([]string, 0, len(seen)) + for term := range seen { + terms = append(terms, term) + } + sort.Strings(terms) + return terms +} + +// Equal reports structural equality (same tree, same identifiers). +func Equal(left, right *Node) bool { + if left == nil || right == nil { + return left == right + } + if left.Kind != right.Kind || left.ID != right.ID || left.OrLater != right.OrLater || left.Exception != right.Exception || len(left.Children) != len(right.Children) { + return false + } + for index := range left.Children { + if !Equal(left.Children[index], right.Children[index]) { + return false + } + } + return true +} diff --git a/internal/supplychain/spdxexpr/spdxexpr_test.go b/internal/supplychain/spdxexpr/spdxexpr_test.go new file mode 100644 index 00000000..7606693c --- /dev/null +++ b/internal/supplychain/spdxexpr/spdxexpr_test.go @@ -0,0 +1,173 @@ +package spdxexpr + +import ( + "strings" + "testing" +) + +func TestPinnedListLoads(t *testing.T) { + licenses, exceptions := Count() + if licenses != 699 || exceptions != 79 { + t.Fatalf("list sizes = %d licenses, %d exceptions; the embedded list must match release %s", licenses, exceptions, ListVersion) + } + if id, deprecated, ok := Pinned().License("mit"); !ok || id != "MIT" || deprecated { + t.Fatalf("MIT lookup = %q %v %v", id, deprecated, ok) + } + if id, deprecated, ok := Pinned().License("GPL-2.0"); !ok || id != "GPL-2.0" || !deprecated { + t.Fatalf("GPL-2.0 must be a deprecated identifier: %q %v %v", id, deprecated, ok) + } + if id, _, ok := Pinned().Exception("classpath-exception-2.0"); !ok || id != "Classpath-exception-2.0" { + t.Fatalf("exception lookup = %q %v", id, ok) + } +} + +func TestParsePreservesStructure(t *testing.T) { + cases := map[string]struct { + normalized string + kind Kind + terms []string + status Status + }{ + "MIT": {normalized: "MIT", kind: KindLicense, terms: []string{"MIT"}, status: StatusParsed}, + "mit": {normalized: "MIT", kind: KindLicense, terms: []string{"MIT"}, status: StatusParsed}, + "MIT OR Apache-2.0": {normalized: "MIT OR Apache-2.0", kind: KindOr, terms: []string{"Apache-2.0", "MIT"}, status: StatusParsed}, + "(MIT OR Apache-2.0)": {normalized: "MIT OR Apache-2.0", kind: KindOr, terms: []string{"Apache-2.0", "MIT"}, status: StatusParsed}, + "MIT AND Apache-2.0": {normalized: "MIT AND Apache-2.0", kind: KindAnd, terms: []string{"Apache-2.0", "MIT"}, status: StatusParsed}, + "GPL-2.0-only WITH Classpath-exception-2.0": {normalized: "GPL-2.0-only WITH Classpath-exception-2.0", kind: KindWith, terms: []string{"GPL-2.0-only WITH Classpath-exception-2.0"}, status: StatusParsed}, + "LGPL-2.1+": {normalized: "LGPL-2.1+", kind: KindLicense, terms: []string{"LGPL-2.1+"}, status: StatusParsed}, + "MIT AND (LGPL-2.1-or-later OR BSD-3-Clause)": {normalized: "MIT AND (LGPL-2.1-or-later OR BSD-3-Clause)", kind: KindAnd, terms: []string{"BSD-3-Clause", "LGPL-2.1-or-later", "MIT"}, status: StatusParsed}, + "(MIT AND LGPL-2.1-or-later) OR BSD-3-Clause": {normalized: "(MIT AND LGPL-2.1-or-later) OR BSD-3-Clause", kind: KindOr, terms: []string{"BSD-3-Clause", "LGPL-2.1-or-later", "MIT"}, status: StatusParsed}, + "MIT and Apache-2.0 or ISC": {normalized: "(MIT AND Apache-2.0) OR ISC", kind: KindOr, terms: []string{"Apache-2.0", "ISC", "MIT"}, status: StatusParsed}, + "LicenseRef-Proprietary-Acme": {normalized: "LicenseRef-Proprietary-Acme", kind: KindReference, terms: []string{"LicenseRef-Proprietary-Acme"}, status: StatusParsed}, + "DocumentRef-sbom:LicenseRef-Custom AND MIT": {normalized: "DocumentRef-sbom:LicenseRef-Custom AND MIT", kind: KindAnd, terms: []string{"DocumentRef-sbom:LicenseRef-Custom", "MIT"}, status: StatusParsed}, + "MIT OR NotARealLicense-1.0": {normalized: "MIT OR NotARealLicense-1.0", kind: KindOr, terms: []string{"MIT", "NotARealLicense-1.0"}, status: StatusUnknownTerms}, + "GPL-2.0-only WITH Made-Up-Exception": {normalized: "GPL-2.0-only WITH Made-Up-Exception", kind: KindWith, terms: []string{"GPL-2.0-only WITH Made-Up-Exception"}, status: StatusUnknownTerms}, + " Apache-2.0 ": {normalized: "Apache-2.0", kind: KindLicense, terms: []string{"Apache-2.0"}, status: StatusParsed}, + "MIT\tAND\nApache-2.0": {normalized: "MIT AND Apache-2.0", kind: KindAnd, terms: []string{"Apache-2.0", "MIT"}, status: StatusParsed}, + "MIT OR (Apache-2.0 AND (ISC OR BSD-2-Clause))": {normalized: "MIT OR (Apache-2.0 AND (ISC OR BSD-2-Clause))", kind: KindOr, terms: []string{"Apache-2.0", "BSD-2-Clause", "ISC", "MIT"}, status: StatusParsed}, + "GPL-2.0": {normalized: "GPL-2.0", kind: KindLicense, terms: []string{"GPL-2.0"}, status: StatusParsed}, + } + for raw, want := range cases { + t.Run(raw, func(t *testing.T) { + got := Parse(raw) + if got.Status != want.status || got.Normalized != want.normalized || got.Expression == nil || got.Expression.Kind != want.kind { + t.Fatalf("Parse(%q) = %s %q kind=%v problem=%q", raw, got.Status, got.Normalized, kindOf(got.Expression), got.Problem) + } + if strings.Join(got.Expression.Terms(), ",") != strings.Join(want.terms, ",") { + t.Fatalf("terms = %v, want %v", got.Expression.Terms(), want.terms) + } + // Round trip: the normalized form parses to an equal tree. + again := Parse(got.Normalized) + if !Equal(again.Expression, got.Expression) { + t.Fatalf("normalized %q did not round-trip: %q", got.Normalized, again.Normalized) + } + }) + } +} + +func kindOf(node *Node) Kind { + if node == nil { + return "" + } + return node.Kind +} + +func TestParseKeepsOrderAndDoesNotFlatten(t *testing.T) { + left := Parse("MIT AND (GPL-2.0-only OR Apache-2.0)") + right := Parse("(MIT AND GPL-2.0-only) OR Apache-2.0") + if Equal(left.Expression, right.Expression) { + t.Fatal("different groupings must not compare equal") + } + if strings.Join(left.Expression.Terms(), ",") != strings.Join(right.Expression.Terms(), ",") { + t.Fatal("the display term set is the same; the trees must carry the difference") + } + // Same associative operator flattens regardless of redundant parentheses; + // mixed operators keep their grouping. + if !Equal(Parse("(MIT OR Apache-2.0) OR ISC").Expression, Parse("MIT OR Apache-2.0 OR ISC").Expression) { + t.Fatal("associative OR grouping must flatten") + } + if Equal(Parse("(MIT AND Apache-2.0) OR ISC").Expression, Parse("MIT AND (Apache-2.0 OR ISC)").Expression) { + t.Fatal("mixed operators must keep grouping") + } + ordered := Parse("Apache-2.0 OR MIT") + if ordered.Expression.Children[0].ID != "Apache-2.0" || ordered.Expression.Children[1].ID != "MIT" { + t.Fatalf("OR branches were reordered: %+v", ordered.Expression) + } +} + +func TestParseSentinelsAreNotLicenses(t *testing.T) { + for raw, want := range map[string]Status{ + "": StatusNoAssertion, + " ": StatusNoAssertion, + "NOASSERTION": StatusNoAssertion, + "noassertion": StatusNoAssertion, + "NONE": StatusNone, + "UNLICENSED": StatusUnlicensed, + } { + got := Parse(raw) + if got.Status != want || got.Expression != nil || got.Normalized != "" { + t.Fatalf("Parse(%q) = %+v, want status %s and no expression", raw, got, want) + } + } +} + +func TestParseRejectsInvalidText(t *testing.T) { + for _, raw := range []string{ + "SEE LICENSE IN LICENSE.txt", "MIT OR", "OR MIT", "MIT AND AND Apache-2.0", "(MIT", "MIT)", "MIT WITH", "MIT WITH (Classpath-exception-2.0)", + "MIT/Apache-2.0", "MIT, Apache-2.0", "Copyright (c) 2020 Someone", "+MIT", "(MIT)+", "MIT OR Apache-2.0 WITH", "MIT AND ()", "()", + strings.Repeat("(", 40) + "MIT" + strings.Repeat(")", 40), "MIT " + strings.Repeat("OR MIT ", 600), strings.Repeat("a", 5000), + "MIT\x00", "MIT \u00e9", + } { + got := Parse(raw) + if got.Status != StatusInvalid || got.Expression != nil { + t.Fatalf("Parse(%q) = %s %q, want invalid", raw, got.Status, got.Normalized) + } + if got.Problem == "" || len(got.Problem) > 200 || strings.Contains(got.Problem, "LICENSE.txt") { + t.Fatalf("problem for %q = %q", raw, got.Problem) + } + } +} + +func TestParseReportsDeprecatedAndReferences(t *testing.T) { + got := Parse("GPL-2.0+ AND LicenseRef-Internal OR AGPL-1.0") + if got.Status != StatusParsed || strings.Join(got.Deprecated, ",") != "GPL-2.0,AGPL-1.0" || strings.Join(got.References, ",") != "LicenseRef-Internal" { + t.Fatalf("result = %+v", got) + } + unknown := Parse("Foo-1.0 OR Bar-2.0 WITH Baz-exception") + if unknown.Status != StatusUnknownTerms || strings.Join(unknown.UnknownTerms, ",") != "Foo-1.0,Bar-2.0,Baz-exception" { + t.Fatalf("unknown = %+v", unknown) + } + // Syntactically valid but with a free-text "exception": kept as unknown + // terms so it is visible for review, never treated as a known license. + loose := Parse("GPL-2.0 with exceptions") + if loose.Status != StatusUnknownTerms || loose.Normalized != "GPL-2.0 WITH exceptions" || strings.Join(loose.UnknownTerms, ",") != "exceptions" { + t.Fatalf("loose = %+v", loose) + } +} + +func FuzzParse(f *testing.F) { + for _, seed := range []string{"MIT", "MIT OR Apache-2.0", "(GPL-2.0-only WITH Classpath-exception-2.0) AND MIT", "LicenseRef-x", "NOASSERTION", "((", "MIT+ OR", strings.Repeat("(MIT OR ", 20)} { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, raw string) { + got := Parse(raw) + switch got.Status { + case StatusParsed, StatusUnknownTerms: + if got.Expression == nil || got.Normalized == "" { + t.Fatalf("parsed without a tree: %+v", got) + } + again := Parse(got.Normalized) + if again.Status == StatusInvalid || !Equal(again.Expression, got.Expression) { + t.Fatalf("normalized %q did not round-trip (%s)", got.Normalized, again.Problem) + } + case StatusInvalid: + if got.Expression != nil || got.Problem == "" { + t.Fatalf("invalid with a tree or no problem: %+v", got) + } + default: + if got.Expression != nil { + t.Fatalf("sentinel with a tree: %+v", got) + } + } + }) +} diff --git a/internal/supplychain/spdxexpr/testdata/fuzz/FuzzParse/d7200aea03672be7 b/internal/supplychain/spdxexpr/testdata/fuzz/FuzzParse/d7200aea03672be7 new file mode 100644 index 00000000..74044ff1 --- /dev/null +++ b/internal/supplychain/spdxexpr/testdata/fuzz/FuzzParse/d7200aea03672be7 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("0 OR(0 OR(0))") diff --git a/internal/webui/supply-chain.html b/internal/webui/supply-chain.html index 16849f37..9bbfe4a2 100644 --- a/internal/webui/supply-chain.html +++ b/internal/webui/supply-chain.html @@ -12,8 +12,8 @@ #sc-shell{min-height:100vh;display:grid;grid-template-columns:216px minmax(0,1fr)}aside{position:sticky;top:0;height:100vh;background:var(--surface);border-right:1px solid var(--border);display:flex;flex-direction:column}.brand{display:flex;gap:9px;align-items:center;padding:14px 16px}.mark{width:26px;height:26px;border-radius:7px;background:linear-gradient(135deg,var(--accent),#48b8c5);display:grid;place-items:center;color:#0a0d15;font:600 11px var(--mono);flex:none}:root.light .mark{color:#fff}.brand>div{display:flex;align-items:center;gap:8px}.brand strong{font:600 14px var(--disp)}.brand small{height:18px;padding:0 6px;display:inline-flex;align-items:center;border-radius:4px;background:var(--surface2);color:var(--muted);font:600 10px var(--mono);letter-spacing:.06em}.nav{display:grid;gap:1px;padding:4px 8px}.nav a{min-height:32px;display:flex;align-items:center;border-radius:6px;padding:0 10px;color:var(--muted);font-size:13px}.nav a:hover{color:var(--ink)}.nav a[aria-current=page]{background:var(--soft);color:var(--accent);font-weight:500}.nav-group{padding:12px 10px 4px;color:var(--faint);font-size:10.5px;font-weight:600;text-transform:uppercase;letter-spacing:.09em}.nav-group:first-child{padding-top:6px}.aside-foot{margin-top:auto;padding:12px 16px;border-top:1px solid var(--border);display:grid;gap:8px;font-size:12px}.aside-foot a{min-height:0;display:flex;align-items:center} .page{min-width:0}.top{height:52px;position:sticky;top:0;z-index:2;background:var(--surface);border-bottom:1px solid var(--border);display:flex;align-items:center;gap:12px;padding:0 24px}.top h1{font:600 15px var(--disp)}.top p{margin:0;color:var(--faint);font-size:12.5px}.top .right{margin-left:auto;display:flex;gap:6px;align-items:center}.top .right button{min-height:32px;border-radius:7px;color:var(--muted);font-size:13px;padding:0 11px}#sc-theme{width:32px;padding:0}main{padding:20px 24px 60px}.screen{display:grid;gap:14px}.cards{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px}.card,.panel{margin:0;background:var(--surface);border:1px solid var(--border);border-radius:10px;padding:14px 16px}.panel{display:grid;gap:10px;align-content:start}.card small,.muted{color:var(--muted)}.card small{font-size:12px}.card strong{display:block;font:600 22px/1.2 var(--disp);margin-top:4px}.card strong.warn{color:var(--warn)}.card strong.err{color:var(--err)} .toolbar{display:flex;gap:8px;align-items:center;flex-wrap:wrap}.toolbar .push{margin-left:auto}#sc-search,#sc-repository,#sc-stream{min-height:34px;border-radius:7px}#sc-search{width:260px} -.table-wrap{overflow:auto;background:var(--surface);border:1px solid var(--border);border-radius:10px}table{border-collapse:collapse;width:100%;min-width:760px}th,td{text-align:left;padding:0 12px;border-bottom:1px solid var(--border);vertical-align:middle}th{height:36px;background:var(--surface2);color:var(--faint);font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.08em;white-space:nowrap}td{height:44px;font-size:12.5px}tr:last-child td{border-bottom:0}.mono{font-family:var(--mono);overflow-wrap:anywhere}.pill{display:inline-flex;align-items:center;gap:6px;font-size:12px;color:var(--muted)}.pill:before{content:"";width:6px;height:6px;border-radius:50%;background:currentColor}.pill.ok{color:var(--ok)}.pill.warn{color:var(--warn)}.pill.err{color:var(--err)}.empty{padding:32px;color:var(--faint);text-align:center;font-size:13px} -.split{display:grid;grid-template-columns:minmax(0,1.5fr) minmax(280px,1fr);gap:14px;align-items:start}.stack{display:grid;gap:12px}.panel p{margin:0;font-size:12.5px}.panel p.mono{font-size:12px}#sc-details{display:grid;grid-template-columns:auto 1fr;gap:6px 16px;margin:0;font-size:12.5px}#sc-details dt{color:var(--muted)}#sc-details dd{margin:0}.notice{background:color-mix(in oklab,var(--warn) 10%,transparent);border:1px solid color-mix(in oklab,var(--warn) 35%,transparent);border-radius:8px;color:var(--warn);padding:8px 12px;margin:0;font-size:12.5px;display:grid;gap:4px}#sc-notes p{margin:0;color:var(--muted);font-size:12.5px}.status{position:fixed;right:18px;bottom:18px;max-width:min(420px,calc(100vw - 36px));background:var(--surface);border:1px solid var(--border);border-radius:8px;padding:10px 12px;z-index:5;font-size:12.5px;box-shadow:0 12px 32px #0005}.status:empty{display:none} +.table-wrap{overflow:auto;background:var(--surface);border:1px solid var(--border);border-radius:10px}table{border-collapse:collapse;width:100%;min-width:760px}th,td{text-align:left;padding:0 12px;border-bottom:1px solid var(--border);vertical-align:middle}th{height:36px;background:var(--surface2);color:var(--faint);font-size:11px;font-weight:600;text-transform:uppercase;letter-spacing:.08em;white-space:nowrap}td{height:44px;font-size:12.5px}tr:last-child td{border-bottom:0}.mono{font-family:var(--mono);overflow-wrap:anywhere}.linkish{border:0;background:none;color:var(--accent);padding:0;min-height:0;text-align:left;font:inherit}.linkish:hover{color:var(--ink);text-decoration:underline}.pill{display:inline-flex;align-items:center;gap:6px;font-size:12px;color:var(--muted)}.pill:before{content:"";width:6px;height:6px;border-radius:50%;background:currentColor}.pill.ok{color:var(--ok)}.pill.warn{color:var(--warn)}.pill.err{color:var(--err)}.empty{padding:32px;color:var(--faint);text-align:center;font-size:13px} +.split{display:grid;grid-template-columns:minmax(0,1.5fr) minmax(280px,1fr);gap:14px;align-items:start}.stack{display:grid;gap:12px}.panel p{margin:0;font-size:12.5px}.panel p.mono{font-size:12px}#sc-details,.kv{display:grid;grid-template-columns:auto 1fr;gap:6px 16px;margin:0;font-size:12.5px}#sc-details dt,.kv dt{color:var(--muted)}#sc-details dd,.kv dd{margin:0}.card .sum{font:600 13px/1.4 var(--disp)}#sc-detail h3{margin:6px 0 0;font:600 12px var(--disp);color:var(--muted);text-transform:uppercase;letter-spacing:.08em}#sc-detail-title{margin-right:auto}#sc-detail p{margin:0;font-size:12.5px}.notice{background:color-mix(in oklab,var(--warn) 10%,transparent);border:1px solid color-mix(in oklab,var(--warn) 35%,transparent);border-radius:8px;color:var(--warn);padding:8px 12px;margin:0;font-size:12.5px;display:grid;gap:4px}#sc-notes p{margin:0;color:var(--muted);font-size:12.5px}.status{position:fixed;right:18px;bottom:18px;max-width:min(420px,calc(100vw - 36px));background:var(--surface);border:1px solid var(--border);border-radius:8px;padding:10px 12px;z-index:5;font-size:12.5px;box-shadow:0 12px 32px #0005}.status:empty{display:none} @media(max-width:900px){#sc-shell{grid-template-columns:minmax(0,1fr)}aside{position:static;height:auto;border-right:0;border-bottom:1px solid var(--border)}.nav{display:flex;overflow:auto}.nav-group{display:none}.nav a{white-space:nowrap}.aside-foot{margin-top:0;display:flex;flex-wrap:wrap;align-items:center}.cards{grid-template-columns:repeat(2,1fr)}.split{grid-template-columns:1fr}.top{position:static}} @media(max-width:520px){main{padding:16px 12px 48px}.top{padding:0 12px}.top p{display:none}.cards{grid-template-columns:1fr}.toolbar>:not(.sr){width:100%}#sc-search{width:100%}.toolbar .push{margin-left:0}.nav{padding:6px}.brand{padding:10px 14px}} @media(pointer:coarse){button,input,select,.nav a,.top .right button{min-height:44px}} @@ -52,12 +52,16 @@

Warnings

-
NameVersionEcosystemPURLScopeDeclared licenseRoot
+
NameVersionEcosystemPURLScopeDeclared licenseAssessed licenseRoot
+

Recent collection attempts

FinishedOutcomeHTTP statusError codeMessageProjection error
@@ -73,41 +77,79 @@ const say=(message,bad=false)=>{status.textContent=message;status.style.borderColor=bad?"var(--err)":"var(--border)"}; const OK=new Set(["current","published","unchanged","succeeded"]),WARN=new Set(["stale","queued","running","rate_limited","transient"]),BAD=new Set(["failed","forbidden","not_found","malformed","too_large","error","cancelled"]); function pill(value,label){const shown=value?String(value):"unknown",node=el("span",shown,"pill"),key=shown.toLowerCase();node.classList.add(OK.has(key)?"ok":WARN.has(key)?"warn":BAD.has(key)?"err":"unknown");if(label)node.setAttribute("aria-label",label+": "+shown);return node} +const LICENSE_TONE={resolved:"ok",conflict:"err",unlicensed:"err"},OUTCOME_TONE={resolved:"ok",unavailable:"warn",not_found:"warn",rejected:"err",too_large:"err",malformed:"err"}; +function tonePill(value,tone,label){const shown=value?String(value):"unknown",node=el("span",shown,"pill");node.classList.add(tone||"unknown");if(label)node.setAttribute("aria-label",label+": "+shown);return node} +const LICENSE_ORDER=["resolved","declared","conflict","unlicensed","unknown","pending","not_applicable"]; function card(label,value){const node=el("div",undefined,"card");node.append(el("small",label));if(value instanceof Node){const strong=el("strong");strong.append(value);node.append(strong)}else node.append(el("strong",value));return node} function cells(row,values){for(const value of values){const cell=el("td");if(value instanceof Node)cell.append(value);else cell.textContent=value;row.append(cell)}} function humanize(seconds){if(seconds===null||seconds===undefined)return "—";const total=Number(seconds);if(!Number.isFinite(total)||total<0)return "—";for(const [unit,size] of [["day",86400],["hour",3600],["minute",60]]){if(total>=size){const count=Math.floor(total/size);return count+" "+unit+(count===1?"":"s")+" ago"}}return "just now"} const STREAM="github:source"; let mode="anonymous",token="",pending="";try{pending=sessionStorage.getItem("graphnest_admin_token")||""}catch(_){} -let state={repositories:[],repository:"",stream:STREAM,query:"",status:null,snapshotID:null,components:[],cursor:"",debounce:0}; +let state={repositories:[],repository:"",stream:STREAM,query:"",status:null,snapshotID:null,components:[],cursor:"",debounce:0,element:""}; async function api(path,options={}){const headers=new Headers(options.headers||{});if(mode==="bearer"&&token)headers.set("Authorization","Bearer "+token);return fetch(path,{...options,headers,cache:"no-store",credentials:"same-origin"})} async function request(path,options={}){const response=await api(path,options);if(response.status===401){lockAccess("Token required or expired.");const error=new Error("Token required or expired.");error.access=401;throw error}if(!response.ok){let message="Request failed with HTTP "+response.status+".",code="";try{const body=await response.json();message=body.error?.message||message;code=body.error?.code||""}catch(_){}const error=new Error(message);error.code=code;error.status=response.status;throw error}return response.status===204?null:response.json()} function showAccess(message){shell.hidden=true;access.hidden=false;if(message)$("access-message").textContent=message;$("token").focus()} -function lockAccess(message){mode="anonymous";token="";$("token").value="";try{sessionStorage.removeItem("graphnest_admin_token")}catch(_){}state={...state,status:null,components:[],cursor:"",snapshotID:null};for(const id of ["sc-cards","sc-details","sc-notes","sc-warnings","sc-rows","sc-collections"])clear($(id));showAccess(message)} -function readHash(){const raw=String(location.hash||"").replace(/^#/,"");for(const part of raw.split("&")){if(!part)continue;const [key,value=""]=part.split("=");const decoded=decodeURIComponent(value);if(key==="repo")state.repository=decoded;else if(key==="stream"&&decoded===STREAM)state.stream=decoded;else if(key==="q")state.query=decoded}} -function writeHash(){location.hash="repo="+encodeURIComponent(state.repository)+"&stream="+state.stream+(state.query?"&q="+encodeURIComponent(state.query):"")} +function lockAccess(message){mode="anonymous";token="";$("token").value="";try{sessionStorage.removeItem("graphnest_admin_token")}catch(_){}state={...state,status:null,components:[],cursor:"",snapshotID:null};for(const id of ["sc-cards","sc-details","sc-notes","sc-warnings","sc-rows","sc-collections","sc-detail-body"])clear($(id));$("sc-detail").hidden=true;showAccess(message)} +function readHash(){const raw=String(location.hash||"").replace(/^#/,"");for(const part of raw.split("&")){if(!part)continue;const [key,value=""]=part.split("=");const decoded=decodeURIComponent(value);if(key==="repo")state.repository=decoded;else if(key==="stream"&&decoded===STREAM)state.stream=decoded;else if(key==="q")state.query=decoded;else if(key==="element")state.element=decoded}} +function writeHash(){location.hash="repo="+encodeURIComponent(state.repository)+"&stream="+state.stream+(state.query?"&q="+encodeURIComponent(state.query):"")+(state.element?"&element="+encodeURIComponent(state.element):"")} const base=()=>"/v1/supply-chain/repositories/"+encodeURIComponent(state.repository); const streamQuery=()=>"stream="+encodeURIComponent(state.stream); async function loadRepositories(){const repositories=[];let cursor="";do{const page=await request("/v1/repositories"+(cursor?"?cursor="+encodeURIComponent(cursor):""));repositories.push(...(Array.isArray(page.repositories)?page.repositories:[]));cursor=page.truncated&&page.next_cursor&&repositories.length<1000?page.next_cursor:""}while(cursor);state.repositories=repositories.slice(0,1000);const picker=$("sc-repository");clear(picker);for(const repository of state.repositories){const option=el("option",repository.name||String(repository.github_id));option.value=String(repository.github_id);picker.append(option)}if(!state.repositories.some(repository=>String(repository.github_id)===state.repository))state.repository=state.repositories.length?String(state.repositories[0].github_id):"";picker.value=state.repository} function renderStatus(){const report=state.status,cards=$("sc-cards"),details=$("sc-details"),notes=$("sc-notes"),notice=$("sc-notice");clear(cards);clear(details);clear(notes);notice.hidden=true;clear(notice);if(!report)return;const snapshot=report.latest_snapshot; -cards.append(card("Collection state",pill(report.collection,"Collection state")),card("Freshness",humanize(report.freshness_seconds)),card("Components",snapshot?String(snapshot.component_count||0):"—"),card("Warnings",snapshot?String(snapshot.warning_count||0):"0")); -const rows=[["Subject assurance","unknown — not bound to a commit"],["Producer",text(report.producer)],["Producer tool",text(snapshot?.producer_tool)],["Observed by GraphNest (collected_at)",when(snapshot?.collected_at)],["Producer-claimed creation time",when(snapshot?.created_at_claimed)],["Enrichment","not configured"],["Snapshot",snapshot?"#"+snapshot.id:"—"]]; +cards.append(card("Collection state",pill(report.collection,"Collection state")),card("Freshness",humanize(report.freshness_seconds)),card("Components",snapshot?String(snapshot.component_count||0):"—"),card("Warnings",snapshot?String(snapshot.warning_count||0):"0"),card("Licenses",el("span",licenseSummaryText(report.license_summary),"sum"))); +const rows=[["Subject assurance","unknown — not bound to a commit"],["Producer",text(report.producer)],["Producer tool",text(snapshot?.producer_tool)],["Observed by GraphNest (collected_at)",when(snapshot?.collected_at)],["Producer-claimed creation time",when(snapshot?.created_at_claimed)],["License enrichment",enrichmentText(report)],["Snapshot",snapshot?"#"+snapshot.id:"—"]]; for(const [label,value] of rows){details.append(el("dt",label),el("dd",value))} const document0=Array.isArray(report.documents)?report.documents[0]:null;$("sc-document").textContent=document0?"sha256 "+document0.sha256+" · "+document0.format+" · "+document0.bytes+" bytes":"No original document is stored.";$("sc-download").disabled=!document0; for(const note of (Array.isArray(report.notes)?report.notes:[]).slice(0,50))notes.append(el("p",note)); if(report.collection==="failed"&&report.latest_snapshot){notice.hidden=false;notice.append(el("span","The inventory shown is the last successful observation; the most recent refresh failed."));const last=report.last_collection;if(last)notice.append(el("span","Last attempt: "+text(last.outcome)+" · HTTP "+text(last.http_status)+" · "+text(last.message)))} renderWarnings(snapshot);} +function enrichmentText(report){if(report.enrichment!=="configured")return "not configured";const ecosystems=Array.isArray(report.enrichment_ecosystems)?report.enrichment_ecosystems.filter(Boolean):[];return ecosystems.length?"configured for "+ecosystems.join(", "):"configured"} +function licenseSummaryText(summary){const parts=[];for(const status of LICENSE_ORDER){const count=Number(summary&&summary[status]);if(Number.isFinite(count)&&count>0)parts.push(count+" "+status)}return parts.length?parts.join(" · "):"no assessments"} +function assessedCell(license){if(!license)return "—";const host=el("span");host.append(tonePill(license.status,LICENSE_TONE[license.status],"Assessed license"));if(license.expression)host.append(el("span"," "+license.expression,"mono"));return host} function renderWarnings(snapshot){const host=$("sc-warnings");clear(host);const warnings=Array.isArray(snapshot?.warnings)?snapshot.warnings:[];$("sc-warning-count").textContent=warnings.length?"("+warnings.length+")":"(0)";if(!warnings.length){host.append(el("p","No parser warnings were recorded.","muted"));return}for(const warning of warnings)host.append(el("p",text(warning.code)+" · "+text(warning.element)+" · "+text(warning.detail)))} -function renderComponents(){const body=$("sc-rows");clear(body);for(const component of state.components){const row=el("tr");cells(row,[text(component.name),text(component.version),text(component.ecosystem),el("span",text(component.purl),"mono"),pill(component.scope||"unknown","Scope"),text(component.license_declared_raw),component.is_root?"yes":"no"]);body.append(row)} +function renderComponents(){const body=$("sc-rows");clear(body);for(const component of state.components){const row=el("tr"),name=el("button",text(component.name),"linkish");name.addEventListener("click",()=>void openDetail(component.element_id));cells(row,[name,text(component.version),text(component.ecosystem),el("span",text(component.purl),"mono"),pill(component.scope||"unknown","Scope"),text(component.license_declared_raw),assessedCell(component.license),component.is_root?"yes":"no"]);body.append(row)} $("sc-shown").textContent=state.components.length?"Showing "+state.components.length+" of snapshot #"+state.snapshotID:"";$("sc-more").hidden=!state.cursor;$("sc-empty").hidden=state.components.length>0} function renderCollections(collections){const body=$("sc-collections");clear(body);for(const collection of collections){const row=el("tr");cells(row,[when(collection.finished_at),pill(collection.outcome,"Outcome"),text(collection.http_status),text(collection.error_code),text(collection.message),text(collection.projection_error)]);body.append(row)}} async function loadComponents(more){const rows=$("sc-rows"),loading=$("sc-loading"),failure=$("sc-error"),empty=$("sc-empty");failure.hidden=true;empty.hidden=true;loading.hidden=false;if(!more){state.components=[];state.cursor="";clear(rows)} try{const query=[streamQuery(),"limit=100"];if(state.query)query.push("q="+encodeURIComponent(state.query));if(more&&state.cursor)query.push("cursor="+encodeURIComponent(state.cursor));const page=await request(base()+"/components?"+query.join("&"));state.snapshotID=page.snapshot_id;state.components=state.components.concat(Array.isArray(page.components)?page.components:[]);state.cursor=page.truncated&&page.next_cursor?page.next_cursor:"";renderComponents()}catch(error){if(error.access===401)return;clear(rows);state.components=[];state.cursor="";renderComponents();if(error.code==="no_inventory"){empty.hidden=false;empty.textContent="No inventory has been collected yet"}else{failure.hidden=false;failure.textContent=error.message}}finally{loading.hidden=true}} +function detailTable(columns,rows,render){const wrap=el("div",undefined,"table-wrap"),table=el("table"),head=el("thead"),headRow=el("tr");for(const column of columns){const cell=el("th",column);cell.setAttribute("scope","col");headRow.append(cell)}head.append(headRow);const body=el("tbody");for(const row of rows){const tr=el("tr");cells(tr,render(row));body.append(tr)}table.append(head,body);wrap.append(table);return wrap} +function resolverText(row){return "resolver v"+text(row.resolver_version)+" \u00b7 SPDX list "+text(row.license_list_version)} +function evidenceLinkText(row){const parts=[];if(row.license_url)parts.push(row.license_url);if(row.license_file_name)parts.push(row.license_file_name);return parts.length?parts.join(" \u00b7 "):"\u2014"} +function renderDetail(detail){const host=$("sc-detail-body");clear(host);const component=detail.component||{},license=component.license; +$("sc-detail-title").textContent="Component evidence \u00b7 "+text(component.name); +const identity=el("dl",undefined,"kv");for(const [label,value] of [["Name",text(component.name)],["Version",text(component.version)],["PURL",text(component.purl)],["Ecosystem",text(component.ecosystem)],["Element",text(component.element_id)],["Scope",text(component.scope)],["Root",component.is_root?"yes":"no"]])identity.append(el("dt",label),el("dd",value));host.append(identity); +host.append(el("h3","Assessment")); +if(!license)host.append(el("p","No assessment has been recorded for this occurrence.","muted")); +else{const assessment=el("dl",undefined,"kv"),statusValue=el("dd");statusValue.append(tonePill(license.status,LICENSE_TONE[license.status],"Assessed license"));assessment.append(el("dt","Status"),statusValue); +if(license.expression)assessment.append(el("dt","Expression"),el("dd",license.expression,"mono")); +if(license.conflict_detail)assessment.append(el("dt","Conflict"),el("dd",license.conflict_detail)); +assessment.append(el("dt","Evidence rows"),el("dd",String(license.evidence_count||0)),el("dt","Assessed at"),el("dd",when(license.assessed_at))); +const fingerprint=String(license.evidence_fingerprint||""),shown=el("dd",fingerprint?fingerprint.slice(0,12):"\u2014","mono");if(fingerprint)shown.setAttribute("title",fingerprint);assessment.append(el("dt","Evidence fingerprint"),shown);host.append(assessment)} +const declarations=Array.isArray(detail.declarations)?detail.declarations:[]; +host.append(el("h3","Producer declarations")); +if(declarations.length)host.append(detailTable(["Source","Raw value","Parse status"],declarations,row=>[text(row.source),el("span",text(row.raw_value),"mono"),text(row.parse_status)])); +else host.append(el("p","No producer declarations for this component.","muted")); +const evidence=Array.isArray(detail.evidence)?detail.evidence:[]; +host.append(el("h3","Registry evidence")); +if(evidence.length)host.append(detailTable(["Fetched","Source","Route","Outcome","Raw kind","Raw value","Expression","Parse status","Message","License link","Resolver"],evidence,row=>[when(row.fetched_at),text(row.source),text(row.route),tonePill(row.outcome,OUTCOME_TONE[row.outcome],"Outcome"),text(row.raw_kind),el("span",text(row.raw_value),"mono"),text(row.expression),text(row.parse_status),text(row.message),el("span",evidenceLinkText(row),"mono"),resolverText(row)])); +else host.append(el("p","No registry evidence for these coordinates.","muted")); +const relationships=Array.isArray(detail.relationships)?detail.relationships:[]; +host.append(el("h3","Relationships")); +if(relationships.length)for(const edge of relationships)host.append(el("p",text(edge.from)+" \u2192 "+text(edge.type)+" \u2192 "+text(edge.to)+(edge.resolved===false?" (unresolved)":""),"mono")); +else host.append(el("p","No relationships reference this component.","muted")); +for(const note of (Array.isArray(detail.notes)?detail.notes:[]).slice(0,50))host.append(el("p",note,"muted")); +if(detail.truncated)host.append(el("p","Evidence list is truncated; not every row is shown.","muted"))} +function closeDetail(){const panel=$("sc-detail");if(panel.hidden)return;panel.hidden=true;clear($("sc-detail-body"));state.element="";writeHash()} +async function openDetail(elementID){if(!elementID)return;state.element=elementID;writeHash();const panel=$("sc-detail"),host=$("sc-detail-body");panel.hidden=false;clear(host);$("sc-detail-title").textContent="Component evidence";host.append(el("p","Loading evidence\u2026","muted"));$("sc-detail-title").focus(); +const query=base()+"/component?element="+encodeURIComponent(elementID)+"&"+streamQuery()+(state.snapshotID?"&snapshot_id="+encodeURIComponent(state.snapshotID):""); +try{renderDetail(await request(query))}catch(error){if(error.access===401)return;clear(host);host.append(el("p",error.message,"muted"))}} async function load(){if(!state.repositories.length){try{await loadRepositories()}catch(error){if(error.access!==401)say(error.message,true);return}} if(!state.repository){say("No repositories are available.",true);return} writeHash();$("sc-search").value=state.query;$("sc-repository").value=state.repository;$("sc-stream").value=state.stream; try{state.status=await request(base()+"?"+streamQuery())}catch(error){if(error.access===401)return;state.status=null;renderStatus();say(error.message,true)} if(state.status)renderStatus(); await loadComponents(false); +if(state.element)await openDetail(state.element); try{const page=await request(base()+"/collections?"+streamQuery());renderCollections(Array.isArray(page.collections)?page.collections:[])}catch(error){if(error.access!==401)say(error.message,true)}} async function refreshNow(){try{const result=await request(base()+"/refresh?"+streamQuery(),{method:"POST"});say("Refresh job #"+result.job.id+" is "+result.job.state+".")}catch(error){if(error.access===401)return;say(error.status===403?"administrator access required":error.message,true)}} async function downloadDocument(){const reference=Array.isArray(state.status?.documents)?state.status.documents[0]:null;if(!reference?.path)return say("No original document is stored.",true);try{const response=await api(reference.path);if(!response.ok)throw new Error("Request failed with HTTP "+response.status+".");const disposition=response.headers?.get?.("Content-Disposition")||"";const match=/filename="?([^";]+)"?/.exec(disposition);const href=URL.createObjectURL(await response.blob()),link=el("a");link.href=href;link.download=match?match[1]:"graphnest-sbom-snapshot"+reference.snapshot_id+".json";link.click();URL.revokeObjectURL(href);say("Original document downloaded.")}catch(error){say(error.message,true)}} @@ -126,6 +168,8 @@ $("sc-refresh").addEventListener("click",()=>void refreshNow()); $("sc-download").addEventListener("click",()=>void downloadDocument()); $("sc-more").addEventListener("click",()=>void loadComponents(true)); +$("sc-detail-close").addEventListener("click",closeDetail); +document.addEventListener("keydown",event=>{if(event.key==="Escape")closeDetail()}); $("sc-search").addEventListener("input",event=>{state.query=event.target.value;clearTimeout(state.debounce);state.debounce=setTimeout(()=>{writeHash();void loadComponents(false)},250)}); void start(); })(); diff --git a/internal/webui/supply_chain_contract_test.go b/internal/webui/supply_chain_contract_test.go index c197430e..ba13608f 100644 --- a/internal/webui/supply_chain_contract_test.go +++ b/internal/webui/supply_chain_contract_test.go @@ -17,6 +17,11 @@ func TestSupplyChainDocumentContract(t *testing.T) { `data-graphnest-supply-chain`, `id="sc-shell"`, `id="access-panel"`, `id="sc-repository"`, `id="sc-stream"`, `id="sc-search"`, `id="sc-rows"`, `id="sc-more"`, `id="sc-collections"`, `id="sc-warnings"`, `id="sc-notice"`, + `id="sc-detail"`, `id="sc-detail-body"`, `id="sc-detail-close"`, + `Assessed license`, `/component?`, `Producer declarations`, `Registry evidence`, + `No registry evidence for these coordinates.`, `Evidence fingerprint`, + `license_summary`, `enrichment_ecosystems`, `conflict_detail`, `no assessments`, + `snapshot_id=`, `(unresolved)`, `resolver v`, `SPDX list `, `github:source`, `GitHub dependency graph (source observation)`, `/v1/repositories`, `/v1/supply-chain/repositories/`, `/components`, `/refresh`, `/collections`, `license_declared_raw`, `created_at_claimed`, diff --git a/internal/webui/supply_chain_dom_test.mjs b/internal/webui/supply_chain_dom_test.mjs index 705e3f78..a3e374ea 100644 --- a/internal/webui/supply_chain_dom_test.mjs +++ b/internal/webui/supply_chain_dom_test.mjs @@ -36,12 +36,14 @@ class FakeNode { const all = []; const ids = new Map(); +const documentListeners = {}; const document = { activeElement: null, documentElement: new FakeNode("html"), createElement(tag) { const node = new FakeNode(tag); all.push(node); return node; }, getElementById(id) { return ids.get(id); }, querySelectorAll() { return []; }, + addEventListener(name, listener) { documentListeners[name] = listener; }, }; globalThis.document = document; globalThis.Node = FakeNode; @@ -60,9 +62,10 @@ for (const id of [ "sc-refresh", "sc-download", "sc-notes", "sc-notice", "sc-cards", "sc-details", "sc-document", "sc-warnings", "sc-warning-count", "sc-search", "sc-shown", "sc-rows", "sc-loading", "sc-empty", "sc-error", "sc-more", "sc-collections", + "sc-detail", "sc-detail-title", "sc-detail-body", "sc-detail-close", ]) { const node = document.createElement(id === "token-form" ? "form" : "div"); - node.hidden = id === "sc-shell" || id === "sc-notice" || id === "sc-more"; + node.hidden = id === "sc-shell" || id === "sc-notice" || id === "sc-more" || id === "sc-detail"; ids.set(id, node); } @@ -81,12 +84,36 @@ const component = (ordinal, overrides = {}) => ({ purl: "pkg:npm/pkg-" + ordinal + "@1." + ordinal + ".0", ecosystem: "npm", license_declared_raw: "MIT", is_root: false, scope: "transitive", ...overrides, }); +const assessment = (status, overrides = {}) => ({ + status, evidence_count: 2, assessed_at: "2026-02-01T11:00:00Z", + evidence_fingerprint: "a1b2c3d4e5f60718293a4b5c6d7e8f90", ...overrides, +}); const firstPage = [ - component(1, {is_root: true, scope: "root", name: "x"}), - component(2, {scope: "direct"}), - component(3, {version: null, purl: null, license_declared_raw: "NOASSERTION", scope: "unknown"}), + component(1, {is_root: true, scope: "root", name: "x", license: assessment("resolved", {expression: "MIT"})}), + component(2, {scope: "direct", license: assessment("conflict", {conflict_detail: "github says MIT, npm says Apache-2.0"})}), + component(3, {version: null, purl: null, license_declared_raw: "NOASSERTION", scope: "unknown", license: null}), component(4), ]; +const evidenceRow = (overrides = {}) => ({ + id: 1, source: "registry", route: "npm:test", ecosystem: "npm", name: "pkg-1", version: "1.1.0", + raw_value: "MIT", raw_kind: "expression", parse_status: "parsed", expression: "MIT", + resolver_version: 1, license_list_version: "3.27.0", fetched_at: "2026-02-01T10:30:00Z", + outcome: "resolved", ...overrides, +}); +const componentDetail = { + component: firstPage[0], snapshot, + declarations: [ + evidenceRow({id: 10, source: "producer", route: "", raw_value: "MIT", raw_kind: "spdx_declared"}), + evidenceRow({id: 11, source: "producer", route: "", raw_value: "MIT", raw_kind: "spdx_declared", parse_status: "unparsed"}), + ], + evidence: [ + evidenceRow({id: 20, license_url: "https://registry.test/license"}), + evidenceRow({id: 21, outcome: "unavailable", route: "npm:test", raw_value: "", expression: "", parse_status: "absent", message: "registry timed out"}), + ], + relationships: [{from: "SPDXRef-1", type: "DEPENDS_ON", to: "SPDXRef-2", resolved: false}], + notes: ["Registry evidence is cached and may lag the registry.", "Declarations are shown verbatim."], + truncated: false, +}; const secondPage = [component(5), component(6), component(7)]; const STREAM = "stream=github%3Asource"; @@ -100,12 +127,14 @@ const responses = { subject: "repository", collection: "failed", freshness_seconds: 7200, latest_snapshot: snapshot, last_collection: {id: 5, outcome: "forbidden", http_status: 403, message: "dependency graph is disabled", finished_at: "2026-02-02T10:00:00Z"}, - active_job: null, enrichment: "not_configured", opt_out: false, + active_job: null, enrichment: "configured", enrichment_ecosystems: ["npm", "maven"], + license_summary: {resolved: 3, conflict: 1, unknown: 2}, opt_out: false, notes: ["GitHub reports the dependency graph for the default branch.", "Subject assurance is unknown on GitHub Enterprise Server."], documents: [{snapshot_id: 11, sha256: "ab".repeat(32), format: "spdx-2.3-json", bytes: 1234, path: "/v1/supply-chain/snapshots/11/document"}], }, ["/v1/supply-chain/repositories/101/components?" + STREAM + "&limit=100"]: {snapshot_id: 11, components: firstPage, truncated: true, next_cursor: "c2"}, ["/v1/supply-chain/repositories/101/components?" + STREAM + "&limit=100&cursor=c2"]: {snapshot_id: 11, components: secondPage, truncated: false}, + ["/v1/supply-chain/repositories/101/component?element=SPDXRef-1&" + STREAM + "&snapshot_id=11"]: componentDetail, ["/v1/supply-chain/repositories/101/collections?" + STREAM]: {collections: [ {id: 5, outcome: "forbidden", http_status: 403, error_code: "forbidden", message: "dependency graph is disabled", finished_at: "2026-02-02T10:00:00Z"}, {id: 4, outcome: "published", http_status: 200, snapshot_id: 11, finished_at: "2026-02-01T10:00:00Z"}, @@ -177,6 +206,49 @@ assert.equal(rows[2][5], "NOASSERTION", "declared license is shown verbatim"); assert.equal(rows[0][0], "x", "component names are rendered as literal text"); assert.equal(all.some(node => node.tagName === "B"), false, "no markup is built from component data"); +// Assessed licenses are shown as a status pill plus the normalized expression. +assert.equal(rows[2][6], "—", "a component without an assessment shows an em dash"); +assert.match(rows[0][6], /resolved/); +assert.match(rows[0][6], /MIT/, "a resolved assessment shows its expression"); +const conflictPill = all.find(node => node.textContent === "conflict" && node.className.includes("pill")); +assert.ok(conflictPill.className.includes("err"), "a conflicting assessment uses the error tone"); + +// Enrichment state and the license summary are reported in the status area. +assert.match(text(ids.get("sc-details")), /License enrichment/); +assert.match(text(ids.get("sc-details")), /configured for npm, maven/); +assert.match(text(ids.get("sc-cards")), /3 resolved · 1 conflict · 2 unknown/); + +// The component name opens the evidence detail panel for that element. +const nameButton = ids.get("sc-rows").children[0].children[0].children[0]; +assert.equal(nameButton.tagName, "BUTTON", "component names open the detail panel"); +await nameButton.dispatch("click"); +await settle(); +const detailRequest = requests.find(({path}) => path.includes("/component?")); +assert.equal(detailRequest.options.headers.get("Authorization"), "Bearer sc-token", "detail fetch must send Authorization"); +assert.match(detailRequest.path, /snapshot_id=11/); +assert.equal(ids.get("sc-detail").hidden, false); +assert.equal(document.activeElement, ids.get("sc-detail-title"), "opening the panel moves focus to its heading"); +const detailText = text(ids.get("sc-detail-body")); +const evidenceTable = ids.get("sc-detail-body").children.filter(node => node.className === "table-wrap"); +assert.equal(evidenceTable.length, 2, "declarations and registry evidence each render a table"); +assert.equal(evidenceTable[1].children[0].children[1].children.length, 2, "both registry evidence rows are rendered"); +assert.match(detailText, /npm:test/); +assert.match(detailText, /resolver v1 · SPDX list 3\.27\.0/); +assert.match(detailText, /a1b2c3d4e5f6/, "the fingerprint is shortened"); +assert.match(detailText, /registry timed out/); +assert.match(detailText, /Registry evidence is cached/, "detail notes are rendered"); +assert.match(detailText, /\(unresolved\)/); +assert.ok(detailText.includes("MIT"), "raw declaration values stay literal text"); +assert.equal(all.some(node => node.tagName === "B"), false, "no markup is built from evidence data"); +const unavailablePill = all.find(node => node.textContent === "unavailable" && node.className.includes("pill")); +assert.ok(unavailablePill.className.includes("warn"), "an unavailable registry fetch uses the warning tone"); +assert.match(location.hash, /element=SPDXRef-1/); + +// Escape closes the panel and drops the element from the hash. +documentListeners.keydown({key: "Escape"}); +assert.equal(ids.get("sc-detail").hidden, true, "Escape closes the evidence panel"); +assert.equal(/element=/.test(location.hash), false); + // Refresh reports the queued job. await ids.get("sc-refresh").dispatch("click"); await settle(); diff --git a/pkg/api/supply_chain.go b/pkg/api/supply_chain.go index a66ec15a..30902b88 100644 --- a/pkg/api/supply_chain.go +++ b/pkg/api/supply_chain.go @@ -17,14 +17,19 @@ type SupplyChainRepositoryStatus struct { Collection string `json:"collection"` // Freshness is the age of the latest successful observation in seconds, or // null when there is none. - FreshnessSeconds *int64 `json:"freshness_seconds"` - LatestSnapshot *SupplyChainSnapshot `json:"latest_snapshot"` - LastCollection *SupplyChainCollection `json:"last_collection"` - ActiveJob *SupplyChainJob `json:"active_job"` - Enrichment string `json:"enrichment"` - OptOut bool `json:"opt_out"` - Notes []string `json:"notes"` - Documents []SupplyChainDocumentRef `json:"documents"` + FreshnessSeconds *int64 `json:"freshness_seconds"` + LatestSnapshot *SupplyChainSnapshot `json:"latest_snapshot"` + LastCollection *SupplyChainCollection `json:"last_collection"` + ActiveJob *SupplyChainJob `json:"active_job"` + // Enrichment is "not_configured" (no registry routes), or "configured". + Enrichment string `json:"enrichment"` + // EnrichmentEcosystems lists ecosystems with a configured registry route. + EnrichmentEcosystems []string `json:"enrichment_ecosystems"` + // LicenseSummary counts the latest snapshot's assessments by status. + LicenseSummary map[string]int `json:"license_summary"` + OptOut bool `json:"opt_out"` + Notes []string `json:"notes"` + Documents []SupplyChainDocumentRef `json:"documents"` } // SupplyChainDocumentRef points at a downloadable original document. @@ -88,6 +93,65 @@ type SupplyChainComponent struct { // Scope is "root", "direct", "transitive", or "unknown" and is derived only // from resolved DEPENDS_ON edges from a root; a flattened list yields unknown. Scope string `json:"scope"` + // License is the derived assessment for this occurrence; nil until an + // assessment exists (enrichment disabled or pending). + License *SupplyChainLicenseAssessment `json:"license"` +} + +// SupplyChainLicenseAssessment is the derived view over declarations and +// registry evidence for one occurrence. It is not an approval. +type SupplyChainLicenseAssessment struct { + // Status is unknown, declared, resolved, conflict, unlicensed, not_applicable, or pending. + Status string `json:"status"` + // Expression is the normalized SPDX expression when status is declared or resolved. + Expression string `json:"expression,omitempty"` + // ConflictDetail lists the disagreeing sources when status is conflict. + ConflictDetail string `json:"conflict_detail,omitempty"` + EvidenceCount int `json:"evidence_count"` + AssessedAt time.Time `json:"assessed_at"` + // EvidenceFingerprint (hex) changes when any considered evidence changes; + // reviews record it to detect stale bases. + EvidenceFingerprint string `json:"evidence_fingerprint"` +} + +// SupplyChainLicenseEvidence is one immutable evidence row as shown in the +// evidence detail view. Raw values are verbatim; nothing is mapped. +type SupplyChainLicenseEvidence struct { + ID int64 `json:"id"` + Source string `json:"source"` + Route string `json:"route,omitempty"` + Ecosystem string `json:"ecosystem"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + ArtifactSHA256 string `json:"artifact_sha256,omitempty"` + RawValue string `json:"raw_value"` + RawKind string `json:"raw_kind"` + ParseStatus string `json:"parse_status"` + Expression string `json:"expression,omitempty"` + UnknownTerms []string `json:"unknown_terms,omitempty"` + LicenseURL string `json:"license_url,omitempty"` + LicenseFileName string `json:"license_file_name,omitempty"` + Detail map[string]any `json:"detail,omitempty"` + ResolverVersion int `json:"resolver_version"` + LicenseListVersion string `json:"license_list_version"` + ContentSHA256 string `json:"content_sha256,omitempty"` + FetchedAt time.Time `json:"fetched_at"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Outcome string `json:"outcome"` + HTTPStatus *int `json:"http_status,omitempty"` + Message string `json:"message,omitempty"` +} + +// SupplyChainComponentDetail is the evidence detail view for one occurrence. +type SupplyChainComponentDetail struct { + Component SupplyChainComponent `json:"component"` + Snapshot SupplyChainSnapshot `json:"snapshot"` + Declarations []SupplyChainLicenseEvidence `json:"declarations"` + Evidence []SupplyChainLicenseEvidence `json:"evidence"` + Relationships []SupplyChainRelationship `json:"relationships"` + Notes []string `json:"notes"` + Truncated bool `json:"truncated"` } type SupplyChainChecksum struct { diff --git a/test/integration/supply_chain_test.go b/test/integration/supply_chain_test.go index 5e1c37bc..7290466d 100644 --- a/test/integration/supply_chain_test.go +++ b/test/integration/supply_chain_test.go @@ -24,6 +24,7 @@ import ( "github.com/balcsida/graphnest/internal/httpapi" "github.com/balcsida/graphnest/internal/postgres" "github.com/balcsida/graphnest/internal/supplychain" + "github.com/balcsida/graphnest/internal/supplychain/license" "github.com/balcsida/graphnest/pkg/api" ) @@ -265,3 +266,132 @@ func TestSupplyChainVerticalSlice(t *testing.T) { _ = gadgets _ = other } + +// TestSupplyChainLicenseEnrichment proves registry evidence flows from a +// configured route through the real worker and store into assessments and +// the REST detail view, that unconfigured ecosystems produce no traffic, and +// that evidence stays reachable only through authorized occurrences. +func TestSupplyChainLicenseEnrichment(t *testing.T) { + h := newPostgresHarness(t) + widgets := h.seedRepository(t, 10, 101) + github, client := newFakeSBOMGitHub(t) + envelope := sbomEnvelope(t) + github.responses["acme/repo-101"] = func(writer http.ResponseWriter) { fmt.Fprint(writer, envelope) } + + var registryCalls atomic.Int32 + registry := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + registryCalls.Add(1) + switch request.URL.EscapedPath() { + case "/npm/@scope%2Fleft-pad/1.3.0": + fmt.Fprint(writer, `{"name":"@scope/left-pad","version":"1.3.0","license":"MIT"}`) + default: + writer.WriteHeader(http.StatusNotFound) + } + })) + defer registry.Close() + base, _ := url.Parse(registry.URL + "/npm/") + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: registry.Certificate().Raw}) + routes, err := license.NewRegistry([]license.Route{{Name: "npm:test", Ecosystem: "npm", BaseURL: base, CAPEM: certificate, AllowPrivateHosts: true, Timeout: 5 * time.Second, MaxResponseBytes: 1 << 20}}) + if err != nil { + t.Fatal(err) + } + enricher := &license.Worker{Store: h.store, Registry: routes, Owner: "enrich"} + collector := &supplychain.Collector{Store: h.store, GitHub: client, Owner: "collect", MaxDocumentBytes: 1 << 20, Enricher: enricher} + if _, _, err := h.store.EnqueueSupplyChainJob(t.Context(), widgets, supplychain.StreamGitHubSource, "manual", "t", 10, time.Now()); err != nil { + t.Fatal(err) + } + if processed, err := collector.RunOnce(t.Context()); err != nil || !processed { + t.Fatalf("collection processed=%v err=%v", processed, err) + } + // Only the npm coordinate has a route; maven, nuget, golang, and githubactions stay unqueued. + depths, err := h.store.EnrichmentQueueDepths(t.Context()) + if err != nil || depths["queued"] != 1 { + t.Fatalf("enrichment depths = %v %v", depths, err) + } + if processed, err := enricher.RunOnce(t.Context()); err != nil || !processed { + t.Fatalf("enrichment processed=%v err=%v", processed, err) + } + if processed, err := enricher.RunOnce(t.Context()); err != nil || processed { + t.Fatalf("enrichment queue not drained: processed=%v err=%v", processed, err) + } + if registryCalls.Load() != 1 { + t.Fatalf("registry calls = %d, want exactly one exact-version lookup", registryCalls.Load()) + } + + service := &supplychain.Service{Store: h.store, Authorizer: authz.NewPostgres(h.store), Interval: time.Hour, MaxResults: 100, License: h.store, EnrichmentEcosystems: routes.Ecosystems()} + authenticator := authn.RequestAuthenticator{Bearer: authn.NewStatic(map[string]authn.Principal{ + "acme": {Subject: "acme", Method: "api_token", InstallationID: 10, RepositoryIDs: []int64{101}}, + "other": {Subject: "other", Method: "api_token", InstallationID: 20, RepositoryIDs: []int64{999}}, + })} + mux := http.NewServeMux() + httpapi.RegisterSupplyChain(mux, authenticator, service, 100, 256<<10) + get := func(token, path string) *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.Header.Set("Authorization", "Bearer "+token) + recorder := httptest.NewRecorder() + mux.ServeHTTP(recorder, request) + return recorder + } + response := get("acme", "/v1/supply-chain/repositories/101") + var status api.SupplyChainRepositoryStatus + if err := json.Unmarshal(response.Body.Bytes(), &status); err != nil || response.Code != http.StatusOK { + t.Fatalf("status = %d %s", response.Code, response.Body.String()) + } + if status.Enrichment != "configured" || len(status.EnrichmentEcosystems) != 1 || status.EnrichmentEcosystems[0] != "npm" || status.LicenseSummary["resolved"] != 1 { + t.Fatalf("status = %+v", status) + } + response = get("acme", "/v1/supply-chain/repositories/101/components?q=left-pad") + var page api.SupplyChainComponentList + if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil || response.Code != http.StatusOK || len(page.Components) != 1 { + t.Fatalf("components = %d %s", response.Code, response.Body.String()) + } + leftPad := page.Components[0] + if leftPad.License == nil || leftPad.License.Status != "resolved" || leftPad.License.Expression != "MIT" || leftPad.License.EvidenceCount != 1 || len(leftPad.License.EvidenceFingerprint) != 64 { + t.Fatalf("left-pad assessment = %+v", leftPad.License) + } + if leftPad.LicenseDeclaredRaw == nil || *leftPad.LicenseDeclaredRaw != "NOASSERTION" { + t.Fatalf("declared raw must remain the producer's NOASSERTION: %+v", leftPad) + } + response = get("acme", "/v1/supply-chain/repositories/101/components?q=core") + if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil || len(page.Components) != 1 || page.Components[0].License != nil { + t.Fatalf("maven component without a route must have no assessment: %s", response.Body.String()) + } + response = get("acme", "/v1/supply-chain/repositories/101/component?element=SPDXRef-npm-scope-left-pad-1.3.0") + var detail api.SupplyChainComponentDetail + if err := json.Unmarshal(response.Body.Bytes(), &detail); err != nil || response.Code != http.StatusOK { + t.Fatalf("detail = %d %s", response.Code, response.Body.String()) + } + if len(detail.Declarations) != 2 || detail.Declarations[0].ParseStatus != "no_assertion" || detail.Declarations[0].RawKind != "sentinel" { + t.Fatalf("declarations = %+v", detail.Declarations) + } + if len(detail.Evidence) != 1 || detail.Evidence[0].Source != "registry_npm" || detail.Evidence[0].Route != "npm:test" || detail.Evidence[0].Expression != "MIT" || detail.Evidence[0].RawValue != "MIT" || + detail.Evidence[0].LicenseListVersion != "3.27.0" || detail.Evidence[0].ResolverVersion != 1 || len(detail.Evidence[0].ContentSHA256) != 64 || detail.Evidence[0].Outcome != "resolved" { + t.Fatalf("evidence = %+v", detail.Evidence) + } + if len(detail.Relationships) != 1 || detail.Relationships[0].Type != "DEPENDS_ON" || detail.Relationships[0].To != "SPDXRef-npm-scope-left-pad-1.3.0" { + t.Fatalf("relationships = %+v", detail.Relationships) + } + // Evidence is reachable only through an authorized occurrence. + if response := get("other", "/v1/supply-chain/repositories/101/component?element=SPDXRef-npm-scope-left-pad-1.3.0"); response.Code != http.StatusNotFound { + t.Fatalf("other principal reached evidence: %d", response.Code) + } + if response := get("acme", "/v1/supply-chain/repositories/101/component?element=SPDXRef-nope"); response.Code != http.StatusNotFound { + t.Fatalf("unknown element = %d", response.Code) + } + if response := get("acme", "/v1/supply-chain/repositories/101/component"); response.Code != http.StatusBadRequest { + t.Fatalf("missing element = %d", response.Code) + } + // A second publication of the same document does not re-enqueue resolved coordinates. + if _, _, err := h.store.EnqueueSupplyChainJob(t.Context(), widgets, supplychain.StreamGitHubSource, "manual", "t", 10, time.Now()); err != nil { + t.Fatal(err) + } + if _, err := collector.RunOnce(t.Context()); err != nil { + t.Fatal(err) + } + if depths, _ := h.store.EnrichmentQueueDepths(t.Context()); depths["queued"] != 0 { + t.Fatalf("resolved coordinates were re-queued: %v", depths) + } + if registryCalls.Load() != 1 { + t.Fatalf("registry calls after unchanged republish = %d", registryCalls.Load()) + } +} diff --git a/test/smoke/supply-chain-screenshots.mjs b/test/smoke/supply-chain-screenshots.mjs index 8bf7d3cf..9ec891f1 100644 --- a/test/smoke/supply-chain-screenshots.mjs +++ b/test/smoke/supply-chain-screenshots.mjs @@ -13,6 +13,26 @@ const html = fs.readFileSync(htmlPath); const fixture = fs.readFileSync(path.resolve("test/fixtures/supplychain/ghes-spdx-2.3.json"), "utf8"); const doc = JSON.parse(fixture); const collected = "2026-09-21T10:00:00Z"; +const assessed = "2026-09-21T10:05:00Z"; +// Registry-derived assessments; the fixture declares NOASSERTION everywhere, so +// nothing here is inferred from the producer's declared value. +// Only ecosystems with a configured route (npm here) can be resolved; the +// root has no version, maven/nuget/golang/actions have no route, and the +// vendored component has no coordinates at all. +const ASSESSMENTS = [ + null, + {status: "resolved", expression: "MIT", evidence_count: 1}, + {status: "conflict", conflict_detail: "producer_declared: Apache-2.0 | registry_maven@maven:internal: MIT", evidence_count: 1}, + null, + null, + null, + {status: "not_applicable", evidence_count: 0}, +]; +function licenseFor(ordinal) { + const assessment = ASSESSMENTS[ordinal]; + if (!assessment) return null; + return {...assessment, assessed_at: assessed, evidence_fingerprint: "c3" + String(ordinal) + "9f41ad7b2e5c81d4a6027f3b9e5148dc6a0b7739ce21845f0db3c6a1e9f472"}; +} const components = doc.packages.map((pkg, ordinal) => { const purl = pkg.externalRefs?.find(ref => ref.referenceType === "purl")?.referenceLocator ?? null; const ecosystem = purl ? purl.slice(4, purl.indexOf("/")) : ""; @@ -20,8 +40,12 @@ const components = doc.packages.map((pkg, ordinal) => { element_id: pkg.SPDXID, ordinal, name: pkg.name, version: pkg.versionInfo || null, purl, ecosystem, license_declared_raw: pkg.licenseDeclared ?? null, license_concluded_raw: pkg.licenseConcluded ?? null, is_root: ordinal === 0, scope: ordinal === 0 ? "root" : "direct", + license: licenseFor(ordinal), }; }); +const detailComponent = components[1]; +const licenseSummary = {}; +for (const item of components) if (item.license) licenseSummary[item.license.status] = (licenseSummary[item.license.status] || 0) + 1; const snapshot = { id: 11, repository_id: 1, stream: "github:source", producer: "github", subject: "source", collected_at: collected, created_at_claimed: doc.creationInfo.created, producer_tool: "GitHub.com-Dependency-Graph", document_name: doc.name, @@ -42,11 +66,25 @@ const responses = { freshness_seconds: 93600, latest_snapshot: snapshot, last_collection: {id: 40, job_id: 9, producer: "github", stream: "github:source", started_at: "2026-09-22T11:59:58Z", finished_at: "2026-09-22T12:00:00Z", outcome: "forbidden", http_status: 403, snapshot_id: null, error_code: "github_forbidden", message: "GitHub returned 403 for the SBOM export: the dependency graph may be disabled, the installation may lack Contents read access, or the endpoint may be unsupported on this GitHub version."}, - active_job: null, enrichment: "not_configured", opt_out: false, + active_job: null, enrichment: "configured", enrichment_ecosystems: ["npm"], license_summary: licenseSummary, opt_out: false, notes: ["The latest refresh failed; the inventory shown is the last successful observation.", "GitHub dependency-graph exports are timestamped observations of the default branch; they are not bound to a commit and carry no license data.", "Normalization reported 3 coverage warning(s)."], documents: [{snapshot_id: 11, sha256: snapshot.document_sha256, format: "spdx-2.3-json", bytes: 4650, path: "/v1/supply-chain/snapshots/11/document"}], }, "/v1/supply-chain/repositories/101/components": {snapshot_id: 11, components, truncated: false}, + "/v1/supply-chain/repositories/101/component": { + component: detailComponent, snapshot, + declarations: [ + {id: 0, source: "producer_concluded", ecosystem: "npm", namespace: "@scope", name: "left-pad", version: "1.3.0", raw_value: "NOASSERTION", raw_kind: "sentinel", parse_status: "no_assertion", resolver_version: 1, license_list_version: "3.27.0", fetched_at: collected, outcome: "resolved"}, + {id: 0, source: "producer_declared", ecosystem: "npm", namespace: "@scope", name: "left-pad", version: "1.3.0", raw_value: "NOASSERTION", raw_kind: "sentinel", parse_status: "no_assertion", resolver_version: 1, license_list_version: "3.27.0", fetched_at: collected, outcome: "resolved"}, + ], + evidence: [ + {id: 21, source: "registry_npm", route: "npm:npm.example.internal", ecosystem: "npm", namespace: "@scope", name: "left-pad", version: "1.3.0", raw_value: "", raw_kind: "missing", parse_status: "not_applicable", resolver_version: 1, license_list_version: "3.27.0", fetched_at: "2026-09-22T09:00:00Z", expires_at: "2026-09-23T09:00:00Z", outcome: "unavailable", http_status: 503, message: "registry returned an unexpected status"}, + {id: 20, source: "registry_npm", route: "npm:npm.example.internal", ecosystem: "npm", namespace: "@scope", name: "left-pad", version: "1.3.0", raw_value: "MIT", raw_kind: "expression", parse_status: "parsed", expression: "MIT", detail: {integrity: "sha512-abc"}, resolver_version: 1, license_list_version: "3.27.0", content_sha256: "9f".repeat(32), fetched_at: assessed, outcome: "resolved"}, + ], + relationships: [{from: doc.packages[0].SPDXID, type: "DEPENDS_ON", to: detailComponent.element_id, resolved: true}], + notes: ["Publisher declarations and registry metadata are evidence, not approval; a human conclusion or policy decision is recorded separately."], + truncated: false, + }, "/v1/supply-chain/repositories/101/collections": {collections: [ {id: 40, job_id: 9, producer: "github", stream: "github:source", started_at: "2026-09-22T11:59:58Z", finished_at: "2026-09-22T12:00:00Z", outcome: "forbidden", http_status: 403, snapshot_id: null, error_code: "github_forbidden", message: "GitHub returned 403 for the SBOM export."}, {id: 39, job_id: 8, producer: "github", stream: "github:source", started_at: "2026-09-21T09:59:57Z", finished_at: collected, outcome: "published", http_status: 200, snapshot_id: 11}, @@ -89,6 +127,9 @@ try { await page.getByLabel("Bearer token").fill("demo"); await page.getByRole("button", {name: "Open inventory"}).click(); await page.locator("#sc-rows tr").first().waitFor(); + await page.getByRole("button", {name: detailComponent.name, exact: true}).first().click(); + await page.locator("#sc-detail").waitFor({state: "visible"}); + await page.getByText("No registry evidence for these coordinates.").waitFor({state: "hidden"}).catch(() => {}); if (theme === "light") await page.locator("#sc-theme").click(); await page.waitForTimeout(150); await page.screenshot({path: path.join(outDir, `supply-chain-${theme}.png`), fullPage: true});