diff --git a/tools/repo-allowlist-sync/README.md b/tools/repo-allowlist-sync/README.md new file mode 100644 index 00000000000..1c8d42e523e --- /dev/null +++ b/tools/repo-allowlist-sync/README.md @@ -0,0 +1,71 @@ +# Repository Allowlist Sync Tool (`repo-allowlist-sync`) + +`repo-allowlist-sync` is a Go command-line tool that synchronizes repository allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoAllowList` entities). + +## Overview + +Gitter has options to enumerate affected commits with `consider_all_branches` and cherrypick detection options (`cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`). This tool manages the Datastore allowlist index (`RepoAllowList`) that controls these behaviors on a repository level. + +> [!NOTE] +> If feature flags are already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist. + +The tool performs a two-way sync: + +- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities. +- **Delete**: Removes entities from Datastore that are no longer present in the YAML file. + +## Allowlist YAML Format + +The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields. +* Supported `type`s: `url`, `regex` + * `url`: A URL to match against the repository URL. + * `regex`: A regex pattern to match against the repository URL (Go RE2 syntax). +* Supported boolean flags: `consider_all_branches`, `cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`, `cherrypicks` (shorthand for all 3 cherrypick flags) + +```yaml +# Examples + +- type: url + value: "https://github.com/google/osv.dev.git" + consider_all_branches: true + cherrypicks: true + +- type: url + value: "https://github.com/google/osv.dev.git" + consider_all_branches: true + cherrypicks: true + cherrypicks_fixed: false # Overrides cherrypicks: true for fixed event + +- type: regex + value: 'github\.com/google/osv-.*' + consider_all_branches: true + cherrypicks_introduced: true + cherrypicks_fixed: true + cherrypicks_limit: true +``` + +> [!TIP] +> Use single quotes for regex values so you don't have to escape backslashes or other special characters. + +### Normalization and Validation + +- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. +- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. +- **`cherrypicks`**: Acts as a shorthand for setting `cherrypicks_introduced`, `cherrypicks_fixed`, and `cherrypicks_limit` simultaneously. Specific `cherrypicks_` fields override the shorthand value if provided. + +## Usage + +Run the tool using `go run`: + +```bash +go run . [flags] +``` + +### Options & Flags + +| Flag | Default | Description | +| ----------- | --------------------- | ----------------------------------------------------------------- | +| `--file` | `repo_allowlist.yaml` | Path to the input YAML allowlist file | +| `--project` | `oss-vdb-test` | Target GCP Project ID | +| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | +| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-allowlist-sync/go.mod b/tools/repo-allowlist-sync/go.mod new file mode 100644 index 00000000000..941de59e832 --- /dev/null +++ b/tools/repo-allowlist-sync/go.mod @@ -0,0 +1,41 @@ +module github.com/google/osv.dev/tools/repo-allowlist-sync + +go 1.26.5 + +require ( + cloud.google.com/go/datastore v1.25.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.287.1 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/grpc v1.82.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/tools/repo-allowlist-sync/go.sum b/tools/repo-allowlist-sync/go.sum new file mode 100644 index 00000000000..7c948c6563f --- /dev/null +++ b/tools/repo-allowlist-sync/go.sum @@ -0,0 +1,101 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/datastore v1.25.0 h1:zUjMnCLCcRZVDSdQIXsbnNCl1SVRNw5Jm0J77gPaPKs= +cloud.google.com/go/datastore v1.25.0/go.mod h1:jvJVNe+S2nHVIndV1H/B4s9K3MLsTMqOKlxSrzHTxB4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/repo-allowlist-sync/main.go b/tools/repo-allowlist-sync/main.go new file mode 100644 index 00000000000..d8a0a0bf2bb --- /dev/null +++ b/tools/repo-allowlist-sync/main.go @@ -0,0 +1,245 @@ +// Package main implements a CLI tool to sync Repository AllowList YAML files to Cloud Datastore. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/url" + "os" + "regexp" + "strings" + + "cloud.google.com/go/datastore" + "gopkg.in/yaml.v3" +) + +// RepoAllowListEntity represents a repository allowlist entity stored in Cloud Datastore +type RepoAllowListEntity struct { + Key *datastore.Key `datastore:"__key__"` + Type string `datastore:"type"` + Value string `datastore:"value"` + ConsiderAllBranches bool `datastore:"consider_all_branches"` + CherrypicksIntroduced bool `datastore:"cherrypicks_introduced"` + CherrypicksFixed bool `datastore:"cherrypicks_fixed"` + CherrypicksLimit bool `datastore:"cherrypicks_limit"` +} + +// rawYAMLEntry represents an unmarshaled entry from the YAML file, including optional shorthand field +type rawYAMLEntry struct { + Type string `yaml:"type"` + Value string `yaml:"value"` + ConsiderAllBranches bool `yaml:"consider_all_branches"` + Cherrypicks *bool `yaml:"cherrypicks"` + CherrypicksIntroduced *bool `yaml:"cherrypicks_introduced"` + CherrypicksFixed *bool `yaml:"cherrypicks_fixed"` + CherrypicksLimit *bool `yaml:"cherrypicks_limit"` +} + +func (e RepoAllowListEntity) matches(other RepoAllowListEntity) bool { + return e.Type == other.Type && + e.ConsiderAllBranches == other.ConsiderAllBranches && + e.CherrypicksIntroduced == other.CherrypicksIntroduced && + e.CherrypicksFixed == other.CherrypicksFixed && + e.CherrypicksLimit == other.CherrypicksLimit +} + +func main() { + filePath := flag.String("file", "repo_allowlist.yaml", "Path to repo_allowlist YAML file") + project := flag.String("project", "oss-vdb-test", "GCP project ID") + dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") + verbose := flag.Bool("verbose", false, "Display verbose sync operations") + + flag.Parse() + + if *filePath == "" { + log.Fatalf("Error: --file argument is required") + } + + if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { + log.Fatalf("Error syncing repo allowlist: %v", err) + } +} + +// normalizeRepo removes the URL scheme, trailing slashes, and .git extensions to standardize repo paths. +func normalizeRepo(repoURL string) string { + repoURL = strings.TrimSpace(repoURL) + if repoURL == "" { + return "" + } + + if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { + log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) + return "" + } + + parsed, err := url.Parse(repoURL) + if err != nil { + return repoURL + } + normalized := parsed.Host + parsed.Path + normalized = strings.TrimRight(normalized, "/") + normalized = strings.TrimSuffix(normalized, ".git") + + return normalized +} + +// parseYAMLEntries parses and validates allowlist YAML content, expanding shorthand fields and normalizing values. +func parseYAMLEntries(data []byte) ([]RepoAllowListEntity, error) { + var rawEntries []rawYAMLEntry + if err := yaml.Unmarshal(data, &rawEntries); err != nil { + return nil, err + } + + var entries []RepoAllowListEntity + for _, raw := range rawEntries { + raw.Type = strings.TrimSpace(strings.ToLower(raw.Type)) + + switch raw.Type { + case "url": + raw.Value = normalizeRepo(raw.Value) + if raw.Value == "" { + continue + } + case "regex": + if raw.Value == "" { + continue + } + if _, err := regexp.Compile(raw.Value); err != nil { + log.Printf("Warning: Skipping invalid regex pattern %q: %v", raw.Value, err) + continue + } + default: + log.Printf("Warning: Skipping unrecognized entry type %q for value %q", raw.Type, raw.Value) + continue + } + + // Process cherrypicks flags: "cherrypicks: bool" acts as a shorthand for all 3 event types, + // specifying "cherrypicks_" fields overrides that. + intro := false + fixed := false + limit := false + + if raw.Cherrypicks != nil { + intro = *raw.Cherrypicks + fixed = *raw.Cherrypicks + limit = *raw.Cherrypicks + } + if raw.CherrypicksIntroduced != nil { + intro = *raw.CherrypicksIntroduced + } + if raw.CherrypicksFixed != nil { + fixed = *raw.CherrypicksFixed + } + if raw.CherrypicksLimit != nil { + limit = *raw.CherrypicksLimit + } + + entries = append(entries, RepoAllowListEntity{ + Type: raw.Type, + Value: raw.Value, + ConsiderAllBranches: raw.ConsiderAllBranches, + CherrypicksIntroduced: intro, + CherrypicksFixed: fixed, + CherrypicksLimit: limit, + }) + } + + return entries, nil +} + +func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed reading file %s: %w", filePath, err) + } + + entries, err := parseYAMLEntries(data) + if err != nil { + return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) + } + + if verbose { + log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) + } + + dsClient, err := datastore.NewClient(ctx, project) + if err != nil { + return fmt.Errorf("failed creating datastore client for project %s: %w", project, err) + } + defer func() { _ = dsClient.Close() }() + + // Fetch existing Datastore entities + query := datastore.NewQuery("RepoAllowList") + var dsEntities []RepoAllowListEntity + if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { + return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) + } + + dsEntitiesMap := make(map[string]RepoAllowListEntity) + for _, entity := range dsEntities { + dsEntitiesMap[entity.Value] = entity + } + + localEntriesMap := make(map[string]RepoAllowListEntity) + for _, item := range entries { + localEntriesMap[item.Value] = item + } + + // Upsert entries in local YAML that are missing from Datastore or modified + for val, item := range localEntriesMap { + existing, exists := dsEntitiesMap[val] + entity := &RepoAllowListEntity{ + Type: item.Type, + Value: item.Value, + ConsiderAllBranches: item.ConsiderAllBranches, + CherrypicksIntroduced: item.CherrypicksIntroduced, + CherrypicksFixed: item.CherrypicksFixed, + CherrypicksLimit: item.CherrypicksLimit, + } + + if !exists { + key := datastore.IncompleteKey("RepoAllowList", nil) + if !dryRun { + if _, err := dsClient.Put(ctx, key, entity); err != nil { + return fmt.Errorf("failed putting entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Creating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } else if !existing.matches(item) { + entity.Key = existing.Key + if !dryRun { + if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { + return fmt.Errorf("failed updating entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Updating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } + } + + // Delete entries in Datastore that are no longer present in local YAML + for val, existing := range dsEntitiesMap { + if _, exists := localEntriesMap[val]; !exists { + if verbose { + log.Printf("Deleting RepoAllowList entity: val=%s", val) + } + if !dryRun { + if err := dsClient.Delete(ctx, existing.Key); err != nil { + return fmt.Errorf("failed deleting entity for %s: %w", val, err) + } + } + } + } + + if dryRun { + log.Println("[DRY RUN] Sync completed successfully.") + } else { + log.Println("Sync completed successfully.") + } + return nil +} diff --git a/tools/repo-allowlist-sync/main_test.go b/tools/repo-allowlist-sync/main_test.go new file mode 100644 index 00000000000..cc5e572538d --- /dev/null +++ b/tools/repo-allowlist-sync/main_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestNormalizeRepo(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "Empty string", + input: "", + expected: "", + }, + { + name: "HTTPS URL with .git", + input: "https://github.com/google/osv.dev.git", + expected: "github.com/google/osv.dev", + }, + { + name: "HTTPS URL without .git", + input: "https://github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + }, + { + name: "URL with trailing slash", + input: "https://github.com/google/osv.dev/", + expected: "github.com/google/osv.dev", + }, + { + name: "No scheme URL", + input: "github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + }, + { + name: "No scheme with .git", + input: "github.com/google/osv-scanner.git", + expected: "github.com/google/osv-scanner", + }, + { + name: "Whitespace in URL", + input: " https://github.com/google/osv.dev.git ", + expected: "github.com/google/osv.dev", + }, + { + name: "SSH URL format git@", + input: "git@github.com:google/osv.dev.git", + expected: "", + }, + { + name: "SSH URL format ssh://", + input: "ssh://git@github.com/google/osv.dev.git", + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := normalizeRepo(tt.input) + if got != tt.expected { + t.Errorf("normalizeRepo(%q) = %q, want %q", tt.input, got, tt.expected) + } + }) + } +} + +func TestParseYAMLEntries(t *testing.T) { + yamlContent := []byte(` +- type: URL + value: " https://github.com/google/osv.dev.git " + consider_all_branches: true + cherrypicks_introduced: true +- type: REGEX + value: 'github\.com/google/osv-.*' + cherrypicks: true +- type: url + value: "https://github.com/noflags/repo.git" +- type: regex + value: '[invalid regex' +- type: unknown + value: "https://github.com/google/osv.dev" +- type: url + value: "git@github.com:ssh/isnot.supported.git" +`) + + want := []RepoAllowListEntity{ + // Normalized URL + { + Type: "url", + Value: "github.com/google/osv.dev", + ConsiderAllBranches: true, + CherrypicksIntroduced: true, + CherrypicksFixed: false, + CherrypicksLimit: false, + }, + // Regex type and cherrypicks: true populates all 3 event types + { + Type: "regex", + Value: `github\.com/google/osv-.*`, + ConsiderAllBranches: false, + CherrypicksIntroduced: true, + CherrypicksFixed: true, + CherrypicksLimit: true, + }, + // No flags set (Shouldn't really happen) + { + Type: "url", + Value: "github.com/noflags/repo", + ConsiderAllBranches: false, + CherrypicksIntroduced: false, + CherrypicksFixed: false, + CherrypicksLimit: false, + }, + } + + got, err := parseYAMLEntries(yamlContent) + if err != nil { + t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) + } + + if len(got) != len(want) { + t.Fatalf("expected %d valid entries, got %d: %+v", len(want), len(got), got) + } + + for i, wantEntry := range want { + if got[i] != wantEntry { + t.Errorf("entry %d mismatch:\n got: %+v\nwant: %+v", i, got[i], wantEntry) + } + } +} + +func TestRun_InvalidFile(t *testing.T) { + err := run(context.Background(), "non_existent_file.yaml", "test-project", true, false) + if err == nil { + t.Error("expected error for non-existent file, got nil") + } + + tmpDir := t.TempDir() + badYAMLPath := filepath.Join(tmpDir, "bad.yaml") + if err := os.WriteFile(badYAMLPath, []byte("invalid: yaml: ["), 0644); err != nil { + t.Fatalf("failed creating bad yaml file: %v", err) + } + + err = run(context.Background(), badYAMLPath, "test-project", true, false) + if err == nil { + t.Error("expected error for invalid YAML file, got nil") + } +} diff --git a/tools/repo-allowlist-sync/repo_allowlist.yaml b/tools/repo-allowlist-sync/repo_allowlist.yaml new file mode 100644 index 00000000000..ec257649e33 --- /dev/null +++ b/tools/repo-allowlist-sync/repo_allowlist.yaml @@ -0,0 +1,15 @@ +# Repository allowlist configuration +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# consider_all_branches: true +# cherrypicks: true # Shorthand: sets introduced, fixed, and limit to true +# - type: url +# value: "https://github.com/apache/hadoop.git" +# consider_all_branches: true +# cherrypicks: true +# cherrypicks_fixed: false # Specific override for fixed +# - type: regex +# value: 'github\.com/google/osv-.*' +# consider_all_branches: true diff --git a/tools/repo-allowlist-sync/repo_allowlist_test.yaml b/tools/repo-allowlist-sync/repo_allowlist_test.yaml new file mode 100644 index 00000000000..4c6c8bfccd3 --- /dev/null +++ b/tools/repo-allowlist-sync/repo_allowlist_test.yaml @@ -0,0 +1,13 @@ +# Repository allowlist configuration +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# consider_all_branches: true +# - type: regex +# value: 'github\.com/google/osv-.*' +# cherrypicks_fixed: true +- type: url + value: 'https://github.com/apache/hadoop.git' + consider_all_branches: true + cherrypicks: true