From 050448fcf4f589c09ab81efa77dbe2bd1cb9157f Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 01:24:45 +0000 Subject: [PATCH 1/6] Add repo-based cab allowlist datastore sync tool --- tools/repo-cab-sync/README.md | 56 ++++++ tools/repo-cab-sync/go.mod | 41 +++++ tools/repo-cab-sync/go.sum | 101 ++++++++++ tools/repo-cab-sync/main.go | 173 ++++++++++++++++++ tools/repo-cab-sync/main_test.go | 105 +++++++++++ tools/repo-cab-sync/repo_cab_allowlist.yaml | 7 + .../repo_cab_allowlist_test.yaml | 7 + 7 files changed, 490 insertions(+) create mode 100644 tools/repo-cab-sync/README.md create mode 100644 tools/repo-cab-sync/go.mod create mode 100644 tools/repo-cab-sync/go.sum create mode 100644 tools/repo-cab-sync/main.go create mode 100644 tools/repo-cab-sync/main_test.go create mode 100644 tools/repo-cab-sync/repo_cab_allowlist.yaml create mode 100644 tools/repo-cab-sync/repo_cab_allowlist_test.yaml diff --git a/tools/repo-cab-sync/README.md b/tools/repo-cab-sync/README.md new file mode 100644 index 00000000000..de24c7f6431 --- /dev/null +++ b/tools/repo-cab-sync/README.md @@ -0,0 +1,56 @@ +# Repo Consider All Branches Allowlist Sync Tool (`repo-cab-sync`) + +`repo-cab-sync` is a Go command-line tool that synchronizes repository "Consider All Branches" (CAB) allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoConsiderAllBranchesAllowList` entities). + +## Overview + +Gitter has the option to enumerate affected commits with `consider_all_branches` enabled or disabled. This tool manages the Datastore allowlist index (`RepoConsiderAllBranchesAllowList`) that controls this behavior on a repository level. + +> [!NOTE] +> If `consider_all_branches` is already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist. + +The tool performs a two-way sync: + +- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities. +- **Delete**: Removes entities from Datastore that are no longer present in the YAML file. + +## Allowlist YAML Format + +The allowlist YAML configuration file accepts a list of entries with `type` and `value` fields: + +```yaml +# Supported entry types: 'url' and 'regex' + +# Exact repository URL match +- type: url + value: "https://github.com/google/osv.dev.git" + +# Regex pattern match (Go RE2 syntax) +- type: regex + value: 'github\.com/google/osv-.*' +``` + +> [!TIP] +> Use single quotes for regex values so you don't have to escape backslashes or other special characters. + +### Normalization and Validation + +- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. +- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. + +## Usage + +Run the tool using `go run`: + +```bash +go run . [flags] +``` + +### Options & Flags + +| Flag | Default | Description | +| ----------- | ------------------------- | ----------------------------------------------------------------- | +| `--file` | `repo_cab_allowlist.yaml` | Path to the input YAML allowlist file | +| `--project` | `oss-vdb-test` | Target GCP Project ID | +| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | +| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-cab-sync/go.mod b/tools/repo-cab-sync/go.mod new file mode 100644 index 00000000000..4595f7b868a --- /dev/null +++ b/tools/repo-cab-sync/go.mod @@ -0,0 +1,41 @@ +module github.com/google/osv.dev/tools/repo-cab-sync + +go 1.26.5 + +require ( + cloud.google.com/go/datastore v1.25.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.287.1 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/grpc v1.82.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/tools/repo-cab-sync/go.sum b/tools/repo-cab-sync/go.sum new file mode 100644 index 00000000000..360fa03b5ce --- /dev/null +++ b/tools/repo-cab-sync/go.sum @@ -0,0 +1,101 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/datastore v1.25.0 h1:zUjMnCLCcRZVDSdQIXsbnNCl1SVRNw5Jm0J77gPaPKs= +cloud.google.com/go/datastore v1.25.0/go.mod h1:jvJVNe+S2nHVIndV1H/B4s9K3MLsTMqOKlxSrzHTxB4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/repo-cab-sync/main.go b/tools/repo-cab-sync/main.go new file mode 100644 index 00000000000..44dcba17a8c --- /dev/null +++ b/tools/repo-cab-sync/main.go @@ -0,0 +1,173 @@ +// Package main implements a CLI tool to sync Repo Consider All Branches (CAB) allowlist YAML files to Cloud Datastore. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/url" + "os" + "regexp" + "strings" + + "cloud.google.com/go/datastore" + "gopkg.in/yaml.v3" +) + +type RepoCABEntity struct { + Key *datastore.Key `yaml:"-" datastore:"__key__"` + Type string `yaml:"type" datastore:"type"` + Value string `yaml:"value" datastore:"value"` +} + +func main() { + filePath := flag.String("file", "repo_cab_allowlist.yaml", "Path to repo_cab_allowlist YAML file") + project := flag.String("project", "oss-vdb-test", "GCP project ID") + dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") + verbose := flag.Bool("verbose", false, "Display verbose sync operations") + + flag.Parse() + + if *filePath == "" { + log.Fatalf("Error: --file argument is required") + } + + if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { + log.Fatalf("Error syncing repo CAB allowlist: %v", err) + } +} + +func normalizeRepo(repoURL string) string { + // Normalize the repo_url to align with matching logic + // Removes the scheme/protocol, the .git extension, and trailing slashes. + if repoURL == "" { + return "" + } + parsed, err := url.Parse(repoURL) + if err != nil { + return repoURL + } + normalized := parsed.Host + parsed.Path + normalized = strings.TrimRight(normalized, "/") + normalized = strings.TrimSuffix(normalized, ".git") + + return normalized +} + +func parseYAMLEntries(data []byte) ([]RepoCABEntity, error) { + var parsed []RepoCABEntity + if err := yaml.Unmarshal(data, &parsed); err != nil { + return nil, err + } + + var entries []RepoCABEntity + for _, entry := range parsed { + if entry.Type == "url" { + // For repo URLs, we normalize the value before inserting to datastore + entry.Value = normalizeRepo(entry.Value) + } else if entry.Type == "regex" { + // For regex, we make sure it compiles + if _, err := regexp.Compile(entry.Value); err != nil { + log.Printf("Warning: Skipping invalid regex pattern %q: %v", entry.Value, err) + continue + } + } + entries = append(entries, entry) + } + + return entries, nil +} + +func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed reading file %s: %w", filePath, err) + } + + entries, err := parseYAMLEntries(data) + if err != nil { + return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) + } + + if verbose { + log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) + } + + dsClient, err := datastore.NewClient(ctx, project) + if err != nil { + return fmt.Errorf("failed creating datastore client for project %s: %w", project, err) + } + defer func() { _ = dsClient.Close() }() + + // Get existing Datastore entities + query := datastore.NewQuery("RepoConsiderAllBranchesAllowList") + var dsEntities []RepoCABEntity + if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { + return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) + } + + dsEntitiesMap := make(map[string]RepoCABEntity) + for _, entity := range dsEntities { + dsEntitiesMap[entity.Value] = entity + } + + localEntriesMap := make(map[string]RepoCABEntity) + for _, item := range entries { + localEntriesMap[item.Value] = item + } + + // 1. Put/Upsert entries in local YAML that are not in Datastore or modified + for val, item := range localEntriesMap { + existing, exists := dsEntitiesMap[val] + if !exists { + key := datastore.IncompleteKey("RepoConsiderAllBranchesAllowList", nil) + entity := &RepoCABEntity{ + Type: item.Type, + Value: item.Value, + } + if !dryRun { + if _, err := dsClient.Put(ctx, key, entity); err != nil { + return fmt.Errorf("failed putting entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Creating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } else if existing.Type != item.Type { + entity := &RepoCABEntity{ + Type: item.Type, + Value: item.Value, + } + if !dryRun { + if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { + return fmt.Errorf("failed updating entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Updating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } + } + + // 2. Delete entries in Datastore that are no longer in local YAML + for val, existing := range dsEntitiesMap { + if _, exists := localEntriesMap[val]; !exists { + if verbose { + log.Printf("Deleting RepoConsiderAllBranchesAllowList entity: val=%s", val) + } + if !dryRun { + if err := dsClient.Delete(ctx, existing.Key); err != nil { + return fmt.Errorf("failed deleting entity for %s: %w", val, err) + } + } + } + } + + if dryRun { + log.Println("[DRY RUN] Sync completed successfully.") + } else { + log.Println("Sync completed successfully.") + } + return nil +} diff --git a/tools/repo-cab-sync/main_test.go b/tools/repo-cab-sync/main_test.go new file mode 100644 index 00000000000..b06be9414ca --- /dev/null +++ b/tools/repo-cab-sync/main_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestNormalizeRepo(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "Empty string", + input: "", + expected: "", + }, + { + name: "HTTPS URL with .git", + input: "https://github.com/google/osv.dev.git", + expected: "github.com/google/osv.dev", + }, + { + name: "HTTPS URL without .git", + input: "https://github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + }, + { + name: "URL with trailing slash", + input: "https://github.com/google/osv.dev/", + expected: "github.com/google/osv.dev", + }, + { + name: "No scheme URL", + input: "github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + }, + { + name: "No scheme with .git", + input: "github.com/google/osv-scanner.git", + expected: "github.com/google/osv-scanner", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := normalizeRepo(tt.input) + if got != tt.expected { + t.Errorf("normalizeRepo(%q) = %q, want %q", tt.input, got, tt.expected) + } + }) + } +} + +func TestParseYAMLEntries(t *testing.T) { + yamlContent := []byte(` +- type: url + value: "https://github.com/google/osv.dev.git" +- type: regex + value: 'github\.com/google/osv-.*' +- type: regex + value: '[invalid regex' +`) + + entries, err := parseYAMLEntries(yamlContent) + if err != nil { + t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) + } + + // Should skip invalid regex and return 2 entries + if len(entries) != 2 { + t.Fatalf("expected 2 valid entries, got %d", len(entries)) + } + + // First entry should be normalized URL + if entries[0].Type != "url" || entries[0].Value != "github.com/google/osv.dev" { + t.Errorf("entry 0 mismatch: got type=%q val=%q, want type=url val=github.com/google/osv.dev", entries[0].Type, entries[0].Value) + } + + // Second entry should preserve regex string + if entries[1].Type != "regex" || entries[1].Value != `github\.com/google/osv-.*` { + t.Errorf("entry 1 mismatch: got type=%q val=%q, want type=regex val=github\\.com/google/osv-.*", entries[1].Type, entries[1].Value) + } +} + +func TestRun_InvalidFile(t *testing.T) { + err := run(context.Background(), "non_existent_file.yaml", "test-project", true, false) + if err == nil { + t.Error("expected error for non-existent file, got nil") + } + + tmpDir := t.TempDir() + badYAMLPath := filepath.Join(tmpDir, "bad.yaml") + if err := os.WriteFile(badYAMLPath, []byte("invalid: yaml: ["), 0644); err != nil { + t.Fatalf("failed creating bad yaml file: %v", err) + } + + err = run(context.Background(), badYAMLPath, "test-project", true, false) + if err == nil { + t.Error("expected error for invalid YAML file, got nil") + } +} diff --git a/tools/repo-cab-sync/repo_cab_allowlist.yaml b/tools/repo-cab-sync/repo_cab_allowlist.yaml new file mode 100644 index 00000000000..2b477302dc5 --- /dev/null +++ b/tools/repo-cab-sync/repo_cab_allowlist.yaml @@ -0,0 +1,7 @@ +# Repository-based consider all branches allowlist +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# - type: regex +# value: 'github\.com/google/osv-.*' diff --git a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml new file mode 100644 index 00000000000..2b477302dc5 --- /dev/null +++ b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml @@ -0,0 +1,7 @@ +# Repository-based consider all branches allowlist +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# - type: regex +# value: 'github\.com/google/osv-.*' From fd946a0e0614e4d77df4bff719bccbcafb1b552b Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 01:50:22 +0000 Subject: [PATCH 2/6] update some dependencies --- tools/repo-cab-sync/go.mod | 10 +++++----- tools/repo-cab-sync/go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/repo-cab-sync/go.mod b/tools/repo-cab-sync/go.mod index 4595f7b868a..c316e0eba69 100644 --- a/tools/repo-cab-sync/go.mod +++ b/tools/repo-cab-sync/go.mod @@ -25,17 +25,17 @@ require ( go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - golang.org/x/crypto v0.53.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.287.1 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect - google.golang.org/grpc v1.82.0 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/tools/repo-cab-sync/go.sum b/tools/repo-cab-sync/go.sum index 360fa03b5ce..7c948c6563f 100644 --- a/tools/repo-cab-sync/go.sum +++ b/tools/repo-cab-sync/go.sum @@ -66,18 +66,18 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -90,8 +90,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 1b321ffcca2b18819832b7fab00dd2f7c963ef65 Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 01:53:00 +0000 Subject: [PATCH 3/6] populate a repo in test --- tools/repo-cab-sync/repo_cab_allowlist_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml index 2b477302dc5..4e83defe770 100644 --- a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml +++ b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml @@ -5,3 +5,5 @@ # value: "https://github.com/google/osv.dev.git" # - type: regex # value: 'github\.com/google/osv-.*' +- type: url + value: 'https://github.com/apache/hadoop.git' From 03dd4983cb966709045ed15aa04125acfeb98456 Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 02:03:49 +0000 Subject: [PATCH 4/6] warn if not url or regex type --- tools/repo-cab-sync/main.go | 24 ++++++++++++++++++++++-- tools/repo-cab-sync/main_test.go | 29 ++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/tools/repo-cab-sync/main.go b/tools/repo-cab-sync/main.go index 44dcba17a8c..7ab771a54c5 100644 --- a/tools/repo-cab-sync/main.go +++ b/tools/repo-cab-sync/main.go @@ -41,9 +41,16 @@ func main() { func normalizeRepo(repoURL string) string { // Normalize the repo_url to align with matching logic // Removes the scheme/protocol, the .git extension, and trailing slashes. + repoURL = strings.TrimSpace(repoURL) if repoURL == "" { return "" } + + if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { + log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) + return "" + } + parsed, err := url.Parse(repoURL) if err != nil { return repoURL @@ -63,16 +70,29 @@ func parseYAMLEntries(data []byte) ([]RepoCABEntity, error) { var entries []RepoCABEntity for _, entry := range parsed { - if entry.Type == "url" { + entry.Type = strings.TrimSpace(strings.ToLower(entry.Type)) + + switch entry.Type { + case "url": // For repo URLs, we normalize the value before inserting to datastore entry.Value = normalizeRepo(entry.Value) - } else if entry.Type == "regex" { + if entry.Value == "" { + continue + } + case "regex": // For regex, we make sure it compiles + if entry.Value == "" { + continue + } if _, err := regexp.Compile(entry.Value); err != nil { log.Printf("Warning: Skipping invalid regex pattern %q: %v", entry.Value, err) continue } + default: + log.Printf("Warning: Skipping unrecognized entry type %q for value %q", entry.Type, entry.Value) + continue } + entries = append(entries, entry) } diff --git a/tools/repo-cab-sync/main_test.go b/tools/repo-cab-sync/main_test.go index b06be9414ca..c7c5a8cca45 100644 --- a/tools/repo-cab-sync/main_test.go +++ b/tools/repo-cab-sync/main_test.go @@ -43,6 +43,21 @@ func TestNormalizeRepo(t *testing.T) { input: "github.com/google/osv-scanner.git", expected: "github.com/google/osv-scanner", }, + { + name: "Whitespace in URL", + input: " https://github.com/google/osv.dev.git ", + expected: "github.com/google/osv.dev", + }, + { + name: "SSH URL format git@", + input: "git@github.com:google/osv.dev.git", + expected: "", + }, + { + name: "SSH URL format ssh://", + input: "ssh://git@github.com/google/osv.dev.git", + expected: "", + }, } for _, tt := range tests { @@ -57,12 +72,16 @@ func TestNormalizeRepo(t *testing.T) { func TestParseYAMLEntries(t *testing.T) { yamlContent := []byte(` -- type: url - value: "https://github.com/google/osv.dev.git" -- type: regex +- type: URL + value: " https://github.com/google/osv.dev.git " +- type: REGEX value: 'github\.com/google/osv-.*' - type: regex value: '[invalid regex' +- type: unknown + value: "https://github.com/google/osv.dev" +- type: url + value: "git@github.com:google/osv.dev.git" `) entries, err := parseYAMLEntries(yamlContent) @@ -70,9 +89,9 @@ func TestParseYAMLEntries(t *testing.T) { t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) } - // Should skip invalid regex and return 2 entries + // Should normalize types, trim values, and skip invalid regex, unrecognized types, and SSH URLs if len(entries) != 2 { - t.Fatalf("expected 2 valid entries, got %d", len(entries)) + t.Fatalf("expected 2 valid entries, got %d: %+v", len(entries), entries) } // First entry should be normalized URL From 8352fb8d030e1c4892324679b3572350c3f93c82 Mon Sep 17 00:00:00 2001 From: Joey L Date: Fri, 7 Aug 2026 05:13:39 +0000 Subject: [PATCH 5/6] Expand cab allowlist to cherrypick options --- tools/repo-allowlist-sync/README.md | 70 +++++ .../go.mod | 2 +- .../go.sum | 0 tools/repo-allowlist-sync/main.go | 245 ++++++++++++++++++ .../main_test.go | 56 +++- tools/repo-allowlist-sync/repo_allowlist.yaml | 15 ++ .../repo_allowlist_test.yaml} | 6 +- tools/repo-cab-sync/README.md | 56 ---- tools/repo-cab-sync/main.go | 193 -------------- tools/repo-cab-sync/repo_cab_allowlist.yaml | 7 - 10 files changed, 379 insertions(+), 271 deletions(-) create mode 100644 tools/repo-allowlist-sync/README.md rename tools/{repo-cab-sync => repo-allowlist-sync}/go.mod (96%) rename tools/{repo-cab-sync => repo-allowlist-sync}/go.sum (100%) create mode 100644 tools/repo-allowlist-sync/main.go rename tools/{repo-cab-sync => repo-allowlist-sync}/main_test.go (65%) create mode 100644 tools/repo-allowlist-sync/repo_allowlist.yaml rename tools/{repo-cab-sync/repo_cab_allowlist_test.yaml => repo-allowlist-sync/repo_allowlist_test.yaml} (65%) delete mode 100644 tools/repo-cab-sync/README.md delete mode 100644 tools/repo-cab-sync/main.go delete mode 100644 tools/repo-cab-sync/repo_cab_allowlist.yaml diff --git a/tools/repo-allowlist-sync/README.md b/tools/repo-allowlist-sync/README.md new file mode 100644 index 00000000000..ebed96ee8a7 --- /dev/null +++ b/tools/repo-allowlist-sync/README.md @@ -0,0 +1,70 @@ +# Repository Allowlist Sync Tool (`repo-allowlist-sync`) + +`repo-allowlist-sync` is a Go command-line tool that synchronizes repository allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoAllowList` entities). + +## Overview + +Gitter has options to enumerate affected commits with `consider_all_branches` and cherrypick detection options (`cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`). This tool manages the Datastore allowlist index (`RepoAllowList`) that controls these behaviors on a repository level. + +> [!NOTE] +> If feature flags are already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist. + +The tool performs a two-way sync: + +- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities. +- **Delete**: Removes entities from Datastore that are no longer present in the YAML file. + +## Allowlist YAML Format + +The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields: + +```yaml +# Supported entry types: 'url' and 'regex' + +# Shorthand: 'cherrypicks: true' applies to all 3 cherrypick flags (introduced, fixed, limit) +- type: url + value: "https://github.com/google/osv.dev.git" + consider_all_branches: true + cherrypicks: true + +# Fine-grained control with specific overrides +- type: url + value: "https://github.com/apache/hadoop.git" + consider_all_branches: true + cherrypicks: true + cherrypicks_fixed: false # Specific override for fixed + +# Regex pattern match (Go RE2 syntax) +- type: regex + value: 'github\.com/google/osv-.*' + consider_all_branches: true + cherrypicks_introduced: true + cherrypicks_fixed: true + cherrypicks_limit: true +``` + +> [!TIP] +> Use single quotes for regex values so you don't have to escape backslashes or other special characters. + +### Normalization and Validation + +- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. +- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. +- **`cherrypicks`**: Acts as a shorthand for setting `cherrypicks_introduced`, `cherrypicks_fixed`, and `cherrypicks_limit` simultaneously. Specific `cherrypicks_` fields override the shorthand value if provided. + +## Usage + +Run the tool using `go run`: + +```bash +go run . [flags] +``` + +### Options & Flags + +| Flag | Default | Description | +| ----------- | --------------------- | ----------------------------------------------------------------- | +| `--file` | `repo_allowlist.yaml` | Path to the input YAML allowlist file | +| `--project` | `oss-vdb-test` | Target GCP Project ID | +| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | +| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-cab-sync/go.mod b/tools/repo-allowlist-sync/go.mod similarity index 96% rename from tools/repo-cab-sync/go.mod rename to tools/repo-allowlist-sync/go.mod index c316e0eba69..941de59e832 100644 --- a/tools/repo-cab-sync/go.mod +++ b/tools/repo-allowlist-sync/go.mod @@ -1,4 +1,4 @@ -module github.com/google/osv.dev/tools/repo-cab-sync +module github.com/google/osv.dev/tools/repo-allowlist-sync go 1.26.5 diff --git a/tools/repo-cab-sync/go.sum b/tools/repo-allowlist-sync/go.sum similarity index 100% rename from tools/repo-cab-sync/go.sum rename to tools/repo-allowlist-sync/go.sum diff --git a/tools/repo-allowlist-sync/main.go b/tools/repo-allowlist-sync/main.go new file mode 100644 index 00000000000..d8a0a0bf2bb --- /dev/null +++ b/tools/repo-allowlist-sync/main.go @@ -0,0 +1,245 @@ +// Package main implements a CLI tool to sync Repository AllowList YAML files to Cloud Datastore. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/url" + "os" + "regexp" + "strings" + + "cloud.google.com/go/datastore" + "gopkg.in/yaml.v3" +) + +// RepoAllowListEntity represents a repository allowlist entity stored in Cloud Datastore +type RepoAllowListEntity struct { + Key *datastore.Key `datastore:"__key__"` + Type string `datastore:"type"` + Value string `datastore:"value"` + ConsiderAllBranches bool `datastore:"consider_all_branches"` + CherrypicksIntroduced bool `datastore:"cherrypicks_introduced"` + CherrypicksFixed bool `datastore:"cherrypicks_fixed"` + CherrypicksLimit bool `datastore:"cherrypicks_limit"` +} + +// rawYAMLEntry represents an unmarshaled entry from the YAML file, including optional shorthand field +type rawYAMLEntry struct { + Type string `yaml:"type"` + Value string `yaml:"value"` + ConsiderAllBranches bool `yaml:"consider_all_branches"` + Cherrypicks *bool `yaml:"cherrypicks"` + CherrypicksIntroduced *bool `yaml:"cherrypicks_introduced"` + CherrypicksFixed *bool `yaml:"cherrypicks_fixed"` + CherrypicksLimit *bool `yaml:"cherrypicks_limit"` +} + +func (e RepoAllowListEntity) matches(other RepoAllowListEntity) bool { + return e.Type == other.Type && + e.ConsiderAllBranches == other.ConsiderAllBranches && + e.CherrypicksIntroduced == other.CherrypicksIntroduced && + e.CherrypicksFixed == other.CherrypicksFixed && + e.CherrypicksLimit == other.CherrypicksLimit +} + +func main() { + filePath := flag.String("file", "repo_allowlist.yaml", "Path to repo_allowlist YAML file") + project := flag.String("project", "oss-vdb-test", "GCP project ID") + dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") + verbose := flag.Bool("verbose", false, "Display verbose sync operations") + + flag.Parse() + + if *filePath == "" { + log.Fatalf("Error: --file argument is required") + } + + if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { + log.Fatalf("Error syncing repo allowlist: %v", err) + } +} + +// normalizeRepo removes the URL scheme, trailing slashes, and .git extensions to standardize repo paths. +func normalizeRepo(repoURL string) string { + repoURL = strings.TrimSpace(repoURL) + if repoURL == "" { + return "" + } + + if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { + log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) + return "" + } + + parsed, err := url.Parse(repoURL) + if err != nil { + return repoURL + } + normalized := parsed.Host + parsed.Path + normalized = strings.TrimRight(normalized, "/") + normalized = strings.TrimSuffix(normalized, ".git") + + return normalized +} + +// parseYAMLEntries parses and validates allowlist YAML content, expanding shorthand fields and normalizing values. +func parseYAMLEntries(data []byte) ([]RepoAllowListEntity, error) { + var rawEntries []rawYAMLEntry + if err := yaml.Unmarshal(data, &rawEntries); err != nil { + return nil, err + } + + var entries []RepoAllowListEntity + for _, raw := range rawEntries { + raw.Type = strings.TrimSpace(strings.ToLower(raw.Type)) + + switch raw.Type { + case "url": + raw.Value = normalizeRepo(raw.Value) + if raw.Value == "" { + continue + } + case "regex": + if raw.Value == "" { + continue + } + if _, err := regexp.Compile(raw.Value); err != nil { + log.Printf("Warning: Skipping invalid regex pattern %q: %v", raw.Value, err) + continue + } + default: + log.Printf("Warning: Skipping unrecognized entry type %q for value %q", raw.Type, raw.Value) + continue + } + + // Process cherrypicks flags: "cherrypicks: bool" acts as a shorthand for all 3 event types, + // specifying "cherrypicks_" fields overrides that. + intro := false + fixed := false + limit := false + + if raw.Cherrypicks != nil { + intro = *raw.Cherrypicks + fixed = *raw.Cherrypicks + limit = *raw.Cherrypicks + } + if raw.CherrypicksIntroduced != nil { + intro = *raw.CherrypicksIntroduced + } + if raw.CherrypicksFixed != nil { + fixed = *raw.CherrypicksFixed + } + if raw.CherrypicksLimit != nil { + limit = *raw.CherrypicksLimit + } + + entries = append(entries, RepoAllowListEntity{ + Type: raw.Type, + Value: raw.Value, + ConsiderAllBranches: raw.ConsiderAllBranches, + CherrypicksIntroduced: intro, + CherrypicksFixed: fixed, + CherrypicksLimit: limit, + }) + } + + return entries, nil +} + +func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed reading file %s: %w", filePath, err) + } + + entries, err := parseYAMLEntries(data) + if err != nil { + return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) + } + + if verbose { + log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) + } + + dsClient, err := datastore.NewClient(ctx, project) + if err != nil { + return fmt.Errorf("failed creating datastore client for project %s: %w", project, err) + } + defer func() { _ = dsClient.Close() }() + + // Fetch existing Datastore entities + query := datastore.NewQuery("RepoAllowList") + var dsEntities []RepoAllowListEntity + if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { + return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) + } + + dsEntitiesMap := make(map[string]RepoAllowListEntity) + for _, entity := range dsEntities { + dsEntitiesMap[entity.Value] = entity + } + + localEntriesMap := make(map[string]RepoAllowListEntity) + for _, item := range entries { + localEntriesMap[item.Value] = item + } + + // Upsert entries in local YAML that are missing from Datastore or modified + for val, item := range localEntriesMap { + existing, exists := dsEntitiesMap[val] + entity := &RepoAllowListEntity{ + Type: item.Type, + Value: item.Value, + ConsiderAllBranches: item.ConsiderAllBranches, + CherrypicksIntroduced: item.CherrypicksIntroduced, + CherrypicksFixed: item.CherrypicksFixed, + CherrypicksLimit: item.CherrypicksLimit, + } + + if !exists { + key := datastore.IncompleteKey("RepoAllowList", nil) + if !dryRun { + if _, err := dsClient.Put(ctx, key, entity); err != nil { + return fmt.Errorf("failed putting entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Creating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } else if !existing.matches(item) { + entity.Key = existing.Key + if !dryRun { + if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { + return fmt.Errorf("failed updating entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Updating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } + } + + // Delete entries in Datastore that are no longer present in local YAML + for val, existing := range dsEntitiesMap { + if _, exists := localEntriesMap[val]; !exists { + if verbose { + log.Printf("Deleting RepoAllowList entity: val=%s", val) + } + if !dryRun { + if err := dsClient.Delete(ctx, existing.Key); err != nil { + return fmt.Errorf("failed deleting entity for %s: %w", val, err) + } + } + } + } + + if dryRun { + log.Println("[DRY RUN] Sync completed successfully.") + } else { + log.Println("Sync completed successfully.") + } + return nil +} diff --git a/tools/repo-cab-sync/main_test.go b/tools/repo-allowlist-sync/main_test.go similarity index 65% rename from tools/repo-cab-sync/main_test.go rename to tools/repo-allowlist-sync/main_test.go index c7c5a8cca45..cc5e572538d 100644 --- a/tools/repo-cab-sync/main_test.go +++ b/tools/repo-allowlist-sync/main_test.go @@ -74,34 +74,64 @@ func TestParseYAMLEntries(t *testing.T) { yamlContent := []byte(` - type: URL value: " https://github.com/google/osv.dev.git " + consider_all_branches: true + cherrypicks_introduced: true - type: REGEX value: 'github\.com/google/osv-.*' + cherrypicks: true +- type: url + value: "https://github.com/noflags/repo.git" - type: regex value: '[invalid regex' - type: unknown value: "https://github.com/google/osv.dev" - type: url - value: "git@github.com:google/osv.dev.git" + value: "git@github.com:ssh/isnot.supported.git" `) - entries, err := parseYAMLEntries(yamlContent) - if err != nil { - t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) + want := []RepoAllowListEntity{ + // Normalized URL + { + Type: "url", + Value: "github.com/google/osv.dev", + ConsiderAllBranches: true, + CherrypicksIntroduced: true, + CherrypicksFixed: false, + CherrypicksLimit: false, + }, + // Regex type and cherrypicks: true populates all 3 event types + { + Type: "regex", + Value: `github\.com/google/osv-.*`, + ConsiderAllBranches: false, + CherrypicksIntroduced: true, + CherrypicksFixed: true, + CherrypicksLimit: true, + }, + // No flags set (Shouldn't really happen) + { + Type: "url", + Value: "github.com/noflags/repo", + ConsiderAllBranches: false, + CherrypicksIntroduced: false, + CherrypicksFixed: false, + CherrypicksLimit: false, + }, } - // Should normalize types, trim values, and skip invalid regex, unrecognized types, and SSH URLs - if len(entries) != 2 { - t.Fatalf("expected 2 valid entries, got %d: %+v", len(entries), entries) + got, err := parseYAMLEntries(yamlContent) + if err != nil { + t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) } - // First entry should be normalized URL - if entries[0].Type != "url" || entries[0].Value != "github.com/google/osv.dev" { - t.Errorf("entry 0 mismatch: got type=%q val=%q, want type=url val=github.com/google/osv.dev", entries[0].Type, entries[0].Value) + if len(got) != len(want) { + t.Fatalf("expected %d valid entries, got %d: %+v", len(want), len(got), got) } - // Second entry should preserve regex string - if entries[1].Type != "regex" || entries[1].Value != `github\.com/google/osv-.*` { - t.Errorf("entry 1 mismatch: got type=%q val=%q, want type=regex val=github\\.com/google/osv-.*", entries[1].Type, entries[1].Value) + for i, wantEntry := range want { + if got[i] != wantEntry { + t.Errorf("entry %d mismatch:\n got: %+v\nwant: %+v", i, got[i], wantEntry) + } } } diff --git a/tools/repo-allowlist-sync/repo_allowlist.yaml b/tools/repo-allowlist-sync/repo_allowlist.yaml new file mode 100644 index 00000000000..ec257649e33 --- /dev/null +++ b/tools/repo-allowlist-sync/repo_allowlist.yaml @@ -0,0 +1,15 @@ +# Repository allowlist configuration +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# consider_all_branches: true +# cherrypicks: true # Shorthand: sets introduced, fixed, and limit to true +# - type: url +# value: "https://github.com/apache/hadoop.git" +# consider_all_branches: true +# cherrypicks: true +# cherrypicks_fixed: false # Specific override for fixed +# - type: regex +# value: 'github\.com/google/osv-.*' +# consider_all_branches: true diff --git a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml b/tools/repo-allowlist-sync/repo_allowlist_test.yaml similarity index 65% rename from tools/repo-cab-sync/repo_cab_allowlist_test.yaml rename to tools/repo-allowlist-sync/repo_allowlist_test.yaml index 4e83defe770..4c6c8bfccd3 100644 --- a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml +++ b/tools/repo-allowlist-sync/repo_allowlist_test.yaml @@ -1,9 +1,13 @@ -# Repository-based consider all branches allowlist +# Repository allowlist configuration # Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). # Example: # - type: url # value: "https://github.com/google/osv.dev.git" +# consider_all_branches: true # - type: regex # value: 'github\.com/google/osv-.*' +# cherrypicks_fixed: true - type: url value: 'https://github.com/apache/hadoop.git' + consider_all_branches: true + cherrypicks: true diff --git a/tools/repo-cab-sync/README.md b/tools/repo-cab-sync/README.md deleted file mode 100644 index de24c7f6431..00000000000 --- a/tools/repo-cab-sync/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Repo Consider All Branches Allowlist Sync Tool (`repo-cab-sync`) - -`repo-cab-sync` is a Go command-line tool that synchronizes repository "Consider All Branches" (CAB) allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoConsiderAllBranchesAllowList` entities). - -## Overview - -Gitter has the option to enumerate affected commits with `consider_all_branches` enabled or disabled. This tool manages the Datastore allowlist index (`RepoConsiderAllBranchesAllowList`) that controls this behavior on a repository level. - -> [!NOTE] -> If `consider_all_branches` is already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist. - -The tool performs a two-way sync: - -- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities. -- **Delete**: Removes entities from Datastore that are no longer present in the YAML file. - -## Allowlist YAML Format - -The allowlist YAML configuration file accepts a list of entries with `type` and `value` fields: - -```yaml -# Supported entry types: 'url' and 'regex' - -# Exact repository URL match -- type: url - value: "https://github.com/google/osv.dev.git" - -# Regex pattern match (Go RE2 syntax) -- type: regex - value: 'github\.com/google/osv-.*' -``` - -> [!TIP] -> Use single quotes for regex values so you don't have to escape backslashes or other special characters. - -### Normalization and Validation - -- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. -- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. - -## Usage - -Run the tool using `go run`: - -```bash -go run . [flags] -``` - -### Options & Flags - -| Flag | Default | Description | -| ----------- | ------------------------- | ----------------------------------------------------------------- | -| `--file` | `repo_cab_allowlist.yaml` | Path to the input YAML allowlist file | -| `--project` | `oss-vdb-test` | Target GCP Project ID | -| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | -| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-cab-sync/main.go b/tools/repo-cab-sync/main.go deleted file mode 100644 index 7ab771a54c5..00000000000 --- a/tools/repo-cab-sync/main.go +++ /dev/null @@ -1,193 +0,0 @@ -// Package main implements a CLI tool to sync Repo Consider All Branches (CAB) allowlist YAML files to Cloud Datastore. -package main - -import ( - "context" - "flag" - "fmt" - "log" - "net/url" - "os" - "regexp" - "strings" - - "cloud.google.com/go/datastore" - "gopkg.in/yaml.v3" -) - -type RepoCABEntity struct { - Key *datastore.Key `yaml:"-" datastore:"__key__"` - Type string `yaml:"type" datastore:"type"` - Value string `yaml:"value" datastore:"value"` -} - -func main() { - filePath := flag.String("file", "repo_cab_allowlist.yaml", "Path to repo_cab_allowlist YAML file") - project := flag.String("project", "oss-vdb-test", "GCP project ID") - dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") - verbose := flag.Bool("verbose", false, "Display verbose sync operations") - - flag.Parse() - - if *filePath == "" { - log.Fatalf("Error: --file argument is required") - } - - if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { - log.Fatalf("Error syncing repo CAB allowlist: %v", err) - } -} - -func normalizeRepo(repoURL string) string { - // Normalize the repo_url to align with matching logic - // Removes the scheme/protocol, the .git extension, and trailing slashes. - repoURL = strings.TrimSpace(repoURL) - if repoURL == "" { - return "" - } - - if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { - log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) - return "" - } - - parsed, err := url.Parse(repoURL) - if err != nil { - return repoURL - } - normalized := parsed.Host + parsed.Path - normalized = strings.TrimRight(normalized, "/") - normalized = strings.TrimSuffix(normalized, ".git") - - return normalized -} - -func parseYAMLEntries(data []byte) ([]RepoCABEntity, error) { - var parsed []RepoCABEntity - if err := yaml.Unmarshal(data, &parsed); err != nil { - return nil, err - } - - var entries []RepoCABEntity - for _, entry := range parsed { - entry.Type = strings.TrimSpace(strings.ToLower(entry.Type)) - - switch entry.Type { - case "url": - // For repo URLs, we normalize the value before inserting to datastore - entry.Value = normalizeRepo(entry.Value) - if entry.Value == "" { - continue - } - case "regex": - // For regex, we make sure it compiles - if entry.Value == "" { - continue - } - if _, err := regexp.Compile(entry.Value); err != nil { - log.Printf("Warning: Skipping invalid regex pattern %q: %v", entry.Value, err) - continue - } - default: - log.Printf("Warning: Skipping unrecognized entry type %q for value %q", entry.Type, entry.Value) - continue - } - - entries = append(entries, entry) - } - - return entries, nil -} - -func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { - data, err := os.ReadFile(filePath) - if err != nil { - return fmt.Errorf("failed reading file %s: %w", filePath, err) - } - - entries, err := parseYAMLEntries(data) - if err != nil { - return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) - } - - if verbose { - log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) - } - - dsClient, err := datastore.NewClient(ctx, project) - if err != nil { - return fmt.Errorf("failed creating datastore client for project %s: %w", project, err) - } - defer func() { _ = dsClient.Close() }() - - // Get existing Datastore entities - query := datastore.NewQuery("RepoConsiderAllBranchesAllowList") - var dsEntities []RepoCABEntity - if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { - return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) - } - - dsEntitiesMap := make(map[string]RepoCABEntity) - for _, entity := range dsEntities { - dsEntitiesMap[entity.Value] = entity - } - - localEntriesMap := make(map[string]RepoCABEntity) - for _, item := range entries { - localEntriesMap[item.Value] = item - } - - // 1. Put/Upsert entries in local YAML that are not in Datastore or modified - for val, item := range localEntriesMap { - existing, exists := dsEntitiesMap[val] - if !exists { - key := datastore.IncompleteKey("RepoConsiderAllBranchesAllowList", nil) - entity := &RepoCABEntity{ - Type: item.Type, - Value: item.Value, - } - if !dryRun { - if _, err := dsClient.Put(ctx, key, entity); err != nil { - return fmt.Errorf("failed putting entity for %s: %w", val, err) - } - } - if verbose { - log.Printf("Creating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) - } - } else if existing.Type != item.Type { - entity := &RepoCABEntity{ - Type: item.Type, - Value: item.Value, - } - if !dryRun { - if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { - return fmt.Errorf("failed updating entity for %s: %w", val, err) - } - } - if verbose { - log.Printf("Updating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) - } - } - } - - // 2. Delete entries in Datastore that are no longer in local YAML - for val, existing := range dsEntitiesMap { - if _, exists := localEntriesMap[val]; !exists { - if verbose { - log.Printf("Deleting RepoConsiderAllBranchesAllowList entity: val=%s", val) - } - if !dryRun { - if err := dsClient.Delete(ctx, existing.Key); err != nil { - return fmt.Errorf("failed deleting entity for %s: %w", val, err) - } - } - } - } - - if dryRun { - log.Println("[DRY RUN] Sync completed successfully.") - } else { - log.Println("Sync completed successfully.") - } - return nil -} diff --git a/tools/repo-cab-sync/repo_cab_allowlist.yaml b/tools/repo-cab-sync/repo_cab_allowlist.yaml deleted file mode 100644 index 2b477302dc5..00000000000 --- a/tools/repo-cab-sync/repo_cab_allowlist.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Repository-based consider all branches allowlist -# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). -# Example: -# - type: url -# value: "https://github.com/google/osv.dev.git" -# - type: regex -# value: 'github\.com/google/osv-.*' From 0c90492507fc8f68c44eca674270b858325f0b8c Mon Sep 17 00:00:00 2001 From: Joey L Date: Fri, 7 Aug 2026 05:41:32 +0000 Subject: [PATCH 6/6] Update readme --- tools/repo-allowlist-sync/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/repo-allowlist-sync/README.md b/tools/repo-allowlist-sync/README.md index ebed96ee8a7..1c8d42e523e 100644 --- a/tools/repo-allowlist-sync/README.md +++ b/tools/repo-allowlist-sync/README.md @@ -16,25 +16,26 @@ The tool performs a two-way sync: ## Allowlist YAML Format -The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields: +The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields. +* Supported `type`s: `url`, `regex` + * `url`: A URL to match against the repository URL. + * `regex`: A regex pattern to match against the repository URL (Go RE2 syntax). +* Supported boolean flags: `consider_all_branches`, `cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`, `cherrypicks` (shorthand for all 3 cherrypick flags) ```yaml -# Supported entry types: 'url' and 'regex' +# Examples -# Shorthand: 'cherrypicks: true' applies to all 3 cherrypick flags (introduced, fixed, limit) - type: url value: "https://github.com/google/osv.dev.git" consider_all_branches: true cherrypicks: true -# Fine-grained control with specific overrides - type: url - value: "https://github.com/apache/hadoop.git" + value: "https://github.com/google/osv.dev.git" consider_all_branches: true cherrypicks: true - cherrypicks_fixed: false # Specific override for fixed + cherrypicks_fixed: false # Overrides cherrypicks: true for fixed event -# Regex pattern match (Go RE2 syntax) - type: regex value: 'github\.com/google/osv-.*' consider_all_branches: true