From e3b63f44f69b5112e82e33e78ca751507a1af13f Mon Sep 17 00:00:00 2001 From: Sibei Chen Date: Fri, 12 Jun 2026 10:57:29 +0800 Subject: [PATCH] Migrate secrets from sops/age to Doppler with per-env R2 and Mongo access Secrets now live in Doppler project api-server (configs dev/prd) instead of a sops/age-encrypted .env.enc: - Container runs `doppler run -- java` (CLI baked into the image); compose feeds DOPPLER_TOKEN from /opt/api-server/doppler.env written by ansible - Playbook fetches the keystore password + nginx proxy secret from Doppler (service token in CI, logged-in CLI on laptops); sops/age tasks removed - CI ships no .env.enc and passes DOPPLER_TOKEN instead of AGE/TLS secrets - Dev and prod use separate Mongo users/databases and bucket-scoped R2 keys (api-server-dev bucket has a 7-day object retention rule) - @Value keys renamed to kebab-case so UPPER_SNAKE env vars bind cleanly - hosts.ini pins ansible_python_interpreter to the distro python (3.14 discovery breaks the apt cryptography stack needed by openssl_pkcs12) Co-Authored-By: Claude Fable 5 --- .env.enc | 31 ----- .githooks/post-checkout | 81 +---------- .githooks/pre-commit | 44 +----- .github/workflows/maven.yml | 18 +-- .sops.yaml | 5 - CLAUDE.md | 8 +- Dockerfile | 16 +-- README.md | 30 ++--- docker-compose.yml | 3 + doppler.yaml | 5 + entrypoint.sh | 12 +- hosts.ini | 4 +- playbook.yml | 127 ++++++------------ requirements.yml | 1 - scripts/age_keygen.sh | 15 --- .../com/chencraft/common/config/S3Config.java | 6 +- .../service/api/CloudflareWebClient.java | 6 +- src/main/resources/application.properties | 6 +- templates/.sops.yaml.template | 4 - 19 files changed, 99 insertions(+), 323 deletions(-) delete mode 100644 .env.enc delete mode 100644 .sops.yaml create mode 100644 doppler.yaml delete mode 100755 scripts/age_keygen.sh delete mode 100644 templates/.sops.yaml.template diff --git a/.env.enc b/.env.enc deleted file mode 100644 index 6cf9f06..0000000 --- a/.env.enc +++ /dev/null @@ -1,31 +0,0 @@ -#ENC[AES256_GCM,data:GfL8IuOvDhNoNtWLBSwtQg==,iv:kBkbQ2d9Ez+GvRQIsjCH5lhqAxrQKl3M9NRr3lhVjEw=,tag:M3tdQnnkRNh95ikWitN+EQ==,type:comment] -github.readonly.token=ENC[AES256_GCM,data:c+GaeHdhMraHSEn/w08LSGFOrMNpCceeIl6JN1kvElF5RJbBbu+wZY7hLiEnAiFJvKuFpShX26ovxq1Nl8gLG/SACYEKfkuKxXmTL4QY5aUrk992zcqqFjKQVLVxQg==,iv:xxCl33+C1T4Nz0HlsGkrhE2+IWom/bUb79YqlCagcLE=,tag:/WIdJ2EkeGhvwJJ8XtdSEA==,type:str] -github.webhook.secret=ENC[AES256_GCM,data:hDm61TkqtQpxY1yxcLtnfqRYGKqjC4TRRCq8ecQorTnWQngq6jPM55FMIxylHB+2Ho+RaSne0/k4T/znHcaF8JQFDu9IcBfX1pNS4Wg16QejFas=,iv:9HA3t3WdXjE6EV0w+PhljzsMU8cno3o+x9PAOEM39os=,tag:kSXBeetMSw8pVfIN+59mpA==,type:str] -#ENC[AES256_GCM,data:CGhI8UyiEia5m4UeIC+scvDGUelidRT2FDXN,iv:nrk6qhnSvQdhl3vjh7IhRx9ea9jpXAlrdUfUFtLtyPE=,tag:wwm/NvjIJMZ8n2HsYeMN/w==,type:comment] -app.tls.keystore=ENC[AES256_GCM,data:+rZCt9tZ8OLYs9GgC5OZU08vllSnjbRameKu,iv:r5y3IxBVAGTgwo31sNL+MKEVNzgaPzbskm8UnPqRMSA=,tag:CXOJlVHf1gLM+EU7TVgAlg==,type:str] -app.tls.keystore-password=ENC[AES256_GCM,data:z95hZadCNekx3p/J1uaHWFVyGz7uRdg2VwjDCCs1OP5K4aRl9T7kKRjEqgfKgtg=,iv:7jPwuSHvK2IRwGohkoGJsoV/orxp0RD0d3gN3CCE5Bk=,tag:mmIUv6CMc5rsc/ceRKllLA==,type:str] -app.tls.keystore-alias=ENC[AES256_GCM,data:kMLLDdaRv2i3j0E=,iv:6tHCidb+DkDTUvNBxsvre/Z+N9LIYqMACmY/uSKTAtM=,tag:5Psuj6aWQsuYrqpGB9bhPg==,type:str] -#ENC[AES256_GCM,data:s8vVHSc3WX8VST3EUTqWapoIrQ==,iv:2DFNvbdbgPM38p6MEgbpXvn6rwzvKlzGbBy24TGuduI=,tag:YECKsD0zzSSq939wHjdBWw==,type:comment] -spring.mail.host=ENC[AES256_GCM,data:dp+ZSFBIuLmNxrwxVVKx2lw=,iv:fyhOXF9FixwswKoHfRpq4qHrl8ITdNP6K63zXAHP4tU=,tag:VpwDfaLT7Ho2xNwXc+uYMQ==,type:str] -spring.mail.port=ENC[AES256_GCM,data:bzu6kA==,iv:fGxBfcDV8OeW/jwUhTfL7nEQE/zC4cVEmENBq8H+Clo=,tag:295VaOKm9bmXsOky9ZGNKA==,type:str] -spring.mail.username=ENC[AES256_GCM,data:vMQjBAAQJo4fx1IeJicA4ECpv3nn80s=,iv:zQQ2HTXJt4KF2al3Fluin5eMxmVqjZiPLFXnve+vI9E=,tag:xEK5GiOHBIRIwvduSZXAEg==,type:str] -spring.mail.password=ENC[AES256_GCM,data:Ulzx0Y68Ai8jIrjOXEP853McQ2A=,iv:2n+P37jnh+I4+MleqOAfW0kv/A71vpNMpfOLunM3VlQ=,tag:RM0/Pl5MjkC8RJNaSFFVgg==,type:str] -#ENC[AES256_GCM,data:dBD+qTpW8lxNW7kc6dX12ccQPMfhQQ==,iv:kXJMDvI88h4ANodJs+Iq4KFSwcz7OnNvzxbv/NDBj6k=,tag:+Kock75Na16H3R72W76UsA==,type:comment] -cloudflare.r2.accountId=ENC[AES256_GCM,data:7L/AT/RjRbxYkOiPQ987X3aj6DGqyYG1/yquCzJXQawk,iv:daU5nSR+QR8U9/BXD189Ab7wjuuPGPGKLM/lSsLQF8c=,tag:LHWcS+X0gkoz2pvobgMgkg==,type:str] -cloudflare.r2.accessKey=ENC[AES256_GCM,data:NBAHppW+I+GrPjzXpNr7QvmeilFPjY8quBGVkhPtS/AX,iv:ojj4f2waVo6AUyrRDEolRHtpSVBfVmF28S4EmtlKhNQ=,tag:O5kff1b00+W0KJydA4fszg==,type:str] -cloudflare.r2.secretKey=ENC[AES256_GCM,data:z1fkqBmoDP7tQUJ2wjRCuHVWmi0lr6gj7iMuW+uYT4VvLs///muVK4C45TiRt0zyZtLIyuIckX42aTFBN2uwZsE=,iv:v1CLNbFh5H/Gva1B40P/1fP38HAPUcMqT+zKSCRULy0=,tag:dlAYY/4za/wOD+ADCoiwsQ==,type:str] -cloudflare.r2.bucket=ENC[AES256_GCM,data:yVA1UEFOB/PqrPw=,iv:XkW5/WQV7yokPeYP4eA9YcBpIwU3bPuoSIMZJBd/RcM=,tag:0+hagHnMyAFr1XPpIB1w5w==,type:str] -#ENC[AES256_GCM,data:OFQpYtRhzcXVjMKYbG5+E6T3gA46cd0=,iv:ladsGIgPgJJrUSlRyUNYbjkLQ+FsGG0eZ7cP/vF9oqw=,tag:W8lrHjWUAJjdWrmN5ml6Ow==,type:comment] -cloudflare.dns.zoneId=ENC[AES256_GCM,data:1UzhXdhFMx4zslcjgAyiEuMxFna3dAqfKuXJSjJdBGE+,iv:z65PYFcnznsFAmLn+KyErL1mIVb/9aRKl9J0yPzqEX8=,tag:FgQnHqeEufQWWLpspMOWSA==,type:str] -cloudflare.dns.apiKey=ENC[AES256_GCM,data:ABforpx9atJzMfeCpiv/n3eQNcEdvD6Oc5qsyQ+hL21w7Yn/R57CXnw=,iv:dCmGjrm3CweBEc9QvUHkCoKjWLtmp3pkppWU+CSdAUI=,tag:0htPGw+AeswisD245+nDAw==,type:str] -#ENC[AES256_GCM,data:2QDxKvvV0GgadVTHGzFmtw==,iv:WRtuaQ4XqDfQcUaClCpd21UGHU0KBG2nCUduJpl2Cmg=,tag:77KrTxaZLBD0uRC8Auwp/A==,type:comment] -spring.mongodb.uri=ENC[AES256_GCM,data:6uzrEHPKqCVa6PrO2zm6aY9NxO+8yWOeYGrR552HDMjKqnfy6nyG7XUgCgV3xwmLWWcR19SfIsl559iF8z2wBmOb8EG8WKAHvazOeRxnF4xsEnBEToZnO1R/nrH8hZgxPSbsn2lmn4NX7rOi8jAUqgs6/hPHjpmlZn30lrjIEK6jAT9O,iv:0hAtqnv0MBAVWZHAzjKk8liPsmo/pgqjjIEaycAGfq4=,tag:EUozhisTCpJ+7FjQgBNUKQ==,type:str] -#ENC[AES256_GCM,data:Fkie16ocYNpPY3jGnfWmpXCWMWeBjmFIS8ryQ6kAYStw8Iozx2HgeudyaalPhVux25+HRbLM5EW6iwUOWceowY+rWqwhs5LOU0Qz8g==,iv:8BaGlhZnGdPx8Tnm8GQhWINKam+4NjdXyjFtmvTp3Ks=,tag:zwvNvUDLqev8U1HL97rLvQ==,type:comment] -#ENC[AES256_GCM,data:SNA2WN3M6WRkYacRkqY7BU0LPT1gz7PijfYhA2NrhSATL2CyKAbfzl6Ij3iCkqMpaylkHg5epqOuEtSiHmH+zDk9cFUCANzOCWNXJ+ozVYL24TIzDQvU0k6j1keKv2PGy1McThOklA7VH3LngNdVCs+dGDOwZAQ=,iv:MjQhqfpHUxFetAyDBWNFQ7k1L2+UAvbfx4Ckm04BEnE=,tag:yQixKrseJtC6GaLGuBjAyg==,type:comment] -APP_MTLS_PROXY_SECRET=ENC[AES256_GCM,data:Nc6DQBNJgRPFEW3LjKI9kMFc0H78oREBeOJWhZk4UUyejHa7kue/XhmgclbKg8hjHxfY9xI4a4HvS4EREbjmN6E=,iv:uccD6VPzKmYpuFrj18sWtLRY+1r9QaB9mOF1L/ZLOhk=,tag:hBwbxAob5XWFHDIQJw4EvQ==,type:str] -sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGQnZBaTlUWFRLL3pMOGdL\naDFPck93a3RrajR6VmxnbUovc2ozZTl6Z1VJCmdFYlphRU5pdlNNMXdmWFlMZlk4\nU0xBVmFRcHFYL3ozMVpnQzdxaW9HeE0KLS0tIHgrbER6YmlndWl0dlRySGliYW5r\nSEZxUzlhRmRaYjFsL05KWnNFNFo3ckkKDfiFekqnRG6bI2LTq5rUoQAYZMPV+l2N\nnvv71za0jQjryVZpep+WhCw2Sf66ZSxj3ALQJod/UW8AiVUSIljmjQ==\n-----END AGE ENCRYPTED FILE-----\n -sops_age__list_0__map_recipient=age1jjpfu24vn3ral5aghj9ztgwnaurg7t7uwprlw3nlzvtk7x4xadlq5xc9ue -sops_lastmodified=2026-05-14T11:50:20Z -sops_mac=ENC[AES256_GCM,data:Kwy/7eGqw0UVsEYKfiu2IitYmg241jl0JzJdlSm5TrcunSj+hp9K5jHaF4Hu/roYymAHBbWZCS8IAEj4nOhsz4iGHjYF4agmB5JSNQV17OzniLP+GJpu2gVFo/AW7bpydaEMjDKmZliSi5uIvQaJ/EcJXyubCAz9R/BSHbfNgv0=,iv:IyK5dTLySGROqZohh4scTaoJAkeZ2liiN+mDyJMqXzo=,tag:jYwg8gEBhknyPyG9T0HrTw==,type:str] -sops_unencrypted_suffix=_unencrypted -sops_version=3.13.0 diff --git a/.githooks/post-checkout b/.githooks/post-checkout index 0477a18..454d27e 100755 --- a/.githooks/post-checkout +++ b/.githooks/post-checkout @@ -1,78 +1,11 @@ #!/bin/bash set -euo pipefail -# Install sops if not exists -# Detect platform -OS="$(uname -s)" -ARCH="$(uname -m)" - -# Only run if sops not installed -if ! command -v sops >/dev/null 2>&1; then - echo "[INFO] sops not found, installing..." - - case "$OS" in - Linux) - if [ "$ARCH" = "x86_64" ]; then - curl -LO https://github.com/getsops/sops/releases/download/v3.11.0/sops-v3.11.0.linux.amd64 - sudo mv sops-v3.11.0.linux.amd64 /usr/local/bin/sops - sudo chmod +x /usr/local/bin/sops - echo "[INFO] sops installed at /usr/local/bin/sops" - else - echo "[ERROR] Unsupported Linux architecture: $ARCH" - exit 1 - fi - ;; - Darwin) # macOS - echo "[INFO] Detected macOS" - if command -v brew >/dev/null 2>&1; then - brew install sops - else - echo "[ERROR] Homebrew not installed. Please install sops manually:" - echo " https://github.com/getsops/sops#installation" - exit 1 - fi - ;; - MINGW*|MSYS*|CYGWIN*) # Git Bash on Windows - echo "[ERROR] Windows detected. Please install sops manually:" - echo " choco install sops # (if using Chocolatey)" - echo " scoop install sops # (if using Scoop)" - exit 1 - ;; - *) - echo "[ERROR] Unsupported OS: $OS" - exit 1 - ;; - esac -fi - -# Only run if age not installed -if ! command -v age >/dev/null 2>&1; then - echo "[INFO] age not found, installing..." - - case "$OS" in - Linux) - sudo apt install age -y - ;; - Darwin) # macOS - echo "[INFO] Detected macOS" - if command -v brew >/dev/null 2>&1; then - brew install age - else - echo "[ERROR] Homebrew not installed. Please install age manually:" - exit 1 - fi - ;; - *) - echo "[ERROR] Unsupported OS: $OS" - exit 1 - ;; - esac +# Secrets come from Doppler (project api-server). Nothing to decrypt on checkout — +# just nudge if the CLI is missing or the repo hasn't been bound yet. +if ! command -v doppler >/dev/null 2>&1; then + echo "[INFO] doppler CLI not found. Install it (https://docs.doppler.com/docs/install-cli)," + echo " then run: doppler login && doppler setup --no-interactive" +elif ! doppler configure get config --plain >/dev/null 2>&1; then + echo "[INFO] Doppler not configured for this repo. Run: doppler setup --no-interactive" fi - - -# Auto-decrypt after checkout -if [ -f .env.enc ]; then - echo "Decrypting secrets after checkout..." - sops -d --input-type dotenv --output-type dotenv .env.enc > .env -fi - diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 509b5c2..6c50658 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,50 +1,8 @@ #!/bin/bash set -euo pipefail -if [ -f .env ]; then - echo "Checking if .env has changed since last encryption..." - - tmp_decrypted=$(mktemp) - if [ -f .env.enc ]; then - # Decrypt the existing .env.enc for comparison - if ! sops --decrypt --input-type dotenv --output-type dotenv .env.enc > "$tmp_decrypted"; then - echo "Warning: Failed to decrypt existing .env.enc, re-encrypting..." - tmp_decrypted="/dev/null" - fi - else - # No .env.enc exists yet - tmp_decrypted="/dev/null" - fi - - # Normalize both files (strip CRLF, trailing spaces, and blank line differences) before comparing - norm_env=$(mktemp) - norm_dec=$(mktemp) - - # Create normalized versions - awk 'NF{print $0}' .env | sed 's/\r$//' | sed 's/[[:space:]]*$//' | sort > "$norm_env" - if [ "$tmp_decrypted" != "/dev/null" ]; then - awk 'NF{print $0}' "$tmp_decrypted" | sed 's/\r$//' | sed 's/[[:space:]]*$//' | sort > "$norm_dec" - else - # force re-encrypt when there is no existing enc file - : > "$norm_dec" - fi - - # Compare normalized content - if ! cmp -s "$norm_dec" "$norm_env"; then - echo "Encrypting updated .env before commit..." - sops --encrypt --input-type dotenv --output-type dotenv .env > .env.enc - git add .env.enc - else - echo ".env unchanged, skipping encryption." - fi - - # Clean up temp files - [ "$tmp_decrypted" != "/dev/null" ] && rm -f "$tmp_decrypted" - rm -f "$norm_env" "$norm_dec" -fi - echo "[pre-commit] Running ansible-lint --fix (non-blocking)..." ansible-lint --fix || { echo "[pre-commit] ansible-lint failed or could not fix all issues, but continuing anyway." } -git add -u \ No newline at end of file +git add -u diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 360925c..9ac9258 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -95,7 +95,6 @@ jobs: sparse-checkout: | Dockerfile entrypoint.sh - .env.enc sparse-checkout-cone-mode: false - name: Download JAR artifact @@ -115,8 +114,6 @@ jobs: run: | echo "REPO=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV echo "IMAGE_TAG=${{ needs.ci-test.outputs.app-version || '0.0.1-SNAPSHOT' }}" >> $GITHUB_ENV - SOPS_VERSION=$(curl -s https://api.github.com/repos/getsops/sops/releases/latest | jq -r .tag_name) - echo "SOPS_VERSION=$SOPS_VERSION" >> $GITHUB_ENV - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -125,8 +122,6 @@ jobs: uses: docker/build-push-action@v7 with: context: . - build-args: | - SOPS_VERSION=${{ env.SOPS_VERSION }} push: true platforms: linux/amd64 tags: | @@ -158,8 +153,6 @@ jobs: hosts.ini requirements.yml docker-compose.yml - .env.enc - .sops.yaml sparse-checkout-cone-mode: false - name: Ship ansible payload to server @@ -169,15 +162,14 @@ jobs: username: ${{ secrets.SERVER_USER }} key: ${{ secrets.SERVER_SSH_KEY }} # Pass `templates` as a directory (not `templates/*`) so subpaths are preserved on the remote. - source: "playbook.yml,templates,hosts.ini,requirements.yml,docker-compose.yml,.env.enc,.sops.yaml" + source: "playbook.yml,templates,hosts.ini,requirements.yml,docker-compose.yml" target: "/home/${{ secrets.SERVER_USER }}/api-server-deploy" rm: true - name: Run ansible + docker compose on server uses: appleboy/ssh-action@v1.2.5 env: - AGE_PRIVATE_KEY: ${{ secrets.AGE_PRIVATE_KEY }} - TLS_KEYSTORE_PASSWORD: ${{ secrets.TLS_KEYSTORE_PASSWORD }} + DOPPLER_TOKEN: ${{ secrets.DOPPLER_TOKEN }} GHCR_PAT_RO: ${{ secrets.GHCR_PAT_RO }} # GHCR_USER must match the account that owns GHCR_PAT_RO — use the repo owner (stable) # rather than github.actor (which varies by who triggers workflow_dispatch / tag push). @@ -186,7 +178,7 @@ jobs: host: ${{ secrets.SERVER_HOST }} username: ${{ secrets.SERVER_USER }} key: ${{ secrets.SERVER_SSH_KEY }} - envs: AGE_PRIVATE_KEY,TLS_KEYSTORE_PASSWORD,GHCR_PAT_RO,GHCR_USER + envs: DOPPLER_TOKEN,GHCR_PAT_RO,GHCR_USER script: | set -euo pipefail cd ~/api-server-deploy @@ -195,9 +187,7 @@ jobs: SECRETS_FILE="$(mktemp /tmp/ansible-secrets.XXXXXX.yml)" trap 'shred -u "$SECRETS_FILE" 2>/dev/null || rm -f "$SECRETS_FILE"' EXIT { - printf 'age_private_key: |\n' - printf '%s\n' "$AGE_PRIVATE_KEY" | sed 's/^/ /' - printf 'tls_keystore_password: "%s"\n' "${TLS_KEYSTORE_PASSWORD//\"/\\\"}" + printf 'doppler_token: "%s"\n' "${DOPPLER_TOKEN//\"/\\\"}" printf 'ghcr_user: "%s"\n' "${GHCR_USER//\"/\\\"}" printf 'ghcr_pat_ro: "%s"\n' "${GHCR_PAT_RO//\"/\\\"}" printf 'docker_deploy: true\n' diff --git a/.sops.yaml b/.sops.yaml deleted file mode 100644 index 2c4938c..0000000 --- a/.sops.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# DO NOT MODIFY, UPDATE USING scripts/age_keygen.sh ONLY -creation_rules: - - path_regex: ^\.env$ - age: age1jjpfu24vn3ral5aghj9ztgwnaurg7t7uwprlw3nlzvtk7x4xadlq5xc9ue diff --git a/CLAUDE.md b/CLAUDE.md index 27536b3..52cfca6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,8 +13,8 @@ mvn -P ci clean verify # Build without tests mvn -P ci -DskipTests=true clean package -# Run locally with dev profile -SPRING_PROFILES_ACTIVE=dev mvn spring-boot:run +# Run locally (Doppler injects dev secrets + SPRING_PROFILES_ACTIVE=dev) +doppler run -- mvn spring-boot:run # Run a single test class mvn -P ci -Dtest=com.chencraft.common.service.HashServiceTest test @@ -56,7 +56,7 @@ mvn -P ci -Dtest=com.chencraft.common.service.HashServiceTest#testValidateHash t ### Configuration - `application.properties` — prod config (port 8080, Prometheus metrics on 8957) - `application-dev.properties` — dev overrides (port 8085, debug logging) -- Environment variables loaded from `.env` file (encrypted as `.env.enc` in Docker, decrypted via sops + age key) +- Secrets arrive as env vars from Doppler (project `api-server`, configs `dev`/`prd` — per-env Mongo URI/database, Cloudflare R2 bucket/keys, mTLS proxy secret). Local binding via committed `doppler.yaml`; prod container runs `doppler run` with a service token from `/opt/api-server/doppler.env`. Tests need no secrets. ### Test Infrastructure (`src/test/`) - Mock/local service implementations: `LocalFileService`, `MockMailService`, `MockCertificateService`, `ImmediateTaskExecutor` @@ -66,7 +66,7 @@ mvn -P ci -Dtest=com.chencraft.common.service.HashServiceTest#testValidateHash t ## Deployment -Tag push (`v*.*.*`) triggers `.github/workflows/maven.yml`: build & test → push image to GHCR → SCP the ansible payload (`playbook.yml`, `templates/`, `docker-compose.yml`, `.env.enc`, etc.) to the server → SSH as `githubdeploy` and run `ansible-playbook` unattended. The playbook converges nginx + PKCS#12 + deploy user, then (when `docker_deploy=true` is passed from CI) runs `docker compose pull && up -d` against `/opt/api-server/docker-compose.yml`. Manual `workflow_dispatch` re-runs the deploy job without rebuilding the image (useful for nginx-only changes). Laptop bootstrap is still `./install.sh`. +Tag push (`v*.*.*`) triggers `.github/workflows/maven.yml`: build & test → push image to GHCR → SCP the ansible payload (`playbook.yml`, `templates/`, `docker-compose.yml`, etc.) to the server → SSH as `githubdeploy` and run `ansible-playbook` unattended with `doppler_token` (the `DOPPLER_TOKEN` repo secret, a read-only `api-server/prd` service token). The playbook fetches the keystore password + nginx proxy secret from Doppler, converges nginx + PKCS#12 + deploy user, writes `/opt/api-server/doppler.env`, then (when `docker_deploy=true` is passed from CI) runs `docker compose pull && up -d` against `/opt/api-server/docker-compose.yml`. Manual `workflow_dispatch` re-runs the deploy job without rebuilding the image (useful for nginx-only changes). Laptop bootstrap is still `./install.sh`. ## Skill maintenance — `api-chencraft` diff --git a/Dockerfile b/Dockerfile index 5f43851..8fe729f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,3 @@ -# ---- Stage 1: Download sops ---- -FROM alpine:3.24 AS sops-downloader - -ARG SOPS_VERSION=v3.11.0 -WORKDIR /tmp - -RUN apk add --no-cache curl \ - && curl -sSL -o sops https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.amd64 \ - && chmod +x sops - -# ---- Stage 2: Runtime ---- FROM eclipse-temurin:25-jre-alpine LABEL org.opencontainers.image.source=https://github.com/SibeiC/api-server/ @@ -18,15 +7,14 @@ LABEL org.opencontainers.image.licenses=GPL-3.0-only # Set working directory WORKDIR /app -# Copy sops binary from downloader stage -COPY --from=sops-downloader /tmp/sops /usr/local/bin/sops +# Doppler CLI injects runtime secrets (DOPPLER_TOKEN supplied via docker-compose env_file) +COPY --from=dopplerhq/cli:3 /bin/doppler /usr/local/bin/doppler # Copy built JAR from CI COPY app/api-server.jar ./api-server.jar # Copy relevant files COPY entrypoint.sh . -COPY .env.enc . # Make entrypoint executable RUN chmod +x entrypoint.sh diff --git a/README.md b/README.md index b5b123c..019bb71 100644 --- a/README.md +++ b/README.md @@ -36,15 +36,16 @@ Environment: - Base config: src/main/resources/application.properties - Dev overrides: src/main/resources/application-dev.properties (activate with SPRING_PROFILES_ACTIVE=dev) -- Env vars: see application.properties and env.sh for Cloudflare/GitHub tokens, S3 endpoint/credentials, iCloud mail - creds +- Secrets (Mongo URI, Cloudflare R2/DNS, GitHub tokens, mail creds, TLS keystore) live in + Doppler — project `api-server`, configs `dev` and `prd`. One-time setup: + `doppler login && doppler setup --no-interactive` (binding committed in `doppler.yaml`). + The dev config also sets SPRING_PROFILES_ACTIVE=dev, so `doppler run` is all you need. Common tasks: -- Build with tests: mvn -P ci clean verify +- Build with tests: mvn -P ci clean verify (no secrets needed — tests use mocks/Testcontainers) - Build only: mvn -P ci -DskipTests=true clean package -- Run app: mvn spring-boot:run -- Run with dev profile: SPRING_PROFILES_ACTIVE=dev mvn spring-boot:run +- Run app (dev secrets + dev profile from Doppler): doppler run -- mvn spring-boot:run - Docker image: docker build -t api-server:local . API docs (local): https://dev.chencraft.com/ @@ -105,16 +106,14 @@ What this playbook does: - Templates and enables an Nginx site for api-server (HTTP 80 redirect to HTTPS 443) - Sets upstream proxy port to 8080 (prod) or 8085 (dev) - Generates a PKCS#12 keystore at /opt/api-server/server.p12 from existing system cert/key -- Optionally stores an age private key in /opt/api-server/age_key (0600) if provided +- Fetches the TLS keystore password and nginx proxy secret from Doppler: with + `-e doppler_token=…` (CI path) it calls the Doppler API; without it (laptop runs) it uses + your logged-in `doppler` CLI against project `api-server`, config `dev` or `prd` +- When `-e doppler_token=…` is supplied, writes /opt/api-server/doppler.env (0600) so + docker-compose can hand DOPPLER_TOKEN to the container - When `-e docker_deploy=true ghcr_user=… ghcr_pat_ro=…` is supplied (CI path), also copies docker-compose.yml to /opt/api-server, logs in to GHCR, and runs `docker compose pull && up -d` -Prompts during execution: - -- TLS keystore password (used to protect server.p12) — skipped when `-e tls_keystore_password=…` is given -- Optional: AGE private key (single line) to write to /opt/api-server/age_key — skipped when `-e age_private_key=…` is given -- Optional (non-dev): GitHub Actions public SSH key for the githubdeploy user - Variables you can override with -e: - app_user, app_group: default to the Ansible remote user (used for file ownership) @@ -122,6 +121,7 @@ Variables you can override with -e: - p12_cert_path, p12_key_path: certificate/key used for PKCS#12 export - server_host: defaults to api.chencraft.com (prod) or dev.chencraft.com (dev) - proxy_port: defaults to 8080 (prod) or 8085 (dev) +- doppler_token: CI-only; read-only Doppler service token for the prd config - docker_deploy, ghcr_user, ghcr_pat_ro: CI-only; gate the container deploy task block CI note: ansible-lint runs in GitHub Actions to validate playbook structure. Ensure requirements.yml is kept in sync @@ -134,7 +134,7 @@ On a fresh host, run this once (SSH in as a user with sudo, e.g. `ubuntu`): 1. `sudo apt-get update && sudo apt-get install -y ansible-core git` 2. `git clone ~/api-server && cd ~/api-server` 3. `./install.sh` — installs collections, creates `githubdeploy` with passwordless sudo and the - GitHub Actions public key, templates nginx, generates the PKCS#12 keystore, places `/opt/api-server/age_key`. + GitHub Actions public key, templates nginx, generates the PKCS#12 keystore. After bootstrap, the GitHub Actions workflow handles every release automatically: it SCPs the playbook payload to `~/api-server-deploy/` on the host and runs `ansible-playbook` as `githubdeploy`. @@ -143,8 +143,8 @@ playbook payload to `~/api-server-deploy/` on the host and runs `ansible-playboo - `SERVER_HOST`, `SERVER_USER` (= `githubdeploy`), `SERVER_SSH_KEY` — SSH transport - `GHCR_PAT_RO` — read-only PAT for `docker login ghcr.io` -- `AGE_PRIVATE_KEY` — multi-line; lets ansible decrypt `.env.enc` to extract the proxy secret -- `TLS_KEYSTORE_PASSWORD` — protects the generated `server.p12` +- `DOPPLER_TOKEN` — read-only Doppler service token for `api-server/prd`; ansible uses it to + fetch the keystore password + proxy secret and hands it to the container for runtime secrets ### Manual re-deploy (no code change) diff --git a/docker-compose.yml b/docker-compose.yml index cf71d6b..b968c01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,9 @@ services: api-server: image: ghcr.io/sibeic/api-server:latest container_name: api-server + # Supplies DOPPLER_TOKEN (read-only prd service token), written by ansible (mode 0600) + env_file: + - /opt/api-server/doppler.env ports: - "127.0.0.1:8080:8080" - "127.0.0.1:8957:8957" diff --git a/doppler.yaml b/doppler.yaml new file mode 100644 index 0000000..a5b6e9c --- /dev/null +++ b/doppler.yaml @@ -0,0 +1,5 @@ +--- +setup: + - project: api-server + config: dev + path: ./ diff --git a/entrypoint.sh b/entrypoint.sh index 3059858..f0f6ff1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,12 +1,6 @@ #!/bin/sh set -eu -# Decrypt and export directly -if [ -f ".env.enc" ]; then - echo "Decrypting .env.enc into environment..." - export SOPS_AGE_KEY_FILE=/opt/api-server/age_key - sops -d --input-type dotenv --output-type dotenv .env.enc > .env -fi - -# Run Java server -java -jar api-server.jar +# Inject secrets from Doppler (project/config implied by DOPPLER_TOKEN service token). +# --fallback keeps restarts working if api.doppler.com is briefly unreachable. +exec doppler run --fallback /opt/api-server/doppler-fallback.json -- java -jar api-server.jar diff --git a/hosts.ini b/hosts.ini index e479079..2d59d4e 100644 --- a/hosts.ini +++ b/hosts.ini @@ -2,4 +2,6 @@ localhost ansible_connection=local ansible_user=sibei [local] -localhost ansible_connection=local ansible_user=ubuntu +# Pin to the distro python: interpreter discovery can pick a newer /usr/bin/python3.X +# that half-imports the apt-installed (cp-version-specific) cryptography stack. +localhost ansible_connection=local ansible_user=ubuntu ansible_python_interpreter=/usr/bin/python3 diff --git a/playbook.yml b/playbook.yml index 49aa236..6d06f89 100644 --- a/playbook.yml +++ b/playbook.yml @@ -30,14 +30,16 @@ p12_cert_path: "/etc/ssl/certs/server.crt" p12_key_path: "/etc/ssl/private/server.key" - # Default proxy secret. Overridden by the value extracted from the decrypted .env when present. - # Empty here keeps the nginx template renderable in dev installs that skip env decryption; - # an empty header is rejected by the Spring filter when APP_MTLS_PROXY_SECRET is set, - # and ignored when it isn't (permissive mode). + # Doppler is the source of truth for runtime secrets. + # CI passes -e doppler_token=; laptop runs leave it + # empty and the logged-in user's doppler CLI session is used instead. + doppler_token: "" + doppler_project: "api-server" + doppler_config: "{{ 'dev' if dev else 'prd' }}" + + # Both are fetched from Doppler below; empty defaults keep the nginx template + # renderable and let the PKCS#12 task self-skip if the fetch is skipped. proxy_secret: "" - - # Empty default lets CI pass -e tls_keystore_password=... directly (skips sops decrypt path). - # When unset, PKCS#12 task self-skips via its own `when:` guard. tls_keystore_password: "" # Gate for the container-deploy task block (set true from CI; laptop runs leave false). @@ -53,12 +55,6 @@ deploy_user: "githubdeploy" deploy_user_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIpJneejmAWqksgGHCcjOqyM4vCcp6edAEnJjo3hm2up ubuntu@DESKTOP-7VSUP4J" - vars_prompt: - - name: age_private_key - prompt: "Enter your AGE private key (single line; leave blank to skip)" - private: true - default: "" - pre_tasks: - name: Detect platform flags ansible.builtin.set_fact: @@ -176,82 +172,47 @@ - { path: "/etc/ssl/certs", mode: "0755" } - { path: "/etc/ssl/private", mode: "0700" } - - name: Age private key is present - when: age_private_key | length > 0 + - name: Fetch runtime secrets from Doppler block: - - name: Ensure age is installed - become: "{{ is_linux }}" - ansible.builtin.package: - name: age - state: present - - - name: Install sops - block: - - name: Install sops for MacOS - when: is_macos - become: false - ansible.builtin.package: - name: sops - state: present - - - name: Install sops on Linux via community.sops.install - when: is_linux - ansible.builtin.include_role: - name: community.sops.install - vars: - sops_install_on_localhost: false - sops_version: "3.11.0" - - - name: Create AGE private key file when provided and missing - become: "{{ is_linux }}" - ansible.builtin.copy: - content: "{{ age_private_key }}\n" - dest: "{{ working_dir }}/age_key" - owner: "{{ app_user }}" - group: "{{ app_group }}" - mode: "0600" - - - name: Check if encrypted env file exists - ansible.builtin.stat: - path: ".env.enc" - register: env_enc_stat + - name: Download secrets with service token (CI) + ansible.builtin.uri: + url: "https://api.doppler.com/v3/configs/config/secrets/download?format=json" + headers: + Authorization: "Bearer {{ doppler_token }}" + return_content: true + register: doppler_api_secrets + no_log: true + when: doppler_token | length > 0 - - name: Decrypt .env.enc to .env using sops and AGE key - ansible.builtin.command: "sops -d --input-type dotenv --output-type dotenv --output {{ working_dir }}/.env .env.enc" - register: decrypt_result - environment: - SOPS_AGE_KEY_FILE: "{{ working_dir }}/age_key" + - name: Download secrets with logged-in doppler CLI (laptop) + become: false + ansible.builtin.command: + cmd: "doppler secrets download --no-file --format json --project {{ doppler_project }} --config {{ doppler_config }}" + register: doppler_cli_secrets changed_when: false - when: - - env_enc_stat.stat.exists - - - name: Read decrypted .env - ansible.builtin.slurp: - path: "{{ working_dir }}/.env" - register: decrypted_env_file - when: - - env_enc_stat is defined and env_enc_stat.stat.exists - - decrypt_result is defined and decrypt_result.rc == 0 - - - name: Extract TLS keystore password from decrypted .env - ansible.builtin.set_fact: - tls_keystore_password: "{{ (decrypted_env_file.content | b64decode | regex_search('(?m)^app\\.tls\\.keystore-password=(.*)$', '\\1') | first | replace('\\r', '') | trim) }}" - when: - - decrypted_env_file is defined + no_log: true + when: doppler_token | length == 0 - - name: Extract proxy secret from decrypted .env (nginx -> Spring shared secret) + - name: Extract TLS keystore password and proxy secret (nginx -> Spring shared secret) ansible.builtin.set_fact: - proxy_secret: "{{ (decrypted_env_file.content | b64decode | regex_search('(?m)^APP_MTLS_PROXY_SECRET=(.*)$', '\\1') | first | replace('\\r', '') | trim) }}" - when: - - decrypted_env_file is defined + tls_keystore_password: "{{ doppler_secrets.APP_TLS_KEYSTORE_PASSWORD | default(tls_keystore_password) }}" + proxy_secret: "{{ doppler_secrets.APP_MTLS_PROXY_SECRET | default(proxy_secret) }}" + vars: + doppler_secrets: >- + {{ (doppler_token | length > 0) + | ternary(doppler_api_secrets.json | default({}), + doppler_cli_secrets.stdout | default('{}') | from_json) }} + no_log: true - - name: Cleanup decrypted .env to avoid leaks - ansible.builtin.file: - path: "{{ working_dir }}/.env" - state: absent - when: - - env_enc_stat is defined and env_enc_stat.stat.exists - changed_when: false + - name: Write Doppler service token env file for docker compose + ansible.builtin.copy: + content: "DOPPLER_TOKEN={{ doppler_token }}\n" + dest: "{{ working_dir }}/doppler.env" + owner: "{{ app_user }}" + group: "{{ app_group }}" + mode: "0600" + no_log: true + when: doppler_token | length > 0 - name: Deploy Nginx site configuration ansible.builtin.template: diff --git a/requirements.yml b/requirements.yml index 0c9724d..0af53f2 100644 --- a/requirements.yml +++ b/requirements.yml @@ -2,4 +2,3 @@ collections: - name: ansible.posix - name: community.crypto - - name: community.sops diff --git a/scripts/age_keygen.sh b/scripts/age_keygen.sh deleted file mode 100755 index f8e923a..0000000 --- a/scripts/age_keygen.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -mkdir -p ~/.config/sops/age - -AGE_KEY="${HOME}/.config/sops/age/keys.txt" -if ! [ -f "${AGE_KEY}" ]; then - age-keygen > "${AGE_KEY}" - chmod 600 "${AGE_KEY}" -fi - -AGE_PUBLIC_KEY=$(age-keygen -y "${AGE_KEY}") -export AGE_PUBLIC_KEY -envsubst < templates/.sops.yaml.template > .sops.yaml \ No newline at end of file diff --git a/src/main/java/com/chencraft/common/config/S3Config.java b/src/main/java/com/chencraft/common/config/S3Config.java index 296a06d..e784f94 100644 --- a/src/main/java/com/chencraft/common/config/S3Config.java +++ b/src/main/java/com/chencraft/common/config/S3Config.java @@ -18,9 +18,9 @@ public class S3Config { private final String endpoint; @Autowired - public S3Config(@Value("${cloudflare.r2.accountId}") String accountId, - @Value("${cloudflare.r2.accessKey}") String accessKey, - @Value("${cloudflare.r2.secretKey}") String secretKey) { + public S3Config(@Value("${cloudflare.r2.account-id}") String accountId, + @Value("${cloudflare.r2.access-key}") String accessKey, + @Value("${cloudflare.r2.secret-key}") String secretKey) { this.accessKey = accessKey; this.secretKey = secretKey; this.endpoint = String.format("https://%s.r2.cloudflarestorage.com", accountId); diff --git a/src/main/java/com/chencraft/common/service/api/CloudflareWebClient.java b/src/main/java/com/chencraft/common/service/api/CloudflareWebClient.java index 1c730ad..7c8e276 100644 --- a/src/main/java/com/chencraft/common/service/api/CloudflareWebClient.java +++ b/src/main/java/com/chencraft/common/service/api/CloudflareWebClient.java @@ -23,7 +23,7 @@ /** * Low-level WebClient wrapper for Cloudflare DNS API. * External IO: HTTPS requests to Cloudflare; maps responses to CloudflareResponse and models. - * Configuration: uses cloudflare.dns.apiKey and cloudflare.dns.zoneId properties. + * Configuration: uses cloudflare.dns.api-key and cloudflare.dns.zone-id properties. */ @Lazy @Slf4j @@ -33,7 +33,7 @@ public class CloudflareWebClient { private final WebClient webClient; - @Value("${cloudflare.dns.zoneId:}") + @Value("${cloudflare.dns.zone-id:}") private String zoneId; /** @@ -43,7 +43,7 @@ public class CloudflareWebClient { * @param cloudflareApiKey API token for Cloudflare (Bearer) */ @Autowired - public CloudflareWebClient(WebClient webClient, @Value("${cloudflare.dns.apiKey:}") String cloudflareApiKey) { + public CloudflareWebClient(WebClient webClient, @Value("${cloudflare.dns.api-key:}") String cloudflareApiKey) { String cloudflareApiBaseUrl = "https://api.cloudflare.com/client/v4"; this.webClient = webClient.mutate() .baseUrl(cloudflareApiBaseUrl) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index c41b2a2..c7a4e24 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -17,10 +17,8 @@ management.endpoint.health.show-details=always management.endpoint.prometheus.access=read_only management.metrics.tags.application=api-server management.server.port=8957 -# Load value from .env -spring.config.import=optional:file:.env[.properties] -# MongoDb config -spring.mongodb.database=api-server +# Secrets (Mongo URI/database, Cloudflare, mail, TLS keystore, ...) arrive as +# environment variables injected by `doppler run` (project api-server, config dev/prd). # Email config spring.mail.properties.mail.smtp.auth=true spring.mail.properties.mail.smtp.starttls.enable=true diff --git a/templates/.sops.yaml.template b/templates/.sops.yaml.template deleted file mode 100644 index 9b3c736..0000000 --- a/templates/.sops.yaml.template +++ /dev/null @@ -1,4 +0,0 @@ -# DO NOT MODIFY, UPDATE USING scripts/age_keygen.sh ONLY -creation_rules: - - path_regex: ^\.env$ - age: $AGE_PUBLIC_KEY \ No newline at end of file