diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a5cd7d8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Build checks hash this upstream snapshot byte-for-byte and compare generated +# CSS; preserve their canonical LF representation in Windows worktrees too. +web/design-system/tokens.json text eol=lf +web/src/styles/design-tokens.generated.css text eol=lf +# CLI contract snapshots also compare exact bytes. +server/cmd/mem/testdata/*.golden.json text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11263b0..b5c0cca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: timeout-minutes: 30 env: MEM_TEST_DB: postgres://mem:mem@localhost:5432/mem_test?sslmode=disable + MEM_OAUTH_TEST_DB: postgres://mem:mem@localhost:5432/mem_oauth_test?sslmode=disable services: postgres: image: pgvector/pgvector:pg16@sha256:00ba258a66dac104fd5171074a0084462a64a1369d8513f3d0a634e2f24d15bc @@ -126,6 +127,9 @@ jobs: "${MEM_TEST_DB}" up + - name: Create isolated GitHub OAuth acceptance database + run: psql "${MEM_TEST_DB}" -v ON_ERROR_STOP=1 -c 'CREATE DATABASE mem_oauth_test;' + - name: Test with race detector and coverage working-directory: server run: | diff --git a/.github/workflows/preview-container-images.yml b/.github/workflows/preview-container-images.yml new file mode 100644 index 0000000..2ba65ef --- /dev/null +++ b/.github/workflows/preview-container-images.yml @@ -0,0 +1,63 @@ +name: Build Sealos preview images + +on: + push: + branches: ["feat/65-github-sealos", "preview/sealos-*"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: mem-preview-images-${{ github.ref }} + cancel-in-progress: true + +jobs: + images: + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: + - image: mem-preview-server + context: server + - image: mem-preview-web + context: web + - image: mem-preview-worker + context: worker + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish commit image + id: build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 + with: + context: ${{ matrix.context }} + platforms: linux/amd64 + push: true + tags: ghcr.io/bytefolk/${{ matrix.image }}:${{ github.sha }} + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + build-args: | + VERSION=sealos-preview + REVISION=${{ github.sha }} + WORKER_EXTRAS= + cache-from: type=gha,scope=mem-preview-${{ matrix.context }} + cache-to: type=gha,scope=mem-preview-${{ matrix.context }},mode=max + - name: Record image digest + env: + IMAGE_NAME: ${{ matrix.image }} + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + run: | + printf '%s\n' "ghcr.io/bytefolk/$IMAGE_NAME@$IMAGE_DIGEST" >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index c3b3334..bcd4ecc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ The project publishes 0.x prerelease versions; a stable release line is not yet ## [Unreleased] +### Personal GitHub preview + +- Add allowlisted GitHub browser sign-in and explicit account binding with + PKCE, single-use browser-bound state, immutable provider subjects, verified + primary email and no automatic email linking. Browser cookies have + server-side expiry/revocation and mutation CSRF/origin checks. Existing + Bearer clients retain their contract. +- Add a small Sealos Helm overlay with closed password enrollment and explicit + allowlisted first-owner GitHub bootstrap. No model Worker is needed for + files and lexical memory recall. Real provider/deployment certification is + separate from the fixture tests. + +### Security + +- Refresh Web development-only transitive dependencies to patched + `brace-expansion` 1.1.21/2.1.7 and `undici` 7.30.0, restoring the unchanged + Linux and Windows real-registry audit gates. Production dependency ranges + and audit severity thresholds are unchanged. + ### Added - Per-agent memory namespace filter on lexical recall (`#227`): `POST /v1/context` @@ -19,6 +38,16 @@ The project publishes 0.x prerelease versions; a stable release line is not yet ### Changed +- Correct native Sealos startup to use the compiled `local-fast-v2` profile, + a private 0700 transfer subdirectory and Worker probe timeouts that accommodate + Python/gRPC startup under the preview CPU limit. +- Render Sealos Web nginx configuration at startup from the shared image + template into its writable `/tmp` claim, avoiding unresolved instance + expressions in ConfigMap block scalars while retaining nonroot/read-only + container security and OAuth callback access-log filtering. +- Restore the Agent-memory integration gates for migration 0027 and build the + disposable MinIO/mc test dependency from pinned official source commits + when the upstream community image registries reject pulls. - The live recall producer (`python3 -m benchmarks.recall produce`) reads engine, profile, provider and embedding dimension from the running memd instead of CLI labels, scores the multilingual v1 set (English and Chinese, including image-description queries), and stays on-demand rather than a CI gate. Refs #175. - Rewrite the root README around a concise product promise, a verified Compose first-run path, source-built MCP setup, current capability and trust boundaries, and links to authoritative reference docs. Remove the unsupported unpublished `@bytefolk/mem-mcp@0.1.2` install path and keep its migration explicitly tracked in #153. Refs #229. - Index generation HTTP create/activate/rollback stay `503 execution_unavailable` with `execution_wired: false` on that body, and the same flag is now on events as well as list/status/cancel/resume/discard. Create still returns `400` for malformed JSON or an empty `profile_id` before the 503. Empty-workspace activate is refused by the same 503. Refs #174. diff --git a/SPEC.md b/SPEC.md index ce49097..7b00e9b 100644 --- a/SPEC.md +++ b/SPEC.md @@ -9,6 +9,18 @@ ## 0. TL;DR +Personal GitHub browser login is optional and requires an exact public HTTPS +origin, OAuth client credentials and numeric GitHub subject allowlist. New +GitHub first-owner bootstrap is explicit and requires disabled public password +registration. HTTP surfaces: `GET /v1/auth/capabilities`, +`POST /v1/auth/github/start`, `GET /v1/auth/github/callback`, +`GET /v1/auth/session`, `POST /v1/auth/logout`, and authenticated +`GET /v1/auth/github/identity`. GitHub identities never link by email alone; +binding requires password reauthentication or a recent browser session and +the callback retains the same active actor/session. See +[`deploy/sealos/README.md`](deploy/sealos/README.md). This personal flow does +not claim the full hosted identity/OIDC/MFA roadmap is implemented. + **mem** 是一个开源、自托管、面向 Agent 的可迁移网盘。 形态像网盘，机器入口像共享 Memory Plane。 diff --git a/deploy/helm/mem/secret.env.example b/deploy/helm/mem/secret.env.example index 590add9..07a5833 100644 --- a/deploy/helm/mem/secret.env.example +++ b/deploy/helm/mem/secret.env.example @@ -14,3 +14,10 @@ MEM_S3_SECRET_KEY=replace-me MEM_S3_USE_SSL=true MEM_WORKER_AUTH_KEY_ID=memd-primary MEM_WORKER_AUTH_KEY_B64=replace-with-standard-base64-for-exactly-32-random-bytes +# Optional personal GitHub login. Use the actual ingress origin and configure +# the OAuth App callback as /v1/auth/github/callback. +# MEM_PUBLIC_URL=https://mem.example.com +# MEM_GITHUB_CLIENT_ID=replace-me +# MEM_GITHUB_CLIENT_SECRET=replace-me +# MEM_GITHUB_ALLOWED_USER_IDS=47820304 +# MEM_GITHUB_BOOTSTRAP=true # requires runtime.registrationMode=disabled diff --git a/deploy/sealos/README.md b/deploy/sealos/README.md new file mode 100644 index 0000000..9af3072 --- /dev/null +++ b/deploy/sealos/README.md @@ -0,0 +1,125 @@ +# Personal mem preview on Sealos + +`template.yaml` is the native Sealos Template API recipe: one Web, one `memd`, +one small CPU/model-free Worker, private pgvector PostgreSQL 16, private Redis, +and a Sealos-managed private object bucket. Files, lexical memory recall, +checkpoints, and workspace transfer work without a model provider. The Worker +is authenticated but no Ollama server, ASR, face or CLIP model is bundled; +vector/image enrichment needs separately configured providers. + +This preview reserves limits of 2.4 CPU, 2048Mi memory and 7Gi RWO storage +(plus managed object storage). Requests follow the Sealos quota ladder. +The actual namespace dry-run, balance and quota decide whether it fits. +Each process that writes uses a StatefulSet claim instead of unsupported +`emptyDir`; PostgreSQL uses a subdirectory so a fresh mounted volume initializes. + +## Inputs + +1. A Sealos namespace with sufficient quota and a registry accepting the built + `server/`, `web/` and model-free `worker/` images for linux/amd64. Build the + worktree revision, not a stale release. The optional preview workflow + publishes commit-SHA tags under `ghcr.io/bytefolk/mem-preview-{server,web,worker}`; + a workflow file is not evidence that the images were built or can be pulled. +2. The template's `pgvector/pgvector:0.8.1-pg16` image supplies the `vector`, + `pgcrypto` and `pg_trgm` extensions. The private Redis URL uses generated + credentials. Bucket keys/endpoint come from Sealos `object-storage-key`, + and the bucket name from its managed per-bucket Secret. No MinIO console or + database port is exposed. mem's MinIO client uses path-style S3 addressing. +3. The actual HTTPS hostname and Sealos ingress/TLS configuration. +4. A GitHub OAuth App owned by the operator, with callback URL exactly + `https:///v1/auth/github/callback`. It needs only + `read:user user:email`; no repository or organization permission is used. + +Supply the required Template inputs from a protected arguments file outside +Git. Generate the Worker auth key as standard Base64 of exactly 32 random +bytes. OAuth secrets and Worker keys enter the runtime Secret only; both +Docker build contexts exclude `.env*`. Do not print a rendered Secret or +commit arguments files. `secret.env.example` lists equivalent Helm keys. +OAuth identities, challenges and sessions are excluded from workspace bundles. + +`MEM_REGISTRATION_MODE=disabled` closes password signup. The explicit +`MEM_GITHUB_BOOTSTRAP=true` switch permits only an allowlisted GitHub subject +to create the first owner atomically. Numeric ID `47820304` is the requested +personal account; usernames/email addresses are not authorization keys. +Further users cannot enroll. Existing email accounts must authenticate and +explicitly link GitHub; matching email never merges accounts. + +## Rollout + +Select the intended instance name and `defaults.app_host`, then read the actual +rendered Ingress hostname before finalizing the GitHub OAuth App callback. +The console hostname does not establish `SEALOS_CLOUD_DOMAIN`; the personal +Hangzhou workspace rendered `mem-peterguy326-f5396ac8.sealoshzh.site`. +The generic template defaults remain random to avoid duplicate instances. +Use public images only after verifying +anonymous pull. For private images, precreate a fixed pull Secret and add its +name to all application workloads/init containers; do not reference a missing +Secret or copy a broad personal GitHub token to the cluster. + +The operator uses the Sealos Template API CLI with the protected input file, +performs its free dry-run/quota preview, then deploys exactly that template +instance. Every resource carries `cloud.sealos.io/deploy-on-sealos` and the +last App CR opens the HTTPS Web entry. A successful API response still needs +pod readiness, URL checks and real GitHub callback/asset acceptance. + +Web renders the image's shared nginx template using only `MEMD_UPSTREAM` and +`MEM_MAX_BODY_SIZE`. The upstream arrives as a single-line Pod environment +value because the Template API leaves instance expressions inside ConfigMap +block scalars unresolved. Generated configs use the `/tmp` claim; the image's +main nginx config is copied there with its include redirected to the generated +server config. UID/GID 101, read-only root filesystem and dropped capabilities +are retained. nginx variables and OAuth callback access-log filtering remain +in the shared template. + +`mem-migrate` runs as the single memd replica's init container, then memd +starts with `MEM_AUTO_MIGRATE=false`. Database-not-ready failures retry with +the Pod; do not bypass migration. This revision adds migration 0027. It +preserves existing users/memberships/machine tokens; downgrade refuses to +orphan a GitHub-only account. Back up an existing deployment first. + +The native profile is `local-fast-v2`. Workspace transfers use +`/tmp/mem-workspace-transfers`; memd creates this private 0700 directory under +the writable claim and rejects a shared temporary root. Worker exec health +checks need time for Python imports and two gRPC waits: startup permits up to +180 seconds, with 15-second startup and 10-second steady-state probe timeouts. +The existing CPU, memory, authentication and container security limits remain. + +The native recipe disables ingress access logging, and the inner nginx filters +OAuth callback query strings. It intentionally has no placeholder NetworkPolicy: +the base Helm chart's example CIDR would block Sealos dependencies/GitHub. +Before broader sharing, use verified namespace/pod labels and real egress +rules; do not mistake absence of a NetworkPolicy for network isolation. + +For operators already running the Helm chart with provisioned dependencies, +`preview-values.yaml` remains a separate model-free/no-Worker overlay: + +```sh +helm upgrade --install mem deploy/helm/mem \ + --namespace "$MEM_NAMESPACE" \ + -f deploy/sealos/preview-values.yaml \ + --set-string images.server.repository="$MEM_SERVER_IMAGE_REPOSITORY" \ + --set-string images.server.tag="$MEM_IMAGE_TAG" \ + --set-string images.web.repository="$MEM_WEB_IMAGE_REPOSITORY" \ + --set-string images.web.tag="$MEM_IMAGE_TAG" \ + --set-string ingress.host="$MEM_PUBLIC_HOST" \ + --set-string ingress.tls.secretName="$MEM_TLS_SECRET" \ + --wait --timeout 10m +``` + +Create the chart's `mem-runtime` Secret from the equivalent runtime keys. +Its pre-install/pre-upgrade migration Job must finish before the rollout. + +## Acceptance + +- Web `/healthz` and internal `memd /readyz` succeed. +- Login offers GitHub and no public account-creation option. +- An anonymous password registration fails; an unlisted GitHub user fails. +- The allowlisted account creates the first workspace and can upload/download + a file and write/recall a sourced memory. +- Account menu shows the linked GitHub identity. +- Cookie-authenticated mutations require the exact origin and CSRF header. +- Logout revokes the server session; stale binding callbacks cannot log a user + back in or change accounts. OAuth callback query strings are not access-logged. + +Unit/HTTP fixtures do not certify real GitHub or Sealos interoperability. +Retain the actual callback/login/asset acceptance result after the rollout. diff --git a/deploy/sealos/VALIDATION.md b/deploy/sealos/VALIDATION.md new file mode 100644 index 0000000..15eff9e --- /dev/null +++ b/deploy/sealos/VALIDATION.md @@ -0,0 +1,301 @@ +# Personal GitHub preview validation ledger + +Local validation on 2026-10-01: Windows worktree mounted in Ubuntu 22.04 WSL, +Go 1.25 toolchain, Node 24.11.1, locked npm dependencies, CPython 3.11.15 and +locked uv dependencies. The original checkout and running user's mem stack +were not modified. PostgreSQL fixtures used a separately named disposable +`pgvector/pgvector:0.8.1-pg16` container on loopback port 55443; every test +database ended in `_test` and was created inside that owned container. + +The requested scope is a personal preview: GitHub numeric allowlist, closed +password enrollment, one atomic first owner, explicit existing-account link, +browser cookies, unchanged machine Bearer API semantics, and a native Sealos +recipe. This does not complete the broader hosted-SaaS authentication roadmap. +The existing auth epic is design context, not a claimed ready issue or approval. +At this initial local validation stage, no PR, production release, merge or +independent end-to-end approval was claimed. Later CI follow-ups are recorded +below; the initial `NOT VERIFIED` rows describe that local validation stage. + +| Check | Result | Evidence and limitations | +| --- | --- | --- | +| Go hermetic regression | PASS | `go test ./... -count=1` across server, CLI and MCP | +| Go static check and compile | PASS | `go vet ./...`, `go build ./...` | +| PostgreSQL integration and race | PASS | `MEM_TEST_DB= MEM_OAUTH_TEST_DB= go test -race -p 1 ./... -count=1` | +| Latest OAuth authorization regression | PASS | Narrow race suite checks invalid Bearer does not fall back to cookie, foreign workspace denial and foreign-origin session denial | +| Populated migration upgrade | PASS | `MEM_MIGRATION_SEQUENCE_TEST_DB= go test -race ./internal/db -count=1` exercises strict upgrades through 0027 and preserves the populated file/chunk/index fixture | +| OAuth-only downgrade guard | PASS | The HTTP fixture runs Goose downgrade, requires the explicit refusal while an OAuth-only user exists, and verifies identities remain readable | +| Provider exchange contract | PASS | PKCE S256, minimal `read:user user:email` scope, primary verified email, fixed endpoints and secret-safe failures use HTTP provider fixtures | +| Browser session state | PASS | Web's 119 tests include 10 new auth/API cases; the final focused 10-test run also passes | +| Web lint | PASS | `npm run lint`, no warnings | +| Web production build | PASS | `npm run build`; existing large-bundle warning remains informational | +| Browser acceptance | PASS | `test:i18n`, `test:theme`, `test:enrichment`, `test:memory`, `test:transfer` with Playwright Chromium and generated ignored MSW worker | +| Worker regression | PASS | `uv sync --locked --extra test`, `uv run pytest -q`; real visual-model evaluation skipped because `MEM_RUN_VISUAL_MODEL_EVAL` is unset | +| Native template syntax/grouping | PASS | 16 YAML documents parse; Template first, App last; deploy grouping labels on every resource and absent from Pod labels | +| GitHub Actions execution | NOT VERIFIED | CI now creates a separate fresh OAuth database; no remote CI success is inferred from local checks | +| Container publishing/anonymous pull | NOT VERIFIED | Runtime secrets are excluded from both changed Docker contexts; built GHCR images and pull access still need actual verification | +| Sealos dry-run, quota, deployment | NOT VERIFIED | Local structure checks do not replace the platform API or actual namespace quota | +| Live GitHub callback/account binding | NOT VERIFIED | All OAuth tests use a fake identity provider; the operator's actual OAuth App is not certified | +| Live file/memory acceptance on Sealos | NOT VERIFIED | After rollout: GitHub login, account identity readback, file upload/download, sourced remember/recall, and logout need real acceptance | +| Model-quality/real enrichment | NOT VERIFIED | The template has one authenticated CPU Worker but bundles no model service or optional AI extras | + +The HTTP/PostgreSQL fixture verifies an anonymous visitor cannot password +register, an unlisted GitHub subject cannot enter, and the allowed owner can +bootstrap and write a sourced memory through the canonical API. It also checks +state tampering/replay, CSRF, cookie versus machine-credential separation, +email collisions without auto-link, identity transfer denial, and stale link +callbacks after logout/account switch or logout during provider exchange. + +Frontend regression covers stale Bearer callback navigation, absent/failed +cookie bootstrap, same-user session replacement across tabs, CSRF/cache +refresh, logout acknowledgement/failure, late session readback, and late +JSON/raw/blob 401 responses from the previous actor. Browser session secrets +are HttpOnly cookies; only public session-generation metadata is persisted. + +The Sealos recipe keeps database/cache ports internal, requests a private +managed bucket, uses managed object-store key references, passes only Worker +keys/Redis credentials to the Worker, and disables outer ingress access logs. +The Web proxy excludes callback codes/state from access logs. The recipe does +not claim verified namespace network isolation: no placeholder NetworkPolicy +is applied. Its limit total is 2.4 CPU/2048Mi and 7Gi RWO, plus object storage. + +Browser artifacts retained outside Git: + +- `/tmp/mem-localization-acceptance-52204` +- `/tmp/mem-theme-acceptance-53188` +- `/tmp/mem-enrichment-acceptance-53616` +- `/tmp/mem-memory-acceptance-54207` +- `/tmp/mem-transfer-acceptance-54648` + +Windows line-ending conversion initially broke the pinned design-token hash +and exact CLI JSON golden comparison. Scoped `.gitattributes` rules preserve +their upstream LF bytes; the content is unchanged. The initial browser attempt +lacked ignored `public/mockServiceWorker.js`; `npx msw init public --save` +generated it locally, and all five browser gates subsequently passed. + +Before any wider rollout, attach actual Sealos/GitHub/asset acceptance and +independent review to the PR validation ledger. Treat every `NOT VERIFIED` +row above as outstanding; source audit is not provider or deployment evidence. + +## PR 231 dependency-audit follow-up + +The first PR CI run at `6e85c9511625e043023dc6dd78ef126e0c8a5e8b` failed +both the [Web audit](https://github.com/bytefolk/mem/actions/runs/36865578915/job/110380137487) +and [Windows audit evidence](https://github.com/bytefolk/mem/actions/runs/36865578915/job/110380137434) +on newly reported development-only transitive dependencies. Production audit +already reported zero vulnerabilities. The Windows process fixture passed and +correctly preserved audit exit 1, so no audit script or severity gate was +weakened. + +The targeted lockfile update preserves parent dependency ranges and changes +only `brace-expansion` 1.1.18 to 1.1.21, 2.1.4 to 2.1.7, and `undici` 7.29.0 +to 7.30.0. npm supplied the updated registry integrity hashes. It addresses the +[brace-expansion advisories](https://github.com/advisories/GHSA-q2hr-2g5m-vwhr) +and [undici advisories](https://github.com/advisories/GHSA-3wwx-pv8p-q78v) +reported by the real registry; the complete registry report now contains no +vulnerabilities. + +- Native Windows Node 24.19.0: `node --test scripts/test_win_audit_verify.mjs` + passed all 4 process tests. +- Native `cmd.exe /d /c ..\scripts\win-audit-verify.bat` passed both the + production-moderate and all-dependencies-high audits with zero vulnerabilities + and final exit 0. Transcript retained outside Git at + `C:\Users\huyz\rw-clone\mem-preview-evidence\win-audit-dependency-refresh.txt`; + it records the tested checkout and exact audit exit status. +- Ubuntu WSL Node 24.11.1: the real `npm run audit` and affected Web checks + passed after `npm ci --ignore-scripts --no-audit --no-fund`: both audit + thresholds report zero vulnerabilities, all 119 unit tests pass, and lint, + typecheck and the production build succeed. The large-bundle warning is + unchanged; the produced JS/CSS asset hashes are unchanged. +- The same original CI run's [Go race/coverage job](https://github.com/bytefolk/mem/actions/runs/36865578915/job/110380137449) + succeeded, including the separately provisioned OAuth database fixture. +- Remote CI must rerun on the committed repair before the PR is declared green. + +## PR 231 PostgreSQL and lifecycle gate follow-up + +The [PostgreSQL job](https://github.com/bytefolk/mem/actions/runs/36865578987/job/110380137083) +at `6e85c9511625e043023dc6dd78ef126e0c8a5e8b` passed every populated migration +upgrade through 0027 and preserved the recorded history/data, then failed +because `scripts/verify.sh` still expected migration head 26. The runner now +expects 27; its rollback, populated-upgrade, HNSW, required-test execution and +race checks are unchanged. The existing isolated OAuth database fixture remains +in the separate Go CI job because it requires a fresh empty database. + +The [HTTP/CLI/MCP job](https://github.com/bytefolk/mem/actions/runs/36865578987/job/110380137349) +failed before building or starting memd: the existing test Compose MinIO image +pull returned `unauthorized`. This Compose reference is identical in the PR's +base commit. Independent anonymous manifest queries reproduced the registry +rejection for the community repositories on Quay and Docker Hub. + +The [official MinIO source-only distribution instructions](https://github.com/minio/minio#source-only-distribution) +describe building from source; the upstream community repository is archived. +The test stack now builds only this disposable dependency from official +[MinIO commit 7aac2a2](https://github.com/minio/minio/commit/7aac2a2c5b7c882e68c1ce017d8256be2feea27f) +and [mc commit 77f82e1](https://github.com/minio/mc/commit/77f82e18b5401a65958f1619df6ebb994634bd88). +`scripts/test-minio/Dockerfile` pins the Docker Official Go and Alpine image +digests, enables the public Go checksum database and retains both licenses. +Its dedicated context denies all files except Dockerfile and `.dockerignore`; +no repository code, runtime secret or provider credential enters it. The +production Sealos managed bucket configuration is unaffected. + +Validation uses a tracked source archive of `6e85c95` plus only the four scoped +runner/test-image file overlays, with shell line endings normalized in the +Linux validation directory. Host Go is 1.25.9, the pinned image builder Go is +1.25.10, and the disposable PostgreSQL image is the same pgvector digest used +by CI. No development/production database is used. + +- `git diff --check`, `bash -n scripts/verify.sh`, and the full e2e Compose + model validation passed. +- The fixed official source image built successfully, and + `scripts/acceptance_agent_memory.sh` exited 0. It preserved the complete HTTP + remember/replay/conflict/model-free recall, CLI/checkpoint path isolation, + sequential MCP lifecycle, and PostgreSQL logical-forget redaction checks. +- `scripts/verify.sh integration` exited 0, including populated migration + upgrades, rollback round trips, HNSW checks and every required PostgreSQL + regression. `scripts/verify.sh integration-race` also exited 0 with all + required PostgreSQL regressions executing and passing under the Go race + detector. +- Test containers, owned databases and networks are removed by their guarded + cleanup paths. Logs are retained outside Git at + `/home/huyz/sealos-preview-build/mem-ci-fixes-audit-acceptance.log` and + `/home/huyz/sealos-preview-build/mem-ci-fixes-audit-integration.log`. + +The [preview image run](https://github.com/bytefolk/mem/actions/runs/36865387084) +successfully published all three images for the original `6e85c95` commit: + +- `mem-preview-server@sha256:a43a5d0e5b310df0cfaf73157d2585609266d354352f8cabbac4cd298d3ae20b` +- `mem-preview-web@sha256:dfb90d483d932c493804f2c72cb30fbbf42188855bf86b6ac22437344bc95727` +- `mem-preview-worker@sha256:7ec03edb854759bb95daedd4db4aae7826542a49f4c59c8e7149c136261258ba` + +These are publication results for that earlier commit. Anonymous registry +pulls, new-head image publication, full remote CI on the committed repair, +and all Sealos/live GitHub acceptance remain unverified. + +## Native Web startup follow-up + +The baseline `58ca323d82cf32d1bdde7260570737b31fda77a1` subsequently passed all +10 [CI jobs](https://github.com/bytefolk/mem/actions/runs/36867557186), all four +[security jobs](https://github.com/bytefolk/mem/actions/runs/36867557222), and +the Agent-memory integration/acceptance and three preview image builds. These +are remote checks for that baseline, not certification of the follow-up below. + +A real Sealos Template API dry-run returned HTTP 200 and the actual deploy +returned HTTP 201 with 15 resources. PostgreSQL and Redis became ready, all six +RWO claims bound (7Gi total), and the managed bucket's policy reads `private`. +Database/cache/Worker services remain ClusterIP. Ingress access logging reads +`false`. Runtime inputs and raw API/log evidence stay outside Git with 0600 +file permissions and 0700 directories; no credentials are included here. + +The platform rendered the actual public domain as +`mem-peterguy326-f5396ac8.sealoshzh.site`, rather than the console's hostname. +The operator's OAuth App callback was updated to that hostname with the +existing `/v1/auth/github/callback` route. + +The first Web Pod reproduced `nginx: [emerg] unexpected "}" in +/etc/nginx/conf.d/default.conf:26`. Its live ConfigMap contained one unresolved +`${{ defaults.app_name }}` in `proxy_pass`, while the namespace expression had +rendered. Pod readback confirmed UID/GID/fsGroup 101, read-only root filesystem, +dropped capabilities and the intended `/tmp`/cache claims; this was a syntax +failure, not a permission failure. A resourceVersion-guarded replacement of +that single upstream expression and a Web-only rollout fixed the live Pod. +It became Ready with no restarts on the new revision; `/healthz` and `/` returned +HTTP 200. No database, runtime key, claim or container privilege was changed. + +The generic recipe now supplies the dynamic upstream as a single-line Pod +environment value. A static ConfigMap startup script renders only the two +application variables in the image's shared nginx template, then copies the +main config to `/tmp` with its include redirected to that generated config. +No instance expression remains in the startup block. + +Validation of this follow-up: + +- A fresh `mem-web:deploy-validation` production image built successfully. + The existing user's `mem-web:local` tag and running Compose stack were not + changed. That older cached image was insufficient to validate the current + OAuth log filter, so the regression uses the freshly built image. +- `scripts/test_sealos_web_startup.sh` passed with the actual startup block, + UID 101, read-only root, dropped capabilities and writable temporary mounts. + Real nginx syntax, HTTP health and SPA login checks passed; the generated + config rendered the upstream/body limit and preserved nginx variables and + the OAuth callback access-log filter. The owned test container was removed. + This regression now runs in the existing `test-deploy-build` CI gate. +- Production Compose/Helm validation passed using a source archive plus the + focused overlays. The archive explicitly disables Windows `core.autocrlf` + conversion; CRLF Helm render lines in a default Windows archive had caused + the shell's exact-kind checks to reject an otherwise valid chart. The + successful log is retained outside Git at + `/home/huyz/sealos-preview-build/mem-nginx-deployment-1790865281589645862/deployment.log`. +- A separate, nondeployed validation instance for the updated native template + passed the real Template API dry-run: HTTP 200, 15 resources. The existing + instance was not redeployed, and its generated DB/Redis keys were retained. +- Shell syntax and `git diff --check` passed. + +At this evidence point, memd's migration image pull has failed with TCP +`connection timed out` and `connection reset by peer`; the Worker image is +still pulling. The final linux/amd64 compressed sizes are approximately +23.94MiB server, 20.07MiB Web and 132.10MiB Worker. Authenticated manifest +access succeeds, and Web layers eventually pulled, but backend/Worker layer +delivery and live login/file/memory acceptance remain outstanding. Web's +auth-capabilities endpoint returns 502 until memd starts. A new follow-up CI +run and the generic startup's real cloud rollout are also not claimed here. + +## Live runtime recovery and personal acceptance + +The application revision `62b47e1affcdf7d7f01ffee49ef72ed6c4a7be5e` +passed all 24 PR checks, including [CI](https://github.com/bytefolk/mem/actions/runs/36877773632), +[security](https://github.com/bytefolk/mem/actions/runs/36877773651), and +[Agent-memory integration/browser acceptance](https://github.com/bytefolk/mem/actions/runs/36877774016). +Its [published images](https://github.com/bytefolk/mem/actions/runs/36877770694) +were copied to an operator-owned regional private registry with every manifest +and blob SHA verified. Anonymous access and the read-only puller's writes were +rejected. The live server/Web/Worker retain that application's image provenance; +this follow-up changes the deployment recipe and its verification only. + +Once the original image pulls completed, actual startup exposed three recipe +errors: the unsupported `local_fast_v2` spelling, `/tmp` failing memd's existing +0700 transfer-directory check, and Worker exec probes timing out at Kubernetes' +default one second. Worker logs showed normal `server.listening`; events showed +timed-out probes and liveness restarts, rather than a processor crash. +The recipe now uses `local-fast-v2`, a private subdirectory created by memd, +and a startup probe plus explicit steady-state timeouts. Focused server-side +dry-runs and spec readbacks verified unchanged runtime keys, claims, resource +limits and security. UID/resourceVersion-guarded replacement of only the +known bad old Pod revisions recovered the blocked rollout; no persistent +resource was deleted. The generic Web startup also rolled out successfully, +and live `nginx -t -c /tmp/nginx.conf` passed. + +Actual platform verification subsequently passed: + +- All five mem workloads Ready 1/1, final server/Worker zero restarts; migration + init exit 0, migration head 27 and pgvector 0.8.1. +- Public Web/health/auth capabilities HTTP 200, GitHub enabled, password + registration disabled, visitor registration HTTP 403 and exact actual + `/v1/auth/github/callback` redirect. +- Worker authenticated bucket HEAD succeeded, anonymous bucket LIST HTTP 403, + managed bucket policy private and dependency services ClusterIP. +- The operator completed real GitHub authorization as `PeterGuy326`, uploaded + a nonsensitive text fixture, reloaded it, previewed its content and saw the + bound identity on Account. Logout and another GitHub login preserved the file. + Read-only database checks confirmed one user, one matching GitHub identity, + one correctly owned fixture and its expected SHA. +- An isolated local Chromium check exercised the actual `downloadFile` module + with a fixture cookie and byte response. The click produced a download event, + the expected filename and a saved file matching the fixture byte for byte + (`0ff3c4c81dbd68c246debf992ec83408795ef3b9ef403b66b3742bfeb5b84146`). + The cloud in-app browser did not expose a download event during this run; + cloud download completion remains unverified. Live memory write/recall was + not exercised in this acceptance. + +Runtime/API evidence and screenshots remain outside Git in protected operator +files. No production credential or user-uploaded private file is recorded here. + +The new `scripts/test_sealos_runtime_startup.py` passed against the exact live +62b server/Worker images. With networking disabled, memd passed production +configuration and its private temporary-directory setup before the expected +DB refusal. An isolated required-auth Worker and disposable Redis passed real +startup/readiness/liveness exec checks at 500m CPU/512Mi and read-only root. +This verifies authenticated configuration can start; the intentionally unsigned +HealthCheck does not certify business HMAC authorization. Owned containers and +network were removed, and the temporary read-only registry login was erased. +The regression is connected to the existing deployment-image CI gate. The +follow-up also passed `git diff --check`; its new-head remote CI is still pending +at this evidence point. diff --git a/deploy/sealos/preview-values.yaml b/deploy/sealos/preview-values.yaml new file mode 100644 index 0000000..9ae8e19 --- /dev/null +++ b/deploy/sealos/preview-values.yaml @@ -0,0 +1,36 @@ +# Personal preview overlay for ../helm/mem. Set registry images and ingress +# host from the actual Sealos account; no credentials belong in this file. +existingSecret: mem-runtime +runtime: + deploymentMode: private + registrationMode: disabled + aiProfiles: local-fast-v2 + logLevel: info +memd: + replicaCount: 1 + resources: + requests: {cpu: 100m, memory: 128Mi} + limits: {cpu: "1", memory: 512Mi} + topologySpreadConstraints: {enabled: false} + podDisruptionBudget: {enabled: false} +worker: + enabled: false +web: + replicaCount: 1 + resources: + requests: {cpu: 50m, memory: 64Mi} + limits: {cpu: 500m, memory: 256Mi} + topologySpreadConstraints: {enabled: false} + podDisruptionBudget: {enabled: false} +ingress: + enabled: true + host: mem.example.com # override with the Sealos-assigned HTTPS hostname + tls: + enabled: true + secretName: mem-tls # override for the actual Sealos ingress/TLS controller +networkPolicy: + # The base chart's 192.0.2.0/24 is a documentation placeholder and blocks + # real dependencies/GitHub. This small private namespace preview disables + # that placeholder policy. Add measured dependency/GitHub egress before + # using this as a shared or hardened environment. + enabled: false diff --git a/deploy/sealos/secret.env.example b/deploy/sealos/secret.env.example new file mode 100644 index 0000000..a73e230 --- /dev/null +++ b/deploy/sealos/secret.env.example @@ -0,0 +1,15 @@ +# Copy OUTSIDE this repository to a protected file. Replace each placeholder. +# Feed it to a Kubernetes Secret, never a Docker build argument or VITE_* env. +MEM_DB_URL=postgres://mem:replace-me@postgres.internal:5432/mem?sslmode=disable +MEM_REDIS_URL=redis://:replace-me@redis.internal:6379/0 +MEM_S3_ENDPOINT=http://minio.internal:9000 +MEM_S3_REGION=us-east-1 +MEM_S3_BUCKET=mem +MEM_S3_ACCESS_KEY=replace-me +MEM_S3_SECRET_KEY=replace-me +MEM_S3_USE_SSL=false +MEM_PUBLIC_URL=https://mem.example.com +MEM_GITHUB_CLIENT_ID=replace-me +MEM_GITHUB_CLIENT_SECRET=replace-me +MEM_GITHUB_ALLOWED_USER_IDS=47820304 +MEM_GITHUB_BOOTSTRAP=true diff --git a/deploy/sealos/template.yaml b/deploy/sealos/template.yaml new file mode 100644 index 0000000..7ed2e75 --- /dev/null +++ b/deploy/sealos/template.yaml @@ -0,0 +1,575 @@ +apiVersion: app.sealos.io/v1 +kind: Template +metadata: + name: mem-personal-preview +spec: + title: 'mem personal preview' + url: 'https://github.com/bytefolk/mem' + gitRepo: 'https://github.com/bytefolk/mem' + author: 'bytefolk' + description: 'Single-owner GitHub preview with private pgvector, Redis and object storage.' + readme: 'Use deploy/sealos/README.md. Supply built public amd64 images and runtime OAuth/Worker secrets. Model providers are not bundled.' + icon: '' + templateType: inline + locale: en + categories: [] + defaults: + app_name: + type: string + value: mem-${{ random(8) }} + app_host: + type: string + value: mem-${{ random(8) }} + db_password: + type: string + value: ${{ random(32) }} + redis_password: + type: string + value: ${{ random(32) }} + inputs: + server_image: + description: 'Public linux/amd64 server image, pinned to the reviewed commit or digest' + type: string + required: true + default: '' + web_image: + description: 'Public linux/amd64 Web image from the same revision' + type: string + required: true + default: '' + worker_image: + description: 'Public linux/amd64 model-free Worker image from the same revision' + type: string + required: true + default: '' + github_client_id: + description: 'Operator-owned GitHub OAuth App client ID' + type: string + required: true + default: '' + github_client_secret: + description: 'Runtime GitHub OAuth App secret; never use a build argument' + type: string + required: true + default: '' + github_allowed_user_ids: + description: 'Comma-separated immutable GitHub numeric subjects' + type: string + required: true + default: '47820304' + worker_auth_key_b64: + description: 'Fresh standard Base64 encoding of exactly 32 random bytes' + type: string + required: true + default: '' +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${{ defaults.app_name }}-runtime + labels: + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +type: Opaque +stringData: + DB_PASSWORD: ${{ defaults.db_password }} + REDIS_PASSWORD: ${{ defaults.redis_password }} + MEM_GITHUB_CLIENT_ID: ${{ inputs.github_client_id }} + MEM_GITHUB_CLIENT_SECRET: ${{ inputs.github_client_secret }} + MEM_WORKER_AUTH_KEY_B64: ${{ inputs.worker_auth_key_b64 }} +--- +apiVersion: objectstorage.sealos.io/v1 +kind: ObjectStorageBucket +metadata: + name: ${{ defaults.app_name }} + labels: + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + policy: private +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ${{ defaults.app_name }}-postgres + annotations: + originImageName: pgvector/pgvector:0.8.1-pg16 + deploy.cloud.sealos.io/minReplicas: '1' + deploy.cloud.sealos.io/maxReplicas: '1' + labels: + app: ${{ defaults.app_name }}-postgres + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-postgres + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + serviceName: ${{ defaults.app_name }}-postgres + replicas: 1 + revisionHistoryLimit: 1 + selector: + matchLabels: { app: '${{ defaults.app_name }}-postgres' } + template: + metadata: + labels: { app: '${{ defaults.app_name }}-postgres' } + spec: + automountServiceAccountToken: false + containers: + - name: ${{ defaults.app_name }}-postgres + image: pgvector/pgvector:0.8.1-pg16 + imagePullPolicy: IfNotPresent + env: + - { name: POSTGRES_USER, value: mem_preview } + - { name: POSTGRES_DB, value: mem_preview } + - { name: PGDATA, value: /var/lib/postgresql/data/pgdata } + - name: POSTGRES_PASSWORD + valueFrom: { secretKeyRef: { name: '${{ defaults.app_name }}-runtime', key: DB_PASSWORD } } + ports: [{ name: postgres, containerPort: 5432 }] + readinessProbe: + exec: { command: [pg_isready, -U, mem_preview, -d, mem_preview] } + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: { cpu: 50m, memory: 51Mi } + limits: { cpu: 500m, memory: 512Mi } + volumeMounts: + - { name: vn-varvn-libvn-postgresqlvn-data, mountPath: /var/lib/postgresql/data } + volumeClaimTemplates: + - metadata: + name: vn-varvn-libvn-postgresqlvn-data + labels: { cloud.sealos.io/deploy-on-sealos: '${{ defaults.app_name }}' } + annotations: { path: /var/lib/postgresql/data, value: '1' } + spec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 2Gi } } +--- +apiVersion: v1 +kind: Service +metadata: + name: ${{ defaults.app_name }}-postgres + labels: + app: ${{ defaults.app_name }}-postgres + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-postgres + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + ports: [{ name: postgres, port: 5432, targetPort: 5432 }] + selector: { app: '${{ defaults.app_name }}-postgres' } +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ${{ defaults.app_name }}-redis + annotations: + originImageName: redis:7.4-alpine + deploy.cloud.sealos.io/minReplicas: '1' + deploy.cloud.sealos.io/maxReplicas: '1' + labels: + app: ${{ defaults.app_name }}-redis + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-redis + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + serviceName: ${{ defaults.app_name }}-redis + replicas: 1 + revisionHistoryLimit: 1 + selector: + matchLabels: { app: '${{ defaults.app_name }}-redis' } + template: + metadata: + labels: { app: '${{ defaults.app_name }}-redis' } + spec: + automountServiceAccountToken: false + containers: + - name: ${{ defaults.app_name }}-redis + image: redis:7.4-alpine + imagePullPolicy: IfNotPresent + args: [redis-server, --requirepass, '$(REDISCLI_AUTH)', --appendonly, 'yes'] + env: + - name: REDISCLI_AUTH + valueFrom: { secretKeyRef: { name: '${{ defaults.app_name }}-runtime', key: REDIS_PASSWORD } } + ports: [{ name: redis, containerPort: 6379 }] + readinessProbe: + exec: { command: [redis-cli, ping] } + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: { cpu: 20m, memory: 25Mi } + limits: { cpu: 200m, memory: 256Mi } + volumeMounts: + - { name: vn-data, mountPath: /data } + volumeClaimTemplates: + - metadata: + name: vn-data + labels: { cloud.sealos.io/deploy-on-sealos: '${{ defaults.app_name }}' } + annotations: { path: /data, value: '1' } + spec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } +--- +apiVersion: v1 +kind: Service +metadata: + name: ${{ defaults.app_name }}-redis + labels: + app: ${{ defaults.app_name }}-redis + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-redis + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + ports: [{ name: redis, port: 6379, targetPort: 6379 }] + selector: { app: '${{ defaults.app_name }}-redis' } +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ${{ defaults.app_name }}-memd + annotations: + originImageName: ${{ inputs.server_image }} + deploy.cloud.sealos.io/minReplicas: '1' + deploy.cloud.sealos.io/maxReplicas: '1' + labels: + app: ${{ defaults.app_name }}-memd + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-memd + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + serviceName: ${{ defaults.app_name }}-memd + replicas: 1 + revisionHistoryLimit: 1 + selector: + matchLabels: { app: '${{ defaults.app_name }}-memd' } + template: + metadata: + labels: { app: '${{ defaults.app_name }}-memd' } + spec: + automountServiceAccountToken: false + securityContext: { runAsNonRoot: true, runAsUser: 65532, runAsGroup: 65532, fsGroup: 65532 } + initContainers: + - name: migrate + image: ${{ inputs.server_image }} + command: [/usr/local/bin/mem-migrate] + env: + - name: DB_PASSWORD + valueFrom: { secretKeyRef: { name: '${{ defaults.app_name }}-runtime', key: DB_PASSWORD } } + - name: MEM_DB_URL + value: postgres://mem_preview:$(DB_PASSWORD)@${{ defaults.app_name }}-postgres.${{ SEALOS_NAMESPACE }}.svc.cluster.local:5432/mem_preview?sslmode=disable + resources: + requests: { cpu: 50m, memory: 25Mi } + limits: { cpu: 500m, memory: 256Mi } + containers: + - name: ${{ defaults.app_name }}-memd + image: ${{ inputs.server_image }} + imagePullPolicy: IfNotPresent + envFrom: [{ secretRef: { name: '${{ defaults.app_name }}-runtime' } }] + env: + - { name: MEM_HTTP_ADDR, value: ':8080' } + - { name: MEM_RUNTIME_PROFILE, value: production } + - { name: MEM_AUTO_MIGRATE, value: 'false' } + - { name: MEM_DEPLOYMENT_MODE, value: private } + - { name: MEM_REGISTRATION_MODE, value: disabled } + - { name: MEM_GITHUB_BOOTSTRAP, value: 'true' } + - { name: MEM_GITHUB_ALLOWED_USER_IDS, value: '${{ inputs.github_allowed_user_ids }}' } + - { name: MEM_PUBLIC_URL, value: 'https://${{ defaults.app_host }}.${{ SEALOS_CLOUD_DOMAIN }}' } + - { name: MEM_AI_PROFILES, value: local-fast-v2 } + - { name: MEM_SESSION_TTL, value: 24h } + - { name: MEM_WORKER_AUTH_KEY_ID, value: preview-v1 } + - { name: MEM_WORKER_GRPC, value: '${{ defaults.app_name }}-worker.${{ SEALOS_NAMESPACE }}.svc.cluster.local:50051' } + - { name: MEM_WORKSPACE_TRANSFER_TMP_DIR, value: /tmp/mem-workspace-transfers } + - { name: MEM_WORKSPACE_BUNDLE_MAX_BYTES, value: '536870912' } + - { name: MEM_WORKSPACE_TRANSFER_MAX_CONCURRENT, value: '1' } + - name: MEM_DB_URL + value: postgres://mem_preview:$(DB_PASSWORD)@${{ defaults.app_name }}-postgres.${{ SEALOS_NAMESPACE }}.svc.cluster.local:5432/mem_preview?sslmode=disable + - name: MEM_REDIS_URL + value: redis://:$(REDIS_PASSWORD)@${{ defaults.app_name }}-redis.${{ SEALOS_NAMESPACE }}.svc.cluster.local:6379/0 + - name: MEM_S3_ACCESS_KEY + valueFrom: { secretKeyRef: { name: object-storage-key, key: accessKey } } + - name: MEM_S3_SECRET_KEY + valueFrom: { secretKeyRef: { name: object-storage-key, key: secretKey } } + - name: S3_ENDPOINT_HOST + valueFrom: { secretKeyRef: { name: object-storage-key, key: external } } + - { name: MEM_S3_ENDPOINT, value: 'https://$(S3_ENDPOINT_HOST)' } + - { name: MEM_S3_USE_SSL, value: 'true' } + - name: MEM_S3_BUCKET + valueFrom: + secretKeyRef: + name: object-storage-key-${{ SEALOS_SERVICE_ACCOUNT }}-${{ defaults.app_name }} + key: bucket + ports: [{ name: http, containerPort: 8080 }] + readinessProbe: + httpGet: { path: /readyz, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /healthz, port: http } + initialDelaySeconds: 20 + periodSeconds: 15 + securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: [ALL] } } + resources: + requests: { cpu: 100m, memory: 51Mi } + limits: { cpu: '1', memory: 512Mi } + volumeMounts: [{ name: vn-tmp, mountPath: /tmp }] + volumeClaimTemplates: + - metadata: + name: vn-tmp + labels: { cloud.sealos.io/deploy-on-sealos: '${{ defaults.app_name }}' } + annotations: { path: /tmp, value: '1' } + spec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } +--- +apiVersion: v1 +kind: Service +metadata: + name: ${{ defaults.app_name }}-memd + labels: + app: ${{ defaults.app_name }}-memd + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-memd + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + ports: [{ name: http, port: 8080, targetPort: http }] + selector: { app: '${{ defaults.app_name }}-memd' } +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ${{ defaults.app_name }}-worker + annotations: + originImageName: ${{ inputs.worker_image }} + deploy.cloud.sealos.io/minReplicas: '1' + deploy.cloud.sealos.io/maxReplicas: '1' + labels: + app: ${{ defaults.app_name }}-worker + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-worker + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + serviceName: ${{ defaults.app_name }}-worker + replicas: 1 + revisionHistoryLimit: 1 + selector: + matchLabels: { app: '${{ defaults.app_name }}-worker' } + template: + metadata: + labels: { app: '${{ defaults.app_name }}-worker' } + spec: + automountServiceAccountToken: false + securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000 } + containers: + - name: ${{ defaults.app_name }}-worker + image: ${{ inputs.worker_image }} + imagePullPolicy: IfNotPresent + env: + - name: REDIS_PASSWORD + valueFrom: { secretKeyRef: { name: '${{ defaults.app_name }}-runtime', key: REDIS_PASSWORD } } + - name: MEM_WORKER_AUTH_KEY_B64 + valueFrom: { secretKeyRef: { name: '${{ defaults.app_name }}-runtime', key: MEM_WORKER_AUTH_KEY_B64 } } + - { name: MEM_DEPLOYMENT_MODE, value: private } + - { name: MEM_WORKER_GRPC_HOST, value: 0.0.0.0 } + - { name: MEM_WORKER_GRPC_PORT, value: '50051' } + - { name: MEM_WORKER_GRPC_MAX_WORKERS, value: '2' } + - { name: MEM_WORKER_AUTH_MODE, value: required } + - { name: MEM_WORKER_AUTH_KEY_ID, value: preview-v1 } + - { name: MEM_WORKER_AUTH_REPLAY_REDIS_URL, value: 'redis://:$(REDIS_PASSWORD)@${{ defaults.app_name }}-redis.${{ SEALOS_NAMESPACE }}.svc.cluster.local:6379/1' } + - { name: MEM_LOG_JSON, value: 'true' } + - { name: XDG_CACHE_HOME, value: /tmp/cache } + - { name: HF_HOME, value: /tmp/huggingface } + - name: MEM_S3_ACCESS_KEY + valueFrom: { secretKeyRef: { name: object-storage-key, key: accessKey } } + - name: MEM_S3_SECRET_KEY + valueFrom: { secretKeyRef: { name: object-storage-key, key: secretKey } } + - name: S3_ENDPOINT_HOST + valueFrom: { secretKeyRef: { name: object-storage-key, key: external } } + - { name: MEM_S3_ENDPOINT, value: 'https://$(S3_ENDPOINT_HOST)' } + - { name: MEM_S3_USE_SSL, value: 'true' } + - name: MEM_S3_BUCKET + valueFrom: + secretKeyRef: + name: object-storage-key-${{ SEALOS_SERVICE_ACCOUNT }}-${{ defaults.app_name }} + key: bucket + ports: [{ name: grpc, containerPort: 50051 }] + startupProbe: + exec: { command: [python, -m, mem_worker.healthcheck] } + timeoutSeconds: 15 + periodSeconds: 15 + failureThreshold: 12 + readinessProbe: + exec: { command: [python, -m, mem_worker.healthcheck] } + timeoutSeconds: 10 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + exec: { command: [python, -m, mem_worker.healthcheck] } + timeoutSeconds: 10 + initialDelaySeconds: 30 + periodSeconds: 15 + securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: [ALL] } } + resources: + requests: { cpu: 50m, memory: 51Mi } + limits: { cpu: 500m, memory: 512Mi } + volumeMounts: [{ name: vn-tmp, mountPath: /tmp }] + volumeClaimTemplates: + - metadata: + name: vn-tmp + labels: { cloud.sealos.io/deploy-on-sealos: '${{ defaults.app_name }}' } + annotations: { path: /tmp, value: '1' } + spec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } +--- +apiVersion: v1 +kind: Service +metadata: + name: ${{ defaults.app_name }}-worker + labels: + app: ${{ defaults.app_name }}-worker + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }}-worker + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + ports: [{ name: grpc, port: 50051, targetPort: grpc }] + selector: { app: '${{ defaults.app_name }}-worker' } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${{ defaults.app_name }} + labels: + app: ${{ defaults.app_name }} + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }} + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +data: + vn-etcvn-memvn-startvn-webvn-sh: | + #!/bin/sh + set -eu + # Only application inputs are substituted; nginx's own variables stay intact. + envsubst '${MEMD_UPSTREAM} ${MEM_MAX_BODY_SIZE}' \ + < /etc/nginx/templates/default.conf.template > /tmp/default.conf + sed 's@/etc/nginx/conf.d/\*.conf@/tmp/default.conf@' \ + /etc/nginx/nginx.conf > /tmp/nginx.conf + nginx -t -c /tmp/nginx.conf + exec nginx -c /tmp/nginx.conf -g 'daemon off;' +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ${{ defaults.app_name }} + annotations: + originImageName: ${{ inputs.web_image }} + deploy.cloud.sealos.io/minReplicas: '1' + deploy.cloud.sealos.io/maxReplicas: '1' + labels: + app: ${{ defaults.app_name }} + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }} + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + serviceName: ${{ defaults.app_name }} + replicas: 1 + revisionHistoryLimit: 1 + selector: + matchLabels: { app: '${{ defaults.app_name }}' } + template: + metadata: + labels: { app: '${{ defaults.app_name }}' } + spec: + automountServiceAccountToken: false + securityContext: { runAsNonRoot: true, runAsUser: 101, runAsGroup: 101, fsGroup: 101 } + containers: + - name: ${{ defaults.app_name }} + image: ${{ inputs.web_image }} + imagePullPolicy: IfNotPresent + # Render the image's shared template into the writable /tmp claim. + command: [/bin/sh, /etc/mem/start-web.sh] + env: + - { name: MEMD_UPSTREAM, value: 'http://${{ defaults.app_name }}-memd.${{ SEALOS_NAMESPACE }}.svc.cluster.local:8080' } + - { name: MEM_MAX_BODY_SIZE, value: 64m } + ports: [{ name: http, containerPort: 8080 }] + readinessProbe: + httpGet: { path: /healthz, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /healthz, port: http } + initialDelaySeconds: 15 + periodSeconds: 15 + securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: [ALL] } } + resources: + requests: { cpu: 20m, memory: 25Mi } + limits: { cpu: 200m, memory: 256Mi } + volumeMounts: + - name: ${{ defaults.app_name }}-cm + mountPath: /etc/mem/start-web.sh + subPath: vn-etcvn-memvn-startvn-webvn-sh + - { name: vn-tmp, mountPath: /tmp } + - { name: vn-varvn-cachevn-nginx, mountPath: /var/cache/nginx } + volumes: + - name: ${{ defaults.app_name }}-cm + configMap: { name: '${{ defaults.app_name }}' } + volumeClaimTemplates: + - metadata: + name: vn-tmp + labels: { cloud.sealos.io/deploy-on-sealos: '${{ defaults.app_name }}' } + annotations: { path: /tmp, value: '1' } + spec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } + - metadata: + name: vn-varvn-cachevn-nginx + labels: { cloud.sealos.io/deploy-on-sealos: '${{ defaults.app_name }}' } + annotations: { path: /var/cache/nginx, value: '1' } + spec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } +--- +apiVersion: v1 +kind: Service +metadata: + name: ${{ defaults.app_name }} + labels: + app: ${{ defaults.app_name }} + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }} + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + ports: [{ name: http, port: 8080, targetPort: http }] + selector: { app: '${{ defaults.app_name }}' } +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ${{ defaults.app_name }} + labels: + app: ${{ defaults.app_name }} + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }} + cloud.sealos.io/app-deploy-manager-domain: ${{ defaults.app_host }} + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/proxy-body-size: 64m + nginx.ingress.kubernetes.io/ssl-redirect: 'true' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/enable-access-log: 'false' + nginx.ingress.kubernetes.io/client-body-buffer-size: 64k + nginx.ingress.kubernetes.io/proxy-buffer-size: 64k + nginx.ingress.kubernetes.io/proxy-send-timeout: '300' + nginx.ingress.kubernetes.io/proxy-read-timeout: '300' + nginx.ingress.kubernetes.io/server-snippet: | + client_header_buffer_size 64k; + large_client_header_buffers 4 128k; +spec: + rules: + - host: ${{ defaults.app_host }}.${{ SEALOS_CLOUD_DOMAIN }} + http: + paths: + - pathType: Prefix + path: / + backend: + service: + name: ${{ defaults.app_name }} + port: { number: 8080 } + tls: + - hosts: ['${{ defaults.app_host }}.${{ SEALOS_CLOUD_DOMAIN }}'] + secretName: ${{ SEALOS_CERT_SECRET_NAME }} +--- +apiVersion: app.sealos.io/v1 +kind: App +metadata: + name: ${{ defaults.app_name }} + labels: + cloud.sealos.io/app-deploy-manager: ${{ defaults.app_name }} + cloud.sealos.io/deploy-on-sealos: ${{ defaults.app_name }} +spec: + data: + url: https://${{ defaults.app_host }}.${{ SEALOS_CLOUD_DOMAIN }} + displayType: normal + icon: '' + name: mem personal preview + type: link diff --git a/docker-compose.test.yml b/docker-compose.test.yml index eea98e1..c05d573 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -22,9 +22,12 @@ services: minio: profiles: ["e2e"] - # MinIO withdrew its Docker Hub repositories in October 2025; quay.io serves - # the same digest-pinned image. - image: quay.io/minio/minio:latest@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + # Official community container pulls are unavailable. Build only the test + # dependency from pinned upstream commits; see scripts/test-minio/Dockerfile. + image: mem-test-minio:7aac2a2c5b7c-77f82e18b540 + build: + context: ./scripts/test-minio + pull_policy: build command: server /data --console-address ":9001" environment: MINIO_ROOT_USER: mem @@ -34,7 +37,7 @@ services: tmpfs: - /data healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"] interval: 2s timeout: 3s retries: 30 @@ -42,7 +45,10 @@ services: minio-init: profiles: ["e2e"] - image: quay.io/minio/mc:latest@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727 + image: mem-test-minio:7aac2a2c5b7c-77f82e18b540 + build: + context: ./scripts/test-minio + pull_policy: build depends_on: minio: condition: service_healthy diff --git a/docs/TESTING.md b/docs/TESTING.md index a3204af..54d886d 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -483,3 +483,29 @@ Use this table in pull requests and add implementation-specific scenarios: - Web builds and test caches remain in ignored repository paths. Browser tests print their retained screenshot directories under the operating-system temp directory; delete only those printed directories after review. +## Personal GitHub fixture acceptance + +Use a fresh, owned PostgreSQL database ending in `_test`, separate from any +running user's mem database: + +```sh +cd server +MEM_OAUTH_TEST_DB=postgres://.../mem_oauth_test go test ./internal/api -run TestGitHubHTTPIntegration -count=1 +go test ./internal/auth ./internal/config ./internal/api ./internal/db ./cmd/memd +cd ../web +npm run build +npm test +npm run lint +``` + +The provider fixture checks PKCE/minimal scopes, primary verified email and +secret-safe errors. HTTP/PostgreSQL acceptance checks closed password signup, +allowlisted bootstrap, state tampering/replay, email collision, CSRF, machine +credential separation, logout revocation, cross-account linking denial and +binding callback rejection after logout/account switch, including logout while +provider exchange is running. Frontend regression checks old-Bearer callback +ordering, failed cookie bootstrap, logout acknowledgement, cross-tab CSRF/cache +refresh and delayed 401s from an earlier account. CI creates a separate fresh +`mem_oauth_test` database for this fixture; ordinary `MEM_TEST_DB` integration +continues against its own database. These tests use no real GitHub credentials +and are not live provider certification. diff --git a/scripts/test-minio/.dockerignore b/scripts/test-minio/.dockerignore new file mode 100644 index 0000000..ed6a2db --- /dev/null +++ b/scripts/test-minio/.dockerignore @@ -0,0 +1,4 @@ +# This context contains no project code or runtime credentials. +** +!Dockerfile +!.dockerignore diff --git a/scripts/test-minio/Dockerfile b/scripts/test-minio/Dockerfile new file mode 100644 index 0000000..50cef59 --- /dev/null +++ b/scripts/test-minio/Dockerfile @@ -0,0 +1,24 @@ +# syntax=docker/dockerfile:1.6 +# Test-only replacement for unavailable MinIO community registry images. +# Official source-only distribution/build instructions: +# https://github.com/minio/minio#source-only-distribution +# https://github.com/minio/mc#install-from-source +# Both source commits and Docker Official Images are immutable; no fork image, +# build secret, repository file, or runtime credential enters this context. +FROM golang:1.25.10-alpine@sha256:8d22e29d960bc50cd025d93d5b7c7d220b1ee9aa7a239b3c8f55a57e987e8d45 AS build +ENV CGO_ENABLED=0 \ + GOTOOLCHAIN=local \ + GOPROXY=https://proxy.golang.org \ + GOSUMDB=sum.golang.org +RUN go install -trimpath github.com/minio/minio@7aac2a2c5b7c882e68c1ce017d8256be2feea27f \ + && go install -trimpath github.com/minio/mc@77f82e18b5401a65958f1619df6ebb994634bd88 \ + && mkdir -p /out/licenses \ + && cp "$(go list -m -f '{{.Dir}}' github.com/minio/minio@7aac2a2c5b7c882e68c1ce017d8256be2feea27f)/LICENSE" /out/licenses/minio-LICENSE \ + && cp "$(go list -m -f '{{.Dir}}' github.com/minio/mc@77f82e18b5401a65958f1619df6ebb994634bd88)/LICENSE" /out/licenses/mc-LICENSE + +FROM alpine:3.22@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 +COPY --from=build /go/bin/minio /go/bin/mc /usr/local/bin/ +COPY --from=build /out/licenses /usr/share/licenses/minio +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +EXPOSE 9000 +ENTRYPOINT ["minio"] diff --git a/scripts/test_sealos_runtime_startup.py b/scripts/test_sealos_runtime_startup.py new file mode 100644 index 0000000..c957d79 --- /dev/null +++ b/scripts/test_sealos_runtime_startup.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Exercise native memd config and Worker probes on the built runtime images.""" +from __future__ import annotations + +import base64 +import json +import os +from pathlib import Path +import re +import subprocess +import time +import uuid + + +REPO = Path(__file__).resolve().parent.parent + + +def run(*args: str, check: bool = True, timeout: int = 45) -> subprocess.CompletedProcess[str]: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + if check and result.returncode: + raise RuntimeError(f"test command failed: {args[0]}") + return result + + +def native_workload(text: str, suffix: str) -> str: + name = " name: ${{ defaults.app_name }}" + suffix + "\n" + matches = [doc for doc in text.split("\n---\n") if "kind: StatefulSet\n" in doc and name in doc] + if len(matches) != 1: + raise AssertionError(f"expected one native {suffix} StatefulSet") + return matches[0] + + +def inline_env(workload: str, name: str) -> str: + values = re.findall(r"name: " + re.escape(name) + r", value: ([^}]+?) \}", workload) + if len(values) != 1: + raise AssertionError(f"expected one inline environment value for {name}") + return values[0].strip().strip("'\"") + + +def worker_probe(workload: str, name: str) -> dict[str, int]: + matches = re.findall(r"(?m)^ " + re.escape(name) + r":\n((?: .+\n)+)", workload) + if len(matches) != 1 or "exec: { command: [python, -m, mem_worker.healthcheck] }" not in matches[0]: + raise AssertionError(f"expected native Worker {name} health command") + return {key: int(value) for key, value in re.findall(r"(timeoutSeconds|periodSeconds|failureThreshold): ([0-9]+)", matches[0])} + + +def main() -> None: + text = (REPO / "deploy/sealos/template.yaml").read_text(encoding="utf-8") + memd = native_workload(text, "-memd") + worker = native_workload(text, "-worker") + probes = {name: worker_probe(worker, name) for name in ("startupProbe", "readinessProbe", "livenessProbe")} + # healthcheck permits two consecutive 3-second gRPC waits plus Python startup. + for name, probe in probes.items(): + if probe.get("timeoutSeconds", 1) < 10: + raise AssertionError(f"{name} cannot accommodate the real healthcheck") + if probes["startupProbe"].get("periodSeconds", 10) * probes["startupProbe"].get("failureThreshold", 3) < 120: + raise AssertionError("Worker startup budget must allow cold imports") + key = base64.b64encode(b"x" * 32).decode() + assert len(base64.b64decode(key)) == 32 + env = {name: inline_env(memd, name) for name in ( + "MEM_RUNTIME_PROFILE", "MEM_AUTO_MIGRATE", "MEM_DEPLOYMENT_MODE", "MEM_REGISTRATION_MODE", + "MEM_GITHUB_BOOTSTRAP", "MEM_AI_PROFILES", "MEM_SESSION_TTL", "MEM_WORKER_AUTH_KEY_ID", + "MEM_WORKSPACE_TRANSFER_TMP_DIR", "MEM_WORKSPACE_BUNDLE_MAX_BYTES", "MEM_WORKSPACE_TRANSFER_MAX_CONCURRENT", + )} + env.update({"MEM_DB_URL": "postgres://fixture:fixture@127.0.0.1:15432/fixture?sslmode=disable", + "MEM_REDIS_URL": "redis://127.0.0.1:16379/0", "MEM_S3_ENDPOINT": "https://storage.example.invalid", + "MEM_S3_ACCESS_KEY": "fixture-access", "MEM_S3_SECRET_KEY": "fixture-secret", "MEM_S3_BUCKET": "fixture", + "MEM_WORKER_GRPC": "127.0.0.1:50051", "MEM_WORKER_AUTH_KEY_B64": key, + "MEM_PUBLIC_URL": "https://preview.example.invalid", "MEM_GITHUB_CLIENT_ID": "fixture-client", + "MEM_GITHUB_CLIENT_SECRET": "fixture-secret", "MEM_GITHUB_ALLOWED_USER_IDS": "1"}) + server_image = os.environ.get("MEM_SEALOS_SERVER_TEST_IMAGE", "mem-server:deploy-validation") + worker_image = os.environ.get("MEM_SEALOS_WORKER_TEST_IMAGE", "mem-worker:deploy-validation") + redis_image = os.environ.get("MEM_SEALOS_REDIS_TEST_IMAGE", "redis:7.4.2-alpine") + owned = "mem-native-runtime-" + uuid.uuid4().hex[:12] + server_name = owned + "-memd" + argv = ["docker", "run", "--rm", "--name", server_name, "--network", "none", "--read-only", "--user", "65532:65532", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--cpus", "1", "--memory", "512m", + "--tmpfs", "/tmp:rw,uid=65532,gid=65532,mode=1777"] + for name, value in env.items(): + argv.extend(["--env", name + "=" + value]) + try: + result = run(*argv, server_image, check=False) + startup_log = result.stdout + result.stderr + if result.returncode != 1 or "db open:" not in startup_log or "config:" in startup_log or "workspace transfer temp dir:" in startup_log: + raise AssertionError("native memd recipe must pass config and private tmp setup before isolated DB refusal") + finally: + run("docker", "rm", "--force", server_name, check=False) + print("PASS: native memd production config and 0700 transfer directory on the actual image") + + redis_name, worker_name = owned + "-redis", owned + "-worker" + created: list[str] = [] + network = False + try: + run("docker", "network", "create", "--internal", owned) + network = True + run("docker", "run", "--detach", "--rm", "--name", redis_name, "--network", owned, + "--label", "com.bytefolk.mem.test=sealos-runtime", redis_image) + created.append(redis_name) + run("docker", "run", "--detach", "--rm", "--name", worker_name, "--network", owned, + "--label", "com.bytefolk.mem.test=sealos-runtime", "--read-only", "--user", "1000:1000", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--cpus", "0.5", "--memory", "512m", + "--tmpfs", "/tmp:rw,uid=1000,gid=1000,mode=1777", + "--env", "MEM_WORKER_AUTH_MODE=" + inline_env(worker, "MEM_WORKER_AUTH_MODE"), + "--env", "MEM_WORKER_AUTH_KEY_ID=" + inline_env(worker, "MEM_WORKER_AUTH_KEY_ID"), + "--env", "MEM_WORKER_AUTH_KEY_B64=" + key, + "--env", "MEM_WORKER_AUTH_REPLAY_REDIS_URL=redis://" + redis_name + ":6379/1", + worker_image) + created.append(worker_name) + startup_budget = probes["startupProbe"]["periodSeconds"] * probes["startupProbe"]["failureThreshold"] + deadline = time.monotonic() + startup_budget + while True: + started = time.monotonic() + try: + result = run("docker", "exec", worker_name, "python", "-m", "mem_worker.healthcheck", check=False, + timeout=probes["startupProbe"]["timeoutSeconds"]) + except subprocess.TimeoutExpired: + result = None + if result is not None and result.returncode == 0: + break + if time.monotonic() >= deadline: + raise AssertionError("native authenticated Worker startup healthcheck failed") + time.sleep(min(max(0, probes["startupProbe"]["periodSeconds"] - (time.monotonic() - started)), max(0, deadline - time.monotonic()))) + for name in ("readinessProbe", "livenessProbe"): + run("docker", "exec", worker_name, "python", "-m", "mem_worker.healthcheck", timeout=probes[name]["timeoutSeconds"]) + state = json.loads(run("docker", "inspect", "--format", "{{json .State}}", worker_name).stdout) + if not state["Running"] or state.get("OOMKilled"): + raise AssertionError("Worker did not remain healthy at native CPU/memory limits") + print("PASS: native Worker startup/readiness/liveness with required auth, 500m CPU and read-only root") + finally: + for name in reversed(created): + run("docker", "rm", "--force", name, check=False) + if network: + run("docker", "network", "rm", owned, check=False) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_sealos_web_startup.sh b/scripts/test_sealos_web_startup.sh new file mode 100644 index 0000000..4a40093 --- /dev/null +++ b/scripts/test_sealos_web_startup.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env sh +# Exercise the native template's startup script on the actual unprivileged image. +set -eu + +repo_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +image=${MEM_SEALOS_WEB_TEST_IMAGE:-mem-web:deploy-validation} +work=$(mktemp -d) +container="mem-sealos-web-startup-$$" +started=0 +cleanup() { + if [ "$started" = 1 ]; then docker rm -f "$container" >/dev/null 2>&1 || true; fi + rm -rf "$work" +} +trap cleanup EXIT HUP INT TERM + +# Read the script that the ConfigMap actually mounts; no duplicate test copy. +python3 - "$repo_root/deploy/sealos/template.yaml" "$work/start-web.sh" <<'PY' +from pathlib import Path +import sys + +text = Path(sys.argv[1]).read_text(encoding="utf-8") +key = " vn-etcvn-memvn-startvn-webvn-sh: |\n" +if text.count(key) != 1: + raise SystemExit("expected one native Web startup script") +block = text.split(key, 1)[1].split("\n---\n", 1)[0] +lines = block.splitlines() +if any(line and not line.startswith(" ") for line in lines): + raise SystemExit("unexpected startup block indentation") +Path(sys.argv[2]).write_text("\n".join(line[4:] if line else "" for line in lines) + "\n", encoding="utf-8") +PY +chmod 644 "$work/start-web.sh" + +docker run --detach --rm --name "$container" \ + --label com.bytefolk.mem.test=sealos-web-startup \ + --read-only --user 101:101 --cap-drop ALL --security-opt no-new-privileges \ + --memory 256m --cpus 0.2 \ + --tmpfs /tmp:rw,uid=101,gid=101,mode=1777 \ + --tmpfs /var/cache/nginx:rw,uid=101,gid=101,mode=1777 \ + --volume "$work/start-web.sh:/etc/mem/start-web.sh:ro" \ + --env MEMD_UPSTREAM=http://127.0.0.1:18081 --env MEM_MAX_BODY_SIZE=64m \ + --publish 127.0.0.1::8080 --entrypoint /bin/sh \ + "$image" /etc/mem/start-web.sh >/dev/null +started=1 +address=$(docker port "$container" 8080/tcp) +case "$address" in 127.0.0.1:*) ;; *) echo "unexpected test bind" >&2; exit 1 ;; esac +attempt=0 +until curl --fail --silent "http://$address/healthz" >"$work/health"; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ]; then + docker logs "$container" >&2 || true + echo "native Sealos Web startup did not become healthy" >&2 + exit 1 + fi + sleep 1 +done +grep -qx 'ok' "$work/health" +curl --fail --silent "http://$address/login" >"$work/login.html" +grep -q '"$work/generated.conf" +python3 - "$work/generated.conf" <<'PY' +from pathlib import Path +import sys + +text = Path(sys.argv[1]).read_text(encoding="utf-8") +assert "proxy_pass http://127.0.0.1:18081;" in text, "runtime upstream did not render" +assert "client_max_body_size 64m;" in text, "runtime upload limit did not render" +assert "${{" not in text and "${MEMD_UPSTREAM}" not in text, "unresolved template expression" +for variable in ("$uri", "$host", "$remote_addr", "$proxy_add_x_forwarded_for", "$mem_auth_loggable"): + assert variable in text, f"nginx variable was substituted: {variable}" +assert "/v1/auth/github/callback 0;" in text, "OAuth access-log filter missing" +PY +echo "PASS: native Sealos Web startup under UID 101, read-only root and runtime rendering" diff --git a/scripts/validate_deploy.sh b/scripts/validate_deploy.sh index 680e4fe..1cefaa9 100755 --- a/scripts/validate_deploy.sh +++ b/scripts/validate_deploy.sh @@ -146,6 +146,8 @@ if [ "${MEM_VALIDATE_BUILD_IMAGES:-0}" = "1" ]; then docker build --tag mem-server:deploy-validation "$repo_root/server" docker build --tag mem-worker:deploy-validation "$repo_root/worker" docker build --tag mem-web:deploy-validation "$repo_root/web" + sh "$repo_root/scripts/test_sealos_web_startup.sh" + python3 "$repo_root/scripts/test_sealos_runtime_startup.py" fi echo "PASS: production Compose and Helm deployment validation" diff --git a/scripts/verify.sh b/scripts/verify.sh index 85dbdd3..8f5a0ac 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -4,7 +4,7 @@ set -euo pipefail REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" MODE="${1:-unit}" -EXPECTED_MIGRATION_HEAD=26 +EXPECTED_MIGRATION_HEAD=27 MIGRATION_ROLLBACK_TARGET=11 MODEL_TEXT_CANONICAL_BASE=15 WORKSPACE_AI_PROFILE_BASE=16 diff --git a/server/.dockerignore b/server/.dockerignore index 888aebf..e7db44a 100644 --- a/server/.dockerignore +++ b/server/.dockerignore @@ -1,4 +1,6 @@ .git +.env* +!.env.example bin coverage.out *.test diff --git a/server/.env.example b/server/.env.example index bd68679..7d35320 100644 --- a/server/.env.example +++ b/server/.env.example @@ -54,3 +54,10 @@ MEM_WORKSPACE_TRANSFER_MAX_CONCURRENT=2 # --- Dev --- # debug|info|warn|error MEM_LOG_LEVEL=info +# Optional personal GitHub browser sign-in (all four inputs are required). +# Keep client secret outside source/build contexts. Public origin must be HTTPS. +# MEM_PUBLIC_URL=https://mem.example.com +# MEM_GITHUB_CLIENT_ID= +# MEM_GITHUB_CLIENT_SECRET= +# MEM_GITHUB_ALLOWED_USER_IDS=47820304 +# MEM_GITHUB_BOOTSTRAP=true # requires MEM_REGISTRATION_MODE=disabled diff --git a/server/cmd/memd/main.go b/server/cmd/memd/main.go index 2cf20cc..d8dd020 100644 --- a/server/cmd/memd/main.go +++ b/server/cmd/memd/main.go @@ -259,8 +259,14 @@ func run() error { RegistrationMode: cfg.RegistrationMode, SessionTTL: cfg.SessionTTL, CORSOrigins: cfg.CORSOrigins, + PublicURL: cfg.PublicURL, + GitHubAllowedUserIDs: cfg.GitHubAllowedUserIDs, + GitHubBootstrap: cfg.GitHubBootstrap, Log: logger, } + if cfg.GitHubClientID != "" { + srv.GitHub = &auth.GitHubOAuth{ClientID: cfg.GitHubClientID, ClientSecret: cfg.GitHubClientSecret, RedirectURL: cfg.PublicURL + "/v1/auth/github/callback"} + } if cfg.DeploymentMode == "saas" { go reconcileManagedEmbeddingReservations( diff --git a/server/internal/api/api.go b/server/internal/api/api.go index d7edd63..7d29d52 100644 --- a/server/internal/api/api.go +++ b/server/internal/api/api.go @@ -197,6 +197,10 @@ type Server struct { RegistrationMode string SessionTTL time.Duration CORSOrigins []string // allowed browser origins; empty disables CORS + PublicURL string + GitHub GitHubIdentityProvider + GitHubAllowedUserIDs []string + GitHubBootstrap bool Log *slog.Logger } @@ -232,6 +236,12 @@ func (s *Server) Router() http.Handler { // Public auth r.Post("/v1/auth/register", s.handleRegister) r.Post("/v1/auth/login", s.handleLogin) + r.Get("/v1/auth/capabilities", s.handleAuthCapabilities) + r.Post("/v1/auth/github/start", s.handleGitHubStart) + r.Get("/v1/auth/github/callback", s.handleGitHubCallback) + r.Get("/v1/auth/session", s.handleBrowserSession) + r.Post("/v1/auth/logout", s.handleBrowserLogout) + r.With(s.authMiddleware).Get("/v1/auth/github/identity", s.handleGitHubIdentity) // Token-authenticated routes r.Group(func(r chi.Router) { @@ -516,6 +526,11 @@ func (s *Server) requestTimeoutMiddleware(next http.Handler) http.Handler { func (s *Server) authMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { header := r.Header.Get("Authorization") + if header == "" && s.GitHub != nil { + if s.authenticateBrowser(w, r, next) { + return + } + } const prefix = "Bearer " if len(header) <= len(prefix) || header[:len(prefix)] != prefix { writeError(w, http.StatusUnauthorized, "missing_bearer", "Authorization: Bearer required") @@ -810,6 +825,7 @@ func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusUnauthorized, "invalid_credentials", "email or password is incorrect") return } + s.clearPriorBrowser(w, r) s.issueSession(w, u, http.StatusOK) } @@ -844,6 +860,7 @@ func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "register_failed", err.Error()) return } + s.clearPriorBrowser(w, r) s.issueSession(w, u, http.StatusCreated) } diff --git a/server/internal/api/github_integration_test.go b/server/internal/api/github_integration_test.go new file mode 100644 index 0000000..c4261af --- /dev/null +++ b/server/internal/api/github_integration_test.go @@ -0,0 +1,284 @@ +package api + +import ( + "context" + "database/sql" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/PeterGuy326/mem/server/internal/auth" + memdb "github.com/PeterGuy326/mem/server/internal/db" + "github.com/PeterGuy326/mem/server/internal/memory" + "github.com/PeterGuy326/mem/server/internal/workspace" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/pressly/goose/v3" +) + +type fakeGitHub struct{ onExchange func() } + +func (fakeGitHub) AuthorizationURL(state, verifier string) string { + return "https://github.com/login/oauth/authorize?state=" + state +} +func (provider fakeGitHub) Exchange(_ context.Context, code, verifier string) (*auth.GitHubIdentity, error) { + if provider.onExchange != nil { + provider.onExchange() + } + identity := &auth.GitHubIdentity{Subject: "47820304", Login: "owner", Email: "owner@example.test"} + switch code { + case "link": + identity.Subject = "222" + identity.Login = "linked" + identity.Email = "linked@example.test" + case "collision": + identity.Subject = "444" + identity.Email = "local@example.test" + case "outsider": + identity.Subject = "555" + identity.Email = "outsider@example.test" + case "second": + identity.Subject = "333" + identity.Email = "second@example.test" + } + return identity, nil +} + +func TestGitHubHTTPIntegration(t *testing.T) { + dsn := os.Getenv("MEM_OAUTH_TEST_DB") + if dsn == "" { + t.Skip("MEM_OAUTH_TEST_DB requires a fresh owned _test database") + } + parsed, err := pgxpool.ParseConfig(dsn) + if err != nil || !strings.HasSuffix(parsed.ConnConfig.Database, "_test") { + t.Fatal("refusing non-test database") + } + ctx := context.Background() + database, err := memdb.Open(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer database.Close() + if err = database.Migrate(ctx); err != nil { + t.Fatal(err) + } + var users int + if err = database.Pool.QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&users); err != nil || users != 0 { + t.Fatal("OAuth acceptance requires a fresh, empty, owned database") + } + s := &Server{Auth: auth.New(database.Pool), Memory: memory.New(database.Pool), Workspace: workspace.New(database.Pool), PublicURL: "https://mem.example", GitHub: fakeGitHub{}, GitHubAllowedUserIDs: []string{"47820304", "222", "333", "444"}, GitHubBootstrap: true, RegistrationMode: "disabled", Log: slog.New(slog.NewTextHandler(io.Discard, nil))} + router := s.Router() + request := func(method, path, body, bearer, csrf string, cookies []*http.Cookie) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, strings.NewReader(body)) + r.Header.Set("Origin", s.PublicURL) + r.Header.Set("Content-Type", "application/json") + if bearer != "" { + r.Header.Set("Authorization", "Bearer "+bearer) + } + if csrf != "" { + r.Header.Set("X-Mem-CSRF", csrf) + } + for _, cookie := range cookies { + r.AddCookie(cookie) + } + w := httptest.NewRecorder() + router.ServeHTTP(w, r) + return w + } + start := func(bearer, password string) (string, []*http.Cookie, string) { + body := `{"intent":"login"}` + if bearer != "" { + value, _ := json.Marshal(map[string]string{"intent": "link", "password": password}) + body = string(value) + } + w := request("POST", "/v1/auth/github/start", body, bearer, "", nil) + if w.Code != 200 { + t.Fatalf("start %d: %s", w.Code, w.Body) + } + var response struct { + URL string `json:"url"` + CSRF string `json:"csrf_token"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + u, _ := url.Parse(response.URL) + return u.Query().Get("state"), w.Result().Cookies(), response.CSRF + } + callback := func(state, code string, cookies []*http.Cookie) *httptest.ResponseRecorder { + return request("GET", "/v1/auth/github/callback?state="+state+"&code="+code, "", "", "", cookies) + } + assertRedirect := func(w *httptest.ResponseRecorder, want string) { + if w.Code != 303 || w.Header().Get("Location") != want { + t.Fatalf("callback %d location %s, want %s", w.Code, w.Header().Get("Location"), want) + } + } + visitor := request("POST", "/v1/auth/register", `{"email":"visitor@example.test","password":"visitor-password"}`, "", "", nil) + if visitor.Code != 403 { + t.Fatalf("public password enrollment must stay closed: %d", visitor.Code) + } + state, cookies, _ := start("", "") + assertRedirect(callback(state, "outsider", cookies), "/login?github=account_denied") + state, cookies, _ = start("", "") + assertRedirect(callback(strings.Repeat("x", 43), "personal", cookies), "/login?github=state_invalid") + accepted := callback(state, "personal", cookies) + assertRedirect(accepted, "/login?github=ok") + assertRedirect(callback(state, "personal", cookies), "/login?github=state_invalid") + var browser []*http.Cookie + for _, cookie := range accepted.Result().Cookies() { + if cookie.Name == s.browserConfig().CookieName { + browser = append(browser, cookie) + if !cookie.HttpOnly || !cookie.Secure || cookie.Path != "/" { + t.Fatal("unsafe browser cookie") + } + } + } + if len(browser) != 1 { + t.Fatal("browser session missing") + } + read := request("GET", "/v1/auth/session", "", "", "", browser) + var session struct { + Authenticated bool `json:"authenticated"` + CSRF string `json:"csrf_token"` + } + if err = json.Unmarshal(read.Body.Bytes(), &session); err != nil || !session.Authenticated { + t.Fatal("browser readback failed") + } + if read.Header().Get("Cache-Control") != "no-store" { + t.Fatal("session response must not be cached") + } + if request("GET", "/v1/workspaces", "", "", "", browser).Code != 200 { + t.Fatal("cookie must pass canonical workspace authorization") + } + if request("GET", "/v1/workspaces", "", browser[0].Value, "", nil).Code != 401 { + t.Fatal("browser cookie cannot be used as an Agent bearer") + } + if request("POST", "/v1/memories", `{}`, "", "", browser).Code != 403 { + t.Fatal("cookie mutation without CSRF must fail") + } + mutation := httptest.NewRequest("POST", "/v1/memories", strings.NewReader(`{"kind":"note","content":"source-backed preview","path":"/","source":{"type":"test"}}`)) + mutation.Header.Set("Origin", s.PublicURL) + mutation.Header.Set("X-Mem-CSRF", session.CSRF) + mutation.Header.Set("Idempotency-Key", "oauth-preview") + mutation.AddCookie(browser[0]) + result := httptest.NewRecorder() + router.ServeHTTP(result, mutation) + if result.Code != 201 { + t.Fatalf("cookie memory write %d: %s", result.Code, result.Body) + } + logout := request("POST", "/v1/auth/logout", `{}`, "", session.CSRF, browser) + if logout.Code != 200 || request("GET", "/v1/workspaces", "", "", "", browser).Code != 401 { + t.Fatal("logout must revoke server-side session") + } + state, cookies, _ = start("", "") + assertRedirect(callback(state, "second", cookies), "/login?github=registration_disabled") + local, err := s.Auth.CreateUser(ctx, "local@example.test", "current-password") + if err != nil { + t.Fatal(err) + } + other, err := s.Auth.CreateUser(ctx, "other@example.test", "other-password") + if err != nil { + t.Fatal(err) + } + bearer, _, err := s.Auth.CreateToken(ctx, local.ID, nil, "link-local", auth.AllScopes, nil, nil, false) + if err != nil { + t.Fatal(err) + } + state, cookies, _ = start("", "") + assertRedirect(callback(state, "collision", cookies), "/login?github=link_required") + state, cookies, csrf := start(bearer, "current-password") + if csrf == "" { + t.Fatal("password reauthentication must establish a bound browser session") + } + if request("POST", "/v1/auth/logout", `{}`, "", csrf, cookies).Code != 200 { + t.Fatal("binding session logout failed") + } + assertRedirect(callback(state, "link", cookies), "/login?github=state_invalid") + state, cookies, csrf = start(bearer, "current-password") + s.GitHub = fakeGitHub{onExchange: func() { + if request("POST", "/v1/auth/logout", `{}`, "", csrf, cookies).Code != 200 { + t.Fatal("exchange-time logout failed") + } + }} + assertRedirect(callback(state, "link", cookies), "/login?github=state_invalid") + if identity, err := s.Auth.GitHubIdentity(ctx, local.ID); err != nil || identity != nil { + t.Fatal("exchange-time logout must prevent binding") + } + s.GitHub = fakeGitHub{} + state, cookies, _ = start(bearer, "current-password") + switchUser := request("POST", "/v1/auth/login", `{"email":"other@example.test","password":"other-password"}`, "", "", cookies) + if switchUser.Code != 200 { + t.Fatalf("switch login %d", switchUser.Code) + } + assertRedirect(callback(state, "link", cookies), "/login?github=state_invalid") + state, cookies, _ = start(bearer, "current-password") + assertRedirect(callback(state, "link", cookies), "/account?github=linked") + identity, err := s.Auth.GitHubIdentity(ctx, local.ID) + if err != nil || identity == nil || identity.Subject != "222" { + t.Fatal("explicit linking failed") + } + otherBearer, _, err := s.Auth.CreateToken(ctx, other.ID, nil, "link-other", auth.AllScopes, nil, nil, false) + if err != nil { + t.Fatal(err) + } + state, cookies, _ = start(otherBearer, "other-password") + assertRedirect(callback(state, "link", cookies), "/login?github=identity_conflict") + state, cookies, _ = start("", "") + returned := callback(state, "personal", cookies) + assertRedirect(returned, "/login?github=ok") + var current *http.Cookie + for _, cookie := range returned.Result().Cookies() { + if cookie.Name == s.browserConfig().CookieName { + current = cookie + } + } + if current == nil { + t.Fatal("returning owner browser session missing") + } + if request("GET", "/v1/workspaces", "", "invalid-old-bearer", "", []*http.Cookie{current}).Code != 401 { + t.Fatal("invalid Bearer must not fall back to a valid browser cookie") + } + var foreignWorkspace uuid.UUID + if err := database.Pool.QueryRow(ctx, `SELECT id FROM workspaces WHERE resource_owner_user_id=$1`, other.ID).Scan(&foreignWorkspace); err != nil { + t.Fatal(err) + } + foreign := httptest.NewRequest("GET", "/v1/memories", nil) + foreign.Header.Set("Origin", s.PublicURL) + foreign.Header.Set("X-Workspace-ID", foreignWorkspace.String()) + foreign.AddCookie(current) + denied := httptest.NewRecorder() + router.ServeHTTP(denied, foreign) + if denied.Code != 403 { + t.Fatalf("browser must not enter another owner's workspace: %d", denied.Code) + } + foreign = httptest.NewRequest("GET", "/v1/auth/session", nil) + foreign.Header.Set("Origin", "https://untrusted.example") + foreign.AddCookie(current) + denied = httptest.NewRecorder() + router.ServeHTTP(denied, foreign) + if denied.Code != 403 { + t.Fatal("foreign browser origin must not read session CSRF") + } + var secrets int + if err = database.Pool.QueryRow(ctx, `SELECT count(*) FROM tokens WHERE hash=$1`, auth.HashToken(browser[0].Value)).Scan(&secrets); err != nil || secrets != 0 { + t.Fatal("browser secret must never become an Agent token") + } + // A production downgrade must refuse to orphan the first OAuth-only user. + sqldb, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatal(err) + } + defer sqldb.Close() + if err := goose.DownContext(ctx, sqldb, "migrations"); err == nil || !strings.Contains(err.Error(), "cannot downgrade while GitHub-only users exist") { + t.Fatalf("unsafe GitHub-only downgrade result: %v", err) + } + if identity, err := s.Auth.GitHubIdentity(ctx, local.ID); err != nil || identity == nil { + t.Fatal("refused downgrade must preserve identities") + } +} diff --git a/server/internal/api/handlers_github.go b/server/internal/api/handlers_github.go new file mode 100644 index 0000000..da08d1a --- /dev/null +++ b/server/internal/api/handlers_github.go @@ -0,0 +1,356 @@ +package api + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "time" + + "github.com/PeterGuy326/mem/server/internal/auth" + "github.com/google/uuid" +) + +type GitHubIdentityProvider interface { + AuthorizationURL(state, verifier string) string + Exchange(context.Context, string, string) (*auth.GitHubIdentity, error) +} + +func (s *Server) handleAuthCapabilities(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, http.StatusOK, map[string]any{"github": s.GitHub != nil, "password": true, "registration": s.RegistrationMode != "disabled"}) +} + +func (s *Server) browserConfig() auth.SessionConfig { + cfg := auth.DefaultSessionConfig() + cfg.CookieName = "__Host-mem_session" + if strings.HasPrefix(s.PublicURL, "http://") { + cfg.Secure = false + cfg.CookieName = "mem_session" + } + if s.SessionTTL > 0 { + cfg.AbsoluteExpiry = s.SessionTTL + } + return cfg +} + +func (s *Server) flowCookieName() string { + if strings.HasPrefix(s.PublicURL, "http://") { + return "mem_github_flow" + } + return "__Host-mem_github_flow" +} + +func randomOAuthValue() (string, error) { + value := make([]byte, 32) + if _, err := rand.Read(value); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(value), nil +} + +func (s *Server) exactBrowserOrigin(r *http.Request) bool { + return s.PublicURL != "" && r.Header.Get("Origin") == s.PublicURL +} + +func (s *Server) handleGitHubStart(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if s.GitHub == nil { + writeError(w, 503, "github_disabled", "GitHub sign-in is not configured") + return + } + if !s.exactBrowserOrigin(r) { + writeError(w, 403, "origin_denied", "Use the configured application origin") + return + } + r.Body = http.MaxBytesReader(w, r.Body, 4096) + var body struct { + Intent string `json:"intent"` + Password string `json:"password"` + } + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&body); err != nil { + writeError(w, 400, "invalid_oauth_request", "Invalid sign-in request") + return + } + var trailing any + if decoder.Decode(&trailing) != io.EOF { + writeError(w, 400, "invalid_oauth_request", "Invalid sign-in request") + return + } + if body.Intent == "" { + body.Intent = "login" + } + if body.Intent != "login" && body.Intent != "link" { + writeError(w, 400, "invalid_oauth_request", "Invalid sign-in intent") + return + } + var userID *uuid.UUID + var sessionID *uuid.UUID + var csrf string + var authGeneration string + if body.Intent == "link" { + if header := r.Header.Get("Authorization"); header != "" && !strings.HasPrefix(header, "Bearer ") { + writeError(w, 401, "reauthentication_required", "Sign in again before linking GitHub") + return + } + // Existing password users must prove the password again. OAuth users + // must have authenticated in this browser within the last ten minutes. + if header := r.Header.Get("Authorization"); strings.HasPrefix(header, "Bearer ") { + user, token, err := s.Auth.ResolveToken(r.Context(), strings.TrimPrefix(header, "Bearer ")) + if err != nil || !auth.HasScope(token, auth.ScopeAdmin) { + writeError(w, 401, "reauthentication_required", "Sign in again before linking GitHub") + return + } + verified, err := s.Auth.VerifyPassword(r.Context(), user.Email, body.Password) + if err != nil || verified.ID != user.ID { + writeError(w, 401, "reauthentication_required", "Confirm your current password before linking GitHub") + return + } + userID = &user.ID + cfg := s.browserConfig() + plain, session, err := s.Auth.CreateSession(r.Context(), user.ID, cfg) + if err != nil { + writeError(w, 503, "github_unavailable", "Try again later") + return + } + if prior, _, err := s.resolveBrowser(r); err == nil { + _ = s.Auth.RevokeSession(r.Context(), prior.ID) + } + http.SetCookie(w, auth.SessionCookie(plain, cfg, cfg.AbsoluteExpiry)) + sessionID = &session.ID + csrf = session.CSRFToken + authGeneration = session.ID.String() + } else { + session, user, err := s.resolveBrowser(r) + if err != nil || time.Since(session.CreatedAt) > 10*time.Minute || !validBrowserCSRF(r, session) { + writeError(w, 401, "reauthentication_required", "Sign in again before linking GitHub") + return + } + userID = &user.ID + sessionID = &session.ID + } + } + state, err := randomOAuthValue() + if err != nil { + writeError(w, 503, "github_unavailable", "Try again later") + return + } + verifier, err := randomOAuthValue() + if err != nil { + writeError(w, 503, "github_unavailable", "Try again later") + return + } + if err := s.Auth.BeginOAuth(r.Context(), state, body.Intent, userID, sessionID); err != nil { + writeError(w, 503, "github_unavailable", "Try again later") + return + } + http.SetCookie(w, &http.Cookie{Name: s.flowCookieName(), Value: state + "." + verifier, Path: "/", MaxAge: 600, HttpOnly: true, Secure: s.browserConfig().Secure, SameSite: http.SameSiteLaxMode}) + writeJSON(w, http.StatusOK, map[string]string{"url": s.GitHub.AuthorizationURL(state, verifier), "csrf_token": csrf, "session_id": authGeneration}) +} + +func (s *Server) handleGitHubCallback(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + http.SetCookie(w, &http.Cookie{Name: s.flowCookieName(), Value: "", Path: "/", MaxAge: -1, HttpOnly: true, Secure: s.browserConfig().Secure, SameSite: http.SameSiteLaxMode}) + fail := func(reason string) { http.Redirect(w, r, "/login?github="+reason, http.StatusSeeOther) } + if s.GitHub == nil { + fail("disabled") + return + } + cookie, err := r.Cookie(s.flowCookieName()) + state := r.URL.Query().Get("state") + if err != nil || len(state) != 43 { + fail("state_invalid") + return + } + parts := strings.Split(cookie.Value, ".") + if len(parts) != 2 || len(parts[1]) != 43 || subtle.ConstantTimeCompare([]byte(parts[0]), []byte(state)) != 1 { + fail("state_invalid") + return + } + intent, userID, sessionID, err := s.Auth.ConsumeOAuth(r.Context(), state) + if err != nil { + fail("state_invalid") + return + } + if intent == "link" { + session, actor, err := s.resolveBrowser(r) + if err != nil || userID == nil || sessionID == nil || actor.ID != *userID || session.ID != *sessionID { + fail("state_invalid") + return + } + } + if r.URL.Query().Get("error") != "" { + fail("denied") + return + } + code := r.URL.Query().Get("code") + if len(code) == 0 || len(code) > 1024 { + fail("provider_unavailable") + return + } + identity, err := s.GitHub.Exchange(r.Context(), code, parts[1]) + if err != nil { + fail("provider_unavailable") + return + } + allowed := false + for _, id := range s.GitHubAllowedUserIDs { + if identity.Subject == id { + allowed = true + break + } + } + if !allowed { + fail("account_denied") + return + } + mode := "disabled" + if s.GitHubBootstrap { + mode = "first_user" + } + user, err := s.Auth.GitHubUser(r.Context(), *identity, userID, sessionID, mode) + if err != nil { + switch { + case errors.Is(err, auth.ErrOAuthState): + fail("state_invalid") + case errors.Is(err, auth.ErrExplicitLinkRequired): + fail("link_required") + case errors.Is(err, auth.ErrIdentityConflict): + fail("identity_conflict") + case errors.Is(err, auth.ErrRegistrationDisabled): + fail("registration_disabled") + default: + fail("provider_unavailable") + } + return + } + if intent == "link" { + if s.Log != nil { + s.Log.Info("github_identity_linked", "user_id", user.ID, "provider_subject", identity.Subject) + } + http.Redirect(w, r, "/account?github=linked", http.StatusSeeOther) + return + } + cfg := s.browserConfig() + plain, _, err := s.Auth.CreateSession(r.Context(), user.ID, cfg) + if err != nil { + fail("provider_unavailable") + return + } + if prior, _, err := s.resolveBrowser(r); err == nil { + _ = s.Auth.RevokeSession(r.Context(), prior.ID) + } + http.SetCookie(w, auth.SessionCookie(plain, cfg, cfg.AbsoluteExpiry)) + if s.Log != nil { + s.Log.Info("github_identity_authenticated", "intent", intent, "user_id", user.ID, "provider_subject", identity.Subject) + } + http.Redirect(w, r, "/login?github=ok", http.StatusSeeOther) +} + +func (s *Server) resolveBrowser(r *http.Request) (*auth.Session, *auth.User, error) { + if s.GitHub == nil { + return nil, nil, auth.ErrSessionNotFound + } + input, err := r.Cookie(s.browserConfig().CookieName) + if err != nil { + return nil, nil, auth.ErrSessionNotFound + } + session, user, _, err := s.Auth.ResolveSession(r.Context(), input.Value, s.browserConfig()) + return session, user, err +} + +func validBrowserCSRF(r *http.Request, session *auth.Session) bool { + return session != nil && subtle.ConstantTimeCompare([]byte(r.Header.Get("X-Mem-CSRF")), []byte(session.CSRFToken)) == 1 +} + +// authenticateBrowser returns false only when no browser cookie was supplied, +// allowing the existing machine Bearer authentication contract to run. +func (s *Server) authenticateBrowser(w http.ResponseWriter, r *http.Request, next http.Handler) bool { + if _, err := r.Cookie(s.browserConfig().CookieName); err != nil { + return false + } + session, user, err := s.resolveBrowser(r) + if err != nil { + writeError(w, 401, "session_expired", "Sign in again") + return true + } + if origin := r.Header.Get("Origin"); origin != "" && origin != s.PublicURL { + writeError(w, 403, "origin_denied", "Use the configured application origin") + return true + } + if r.Method != http.MethodGet && r.Method != http.MethodHead && r.Method != http.MethodOptions { + if !s.exactBrowserOrigin(r) || !validBrowserCSRF(r, session) { + writeError(w, 403, "csrf_denied", "Reload the application and try again") + return true + } + } + // This projection is internal authorization metadata, never a minted API + // token. Existing workspace membership/path middleware still applies. + projection := &auth.Token{ID: session.ID, UserID: user.ID, Name: "browser-session", Scopes: auth.AllScopes, Paths: []string{}, ExpiresAt: &session.ExpiresAt} + ctx := context.WithValue(r.Context(), ctxActor, user) + ctx = context.WithValue(ctx, ctxUser, user) + ctx = context.WithValue(ctx, ctxToken, projection) + next.ServeHTTP(w, r.WithContext(ctx)) + return true +} + +func (s *Server) handleBrowserSession(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if origin := r.Header.Get("Origin"); origin != "" && origin != s.PublicURL { + writeError(w, 403, "origin_denied", "Use the configured application origin") + return + } + session, user, err := s.resolveBrowser(r) + if err != nil { + writeJSON(w, 200, map[string]any{"authenticated": false}) + return + } + writeJSON(w, 200, map[string]any{"authenticated": true, "session_id": session.ID, "user": map[string]any{"id": user.ID, "email": user.Email}, "csrf_token": session.CSRFToken, "expires_at": session.ExpiresAt}) +} + +func (s *Server) handleBrowserLogout(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + session, _, err := s.resolveBrowser(r) + if err != nil { + writeError(w, 401, "session_expired", "Sign in again") + return + } + if !s.exactBrowserOrigin(r) || !validBrowserCSRF(r, session) { + writeError(w, 403, "csrf_denied", "Reload the application and try again") + return + } + if err := s.Auth.RevokeSession(r.Context(), session.ID); err != nil { + writeError(w, 503, "logout_unavailable", "Try again later") + return + } + http.SetCookie(w, auth.ClearSessionCookie(s.browserConfig())) + writeJSON(w, 200, map[string]any{"ok": true}) +} + +func (s *Server) handleGitHubIdentity(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + user := r.Context().Value(ctxActor).(*auth.User) + identity, err := s.Auth.GitHubIdentity(r.Context(), user.ID) + if err != nil { + writeError(w, 503, "identity_unavailable", "Try again later") + return + } + writeJSON(w, 200, map[string]any{"github": identity}) +} + +func (s *Server) clearPriorBrowser(w http.ResponseWriter, r *http.Request) { + if s.GitHub == nil { + return + } + if session, _, err := s.resolveBrowser(r); err == nil { + _ = s.Auth.RevokeSession(r.Context(), session.ID) + } + http.SetCookie(w, auth.ClearSessionCookie(s.browserConfig())) +} diff --git a/server/internal/auth/github.go b/server/internal/auth/github.go new file mode 100644 index 0000000..9cb75f4 --- /dev/null +++ b/server/internal/auth/github.go @@ -0,0 +1,129 @@ +package auth + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +var ErrGitHubUnavailable = errors.New("github authentication unavailable") + +// GitHubIdentity uses the immutable numeric subject; login/email are metadata. +type GitHubIdentity struct { + Subject string `json:"subject"` + Login string `json:"login"` + Email string `json:"email"` +} + +// GitHubOAuth deliberately requests no repository or organization access. +// Provider tokens exist only during this exchange and are never persisted. +type GitHubOAuth struct { + ClientID, ClientSecret, RedirectURL string + HTTPClient *http.Client + tokenURL, apiURL string // test seams; production endpoints are fixed +} + +func (g *GitHubOAuth) AuthorizationURL(state, verifier string) string { + digest := sha256.Sum256([]byte(verifier)) + query := url.Values{ + "client_id": {g.ClientID}, "redirect_uri": {g.RedirectURL}, + "scope": {"read:user user:email"}, "state": {state}, + "code_challenge": {base64.RawURLEncoding.EncodeToString(digest[:])}, + "code_challenge_method": {"S256"}, "allow_signup": {"false"}, + "prompt": {"select_account"}, + } + return "https://github.com/login/oauth/authorize?" + query.Encode() +} + +func (g *GitHubOAuth) Exchange(ctx context.Context, code, verifier string) (*GitHubIdentity, error) { + if code == "" || verifier == "" { + return nil, ErrGitHubUnavailable + } + client := g.HTTPClient + if client == nil { + client = &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + } + tokenURL := g.tokenURL + if tokenURL == "" { + tokenURL = "https://github.com/login/oauth/access_token" + } + apiURL := g.apiURL + if apiURL == "" { + apiURL = "https://api.github.com" + } + form := url.Values{"client_id": {g.ClientID}, "client_secret": {g.ClientSecret}, + "redirect_uri": {g.RedirectURL}, "code": {code}, "code_verifier": {verifier}} + req, err := http.NewRequestWithContext(ctx, http.MethodPost, tokenURL, strings.NewReader(form.Encode())) + if err != nil { + return nil, ErrGitHubUnavailable + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Accept", "application/json") + var token struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + Error string `json:"error"` + } + if err := githubJSON(client, req, &token); err != nil || token.AccessToken == "" || token.Error != "" || !strings.EqualFold(token.TokenType, "bearer") { + return nil, ErrGitHubUnavailable + } + get := func(path string, output any) error { + r, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL+path, nil) + if err != nil { + return ErrGitHubUnavailable + } + r.Header.Set("Authorization", "Bearer "+token.AccessToken) + r.Header.Set("Accept", "application/vnd.github+json") + r.Header.Set("X-GitHub-Api-Version", "2022-11-28") + return githubJSON(client, r, output) + } + var user struct { + ID int64 `json:"id"` + Login string `json:"login"` + } + if err := get("/user", &user); err != nil || user.ID <= 0 || user.Login == "" { + return nil, ErrGitHubUnavailable + } + var emails []struct { + Email string `json:"email"` + Primary bool `json:"primary"` + Verified bool `json:"verified"` + } + if err := get("/user/emails", &emails); err != nil { + return nil, ErrGitHubUnavailable + } + for _, email := range emails { + if email.Primary && email.Verified && strings.Contains(email.Email, "@") { + return &GitHubIdentity{Subject: strconv.FormatInt(user.ID, 10), Login: user.Login, Email: strings.ToLower(strings.TrimSpace(email.Email))}, nil + } + } + return nil, ErrGitHubUnavailable +} + +func githubJSON(client *http.Client, req *http.Request, output any) error { + resp, err := client.Do(req) + if err != nil { + return ErrGitHubUnavailable + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ErrGitHubUnavailable + } + decoder := json.NewDecoder(io.LimitReader(resp.Body, 65537)) + if err := decoder.Decode(output); err != nil { + return ErrGitHubUnavailable + } + var extra any + if decoder.Decode(&extra) != io.EOF { + return ErrGitHubUnavailable + } + return nil +} diff --git a/server/internal/auth/github_test.go b/server/internal/auth/github_test.go new file mode 100644 index 0000000..6c67535 --- /dev/null +++ b/server/internal/auth/github_test.go @@ -0,0 +1,77 @@ +package auth + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestGitHubAuthorizationPKCEAndMinimalScopes(t *testing.T) { + g := &GitHubOAuth{ClientID: "client", RedirectURL: "https://mem.example/v1/auth/github/callback"} + u, _ := url.Parse(g.AuthorizationURL("state", "verifier")) + q := u.Query() + digest := sha256.Sum256([]byte("verifier")) + if u.Host != "github.com" || q.Get("scope") != "read:user user:email" || q.Get("state") != "state" || q.Get("code_challenge_method") != "S256" || q.Get("code_challenge") != base64.RawURLEncoding.EncodeToString(digest[:]) { + t.Fatalf("incorrect OAuth request: %s", u) + } + if strings.Contains(q.Get("scope"), "repo") { + t.Fatal("sign-in must not request repository access") + } +} + +func TestGitHubExchangeVerifiedPrimaryAndSecretSafeErrors(t *testing.T) { + for _, tc := range []struct { + name, emails string + status int + want bool + }{ + {"verified", `[{"email":"Primary@Example.test","primary":true,"verified":true}]`, 200, true}, + {"unverified", `[{"email":"primary@example.test","primary":true,"verified":false}]`, 200, false}, + {"non-primary", `[{"email":"secondary@example.test","primary":false,"verified":true}]`, 200, false}, + {"outage", `provider secret should never escape`, 500, false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/token": + if err := r.ParseForm(); err != nil { + t.Error(err) + } + if r.Form.Get("code_verifier") != "verifier" || r.Form.Get("redirect_uri") != "https://mem.example/callback" || r.Form.Get("client_secret") != "client-secret" { + t.Error("missing PKCE or exact redirect") + } + fmt.Fprint(w, `{"access_token":"provider-secret","token_type":"bearer"}`) + case "/user": + if r.Header.Get("Authorization") != "Bearer provider-secret" { + t.Error("missing provider authorization") + } + fmt.Fprint(w, `{"id":47820304,"login":"PeterGuy326"}`) + case "/user/emails": + w.WriteHeader(tc.status) + fmt.Fprint(w, tc.emails) + default: + t.Error("unexpected request") + w.WriteHeader(404) + } + })) + defer server.Close() + g := &GitHubOAuth{ClientID: "client", ClientSecret: "client-secret", RedirectURL: "https://mem.example/callback", HTTPClient: server.Client(), tokenURL: server.URL + "/token", apiURL: server.URL} + identity, err := g.Exchange(context.Background(), "code", "verifier") + if (err == nil) != tc.want { + t.Fatalf("identity=%v error=%v", identity, err) + } + if err != nil && (strings.Contains(err.Error(), "provider-secret") || strings.Contains(err.Error(), "client-secret")) { + t.Fatal("provider secret leaked") + } + if tc.want && (identity.Subject != "47820304" || identity.Email != "primary@example.test") { + t.Fatalf("unexpected identity: %#v", identity) + } + }) + } +} diff --git a/server/internal/auth/oauth.go b/server/internal/auth/oauth.go new file mode 100644 index 0000000..a91c9b6 --- /dev/null +++ b/server/internal/auth/oauth.go @@ -0,0 +1,141 @@ +package auth + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +var ( + ErrOAuthState = errors.New("invalid or expired oauth state") + ErrIdentityConflict = errors.New("identity already belongs to another account") + ErrExplicitLinkRequired = errors.New("sign in to your existing account and explicitly link github") +) + +func (s *Service) BeginOAuth(ctx context.Context, state, intent string, userID, sessionID *uuid.UUID) error { + // Bound global state and discard expired challenges. Secrets stay in an + // HttpOnly browser cookie; only the one-use state digest is stored here. + _, err := s.pool.Exec(ctx, `DELETE FROM oauth_challenges WHERE expires_at < now()`) + if err != nil { + return err + } + tag, err := s.pool.Exec(ctx, `INSERT INTO oauth_challenges(state_hash, intent, user_id, session_id, expires_at) + SELECT $1, $2, $3, $4, $5 WHERE (SELECT count(*) FROM oauth_challenges) < 10000`, HashToken(state), intent, userID, sessionID, time.Now().UTC().Add(10*time.Minute)) + if err == nil && tag.RowsAffected() != 1 { + return ErrOAuthState + } + return err +} + +func (s *Service) ConsumeOAuth(ctx context.Context, state string) (string, *uuid.UUID, *uuid.UUID, error) { + var intent string + var userID *uuid.UUID + var sessionID *uuid.UUID + err := s.pool.QueryRow(ctx, `DELETE FROM oauth_challenges WHERE state_hash=$1 AND expires_at > now() RETURNING intent, user_id, session_id`, HashToken(state)).Scan(&intent, &userID, &sessionID) + if errors.Is(err, pgx.ErrNoRows) { + return "", nil, nil, ErrOAuthState + } + return intent, userID, sessionID, err +} + +// GitHubUser serializes enrollment/linking with local first-user bootstrap. +// Email never authorizes a link, and an existing provider subject never moves. +func (s *Service) GitHubUser(ctx context.Context, identity GitHubIdentity, linkUserID, linkSessionID *uuid.UUID, mode string) (*User, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(72466301)`); err != nil { + return nil, err + } + if linkUserID != nil { + if linkSessionID == nil { + return nil, ErrOAuthState + } + var proof uuid.UUID + err = tx.QueryRow(ctx, `SELECT id FROM sessions WHERE id=$1 AND user_id=$2 AND revoked_at IS NULL AND expires_at > now() AND last_active > now()-interval '2 hours' FOR UPDATE`, *linkSessionID, *linkUserID).Scan(&proof) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrOAuthState + } + if err != nil { + return nil, err + } + } + var user User + err = tx.QueryRow(ctx, `SELECT u.id, u.email, u.created_at FROM external_identities e JOIN users u ON u.id=e.user_id WHERE e.provider='github' AND e.subject=$1`, identity.Subject).Scan(&user.ID, &user.Email, &user.CreatedAt) + if err == nil { + if linkUserID != nil && user.ID != *linkUserID { + return nil, ErrIdentityConflict + } + _, err = tx.Exec(ctx, `UPDATE external_identities SET provider_login=$2, provider_email=$3, updated_at=now() WHERE provider='github' AND subject=$1`, identity.Subject, identity.Login, identity.Email) + if err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return &user, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, err + } + if linkUserID != nil { + err = tx.QueryRow(ctx, `SELECT id,email,created_at FROM users WHERE id=$1`, *linkUserID).Scan(&user.ID, &user.Email, &user.CreatedAt) + if err != nil { + return nil, err + } + var linked bool + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM external_identities WHERE provider='github' AND user_id=$1)`, user.ID).Scan(&linked); err != nil { + return nil, err + } + if linked { + return nil, ErrIdentityConflict + } + } else { + if mode != "first_user" && mode != "open" { + return nil, ErrRegistrationDisabled + } + var existingEmail, anyUser bool + if err = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM users WHERE email=$1), EXISTS(SELECT 1 FROM users)`, identity.Email).Scan(&existingEmail, &anyUser); err != nil { + return nil, err + } + if existingEmail { + return nil, ErrExplicitLinkRequired + } + if mode == "first_user" && anyUser { + return nil, ErrRegistrationDisabled + } + user = User{ID: uuid.New(), Email: identity.Email, CreatedAt: time.Now().UTC()} + if _, err = tx.Exec(ctx, `INSERT INTO users(id,email,password_hash,created_at) VALUES($1,$2,'!github-only',$3)`, user.ID, user.Email, user.CreatedAt); err != nil { + return nil, err + } + workspaceID := uuid.New() + if _, err = tx.Exec(ctx, `INSERT INTO workspaces(id,name,resource_owner_user_id,created_at) VALUES($1,$2,$3,$4)`, workspaceID, identity.Login+"'s workspace", user.ID, user.CreatedAt); err != nil { + return nil, err + } + if _, err = tx.Exec(ctx, `INSERT INTO workspace_memberships(workspace_id,user_id,role,created_at) VALUES($1,$2,'owner',$3)`, workspaceID, user.ID, user.CreatedAt); err != nil { + return nil, err + } + } + if _, err = tx.Exec(ctx, `INSERT INTO external_identities(provider,subject,user_id,provider_login,provider_email) VALUES('github',$1,$2,$3,$4)`, identity.Subject, user.ID, identity.Login, identity.Email); err != nil { + return nil, fmt.Errorf("link identity: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return &user, nil +} + +func (s *Service) GitHubIdentity(ctx context.Context, userID uuid.UUID) (*GitHubIdentity, error) { + var identity GitHubIdentity + err := s.pool.QueryRow(ctx, `SELECT subject,provider_login,provider_email FROM external_identities WHERE provider='github' AND user_id=$1`, userID).Scan(&identity.Subject, &identity.Login, &identity.Email) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return &identity, err +} diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 7fee5b0..5686f11 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -59,10 +59,15 @@ type Config struct { WorkerAuthKey []byte // Deployment / Auth - DeploymentMode string // private|saas - RegistrationMode string // open|first_user|disabled - SessionTTL time.Duration - CORSOrigins []string // allowed browser origins; empty disables CORS (same-origin only) + DeploymentMode string // private|saas + RegistrationMode string // open|first_user|disabled + SessionTTL time.Duration + CORSOrigins []string // allowed browser origins; empty disables CORS (same-origin only) + PublicURL string + GitHubClientID string + GitHubClientSecret string + GitHubAllowedUserIDs []string + GitHubBootstrap bool // ManagedEmbeddingProvider is the operator-selected primary exact Worker // provider spec. ManagedEmbeddingProviders is the complete exact allow-set // derived from that primary and the enabled immutable profile generations. @@ -336,6 +341,9 @@ func Load() (*Config, error) { return nil, err } } + if err := loadGitHub(cfg); err != nil { + return nil, err + } return cfg, nil } diff --git a/server/internal/config/github.go b/server/internal/config/github.go new file mode 100644 index 0000000..9d96797 --- /dev/null +++ b/server/internal/config/github.go @@ -0,0 +1,54 @@ +package config + +import ( + "errors" + "net/url" + "os" + "regexp" + "strconv" + "strings" +) + +func loadGitHub(cfg *Config) error { + if raw := os.Getenv("MEM_GITHUB_BOOTSTRAP"); raw != "" { + value, err := strconv.ParseBool(raw) + if err != nil { + return errors.New("MEM_GITHUB_BOOTSTRAP must be true or false") + } + cfg.GitHubBootstrap = value + } + cfg.PublicURL = strings.TrimRight(strings.TrimSpace(os.Getenv("MEM_PUBLIC_URL")), "/") + cfg.GitHubClientID = strings.TrimSpace(os.Getenv("MEM_GITHUB_CLIENT_ID")) + cfg.GitHubClientSecret = os.Getenv("MEM_GITHUB_CLIENT_SECRET") + raw := strings.TrimSpace(os.Getenv("MEM_GITHUB_ALLOWED_USER_IDS")) + if cfg.GitHubClientID == "" && cfg.GitHubClientSecret == "" && raw == "" { + if cfg.GitHubBootstrap { + return errors.New("GitHub bootstrap requires configured GitHub OAuth") + } + return nil + } + if cfg.PublicURL == "" || cfg.GitHubClientID == "" || strings.TrimSpace(cfg.GitHubClientSecret) == "" || raw == "" { + return errors.New("GitHub OAuth requires MEM_PUBLIC_URL, MEM_GITHUB_CLIENT_ID, MEM_GITHUB_CLIENT_SECRET and MEM_GITHUB_ALLOWED_USER_IDS") + } + u, err := url.Parse(cfg.PublicURL) + if cfg.GitHubBootstrap && cfg.RegistrationMode != "disabled" { + return errors.New("GitHub bootstrap requires MEM_REGISTRATION_MODE=disabled") + } + if err != nil || u.Host == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return errors.New("MEM_PUBLIC_URL must be an exact origin without credentials, path, query or fragment") + } + if u.Scheme != "https" && !(cfg.RuntimeProfile == "development" && u.Scheme == "http" && (u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1")) { + return errors.New("MEM_PUBLIC_URL must use HTTPS (development may use localhost HTTP)") + } + pattern := regexp.MustCompile(`^[1-9][0-9]{0,19}$`) + seen := map[string]bool{} + for _, value := range strings.Split(raw, ",") { + id := strings.TrimSpace(value) + if !pattern.MatchString(id) || seen[id] { + return errors.New("MEM_GITHUB_ALLOWED_USER_IDS must contain unique numeric GitHub user IDs") + } + seen[id] = true + cfg.GitHubAllowedUserIDs = append(cfg.GitHubAllowedUserIDs, id) + } + return nil +} diff --git a/server/internal/config/github_test.go b/server/internal/config/github_test.go new file mode 100644 index 0000000..628ca30 --- /dev/null +++ b/server/internal/config/github_test.go @@ -0,0 +1,32 @@ +package config + +import "testing" + +func TestGitHubConfigurationFailsClosed(t *testing.T) { + for _, tc := range []struct { + name, url, ids, registration, bootstrap string + want bool + }{ + {"personal", "https://mem.example", "47820304", "disabled", "true", true}, + {"missing-id", "https://mem.example", "", "disabled", "true", false}, + {"duplicate-id", "https://mem.example", "47820304,47820304", "disabled", "false", false}, + {"username-not-subject", "https://mem.example", "PeterGuy326", "disabled", "false", false}, + {"insecure-url", "http://mem.example", "47820304", "disabled", "false", false}, + {"url-path", "https://mem.example/path", "47820304", "disabled", "false", false}, + {"url-credentials", "https://secret@mem.example", "47820304", "disabled", "false", false}, + {"public-password-bootstrap", "https://mem.example", "47820304", "first_user", "true", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("MEM_PUBLIC_URL", tc.url) + t.Setenv("MEM_GITHUB_CLIENT_ID", "client") + t.Setenv("MEM_GITHUB_CLIENT_SECRET", "secret") + t.Setenv("MEM_GITHUB_ALLOWED_USER_IDS", tc.ids) + t.Setenv("MEM_GITHUB_BOOTSTRAP", tc.bootstrap) + cfg := &Config{RuntimeProfile: "production", RegistrationMode: tc.registration} + err := loadGitHub(cfg) + if (err == nil) != tc.want { + t.Fatalf("configuration result: %v", err) + } + }) + } +} diff --git a/server/internal/db/migrations/0027_github_identity.sql b/server/internal/db/migrations/0027_github_identity.sql new file mode 100644 index 0000000..797b26d --- /dev/null +++ b/server/internal/db/migrations/0027_github_identity.sql @@ -0,0 +1,38 @@ +-- +goose Up +-- Authentication policy and provider identities are not workspace assets and +-- must never be exported in a workspace bundle. Provider tokens are not stored. +-- +goose StatementBegin +CREATE TABLE external_identities ( + provider text NOT NULL CHECK (provider = 'github'), + subject text NOT NULL CHECK (subject ~ '^[1-9][0-9]{0,19}$'), + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + provider_login text NOT NULL, + provider_email text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (provider, subject), + UNIQUE (user_id, provider) +); +CREATE TABLE oauth_challenges ( + state_hash text PRIMARY KEY CHECK (state_hash ~ '^[0-9a-f]{64}$'), + intent text NOT NULL CHECK (intent IN ('login', 'link')), + user_id uuid REFERENCES users(id) ON DELETE CASCADE, + session_id uuid REFERENCES sessions(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + CHECK ((intent = 'login' AND user_id IS NULL AND session_id IS NULL) OR (intent = 'link' AND user_id IS NOT NULL AND session_id IS NOT NULL)) +); +CREATE INDEX oauth_challenges_expiry ON oauth_challenges(expires_at); +-- +goose StatementEnd + +-- +goose Down +-- A downgrade cannot silently orphan OAuth-only users. Remove/migrate their +-- credentials explicitly before attempting rollback. +-- +goose StatementBegin +DO $$ BEGIN + IF EXISTS (SELECT 1 FROM users WHERE password_hash = '!github-only') THEN + RAISE EXCEPTION 'cannot downgrade while GitHub-only users exist'; + END IF; +END $$; +DROP TABLE oauth_challenges; +DROP TABLE external_identities; +-- +goose StatementEnd diff --git a/web/.dockerignore b/web/.dockerignore index a055dcc..be16008 100644 --- a/web/.dockerignore +++ b/web/.dockerignore @@ -1,4 +1,6 @@ .git +.env* +!.env.example node_modules dist playwright-report diff --git a/web/nginx/default.conf.template b/web/nginx/default.conf.template index 72de1db..5ceb7fa 100644 --- a/web/nginx/default.conf.template +++ b/web/nginx/default.conf.template @@ -1,7 +1,13 @@ +map $uri $mem_auth_loggable { + default 1; + /v1/auth/github/callback 0; +} server { listen 8080; server_name _; server_tokens off; + # OAuth callback codes and state must not enter access logs. + access_log /dev/stdout combined if=$mem_auth_loggable; root /usr/share/nginx/html; index index.html; diff --git a/web/package-lock.json b/web/package-lock.json index ce35cc0..5f99b79 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -396,9 +396,9 @@ } }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "license": "MIT", "dependencies": { @@ -502,9 +502,9 @@ } }, "node_modules/@humanwhocodes/config-array/node_modules/brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "license": "MIT", "dependencies": { @@ -2497,9 +2497,9 @@ } }, "node_modules/brace-expansion": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", - "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", "dev": true, "license": "MIT", "dependencies": { @@ -3111,9 +3111,9 @@ } }, "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "license": "MIT", "dependencies": { @@ -3472,9 +3472,9 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.18", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", - "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", "dev": true, "license": "MIT", "dependencies": { @@ -5800,9 +5800,9 @@ } }, "node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "version": "7.30.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.0.tgz", + "integrity": "sha512-dkrQXeHSaoamnItlYbmzG0wFYrM0ZwDxCIg0A7aKjTyyhh9svRzCNFEzV+Vm05/yehjCzjDZ31KXfGEjYSztDQ==", "dev": true, "license": "MIT", "engines": { diff --git a/web/src/components/layout/TopBar.tsx b/web/src/components/layout/TopBar.tsx index 066d336..496cffe 100644 --- a/web/src/components/layout/TopBar.tsx +++ b/web/src/components/layout/TopBar.tsx @@ -182,14 +182,28 @@ export function TopBar({ children }: { children?: React.ReactNode }) { ))} + + + + + ) : ( +

{t('github.disabled')}

+ )} + {error && ( +

+ {error} +

+ )} + + + ); +} diff --git a/web/src/pages/LoginPage.tsx b/web/src/pages/LoginPage.tsx index de4493b..3862436 100644 --- a/web/src/pages/LoginPage.tsx +++ b/web/src/pages/LoginPage.tsx @@ -4,7 +4,7 @@ import { Button } from '@/components/ui/Button'; import { Input } from '@/components/ui/Input'; import { Logo } from '@/components/layout/Logo'; import { useAuth } from '@/hooks/useAuth'; -import { ApiException } from '@/lib/api'; +import { api, ApiException } from '@/lib/api'; import { toast } from 'sonner'; import { useT } from '@/i18n'; @@ -18,11 +18,44 @@ export function LoginPage() { const [email, setEmail] = React.useState(''); const [password, setPassword] = React.useState(''); const [error, setError] = React.useState(null); + const [github, setGithub] = React.useState(false); + const [canRegister, setCanRegister] = React.useState(true); + const [githubLoading, setGithubLoading] = React.useState(false); + React.useEffect(() => { + api + .get<{ github: boolean; registration: boolean }>('/auth/capabilities') + .then((res) => { + setGithub(res.github); + setCanRegister(res.registration); + }) + .catch(() => setGithub(false)); + const result = new URLSearchParams(window.location.search).get('github'); + if (result && !['ok', 'linked'].includes(result)) + setError(t(result === 'link_required' ? 'github.linkRequired' : 'github.failed')); + }, [t]); + if (loading) + return ( +
+ … +
+ ); if (token) return ; const isRegister = mode === 'register'; + async function signInWithGitHub() { + setGithubLoading(true); + setError(null); + try { + const result = await api.post<{ url: string }>('/auth/github/start', { intent: 'login' }); + window.location.assign(result.url); + } catch { + setError(t('github.failed')); + setGithubLoading(false); + } + } + async function onSubmit(e: React.FormEvent) { e.preventDefault(); setError(null); @@ -43,7 +76,9 @@ export function LoginPage() { ? err.message : t('toast.retryLater'); setError(msg); - toast.error(isRegister ? t('login.signUpFailed') : t('login.signInFailed'), { description: msg }); + toast.error(isRegister ? t('login.signUpFailed') : t('login.signInFailed'), { + description: msg, + }); } } @@ -61,9 +96,20 @@ export function LoginPage() {

{t('login.tagline')}

+ {github && ( + + )} {/* Mode switch */}
- {(['login', 'register'] as Mode[]).map((m) => ( + {((canRegister ? ['login', 'register'] : ['login']) as Mode[]).map((m) => (
-

- {t('login.footer')} -

+

{t('login.footer')}

); diff --git a/web/src/routes/LoginGate.tsx b/web/src/routes/LoginGate.tsx index 330379e..11c8a25 100644 --- a/web/src/routes/LoginGate.tsx +++ b/web/src/routes/LoginGate.tsx @@ -3,8 +3,14 @@ import { useAuth } from '@/hooks/useAuth'; /** Wrap any private route subtree. Bounces to /login when unauthenticated. */ export function LoginGate({ children }: { children: React.ReactNode }) { - const { token } = useAuth(); + const { token, loading } = useAuth(); const location = useLocation(); + if (loading) + return ( +
+ … +
+ ); if (!token) { return ; } diff --git a/web/src/routes/router.tsx b/web/src/routes/router.tsx index d250905..18ab384 100644 --- a/web/src/routes/router.tsx +++ b/web/src/routes/router.tsx @@ -2,6 +2,7 @@ import { createBrowserRouter, Navigate } from 'react-router'; import { LoginGate } from './LoginGate'; import { AppLayout } from '@/components/layout/AppLayout'; import { LoginPage } from '@/pages/LoginPage'; +import { AccountPage } from '@/pages/AccountPage'; import { ExplorerPage } from '@/pages/ExplorerPage'; import { FileDetailPage } from '@/pages/FileDetailPage'; import { NotFoundPage } from '@/pages/NotFoundPage'; @@ -43,6 +44,7 @@ export const router = createBrowserRouter([ { path: '/faces', element: }, { path: '/providers', element: }, { path: '/permissions', element: }, + { path: '/account', element: }, { path: '/memories', element: }, { path: '/memories/:memoryId', element: }, { path: '/transfer', element: },