feat: declarative database migrations with pgschema, and queries in sql files

.github/workflows/ci.yml

@@ -33,13 +33,31 @@ jobs:
       DATABASE_URL: postgresql://hackproof:hackproof@localhost:5432/hackproof
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v6
         with:
           enable-cache: true
           cache-dependency-glob: apps/api/uv.lock
       - run: uv sync --frozen
       - run: uv run ruff check .
       - run: uv run ruff format --check .
+      - run: uv run sqlfluff lint db/schema src/hackproof/sql
+      - name: A schema change needs a migration file
+        if: github.event_name == 'pull_request'
+        working-directory: .
+        run: |
+          base="origin/${{ github.base_ref }}"
+          changed=$(git diff --name-only "$base...HEAD" -- apps/api/db/schema ':!apps/api/db/schema/manifest.sql')
+          added=$(git diff --name-only --diff-filter=A "$base...HEAD" -- apps/api/db/migrations)
+          if [ -n "$changed" ] && [ -z "$added" ]; then
+            echo "::error::The schema files changed but no migration file was added. Run 'hackproof schema plan' and commit the file it writes."
+            exit 1
+          fi
+      - name: Build the schema on an empty database
+        run: uv run hackproof schema apply
+      - name: The schema files and the database agree
+        run: uv run hackproof schema verify
       - run: uv run pytest
 
   web:
@@ -56,10 +74,46 @@ jobs:
       - run: pnpm format:check
       - run: pnpm build
 
-  deploy-api:
+  migrate:
     needs: api
     if: github.event_name == 'push'
     runs-on: ubuntu-latest
+    environment: production
+    concurrency:
+      group: migrate-production
+      cancel-in-progress: false
+    defaults:
+      run:
+        working-directory: apps/api
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: apps/api/uv.lock
+      - run: uv sync --frozen
+      - name: Apply the schema to production
+        env:
+          DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }}
+        run: |
+          if [ -z "$DATABASE_URL" ]; then
+            echo "::error::Add the PRODUCTION_DATABASE_URL secret. The schema must be migrated before the API deploys."
+            exit 1
+          fi
+          mkdir -p "$RUNNER_TEMP/plan"
+          uv run hackproof schema apply --remote --out-dir "$RUNNER_TEMP/plan"
+      - name: Keep the plan that ran
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: production-plan
+          path: ${{ runner.temp }}/plan
+          if-no-files-found: ignore
+
+  deploy-api:
+    needs: [api, migrate]
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: dorny/paths-filter@v3

.github/workflows/data-sync.yml

@@ -0,0 +1,57 @@
+name: data-sync
+
+on:
+  schedule:
+    - cron: "17 */6 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: data-sync
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    environment: production
+    defaults:
+      run:
+        working-directory: apps/api
+    env:
+      DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }}
+    steps:
+      - name: Require the database secret
+        run: |
+          if [ -z "$DATABASE_URL" ]; then
+            echo "::error::Add the PRODUCTION_DATABASE_URL secret"
+            exit 1
+          fi
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - run: uv sync --frozen
+      - name: Mask the connection details
+        run: >
+          uv run python -c "import os;
+          from hackproof.db_utils.connection import Connection;
+          print('\n'.join(Connection.from_url(os.environ['DATABASE_URL']).mask_lines()))"
+      - name: Regenerate the data files from the database
+        run: uv run hackproof export
+      - uses: peter-evans/create-pull-request@v7
+        with:
+          branch: chore/sync-data
+          delete-branch: true
+          commit-message: "fix: sync the data files with the database"
+          title: "fix: sync the data files with the database"
+          body: |
+            The database and the committed data files had drifted apart. This regenerates the files from the database so they match again.
+
+            - the files are rewritten by `hackproof export`, the same command that produced them
+            - review the diff, then merge to bring the repo back in sync
+            - this branch is refreshed on every scheduled run until it is merged
+          labels: data
+          add-paths: data/**

CONTRIBUTING.md

@@ -22,7 +22,7 @@ The README has the commands to run everything locally.
 - Pydantic models for anything that crosses a module boundary. No dataclasses, NamedTuple or TypedDict.
 - Models for upstream responses use `extra="allow"` so new fields are not lost. Models we derive use `extra="forbid"`.
 - Tests run against the scratch Postgres database from `tests/conftest.py` and never touch the live network. Anything that needs an upstream response uses a recorded fixture in `apps/api/tests/fixtures/`.
-- New pipeline steps follow the existing shape: a `run_*` job in the relevant package, a subcommand in `cli.py`, and tables added to `db.py`.
+- New pipeline steps follow the existing shape: a `run_*` job in the relevant package, a subcommand in `cli.py`, and any schema change as files in `db/schema` with a migration from `hackproof schema plan`. SQL goes in `src/hackproof/sql`, loaded with `load_sql`, never in Python strings.
 
 ## React
 

README.md

@@ -40,9 +40,21 @@ uv run hackproof reconcile
 uv run hackproof export
 ```
 
+## Database
+
+The schema lives in `apps/api/db/schema`, one file per table, index, view and function, and is applied with [pgschema](https://github.com/pgplex/pgschema). Queries live in `apps/api/src/hackproof/sql`. Run from `apps/api`.
+
+```sh
+uv run hackproof schema apply
+uv run hackproof schema plan
+uv run hackproof schema verify
+```
+
+To change the schema, edit the files in `db/schema`, run `schema plan`, and commit the numbered file it writes to `db/migrations` with your change. `apply` refuses a plan that drops data. CI checks every pull request against an empty database and migrates production when the pull request merges to `main`.
+
 ## Checks
 
 ```sh
-cd apps/api && uv run ruff check . && uv run ruff format --check . && uv run pytest
+cd apps/api && uv run ruff check . && uv run ruff format --check . && uv run sqlfluff lint db/schema src/hackproof/sql && uv run pytest
 pnpm lint && pnpm format:check && pnpm build
 ```

apps/api/.sqlfluffignore

@@ -0,0 +1,2 @@
+manifest.sql
+migrations/

apps/api/db/migrations/000001.sql

@@ -0,0 +1,25 @@
+CREATE OR REPLACE FUNCTION method_rank(
+    method text
+)
+RETURNS integer
+LANGUAGE sql
+IMMUTABLE
+AS $$
+    SELECT array_position(ARRAY[
+        'contract_in_repo',
+        'wasm_hash_match',
+        'wasm_meta',
+        'asset_in_repo',
+        'deployer_in_repo',
+        'footprint'
+    ], method)
+$$;
+
+CREATE OR REPLACE VIEW attribution AS
+ SELECT DISTINCT ON (l.subject_kind, l.subject) l.subject_kind,
+    l.subject,
+    l.full_name,
+    l.method
+   FROM links l
+     JOIN repos r ON l.full_name = r.full_name
+  ORDER BY l.subject_kind, l.subject, (method_rank(l.method)), r.created_at, l.full_name;

apps/api/db/migrations/000002.sql

@@ -0,0 +1,14 @@
+CREATE OR REPLACE FUNCTION hackers(
+    window_start timestamptz,
+    window_end timestamptz,
+    friendbot text
+)
+RETURNS SETOF text
+LANGUAGE sql
+STABLE
+AS $$
+    SELECT account FROM accounts
+    WHERE funder = friendbot AND funded_at BETWEEN window_start AND window_end
+    UNION SELECT deployer FROM contracts
+    UNION SELECT uploader FROM wasm_uploads
+$$;

apps/api/db/schema/functions/hackers.sql

@@ -0,0 +1,14 @@
+CREATE OR REPLACE FUNCTION hackers(
+    window_start timestamptz,
+    window_end timestamptz,
+    friendbot text
+)
+RETURNS SETOF text
+LANGUAGE sql
+STABLE
+AS $$
+    SELECT account FROM accounts
+    WHERE funder = friendbot AND funded_at BETWEEN window_start AND window_end
+    UNION SELECT deployer FROM contracts
+    UNION SELECT uploader FROM wasm_uploads
+$$;

apps/api/db/schema/functions/method_rank.sql

@@ -0,0 +1,14 @@
+CREATE OR REPLACE FUNCTION method_rank(method text)
+RETURNS integer
+LANGUAGE sql
+IMMUTABLE
+AS $$
+    SELECT array_position(ARRAY[
+        'contract_in_repo',
+        'wasm_hash_match',
+        'wasm_meta',
+        'asset_in_repo',
+        'deployer_in_repo',
+        'footprint'
+    ], method)
+$$;

apps/api/db/schema/indexes/accounts_funder.sql

@@ -0,0 +1 @@
+CREATE INDEX accounts_funder ON accounts (funder);

apps/api/db/schema/indexes/classic_ops_asset.sql

@@ -0,0 +1 @@
+CREATE INDEX classic_ops_asset ON classic_ops (asset) WHERE (asset IS NOT NULL);

apps/api/db/schema/indexes/classic_ops_source.sql

@@ -0,0 +1 @@
+CREATE INDEX classic_ops_source ON classic_ops (source_account);

apps/api/db/schema/indexes/contracts_deployed_at.sql

@@ -0,0 +1 @@
+CREATE INDEX contracts_deployed_at ON contracts (deployed_at);

apps/api/db/schema/indexes/contracts_deployer.sql

@@ -0,0 +1 @@
+CREATE INDEX contracts_deployer ON contracts (deployer);

apps/api/db/schema/indexes/contracts_wasm.sql

@@ -0,0 +1 @@
+CREATE INDEX contracts_wasm ON contracts (wasm_hash);

apps/api/db/schema/indexes/failed_txs_source.sql

@@ -0,0 +1 @@
+CREATE INDEX failed_txs_source ON failed_txs (source_account);

apps/api/db/schema/indexes/repo_agents_agent.sql

@@ -0,0 +1 @@
+CREATE INDEX repo_agents_agent ON repo_agents (agent);

apps/api/db/schema/indexes/repo_contributors_login.sql

@@ -0,0 +1 @@
+CREATE INDEX repo_contributors_login ON repo_contributors (login);

apps/api/db/schema/indexes/repo_integrations_partner.sql

@@ -0,0 +1 @@
+CREATE INDEX repo_integrations_partner ON repo_integrations (partner);

apps/api/db/schema/indexes/repo_refs_value.sql

@@ -0,0 +1 @@
+CREATE INDEX repo_refs_value ON repo_refs (kind, value);

apps/api/db/schema/indexes/repo_tools_tool.sql

@@ -0,0 +1 @@
+CREATE INDEX repo_tools_tool ON repo_tools (tool);

apps/api/db/schema/indexes/wasm_uploads_hash.sql

@@ -0,0 +1 @@
+CREATE INDEX wasm_uploads_hash ON wasm_uploads (wasm_hash);

apps/api/db/schema/indexes/wasm_uploads_uploader.sql

@@ -0,0 +1 @@
+CREATE INDEX wasm_uploads_uploader ON wasm_uploads (uploader);

apps/api/db/schema/manifest.sql

@@ -0,0 +1,33 @@
+\i functions/hackers.sql
+\i functions/method_rank.sql
+\i tables/accounts.sql
+\i tables/classic_ops.sql
+\i tables/contracts.sql
+\i tables/failed_txs.sql
+\i tables/github_users.sql
+\i tables/links.sql
+\i tables/projects.sql
+\i tables/repo_agents.sql
+\i tables/repo_contributors.sql
+\i tables/repo_integrations.sql
+\i tables/repo_refs.sql
+\i tables/repo_tools.sql
+\i tables/repos.sql
+\i tables/sync_state.sql
+\i tables/wasm_uploads.sql
+\i indexes/accounts_funder.sql
+\i indexes/classic_ops_asset.sql
+\i indexes/classic_ops_source.sql
+\i indexes/contracts_deployed_at.sql
+\i indexes/contracts_deployer.sql
+\i indexes/contracts_wasm.sql
+\i indexes/failed_txs_source.sql
+\i indexes/repo_agents_agent.sql
+\i indexes/repo_contributors_login.sql
+\i indexes/repo_integrations_partner.sql
+\i indexes/repo_refs_value.sql
+\i indexes/repo_tools_tool.sql
+\i indexes/wasm_uploads_hash.sql
+\i indexes/wasm_uploads_uploader.sql
+\i views/attribution.sql
+\i views/account_projects.sql

apps/api/db/schema/tables/accounts.sql

@@ -0,0 +1,8 @@
+CREATE TABLE accounts (
+    account text,
+    funder text NOT NULL,
+    tx_hash text NOT NULL,
+    ledger bigint NOT NULL,
+    funded_at timestamptz NOT NULL,
+    CONSTRAINT accounts_pkey PRIMARY KEY (account)
+);

apps/api/db/schema/tables/classic_ops.sql

@@ -0,0 +1,11 @@
+CREATE TABLE classic_ops (
+    tx_hash text,
+    op_index integer,
+    source_account text NOT NULL,
+    op_type text NOT NULL,
+    asset text,
+    counterparty text,
+    ledger bigint NOT NULL,
+    occurred_at timestamptz NOT NULL,
+    CONSTRAINT classic_ops_pkey PRIMARY KEY (tx_hash, op_index)
+);

apps/api/db/schema/tables/contracts.sql

@@ -0,0 +1,9 @@
+CREATE TABLE contracts (
+    contract_id text,
+    deployer text NOT NULL,
+    wasm_hash text NOT NULL,
+    tx_hash text NOT NULL,
+    ledger bigint NOT NULL,
+    deployed_at timestamptz NOT NULL,
+    CONSTRAINT contracts_pkey PRIMARY KEY (contract_id)
+);

apps/api/db/schema/tables/failed_txs.sql

@@ -0,0 +1,9 @@
+CREATE TABLE failed_txs (
+    tx_hash text,
+    source_account text NOT NULL,
+    kind text NOT NULL,
+    result_code text NOT NULL,
+    ledger bigint NOT NULL,
+    created_at timestamptz NOT NULL,
+    CONSTRAINT failed_txs_pkey PRIMARY KEY (tx_hash)
+);

apps/api/db/schema/tables/github_users.sql

@@ -0,0 +1,7 @@
+CREATE TABLE github_users (
+    login text,
+    location text,
+    country text,
+    fetched_at timestamptz DEFAULT now() NOT NULL,
+    CONSTRAINT github_users_pkey PRIMARY KEY (login)
+);

apps/api/db/schema/tables/links.sql

@@ -0,0 +1,7 @@
+CREATE TABLE links (
+    full_name text,
+    subject_kind text,
+    subject text,
+    method text NOT NULL,
+    CONSTRAINT links_pkey PRIMARY KEY (subject_kind, subject, full_name)
+);

apps/api/db/schema/tables/projects.sql

@@ -0,0 +1,15 @@
+CREATE TABLE projects (
+    project_id text,
+    full_name text,
+    footprint text NOT NULL,
+    tier text NOT NULL,
+    contracts integer NOT NULL,
+    classic_ops integer NOT NULL,
+    assets integer NOT NULL,
+    wasm_uploads integer NOT NULL,
+    failed_txs integer NOT NULL,
+    builders integer NOT NULL,
+    first_activity timestamptz,
+    last_activity timestamptz,
+    CONSTRAINT projects_pkey PRIMARY KEY (project_id)
+);

apps/api/db/schema/tables/repo_agents.sql

@@ -0,0 +1,8 @@
+CREATE TABLE repo_agents (
+    full_name text,
+    agent text,
+    method text,
+    evidence text,
+    path text NOT NULL,
+    CONSTRAINT repo_agents_pkey PRIMARY KEY (full_name, agent, method, evidence)
+);

apps/api/db/schema/tables/repo_contributors.sql

@@ -0,0 +1,8 @@
+CREATE TABLE repo_contributors (
+    full_name text,
+    login text,
+    url text NOT NULL,
+    avatar_url text,
+    contributions integer NOT NULL,
+    CONSTRAINT repo_contributors_pkey PRIMARY KEY (full_name, login)
+);