From 15b018d6aed1a7e9c8b345e96631e45a9e1a2804 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:18:35 +0100 Subject: [PATCH 01/18] style: adopt the shared ruff config --- apps/api/pyproject.toml | 18 ++++- apps/api/src/hackproof/chat/briefs.py | 39 ++++++--- apps/api/src/hackproof/chat/ratelimit.py | 4 +- apps/api/src/hackproof/chat/routes.py | 4 +- apps/api/src/hackproof/chat/stream.py | 30 +++++-- apps/api/src/hackproof/chat/tools.py | 65 +++++++++++---- apps/api/src/hackproof/cli.py | 27 +++++-- apps/api/src/hackproof/config.py | 8 +- apps/api/src/hackproof/export/build.py | 51 +++++++++--- apps/api/src/hackproof/export/files.py | 46 ++++++++--- apps/api/src/hackproof/export/jobs.py | 13 ++- apps/api/src/hackproof/export/models.py | 80 ++++++++++++++----- apps/api/src/hackproof/github/client.py | 36 +++++++-- apps/api/src/hackproof/github/demo.py | 4 +- apps/api/src/hackproof/github/dependencies.py | 26 +++++- apps/api/src/hackproof/github/discovery.py | 43 +++++++--- apps/api/src/hackproof/github/files.py | 4 +- apps/api/src/hackproof/github/history.py | 14 +++- apps/api/src/hackproof/github/jobs.py | 4 +- apps/api/src/hackproof/github/profiles.py | 10 ++- apps/api/src/hackproof/github/repositories.py | 18 +++-- apps/api/src/hackproof/github/scan.py | 57 ++++++++++--- apps/api/src/hackproof/github/track.py | 5 +- apps/api/src/hackproof/indexer/events.py | 9 ++- apps/api/src/hackproof/indexer/horizon.py | 4 +- apps/api/src/hackproof/indexer/parser.py | 37 +++++++-- apps/api/src/hackproof/indexer/rpc.py | 10 ++- apps/api/src/hackproof/indexer/sync.py | 19 +++-- apps/api/src/hackproof/integrations/detect.py | 44 +++++++--- apps/api/src/hackproof/linking/reconcile.py | 16 +++- apps/api/src/hackproof/queries.py | 50 +++++++++--- apps/api/src/hackproof/routes.py | 40 +++++++--- apps/api/tests/conftest.py | 4 +- apps/api/tests/helpers.py | 26 ++++-- apps/api/tests/test_api.py | 35 +++++--- apps/api/tests/test_chat.py | 75 +++++++++++++---- apps/api/tests/test_dependencies.py | 10 ++- apps/api/tests/test_discovery.py | 12 ++- apps/api/tests/test_export.py | 19 ++++- apps/api/tests/test_export_check.py | 8 +- apps/api/tests/test_export_files.py | 12 ++- apps/api/tests/test_files.py | 4 +- apps/api/tests/test_hackathon.py | 4 +- apps/api/tests/test_history.py | 8 +- apps/api/tests/test_integrations.py | 35 ++++++-- apps/api/tests/test_reconcile.py | 58 ++++++++++---- apps/api/tests/test_repositories.py | 22 ++++- apps/api/tests/test_scan.py | 21 +++-- apps/api/tests/test_urls.py | 5 +- apps/api/tests/test_wasm.py | 15 +++- 50 files changed, 932 insertions(+), 276 deletions(-) diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 3327029..9e8ccd0 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -37,7 +37,23 @@ asyncio_mode = "auto" testpaths = ["tests"] [tool.ruff] -line-length = 100 +line-length = 88 [tool.ruff.lint] select = ["E", "F", "I", "UP", "B"] +# E501: the formatter wraps what it can, long SQL and message strings may stay long. +ignore = ["E501"] +# D209: closing """ goes on its own line (https://docs.astral.sh/ruff/rules/new-line-after-last-paragraph/) +# D213: docstring summary starts on the line after the opening """ (https://docs.astral.sh/ruff/rules/multi-line-summary-second-line/) +extend-select = ["TID", "D209", "D213"] + +[tool.ruff.lint.isort] +# Keep `from x import a, b as c` on one line instead of splitting the alias out. +combine-as-imports = true + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "parents" + +[tool.ruff.format] +# Add a trailing comma to function defs with two or more args to enforce multiline formatting in ruff. +skip-magic-trailing-comma = false diff --git a/apps/api/src/hackproof/chat/briefs.py b/apps/api/src/hackproof/chat/briefs.py index 7ca0416..008f9c3 100644 --- a/apps/api/src/hackproof/chat/briefs.py +++ b/apps/api/src/hackproof/chat/briefs.py @@ -26,7 +26,8 @@ class Page(BaseModel): class ProjectFilters(BaseModel): tiers: list[Tier] | None = Field( - default=None, description="Evidence tiers to include; null means verified and likely" + default=None, + description="Evidence tiers to include; null means verified and likely", ) track: Track | None = Field( default=None, @@ -56,7 +57,8 @@ class ProjectFilters(BaseModel): "with failed transactions", ) awards: list[Place] | None = Field( - default=None, description="first or runner-up keeps projects that won that award" + default=None, + description="first or runner-up keeps projects that won that award", ) def list_args(self) -> dict[str, Any]: @@ -128,8 +130,12 @@ async def project_brief(project_id: str) -> dict[str, Any] | None: async def builder_brief(address: str) -> dict[str, Any] | None: - projects, project_total = await queries.list_page("projects", account=address, limit=SAMPLE) - contracts, contract_total = await queries.list_page("contracts", account=address, limit=SAMPLE) + projects, project_total = await queries.list_page( + "projects", account=address, limit=SAMPLE + ) + contracts, contract_total = await queries.list_page( + "contracts", account=address, limit=SAMPLE + ) _, wasm_total = await queries.list_page("wasm", account=address, limit=1) _, failed_total = await queries.list_page("failed", account=address, limit=1) if not (project_total or contract_total or wasm_total or failed_total): @@ -146,11 +152,15 @@ async def builder_brief(address: str) -> dict[str, Any] | None: def _top(counter: Counter[str], size: int) -> list[dict[str, Any]]: - return [{"name": name, "projects": count} for name, count in counter.most_common(size)] + return [ + {"name": name, "projects": count} for name, count in counter.most_common(size) + ] async def summarize_view(filters: ProjectFilters) -> dict[str, Any]: - items, total = await queries.list_page("projects", **filters.list_args(), limit=10_000) + items, total = await queries.list_page( + "projects", **filters.list_args(), limit=10_000 + ) topics: Counter[str] = Counter() stack: Counter[str] = Counter() partners: Counter[str] = Counter() @@ -168,13 +178,17 @@ async def summarize_view(filters: ProjectFilters) -> dict[str, Any]: "by_track": dict(Counter(item["track"] for item in items)), "contracts": sum(item["contracts"] for item in items), "wasm_uploads": sum(item["wasm_uploads"] for item in items), - "languages": _top(Counter(item["language"] for item in items if item["language"]), 10), + "languages": _top( + Counter(item["language"] for item in items if item["language"]), 10 + ), "topics": _top(topics, 20), "stellar_packages": _top(stack, 15), "integrations": _top(partners, 15), "stellar_tooling": _top(tooling, 10), "ai_agents": _top(agents, 10), - "countries": _top(Counter(item["country"] for item in items if item["country"]), 10), + "countries": _top( + Counter(item["country"] for item in items if item["country"]), 10 + ), "descriptions": [ {"project_id": item["project_id"], "description": item["description"][:140]} for item in items @@ -185,7 +199,9 @@ async def summarize_view(filters: ProjectFilters) -> dict[str, Any]: def view_filters(selected: dict[str, list[str]]) -> ProjectFilters: known = { - PAGE_FILTERS[key]: values[:20] for key, values in selected.items() if key in PAGE_FILTERS + PAGE_FILTERS[key]: values[:20] + for key, values in selected.items() + if key in PAGE_FILTERS } tracks = selected.get("track", []) return ProjectFilters.model_validate( @@ -222,5 +238,8 @@ async def describe_page(page: Page | None) -> str: if brief is None: return prompt("page-empty", subject=subject) return prompt( - "page-entity", subject=subject, pronoun=pronoun, data=json.dumps(brief, default=str) + "page-entity", + subject=subject, + pronoun=pronoun, + data=json.dumps(brief, default=str), ) diff --git a/apps/api/src/hackproof/chat/ratelimit.py b/apps/api/src/hackproof/chat/ratelimit.py index 59ba06c..ea45ab1 100644 --- a/apps/api/src/hackproof/chat/ratelimit.py +++ b/apps/api/src/hackproof/chat/ratelimit.py @@ -45,7 +45,9 @@ def check(self, key: str) -> Denied | None: if len(recent) >= window.limit: retry_after = math.ceil(recent[0] + window.seconds - now) denials.append( - Denied(window=window.name, limit=window.limit, retry_after=retry_after) + Denied( + window=window.name, limit=window.limit, retry_after=retry_after + ) ) if denials: return max(denials, key=lambda denied: denied.retry_after) diff --git a/apps/api/src/hackproof/chat/routes.py b/apps/api/src/hackproof/chat/routes.py index 129de76..f0c95ad 100644 --- a/apps/api/src/hackproof/chat/routes.py +++ b/apps/api/src/hackproof/chat/routes.py @@ -39,7 +39,9 @@ async def chat(body: ChatRequest, request: Request): messages = to_anthropic(body.messages) if not messages or messages[-1]["role"] != "user": - raise HTTPException(status_code=422, detail="last message must be from the user") + raise HTTPException( + status_code=422, detail="last message must be from the user" + ) return StreamingResponse( stream_reply( diff --git a/apps/api/src/hackproof/chat/stream.py b/apps/api/src/hackproof/chat/stream.py index aa97300..2899c27 100644 --- a/apps/api/src/hackproof/chat/stream.py +++ b/apps/api/src/hackproof/chat/stream.py @@ -37,7 +37,9 @@ def event(payload: dict[str, Any]) -> str: def to_anthropic(messages: list[Message]) -> list[dict[str, Any]]: converted: list[dict[str, Any]] = [] for message in messages: - text = "".join(part.text or "" for part in message.parts if part.type == "text").strip() + text = "".join( + part.text or "" for part in message.parts if part.type == "text" + ).strip() if message.role in ("user", "assistant") and text: converted.append({"role": message.role, "content": text}) return converted @@ -74,7 +76,9 @@ async def stream_reply( client: AsyncAnthropic, model: str, messages: list[dict[str, Any]], page: str = "" ) -> AsyncIterator[str]: window = get_hackathon().window - system = prompt("system", start=window.start.isoformat(), end=window.end.isoformat(), page=page) + system = prompt( + "system", start=window.start.isoformat(), end=window.end.isoformat(), page=page + ) yield event({"type": "start", "messageId": uuid4().hex}) try: @@ -95,7 +99,9 @@ async def stream_reply( if not text_open: yield event({"type": "text-start", "id": text_id}) text_open = True - yield event({"type": "text-delta", "id": text_id, "delta": chunk.text}) + yield event( + {"type": "text-delta", "id": text_id, "delta": chunk.text} + ) elif ( chunk.type == "content_block_start" and chunk.content_block.type == "tool_use" @@ -129,7 +135,10 @@ async def stream_reply( calls = [block for block in final.content if block.type == "tool_use"] if final.stop_reason == "pause_turn": - messages = [*messages, {"role": "assistant", "content": dump(final.content)}] + messages = [ + *messages, + {"role": "assistant", "content": dump(final.content)}, + ] yield event({"type": "finish-step"}) continue if final.stop_reason != "tool_use" or not calls: @@ -149,7 +158,11 @@ async def stream_reply( try: output = await run_tool(call.name, call.input) yield event( - {"type": "tool-output-available", "toolCallId": call.id, "output": output} + { + "type": "tool-output-available", + "toolCallId": call.id, + "output": output, + } ) content, is_error = json.dumps(output, default=str), False except Exception as error: @@ -177,7 +190,12 @@ async def stream_reply( ] yield event({"type": "finish-step"}) except Exception: - yield event({"type": "error", "errorText": "The assistant ran into a problem. Try again."}) + yield event( + { + "type": "error", + "errorText": "The assistant ran into a problem. Try again.", + } + ) yield event({"type": "finish"}) yield "data: [DONE]\n\n" diff --git a/apps/api/src/hackproof/chat/tools.py b/apps/api/src/hackproof/chat/tools.py index d6ce2e9..82e42b9 100644 --- a/apps/api/src/hackproof/chat/tools.py +++ b/apps/api/src/hackproof/chat/tools.py @@ -4,20 +4,29 @@ from pydantic_core import to_jsonable_python from hackproof import queries -from hackproof.chat.briefs import ProjectFilters, builder_brief, project_brief, slim, summarize_view +from hackproof.chat.briefs import ( + ProjectFilters, + builder_brief, + project_brief, + slim, + summarize_view, +) WEB_SEARCH = {"type": "web_search_20250305", "name": "web_search", "max_uses": 3} Limit = Annotated[int, Field(ge=1, le=25, description="Rows to return")] Offset = Annotated[int, Field(ge=0, le=5000, description="Rows to skip, for paging")] Search = Annotated[ - str | None, Field(max_length=120, description="Case-insensitive text to match, or null") + str | None, + Field(max_length=120, description="Case-insensitive text to match, or null"), ] class QueryProjects(ProjectFilters): - """Find HackMeridian projects. Ordered by evidence strength, then on-chain activity. Each - project has an award field: first, runner-up, or null.""" + """ + Find HackMeridian projects. Ordered by evidence strength, then on-chain activity. Each + project has an award field: first, runner-up, or null. + """ q: Search = None winners: bool = Field( @@ -33,7 +42,9 @@ class QueryContracts(BaseModel): q: Search = None deployer: str | None = Field(default=None, description="Deploy account (G...)") - project_id: str | None = Field(default=None, description="A project id such as repo:owner/name") + project_id: str | None = Field( + default=None, description="A project id such as repo:owner/name" + ) limit: Limit = 10 offset: Offset = 0 @@ -57,9 +68,11 @@ class QueryFailed(BaseModel): class SummarizeProjects(ProjectFilters): - """Aggregate every matching project: counts by tier and track, on chain totals, top + """ + Aggregate every matching project: counts by tier and track, on chain totals, top languages, topics, Stellar packages, integrations, tooling, AI agents and countries, plus a - sample of descriptions. Use this for any breakdown or "what are they building" question.""" + sample of descriptions. Use this for any breakdown or "what are they building" question. + """ class GetProject(BaseModel): @@ -75,20 +88,26 @@ class GetBuilder(BaseModel): class GetIntegrations(BaseModel): - """List the Stellar ecosystem partners we detect (slug, name, category) with how many + """ + List the Stellar ecosystem partners we detect (slug, name, category) with how many verified and likely projects use each. Detection comes from dependency manifests and API - hosts in code, not from READMEs.""" + hosts in code, not from READMEs. + """ class GetTools(BaseModel): - """List the Stellar developer tools we detect (such as Scaffold Stellar, Stellar Registry, - Stellar Raven and Stellar Skills) with how many verified and likely projects use each.""" + """ + List the Stellar developer tools we detect (such as Scaffold Stellar, Stellar Registry, + Stellar Raven and Stellar Skills) with how many verified and likely projects use each. + """ class GetAgents(BaseModel): - """List the AI coding agents we detect (slug, name) with how many verified and likely projects + """ + List the AI coding agents we detect (slug, name) with how many verified and likely projects show each. Detection reads agent config files such as CLAUDE.md or .cursor and commit authors, - so it is a lower bound: a tool that leaves no trace is not counted.""" + so it is a lower bound: a tool that leaves no trace is not counted. + """ class GetStats(BaseModel): @@ -152,18 +171,30 @@ async def run_tool(name: str, raw: dict[str, Any]) -> Any: ) case QueryWasm(): items, total = await queries.list_page( - "wasm", q=args.q, account=args.uploader, limit=args.limit, offset=args.offset + "wasm", + q=args.q, + account=args.uploader, + limit=args.limit, + offset=args.offset, ) case QueryFailed(): items, total = await queries.list_page( - "failed", q=args.q, account=args.account, limit=args.limit, offset=args.offset + "failed", + q=args.q, + account=args.account, + limit=args.limit, + offset=args.offset, ) case SummarizeProjects(): return to_jsonable_python(await summarize_view(args)) case GetProject(): - return await project_brief(args.project_id) or {"error": "project not found"} + return await project_brief(args.project_id) or { + "error": "project not found" + } case GetBuilder(): - return await builder_brief(args.address) or {"error": "no activity in the hack window"} + return await builder_brief(args.address) or { + "error": "no activity in the hack window" + } case GetIntegrations(): return to_jsonable_python(await queries.fetch_integrations()) case GetTools(): diff --git a/apps/api/src/hackproof/cli.py b/apps/api/src/hackproof/cli.py index 1ab4ed8..62c9715 100644 --- a/apps/api/src/hackproof/cli.py +++ b/apps/api/src/hackproof/cli.py @@ -14,17 +14,30 @@ def main() -> None: commands.add_parser("sync", help="index the hack window from testnet") discover = commands.add_parser("discover", help="find candidate repos on github") discover.add_argument( - "--strings", action="store_true", help="also run the network string code queries" + "--strings", + action="store_true", + help="also run the network string code queries", + ) + scan = commands.add_parser( + "scan", help="scan discovered repos for chain fingerprints" + ) + scan.add_argument( + "--force", action="store_true", help="rescan repos already scanned" ) - scan = commands.add_parser("scan", help="scan discovered repos for chain fingerprints") - scan.add_argument("--force", action="store_true", help="rescan repos already scanned") scan.add_argument( - "--projects", action="store_true", help="only scan repos that are already projects" + "--projects", + action="store_true", + help="only scan repos that are already projects", + ) + commands.add_parser( + "reconcile", help="link chain activity to repos and build projects" + ) + commands.add_parser( + "export", help="write one yaml file per project into data and its schema" ) - commands.add_parser("reconcile", help="link chain activity to repos and build projects") - commands.add_parser("export", help="write one yaml file per project into data and its schema") commands.add_parser( - "check", help="check, read-only, that the database matches the committed data files" + "check", + help="check, read-only, that the database matches the committed data files", ) args = parser.parse_args() diff --git a/apps/api/src/hackproof/config.py b/apps/api/src/hackproof/config.py index 45b04fa..616c222 100644 --- a/apps/api/src/hackproof/config.py +++ b/apps/api/src/hackproof/config.py @@ -10,7 +10,9 @@ class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=ROOT_ENV, env_prefix="HACKPROOF_", extra="ignore") + model_config = SettingsConfigDict( + env_file=ROOT_ENV, env_prefix="HACKPROOF_", extra="ignore" + ) database_url: str = Field( default="postgresql://hackproof:hackproof@localhost:5440/hackproof", @@ -25,7 +27,9 @@ class Settings(BaseSettings): rpc_concurrency: int = 6 anthropic_api_key: str | None = Field( default=None, - validation_alias=AliasChoices("HACKPROOF_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY"), + validation_alias=AliasChoices( + "HACKPROOF_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY" + ), ) chat_model: str = "claude-sonnet-5-5" github_tokens: Annotated[list[str], NoDecode] = Field( diff --git a/apps/api/src/hackproof/export/build.py b/apps/api/src/hackproof/export/build.py index 2e79c49..1517423 100644 --- a/apps/api/src/hackproof/export/build.py +++ b/apps/api/src/hackproof/export/build.py @@ -13,6 +13,7 @@ Contributor, EcosystemTool, Evidence, + Hackathon as ExportedHackathon, HackathonExport, IntegrationEvidence, IntegrationPartner, @@ -20,7 +21,6 @@ Project, WasmUpload, ) -from hackproof.export.models import Hackathon as ExportedHackathon from hackproof.export.urls import ( account_url, asset_url, @@ -120,7 +120,9 @@ def _evidence_url(kind: str, subject: str) -> str | None: return None -def _group(rows: Iterable[asyncpg.Record], key: str = "full_name") -> dict[str, list[Any]]: +def _group( + rows: Iterable[asyncpg.Record], key: str = "full_name" +) -> dict[str, list[Any]]: grouped: dict[str, list[Any]] = defaultdict(list) for row in rows: grouped[row[key]].append(row) @@ -142,27 +144,40 @@ def _found( return found -def _agents(catalog: Catalog, rows: list[asyncpg.Record], repo_url: str) -> list[AiAgent]: +def _agents( + catalog: Catalog, rows: list[asyncpg.Record], repo_url: str +) -> list[AiAgent]: found: dict[str, list[AgentEvidence]] = defaultdict(list) for row in rows: found[row["agent"]].append( AgentEvidence( method=row["method"], detail=row["evidence"], - repo_file_url=repo_file_url(repo_url, row["path"]) if row["path"] else None, + repo_file_url=repo_file_url(repo_url, row["path"]) + if row["path"] + else None, ) ) return [ - AiAgent(slug=agent.slug, name=agent.name, website=agent.site, evidence=found[agent.slug]) + AiAgent( + slug=agent.slug, + name=agent.name, + website=agent.site, + evidence=found[agent.slug], + ) for agent in catalog.agents if agent.slug in found ] -def _tools(catalog: Catalog, rows: list[asyncpg.Record], repo_url: str) -> list[EcosystemTool]: +def _tools( + catalog: Catalog, rows: list[asyncpg.Record], repo_url: str +) -> list[EcosystemTool]: found = _found(rows, "tool", repo_url) return [ - EcosystemTool(slug=tool.slug, name=tool.name, website=tool.site, evidence=found[tool.slug]) + EcosystemTool( + slug=tool.slug, name=tool.name, website=tool.site, evidence=found[tool.slug] + ) for tool in catalog.tools if tool.slug in found ] @@ -186,7 +201,9 @@ def _partners( async def build_export( - connection: asyncpg.Connection, hackathon: Hackathon, tiers: Iterable[str] = STRICT_TIERS + connection: asyncpg.Connection, + hackathon: Hackathon, + tiers: Iterable[str] = STRICT_TIERS, ) -> HackathonExport: window = hackathon.window projects = await connection.fetch(PROJECTS_SQL, list(tiers), list(TIER_ORDER)) @@ -226,10 +243,16 @@ async def build_export( total_commits=row["total_commits"], track=row["declared_track"] or "genesis", track_declared=row["declared_track"] is not None, - integration_partners=_partners(catalog, integrations[row["full_name"]], row["url"]), + integration_partners=_partners( + catalog, integrations[row["full_name"]], row["url"] + ), ecosystem_tools=_tools(catalog, tools[row["full_name"]], row["url"]), ai_agents=_agents(catalog, agents[row["full_name"]], row["url"]), - owner=Owner(login=row["owner"], avatar_url=row["owner_avatar"], country=row["country"]), + owner=Owner( + login=row["owner"], + avatar_url=row["owner_avatar"], + country=row["country"], + ), contributors=[ Contributor( login=c["login"], @@ -247,7 +270,9 @@ async def build_export( asset_count=row["assets"], failed_tx_count=row["failed_txs"], accounts=[ - Account(address=a["subject"], explorer_url=account_url(a["subject"])) + Account( + address=a["subject"], explorer_url=account_url(a["subject"]) + ) for a in accounts[row["full_name"]] ], wasm_uploads=[ @@ -277,7 +302,9 @@ async def build_export( subject=e["subject"], method=e["method"], explorer_url=_evidence_url(e["subject_kind"], e["subject"]), - repo_file_url=(repo_file_url(row["url"], e["path"]) if e["path"] else None), + repo_file_url=( + repo_file_url(row["url"], e["path"]) if e["path"] else None + ), ) for e in evidence[row["full_name"]] ], diff --git a/apps/api/src/hackproof/export/files.py b/apps/api/src/hackproof/export/files.py index 7801da9..eb615eb 100644 --- a/apps/api/src/hackproof/export/files.py +++ b/apps/api/src/hackproof/export/files.py @@ -26,7 +26,11 @@ class FileEvidence(FileModel): kind: Literal["contract", "wasm", "account", "asset"] subject: str method: Literal[ - "contract_in_repo", "wasm_hash_match", "wasm_meta", "deployer_in_repo", "asset_in_repo" + "contract_in_repo", + "wasm_hash_match", + "wasm_meta", + "deployer_in_repo", + "asset_in_repo", ] explorer_url: str | None = None file_url: str | None = None @@ -79,20 +83,28 @@ class EventFile(FileModel): ) starts_at: datetime = Field(description="Official start, UTC") ends_at: datetime = Field(description="Official submission deadline, UTC") - source_url: str | None = Field(default=None, description="Where the dataset is produced") + source_url: str | None = Field( + default=None, description="Where the dataset is produced" + ) class ProjectFile(FileModel): event: str = Field(description="Hackathon slug") - canonical_id: str = Field(description="pkg:github//, the join key for repos") + canonical_id: str = Field( + description="pkg:github//, the join key for repos" + ) repo_url: str name: str = Field(description="Repository name") description: str | None = None homepage: str | None = None language: str | None = None license: str | None = None - topics: list[str] = Field(default_factory=list, description="Tags the team set on the repo") - created_at: datetime | None = Field(default=None, description="Repo creation time, UTC") + topics: list[str] = Field( + default_factory=list, description="Tags the team set on the repo" + ) + created_at: datetime | None = Field( + default=None, description="Repo creation time, UTC" + ) owner: str = Field(description="GitHub login of the repo owner") track: Track track_declared: bool = Field( @@ -102,7 +114,8 @@ class ProjectFile(FileModel): evidence_tier: Tier commits_in_window: int | None = None stack: dict[str, str | None] = Field( - default_factory=dict, description="Stellar package to declared version, null when unknown" + default_factory=dict, + description="Stellar package to declared version, null when unknown", ) integrations: list[FileIntegration] = Field(default_factory=list) tools: list[FileIntegration] = Field( @@ -130,7 +143,9 @@ def file_names(projects: list[Project]) -> dict[str, str]: name: f"{slugify(name.split('/')[0])}-{slug}" if counts[slug] > 1 else slug for name, slug in slugs.items() } - clashes = sorted(name for name, count in Counter(names.values()).items() if count > 1) + clashes = sorted( + name for name, count in Counter(names.values()).items() if count > 1 + ) if clashes: raise ValueError(f"file names still collide: {', '.join(clashes)}") @@ -189,7 +204,9 @@ def project_file(event: str, project: Project) -> ProjectFile: name=partner.name, evidence=[ FileIntegrationEvidence( - method=item.method, detail=item.detail, file_url=item.repo_file_url + method=item.method, + detail=item.detail, + file_url=item.repo_file_url, ) for item in partner.evidence ], @@ -202,7 +219,9 @@ def project_file(event: str, project: Project) -> ProjectFile: name=tool.name, evidence=[ FileIntegrationEvidence( - method=item.method, detail=item.detail, file_url=item.repo_file_url + method=item.method, + detail=item.detail, + file_url=item.repo_file_url, ) for item in tool.evidence ], @@ -215,7 +234,9 @@ def project_file(event: str, project: Project) -> ProjectFile: name=agent.name, evidence=[ FileAgentEvidence( - method=item.method, detail=item.detail, file_url=item.repo_file_url + method=item.method, + detail=item.detail, + file_url=item.repo_file_url, ) for item in agent.evidence ], @@ -273,7 +294,10 @@ def write_project_files(export: HackathonExport, root: Path) -> int: for stale in directory.glob("*.yml"): if stale.name not in wanted: stale.unlink() - for name, schema in ((SCHEMA_FILE, file_schema()), (EVENT_SCHEMA_FILE, event_schema())): + for name, schema in ( + (SCHEMA_FILE, file_schema()), + (EVENT_SCHEMA_FILE, event_schema()), + ): (root / name).write_text( yaml.safe_dump(schema, sort_keys=False, allow_unicode=True, width=120) ) diff --git a/apps/api/src/hackproof/export/jobs.py b/apps/api/src/hackproof/export/jobs.py index f5c4cb4..5743c6c 100644 --- a/apps/api/src/hackproof/export/jobs.py +++ b/apps/api/src/hackproof/export/jobs.py @@ -24,13 +24,18 @@ async def run_export() -> None: export = await build_export(connection, hackathon) count = write_project_files(export, DATA) - log.info("wrote %s project files to %s", count, DATA / str(export.hackathon.starts_at.year)) + log.info( + "wrote %s project files to %s", + count, + DATA / str(export.hackathon.starts_at.year), + ) async def run_check() -> int: hackathon = get_hackathon() connection = await asyncpg.connect( - get_settings().database_url, server_settings={"default_transaction_read_only": "on"} + get_settings().database_url, + server_settings={"default_transaction_read_only": "on"}, ) await connection.set_type_codec( "jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog" @@ -49,5 +54,7 @@ async def run_check() -> int: if found: log.warning("%s files differ from the database", len(found)) else: - log.info("in sync: the database matches all %s project files", len(export.projects)) + log.info( + "in sync: the database matches all %s project files", len(export.projects) + ) return 1 if found else 0 diff --git a/apps/api/src/hackproof/export/models.py b/apps/api/src/hackproof/export/models.py index 06abc99..61da780 100644 --- a/apps/api/src/hackproof/export/models.py +++ b/apps/api/src/hackproof/export/models.py @@ -25,7 +25,9 @@ class Contributor(Model): login: str = Field(description="GitHub login") url: str = Field(description="GitHub profile url") avatar_url: str | None = Field(description="Avatar image url") - contributions: int = Field(description="Commits to the default branch, as GitHub counts them") + contributions: int = Field( + description="Commits to the default branch, as GitHub counts them" + ) country: str | None = Field( description="ISO 3166-1 alpha-2 country derived from the public GitHub location" ) @@ -33,23 +35,31 @@ class Contributor(Model): class Account(Model): address: str = Field(description="Stellar account (G...)") - explorer_url: str = Field(description="Stellar Expert page for the account on testnet") + explorer_url: str = Field( + description="Stellar Expert page for the account on testnet" + ) class WasmUpload(Model): wasm_hash: str = Field(description="sha256 of the contract wasm, hex") upload_tx: str = Field(description="Hash of the transaction that uploaded the wasm") - explorer_url: str = Field(description="Stellar Expert page for the upload transaction") + explorer_url: str = Field( + description="Stellar Expert page for the upload transaction" + ) class Contract(Model): contract_id: str = Field(description="Soroban contract id (C...)") - deployer: str = Field(description="Stellar account (G...) that deployed the contract") + deployer: str = Field( + description="Stellar account (G...) that deployed the contract" + ) wasm_hash: str = Field(description="sha256 of the contract wasm, hex") deployed_at: datetime = Field(description="Ledger close time of the deploy, UTC") ledger: int = Field(description="Ledger sequence of the deploy") tx_hash: str = Field(description="Hash of the deploy transaction") - explorer_url: str = Field(description="Stellar Expert page for the contract on testnet") + explorer_url: str = Field( + description="Stellar Expert page for the contract on testnet" + ) tx_url: str = Field(description="Stellar Expert page for the deploy transaction") @@ -57,14 +67,18 @@ class Evidence(Model): kind: Literal["contract", "wasm", "account", "asset"] = Field( description="What was matched between the repo and the chain" ) - subject: str = Field(description="The matched contract id, wasm hash, account or CODE:ISSUER") + subject: str = Field( + description="The matched contract id, wasm hash, account or CODE:ISSUER" + ) method: Literal[ "contract_in_repo", "wasm_hash_match", "wasm_meta", "deployer_in_repo", "asset_in_repo", - ] = Field(description="How the match was made; every method is an exact string match") + ] = Field( + description="How the match was made; every method is an exact string match" + ) explorer_url: str | None = Field( description="Stellar Expert page for the matched item; null for a wasm hash" ) @@ -74,19 +88,33 @@ class Evidence(Model): class Chain(Model): - contract_count: int = Field(description="Contracts deployed inside the official window") - wasm_upload_count: int = Field(description="Distinct wasm uploads inside the window") - classic_op_count: int = Field(description="Classic operations by the project's accounts") + contract_count: int = Field( + description="Contracts deployed inside the official window" + ) + wasm_upload_count: int = Field( + description="Distinct wasm uploads inside the window" + ) + classic_op_count: int = Field( + description="Classic operations by the project's accounts" + ) asset_count: int = Field( description="Assets issued inside the window by the project's accounts" ) failed_tx_count: int = Field( description="Failed Soroban transactions by the project's accounts" ) - accounts: list[Account] = Field(description="Stellar accounts attributed to the project") - wasm_uploads: list[WasmUpload] = Field(description="Wasm uploaded by the project's accounts") - contracts: list[Contract] = Field(description="Contracts deployed inside the window") - evidence: list[Evidence] = Field(description="Direct repo-to-chain matches behind the tier") + accounts: list[Account] = Field( + description="Stellar accounts attributed to the project" + ) + wasm_uploads: list[WasmUpload] = Field( + description="Wasm uploaded by the project's accounts" + ) + contracts: list[Contract] = Field( + description="Contracts deployed inside the window" + ) + evidence: list[Evidence] = Field( + description="Direct repo-to-chain matches behind the tier" + ) class IntegrationEvidence(Model): @@ -133,11 +161,15 @@ class IntegrationPartner(Model): name: str = Field(description="Partner name") category: str = Field(description="payments, wallets, defi or cross-chain") website: str = Field(description="Partner website") - evidence: list[IntegrationEvidence] = Field(description="Where the integration was found") + evidence: list[IntegrationEvidence] = Field( + description="Where the integration was found" + ) class Project(Model): - repo_full_name: str = Field(description="GitHub owner/name; the stable key for a project") + repo_full_name: str = Field( + description="GitHub owner/name; the stable key for a project" + ) repo_url: str = Field(description="GitHub url") name: str = Field(description="Repository name") description: str | None = Field(description="GitHub repo description") @@ -168,13 +200,19 @@ class Project(Model): stars: int = Field(description="Stargazers") forks: int = Field(description="Forks") self_tags: list[str] = Field(description="GitHub topics set by the team") - stack: list[str] = Field(description="Stellar packages the repo's manifests declare") + stack: list[str] = Field( + description="Stellar packages the repo's manifests declare" + ) stack_versions: dict[str, str | None] = Field( description="Declared version for each stack package, as written in the manifest" ) repo_created_at: datetime | None = Field(description="Repo creation time, UTC") - repo_last_commit_at: datetime | None = Field(description="Last push to the repo, UTC") - commits_in_window: int | None = Field(description="Commits inside the official window") + repo_last_commit_at: datetime | None = Field( + description="Last push to the repo, UTC" + ) + commits_in_window: int | None = Field( + description="Commits inside the official window" + ) total_commits: int | None = Field(description="Commits on the default branch") integration_partners: list[IntegrationPartner] = Field( description="Stellar ecosystem partners the repo integrates, from the SCF integration list" @@ -195,7 +233,9 @@ class Hackathon(Model): name: str network: Literal["testnet"] starts_at: datetime = Field(description="Official start, UTC") - submission_deadline: datetime = Field(description="Official submission deadline, UTC") + submission_deadline: datetime = Field( + description="Official submission deadline, UTC" + ) class HackathonExport(Model): diff --git a/apps/api/src/hackproof/github/client.py b/apps/api/src/hackproof/github/client.py index 2343a57..b0339c1 100644 --- a/apps/api/src/hackproof/github/client.py +++ b/apps/api/src/hackproof/github/client.py @@ -22,7 +22,9 @@ class GithubUnavailable(Exception): class Token(BaseModel): value: str | None - budget: dict[str, int] = Field(default_factory=lambda: {"core": 5000, "graphql": 5000}) + budget: dict[str, int] = Field( + default_factory=lambda: {"core": 5000, "graphql": 5000} + ) blocked_until: float = 0.0 next_search_at: dict[str, float] = Field( default_factory=lambda: {"code": 0.0, "repositories": 0.0} @@ -41,7 +43,9 @@ async def _acquire(self, search: Search | None, resource: Resource) -> Token: async with self._lock: now = time.monotonic() usable = [ - t for t in self._tokens if t.blocked_until <= now and t.budget[resource] > 20 + t + for t in self._tokens + if t.blocked_until <= now and t.budget[resource] > 20 ] if usable and search is None: return max(usable, key=lambda t: t.budget[resource]) @@ -67,7 +71,9 @@ def _record(token: Token, response: httpx.Response) -> None: reset = int(response.headers.get("x-ratelimit-reset", time.time() + 60)) token.blocked_until = time.monotonic() + max(reset - time.time(), 1) elif response.status_code in (403, 429) and "retry-after" in response.headers: - token.blocked_until = time.monotonic() + int(response.headers["retry-after"]) + token.blocked_until = time.monotonic() + int( + response.headers["retry-after"] + ) async def _send( self, @@ -84,7 +90,9 @@ async def _send( if token.value: headers["Authorization"] = f"Bearer {token.value}" try: - response = await self._http.request(method, url, headers=headers, **options) + response = await self._http.request( + method, url, headers=headers, **options + ) except httpx.TransportError: await asyncio.sleep(2**attempt / 2) continue @@ -96,7 +104,11 @@ async def _send( raise GithubUnavailable(url) async def get( - self, path: str, params: dict[str, Any] | None = None, *, search: Search | None = None + self, + path: str, + params: dict[str, Any] | None = None, + *, + search: Search | None = None, ) -> httpx.Response | None: response = await self._send("GET", f"{API}{path}", search=search, params=params) if response.status_code in ABSENT: @@ -105,7 +117,11 @@ async def get( return response async def json( - self, path: str, params: dict[str, Any] | None = None, *, search: Search | None = None + self, + path: str, + params: dict[str, Any] | None = None, + *, + search: Search | None = None, ) -> Any | None: response = await self.get(path, params, search=search) if response is None or response.status_code == 204: @@ -122,14 +138,18 @@ async def graphql(self, query: str) -> dict[str, Any]: except ValueError: await asyncio.sleep(2**attempt / 2) continue - errors = [e for e in payload.get("errors", []) if e.get("type") != "NOT_FOUND"] + errors = [ + e for e in payload.get("errors", []) if e.get("type") != "NOT_FOUND" + ] if errors or "data" not in payload: await asyncio.sleep(2**attempt / 2) continue return payload["data"] raise GithubUnavailable("graphql") - async def raw(self, full_name: str, ref: str, path: str, limit: int = 512_000) -> bytes | None: + async def raw( + self, full_name: str, ref: str, path: str, limit: int = 512_000 + ) -> bytes | None: url = f"{RAW}/{full_name}/{ref}/{path}" for attempt in range(ATTEMPTS): await asyncio.sleep(max(self._raw_resume_at - time.monotonic(), 0)) diff --git a/apps/api/src/hackproof/github/demo.py b/apps/api/src/hackproof/github/demo.py index c89f4a9..aa76006 100644 --- a/apps/api/src/hackproof/github/demo.py +++ b/apps/api/src/hackproof/github/demo.py @@ -8,7 +8,9 @@ from hackproof.github.models import Ref from hackproof.github.refs import extract_refs -SCRIPT_SRC = re.compile(r"""]+src=["']([^"']+\.m?js)[^"']*["']""", re.IGNORECASE) +SCRIPT_SRC = re.compile( + r"""]+src=["']([^"']+\.m?js)[^"']*["']""", re.IGNORECASE +) MAX_BUNDLES = 5 MAX_BYTES = 3_000_000 TIMEOUT = 8 diff --git a/apps/api/src/hackproof/github/dependencies.py b/apps/api/src/hackproof/github/dependencies.py index 6b9c5ee..5d09523 100644 --- a/apps/api/src/hackproof/github/dependencies.py +++ b/apps/api/src/hackproof/github/dependencies.py @@ -5,8 +5,22 @@ from hackproof.github.models import DependencyFingerprint -MANIFESTS = {"Cargo.toml", "package.json", "requirements.txt", "pyproject.toml", "go.mod"} -SKIP_DIRECTORIES = {"node_modules", "dist", "target", ".git", "build", "vendor", ".next"} +MANIFESTS = { + "Cargo.toml", + "package.json", + "requirements.txt", + "pyproject.toml", + "go.mod", +} +SKIP_DIRECTORIES = { + "node_modules", + "dist", + "target", + ".git", + "build", + "vendor", + ".next", +} ECOSYSTEM = { "Cargo.toml": "rust", "package.json": "js", @@ -48,7 +62,9 @@ def _requirements(text: str) -> dict[str, str | None]: def _pyproject(text: str) -> dict[str, str | None]: data = tomllib.loads(text) found = _requirements("\n".join(data.get("project", {}).get("dependencies", []))) - for name, spec in data.get("tool", {}).get("poetry", {}).get("dependencies", {}).items(): + for name, spec in ( + data.get("tool", {}).get("poetry", {}).get("dependencies", {}).items() + ): found[name] = spec if isinstance(spec, str) else (spec or {}).get("version") return found @@ -66,7 +82,9 @@ def _go_mod(text: str) -> dict[str, str | None]: } -def fingerprint(files: dict[str, str], wanted: dict[str, list[str]]) -> DependencyFingerprint: +def fingerprint( + files: dict[str, str], wanted: dict[str, list[str]] +) -> DependencyFingerprint: result = DependencyFingerprint() for path, text in files.items(): name = PurePosixPath(path).name diff --git a/apps/api/src/hackproof/github/discovery.py b/apps/api/src/hackproof/github/discovery.py index 26cebb5..edb06bb 100644 --- a/apps/api/src/hackproof/github/discovery.py +++ b/apps/api/src/hackproof/github/discovery.py @@ -30,14 +30,19 @@ def is_pinned(candidate: Candidate) -> bool: def should_keep(candidate: Candidate, window: Window, discovery: Discovery) -> bool: - if candidate.full_name.lower() in {name.lower() for name in discovery.exclude_repos}: + if candidate.full_name.lower() in { + name.lower() for name in discovery.exclude_repos + }: return False if is_pinned(candidate): return True - if candidate.fork or candidate.owner.lower() in {o.lower() for o in discovery.exclude_owners}: + if candidate.fork or candidate.owner.lower() in { + o.lower() for o in discovery.exclude_owners + }: return False - return candidate.pushed_at is not None and candidate.pushed_at >= window.start - timedelta( - days=1 + return ( + candidate.pushed_at is not None + and candidate.pushed_at >= window.start - timedelta(days=1) ) @@ -67,7 +72,9 @@ async def repository_slices( ) -> list[str]: async def count(lo: int, hi: int) -> int: pushed = f"pushed:{_stamp(lo)}..{_stamp(hi)}" - return await _total(github, "/search/repositories", f"{query} {pushed}", "repositories") + return await _total( + github, "/search/repositories", f"{query} {pushed}", "repositories" + ) ranges = await split_range(int(since.timestamp()), int(until.timestamp()), count) return [f"{query} pushed:{_stamp(lo)}..{_stamp(hi)}" for lo, hi in ranges] @@ -80,10 +87,15 @@ async def code_slices(github: GithubClient, query: str) -> list[str]: async def count(lo: int, hi: int) -> int: return await _total(github, "/search/code", f"{query} size:{lo}..{hi}", "code") - return [f"{query} size:{lo}..{hi}" for lo, hi in await split_range(0, SIZE_CEILING, count)] + return [ + f"{query} size:{lo}..{hi}" + for lo, hi in await split_range(0, SIZE_CEILING, count) + ] -async def _hits(github: GithubClient, path: str, query: str, kind: Search) -> list[dict[str, Any]]: +async def _hits( + github: GithubClient, path: str, query: str, kind: Search +) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] for page in range(1, RESULT_CAP // PAGE_SIZE + 1): data = await github.json( @@ -121,7 +133,9 @@ def note(name: str, via: str) -> None: async def repository_query(query: str) -> None: slices = await repository_slices(github, query, since, until) for sliced in slices: - for item in await _hits(github, "/search/repositories", sliced, "repositories"): + for item in await _hits( + github, "/search/repositories", sliced, "repositories" + ): note(item["full_name"], f"search:{query}") log.info("repo search done: %s slices, %s", len(slices), query) @@ -166,7 +180,10 @@ async def discover( for name in wasm_meta_repos: pinned.setdefault(name.lower(), set()).add("wasm_meta") - stages = [(discovery.github_queries, [], pinned), ([], discovery.github_code_queries, {})] + stages = [ + (discovery.github_queries, [], pinned), + ([], discovery.github_code_queries, {}), + ] if include_strings: stages.append(([], discovery.github_string_queries, {})) for repository_queries, code_queries, extra in stages: @@ -175,7 +192,9 @@ async def discover( yield await _resolve(github, found, window, discovery) -async def store_candidates(connection: asyncpg.Connection, candidates: list[Candidate]) -> None: +async def store_candidates( + connection: asyncpg.Connection, candidates: list[Candidate] +) -> None: await connection.executemany( """ INSERT INTO repos (full_name, url, description, stars, owner, owner_avatar, @@ -228,4 +247,6 @@ async def wasm_meta_repos(connection: asyncpg.Connection) -> list[str]: rows = await connection.fetch( "SELECT DISTINCT source_repo FROM wasm_uploads WHERE source_repo IS NOT NULL" ) - return sorted({name for row in rows if (name := normalize_repo(row["source_repo"]))}) + return sorted( + {name for row in rows if (name := normalize_repo(row["source_repo"]))} + ) diff --git a/apps/api/src/hackproof/github/files.py b/apps/api/src/hackproof/github/files.py index f8a65e7..c98cd46 100644 --- a/apps/api/src/hackproof/github/files.py +++ b/apps/api/src/hackproof/github/files.py @@ -30,7 +30,9 @@ async def fetch(batch: list[str]) -> dict[str, str]: if len(batch) == 1: raise half = len(batch) // 2 - first, second = await asyncio.gather(fetch(batch[:half]), fetch(batch[half:])) + first, second = await asyncio.gather( + fetch(batch[:half]), fetch(batch[half:]) + ) return first | second repository = data["repository"] or {} return { diff --git a/apps/api/src/hackproof/github/history.py b/apps/api/src/hackproof/github/history.py index 55f857c..60132e7 100644 --- a/apps/api/src/hackproof/github/history.py +++ b/apps/api/src/hackproof/github/history.py @@ -22,7 +22,9 @@ async def _stellar_packages_at( ), key=lambda path: (len(PurePosixPath(path).parts), path), )[:MAX_MANIFESTS] - contents = await asyncio.gather(*(github.raw(full_name, sha, path) for path in paths)) + contents = await asyncio.gather( + *(github.raw(full_name, sha, path) for path in paths) + ) manifests = { path: content.decode(errors="ignore") for path, content in zip(paths, contents, strict=True) @@ -39,9 +41,15 @@ async def stellar_added_in_window( window_end: datetime, ) -> bool: until = window_end.strftime("%Y-%m-%dT%H:%M:%SZ") - commits = await github.json(f"/repos/{full_name}/commits", {"until": until, "per_page": 1}) + commits = await github.json( + f"/repos/{full_name}/commits", {"until": until, "per_page": 1} + ) if not commits: return False - before = await _stellar_packages_at(github, full_name, base_sha, wanted) if base_sha else {} + before = ( + await _stellar_packages_at(github, full_name, base_sha, wanted) + if base_sha + else {} + ) after = await _stellar_packages_at(github, full_name, commits[0]["sha"], wanted) return not before and bool(after) diff --git a/apps/api/src/hackproof/github/jobs.py b/apps/api/src/hackproof/github/jobs.py index 27860ee..df920e7 100644 --- a/apps/api/src/hackproof/github/jobs.py +++ b/apps/api/src/hackproof/github/jobs.py @@ -38,7 +38,9 @@ async def run_discover(include_strings: bool) -> None: log.info("stored %s candidate repos", len(candidates)) -PROJECT_REPOS = "full_name IN (SELECT full_name FROM projects WHERE full_name IS NOT NULL)" +PROJECT_REPOS = ( + "full_name IN (SELECT full_name FROM projects WHERE full_name IS NOT NULL)" +) async def run_scan(force: bool, projects_only: bool = False) -> None: diff --git a/apps/api/src/hackproof/github/profiles.py b/apps/api/src/hackproof/github/profiles.py index de6797e..c8abdad 100644 --- a/apps/api/src/hackproof/github/profiles.py +++ b/apps/api/src/hackproof/github/profiles.py @@ -27,12 +27,18 @@ async def resolve_profiles(github: GithubClient, connection: asyncpg.Connection) for start in range(0, len(logins), BATCH): batch = logins[start : start + BATCH] data = await github.graphql(_query(batch)) - locations = {node["login"].lower(): node["location"] for node in data.values() if node} + locations = { + node["login"].lower(): node["location"] for node in data.values() if node + } await connection.executemany( "INSERT INTO github_users (login, location, country) VALUES ($1, $2, $3)" " ON CONFLICT (login) DO NOTHING", [ - (login, locations.get(login.lower()), resolve_country(locations.get(login.lower()))) + ( + login, + locations.get(login.lower()), + resolve_country(locations.get(login.lower())), + ) for login in batch ], ) diff --git a/apps/api/src/hackproof/github/repositories.py b/apps/api/src/hackproof/github/repositories.py index f183a1c..915934a 100644 --- a/apps/api/src/hackproof/github/repositories.py +++ b/apps/api/src/hackproof/github/repositories.py @@ -56,14 +56,16 @@ def candidate_from_node(node: dict[str, Any]) -> Candidate: def _lookup(index: int, name: str, fields: str) -> str: owner, repo = name.split("/") - return ( - f"r{index}: repository(owner: {json.dumps(owner)}, name: {json.dumps(repo)}) {{{fields}}}" - ) + return f"r{index}: repository(owner: {json.dumps(owner)}, name: {json.dumps(repo)}) {{{fields}}}" def _query(names: list[str], window: Window) -> str: fields = FIELDS.format(start=window.start.isoformat(), end=window.end.isoformat()) - return "query {" + " ".join(_lookup(i, name, fields) for i, name in enumerate(names)) + "}" + return ( + "query {" + + " ".join(_lookup(i, name, fields) for i, name in enumerate(names)) + + "}" + ) async def fetch_repositories( @@ -81,8 +83,12 @@ async def fetch(batch: list[str]) -> list[Candidate]: if len(batch) == 1: return [] half = len(batch) // 2 - first, second = await asyncio.gather(fetch(batch[:half]), fetch(batch[half:])) + first, second = await asyncio.gather( + fetch(batch[:half]), fetch(batch[half:]) + ) return first + second return [candidate_from_node(node) for node in data.values() if node] - return [c for found in await asyncio.gather(*(fetch(b) for b in batches)) for c in found] + return [ + c for found in await asyncio.gather(*(fetch(b) for b in batches)) for c in found + ] diff --git a/apps/api/src/hackproof/github/scan.py b/apps/api/src/hackproof/github/scan.py index f69fe4f..13b7734 100644 --- a/apps/api/src/hackproof/github/scan.py +++ b/apps/api/src/hackproof/github/scan.py @@ -22,7 +22,13 @@ ".rs", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".toml", ".json", ".md", ".yml", ".yaml", ".sh", ".txt", ".html", ".env", ".sol", ".vue", ".svelte", } # fmt: skip -SKIP_FILES = {"package-lock.json", "pnpm-lock.yaml", "yarn.lock", "Cargo.lock", "poetry.lock"} +SKIP_FILES = { + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "Cargo.lock", + "poetry.lock", +} FILE_CONCURRENCY = 100 MAX_TEXT_FILES = 60 MAX_WASM_FILES = 5 @@ -37,13 +43,19 @@ def _is_text(path: str) -> bool: name = PurePosixPath(path).name return ( - PurePosixPath(path).suffix in TEXT_SUFFIXES or name in MANIFESTS or name.startswith(".env") + PurePosixPath(path).suffix in TEXT_SUFFIXES + or name in MANIFESTS + or name.startswith(".env") ) def _priority(path: str) -> tuple[int, int, str]: name = PurePosixPath(path).name - important = name in MANIFESTS or name.lower().startswith("readme") or name.startswith(".env") + important = ( + name in MANIFESTS + or name.lower().startswith("readme") + or name.startswith(".env") + ) return (0 if important else 1, len(PurePosixPath(path).parts), path) @@ -90,7 +102,9 @@ def __init__( async def _wasm_ref(self, repo: Candidate, branch: str, path: str) -> Ref | None: async with self._files: - content = await self._github.raw(repo.full_name, branch, path, limit=MAX_WASM_BYTES) + content = await self._github.raw( + repo.full_name, branch, path, limit=MAX_WASM_BYTES + ) if content is None: return None return Ref(kind="wasm", value=hashlib.sha256(content).hexdigest(), path=path) @@ -105,14 +119,20 @@ async def scan(self, repo: Candidate) -> RepoScan | None: text_paths, wasm_paths = select_files(tree["tree"]) texts, wasms, people = await asyncio.gather( fetch_texts(self._github, repo.full_name, branch, text_paths), - asyncio.gather(*(self._wasm_ref(repo, branch, path) for path in wasm_paths)), - self._github.json(f"/repos/{repo.full_name}/contributors", {"per_page": 100}), + asyncio.gather( + *(self._wasm_ref(repo, branch, path) for path in wasm_paths) + ), + self._github.json( + f"/repos/{repo.full_name}/contributors", {"per_page": 100} + ), ) files = texts refs = [ref for path, text in files.items() for ref in extract_refs(text, path)] refs.extend(ref for ref in wasms if ref) readme = next((text for path, text in files.items() if _is_readme(path)), "") - demo = repo.homepage or (match[0] if (match := DEMO_URL.search(readme)) else None) + demo = repo.homepage or ( + match[0] if (match := DEMO_URL.search(readme)) else None + ) if demo: refs.extend(await scan_demo(self._http, demo)) wanted = self._discovery.dependency_fingerprints @@ -181,17 +201,30 @@ async def store_scan(connection: asyncpg.Connection, scan: RepoScan) -> None: ) await connection.executemany( "INSERT INTO repo_integrations VALUES ($1, $2, $3, $4, $5)", - [(scan.full_name, i.slug, i.method, i.evidence, i.path) for i in scan.integrations], + [ + (scan.full_name, i.slug, i.method, i.evidence, i.path) + for i in scan.integrations + ], + ) + await connection.execute( + "DELETE FROM repo_tools WHERE full_name = $1", scan.full_name ) - await connection.execute("DELETE FROM repo_tools WHERE full_name = $1", scan.full_name) await connection.executemany( "INSERT INTO repo_tools VALUES ($1, $2, $3, $4, $5)", - [(scan.full_name, t.slug, t.method, t.evidence, t.path) for t in scan.tools], + [ + (scan.full_name, t.slug, t.method, t.evidence, t.path) + for t in scan.tools + ], + ) + await connection.execute( + "DELETE FROM repo_agents WHERE full_name = $1", scan.full_name ) - await connection.execute("DELETE FROM repo_agents WHERE full_name = $1", scan.full_name) await connection.executemany( "INSERT INTO repo_agents VALUES ($1, $2, $3, $4, $5)", - [(scan.full_name, a.slug, a.method, a.evidence, a.path) for a in scan.agents], + [ + (scan.full_name, a.slug, a.method, a.evidence, a.path) + for a in scan.agents + ], ) await connection.execute( """ diff --git a/apps/api/src/hackproof/github/track.py b/apps/api/src/hackproof/github/track.py index 54ce483..851bf53 100644 --- a/apps/api/src/hackproof/github/track.py +++ b/apps/api/src/hackproof/github/track.py @@ -15,7 +15,10 @@ def _declared(text: str) -> Track | None: found = { - value.lower() for match in DECLARATION.finditer(text) for value in match.groups() if value + value.lower() + for match in DECLARATION.finditer(text) + for value in match.groups() + if value } return found.pop() if len(found) == 1 else None # type: ignore[return-value] diff --git a/apps/api/src/hackproof/indexer/events.py b/apps/api/src/hackproof/indexer/events.py index 2cbc522..e5e40ea 100644 --- a/apps/api/src/hackproof/indexer/events.py +++ b/apps/api/src/hackproof/indexer/events.py @@ -22,7 +22,14 @@ class Deploy(Event): at: datetime def row(self) -> tuple[Any, ...]: - return (self.contract_id, self.deployer, self.wasm_hash, self.tx_hash, self.ledger, self.at) + return ( + self.contract_id, + self.deployer, + self.wasm_hash, + self.tx_hash, + self.ledger, + self.at, + ) class Upload(Event): diff --git a/apps/api/src/hackproof/indexer/horizon.py b/apps/api/src/hackproof/indexer/horizon.py index c65cb36..2e84704 100644 --- a/apps/api/src/hackproof/indexer/horizon.py +++ b/apps/api/src/hackproof/indexer/horizon.py @@ -14,7 +14,9 @@ async def closed_at(self, sequence: int) -> datetime: raw = response.json()["closed_at"] return datetime.fromisoformat(raw.replace("Z", "+00:00")).astimezone(UTC) - async def first_ledger_at_or_after(self, moment: datetime, low: int, high: int) -> int: + async def first_ledger_at_or_after( + self, moment: datetime, low: int, high: int + ) -> int: while low < high: middle = (low + high) // 2 if await self.closed_at(middle) < moment: diff --git a/apps/api/src/hackproof/indexer/parser.py b/apps/api/src/hackproof/indexer/parser.py index 7bfc992..c1141c5 100644 --- a/apps/api/src/hackproof/indexer/parser.py +++ b/apps/api/src/hackproof/indexer/parser.py @@ -21,7 +21,10 @@ OP.CHANGE_TRUST: ("change_trust_op", ("line",)), OP.MANAGE_SELL_OFFER: ("manage_sell_offer_op", ("selling", "buying")), OP.MANAGE_BUY_OFFER: ("manage_buy_offer_op", ("selling", "buying")), - OP.CREATE_PASSIVE_SELL_OFFER: ("create_passive_sell_offer_op", ("selling", "buying")), + OP.CREATE_PASSIVE_SELL_OFFER: ( + "create_passive_sell_offer_op", + ("selling", "buying"), + ), OP.CREATE_CLAIMABLE_BALANCE: ("create_claimable_balance_op", ("asset",)), OP.SET_TRUST_LINE_FLAGS: ("set_trust_line_flags_op", ("asset",)), } @@ -51,7 +54,9 @@ def _inner_transaction(envelope: xdr.TransactionEnvelope) -> Any: def _transaction_source(transaction: Any) -> str: if hasattr(transaction, "source_account_ed25519"): - return StrKey.encode_ed25519_public_key(transaction.source_account_ed25519.uint256) + return StrKey.encode_ed25519_public_key( + transaction.source_account_ed25519.uint256 + ) return MuxedAccount.from_xdr_object(transaction.source_account).account_id @@ -89,7 +94,9 @@ def _counterparty(operation: xdr.Operation) -> str | None: field = RECIPIENT_FIELDS.get(operation.body.type) if field is None: return None - return MuxedAccount.from_xdr_object(getattr(operation.body, field).destination).account_id + return MuxedAccount.from_xdr_object( + getattr(operation.body, field).destination + ).account_id def _contract_id(preimage: xdr.ContractIDPreimage, network_id: bytes) -> str: @@ -110,7 +117,11 @@ def _result_code(result_xdr: str) -> str: if result.result.inner_result_pair is not None else None ) - if inner and inner[0].tr is not None and inner[0].tr.invoke_host_function_result is not None: + if ( + inner + and inner[0].tr is not None + and inner[0].tr.invoke_host_function_result is not None + ): return inner[0].tr.invoke_host_function_result.code.name.removeprefix( "INVOKE_HOST_FUNCTION_" ) @@ -145,7 +156,11 @@ def _host_function_event( at=at, ) if function.type in (CREATE, CREATE_V2): - args = function.create_contract if function.type == CREATE else function.create_contract_v2 + args = ( + function.create_contract + if function.type == CREATE + else function.create_contract_v2 + ) if ( args.contract_id_preimage.type != FROM_ADDRESS or args.executable.type != WASM_EXECUTABLE @@ -181,7 +196,9 @@ def parse_transaction(raw: dict[str, Any], network_id: bytes) -> list[Event]: function = operation.body.invoke_host_function_op.host_function failure_kind = failure_kind or _kind(function.type) if not failed and ( - event := _host_function_event(function, source, network_id, tx_hash, ledger, at) + event := _host_function_event( + function, source, network_id, tx_hash, ledger, at + ) ): events.append(event) elif failed: @@ -189,7 +206,13 @@ def parse_transaction(raw: dict[str, Any], network_id: bytes) -> list[Event]: elif op_type == OP.CREATE_ACCOUNT: destination = _account_id(operation.body.create_account_op.destination) events.append( - Funding(account=destination, funder=source, tx_hash=tx_hash, ledger=ledger, at=at) + Funding( + account=destination, + funder=source, + tx_hash=tx_hash, + ledger=ledger, + at=at, + ) ) elif op_type in ASSET_FIELDS or op_type in PLAIN_CLASSIC: events.append( diff --git a/apps/api/src/hackproof/indexer/rpc.py b/apps/api/src/hackproof/indexer/rpc.py index af01f61..d7a2c89 100644 --- a/apps/api/src/hackproof/indexer/rpc.py +++ b/apps/api/src/hackproof/indexer/rpc.py @@ -15,7 +15,9 @@ def __init__(self, client: httpx.AsyncClient, url: str) -> None: self._client = client self._url = url - async def call(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + async def call( + self, method: str, params: dict[str, Any] | None = None + ) -> dict[str, Any]: body = {"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}} for attempt in range(8): try: @@ -34,7 +36,11 @@ async def call(self, method: str, params: dict[str, Any] | None = None) -> dict[ raise RpcError(f"{method} failed after retries") async def transactions( - self, *, start_ledger: int | None = None, cursor: str | None = None, limit: int = 200 + self, + *, + start_ledger: int | None = None, + cursor: str | None = None, + limit: int = 200, ) -> dict[str, Any]: pagination: dict[str, Any] = {"limit": limit} params: dict[str, Any] = {"pagination": pagination} diff --git a/apps/api/src/hackproof/indexer/sync.py b/apps/api/src/hackproof/indexer/sync.py index 5e47303..aee5312 100644 --- a/apps/api/src/hackproof/indexer/sync.py +++ b/apps/api/src/hackproof/indexer/sync.py @@ -31,7 +31,9 @@ def ensure_within_retention(pull_start: datetime, oldest_closed_at: datetime) -> ) -async def _fetch_chunk(rpc: SorobanRpc, network_id: bytes, start: int, end: int) -> list[Event]: +async def _fetch_chunk( + rpc: SorobanRpc, network_id: bytes, start: int, end: int +) -> list[Event]: events: list[Event] = [] page = await rpc.transactions(start_ledger=start) while page["transactions"]: @@ -54,7 +56,9 @@ async def _store(events: list[Event], chunk_key: str) -> None: raise RuntimeError(f"{chunk_key} kept deadlocking") -async def _insert(connection: asyncpg.Connection, events: list[Event], chunk_key: str) -> None: +async def _insert( + connection: asyncpg.Connection, events: list[Event], chunk_key: str +) -> None: by_table: dict[str, list[Event]] = defaultdict(list) for event in events: by_table[event.table].append(event) @@ -66,7 +70,8 @@ async def _insert(connection: asyncpg.Connection, events: list[Event], chunk_key [event.row() for event in rows], ) await connection.execute( - "INSERT INTO sync_state VALUES ($1, 'done') ON CONFLICT DO NOTHING", chunk_key + "INSERT INTO sync_state VALUES ($1, 'done') ON CONFLICT DO NOTHING", + chunk_key, ) @@ -86,7 +91,9 @@ async def run_sync() -> None: window.pull_start, datetime.fromtimestamp(int(health["oldestLedgerCloseTime"]), tz=UTC), ) - first = await horizon.first_ledger_at_or_after(window.pull_start, oldest, latest) + first = await horizon.first_ledger_at_or_after( + window.pull_start, oldest, latest + ) last = await horizon.first_ledger_at_or_after(window.end, first, latest) - 1 log.info("syncing ledgers %s to %s", first, last) @@ -112,6 +119,8 @@ async def process(start: int) -> None: ) await _store(events, f"{CHUNK_PREFIX}{start}") completed += 1 - log.info("chunk %s/%s done (%s events)", completed, len(pending), len(events)) + log.info( + "chunk %s/%s done (%s events)", completed, len(pending), len(events) + ) await asyncio.gather(*(process(s) for s in pending)) diff --git a/apps/api/src/hackproof/integrations/detect.py b/apps/api/src/hackproof/integrations/detect.py index 8fe000e..26dbc10 100644 --- a/apps/api/src/hackproof/integrations/detect.py +++ b/apps/api/src/hackproof/integrations/detect.py @@ -14,8 +14,12 @@ def _normalize(ecosystem: str, name: str) -> str: return name.lower().replace("_", "-") if ecosystem == "python" else name -def _declares(ecosystem: str, partner: Product, declared: dict[str, str | None]) -> list[str]: - wanted = [_normalize(ecosystem, name) for name in partner.packages.get(ecosystem, [])] +def _declares( + ecosystem: str, partner: Product, declared: dict[str, str | None] +) -> list[str]: + wanted = [ + _normalize(ecosystem, name) for name in partner.packages.get(ecosystem, []) + ] found = [] for name in declared: normal = _normalize(ecosystem, name) @@ -35,11 +39,15 @@ def detect(files: dict[str, str], catalog: Catalog, owner: str) -> list[Detectio return _detect(files, catalog.partners, owner) -def detect_tools(files: dict[str, str], catalog: Catalog, owner: str) -> list[Detection]: +def detect_tools( + files: dict[str, str], catalog: Catalog, owner: str +) -> list[Detection]: return _detect(files, catalog.tools, owner) -def _detect(files: dict[str, str], products: Sequence[Product], owner: str) -> list[Detection]: +def _detect( + files: dict[str, str], products: Sequence[Product], owner: str +) -> list[Detection]: partners = [ partner for partner in products @@ -61,16 +69,28 @@ def _detect(files: dict[str, str], products: Sequence[Product], owner: str) -> l for package in _declares(ECOSYSTEM[name], partner, declared): found.setdefault( (partner.slug, "package", package), - Detection(slug=partner.slug, method="package", evidence=package, path=path), + Detection( + slug=partner.slug, + method="package", + evidence=package, + path=path, + ), ) - if PurePosixPath(path).suffix in PROSE_SUFFIXES or name.lower().startswith("readme"): + if PurePosixPath(path).suffix in PROSE_SUFFIXES or name.lower().startswith( + "readme" + ): continue for partner in partners: for host, pattern in endpoints[partner.slug]: if pattern.search(text): found.setdefault( (partner.slug, "endpoint", host), - Detection(slug=partner.slug, method="endpoint", evidence=host, path=path), + Detection( + slug=partner.slug, + method="endpoint", + evidence=host, + path=path, + ), ) return list(found.values()) @@ -82,7 +102,11 @@ def _matches(path: str, pattern: str) -> bool: def detect_agents( paths: Iterable[str], logins: Iterable[str], catalog: Catalog ) -> list[AgentDetection]: - tracked = [path for path in paths if not SKIP_DIRECTORIES & set(PurePosixPath(path).parts[:-1])] + tracked = [ + path + for path in paths + if not SKIP_DIRECTORIES & set(PurePosixPath(path).parts[:-1]) + ] people = {login.lower() for login in logins} found: list[AgentDetection] = [] for agent in catalog.agents: @@ -90,7 +114,9 @@ def detect_agents( match = next((path for path in tracked if _matches(path, pattern)), None) if match: found.append( - AgentDetection(slug=agent.slug, method="file", evidence=pattern, path=match) + AgentDetection( + slug=agent.slug, method="file", evidence=pattern, path=match + ) ) found.extend( AgentDetection(slug=agent.slug, method="author", evidence=login, path="") diff --git a/apps/api/src/hackproof/linking/reconcile.py b/apps/api/src/hackproof/linking/reconcile.py index 6b98c95..9a4be19 100644 --- a/apps/api/src/hackproof/linking/reconcile.py +++ b/apps/api/src/hackproof/linking/reconcile.py @@ -245,18 +245,26 @@ async def reconcile(connection: asyncpg.Connection) -> Report: for select in FOOTPRINT: await _insert_links(connection, LinkMethod.FOOTPRINT, select) official = (window.start, window.end) - await connection.execute(REPO_PROJECTS, *official, MIN_WINDOW_COMMITS, MIN_LIKELY_COMMITS) - await connection.execute(ACCOUNT_PROJECTS, *official, settings.friendbot_account) + await connection.execute( + REPO_PROJECTS, *official, MIN_WINDOW_COMMITS, MIN_LIKELY_COMMITS + ) + await connection.execute( + ACCOUNT_PROJECTS, *official, settings.friendbot_account + ) await connection.execute( "DELETE FROM projects WHERE lower(project_id) = ANY($1)", [f"repo:{name.lower()}" for name in hackathon.discovery.exclude_repos], ) - methods = await connection.fetch("SELECT method, count(*) AS n FROM links GROUP BY method") + methods = await connection.fetch( + "SELECT method, count(*) AS n FROM links GROUP BY method" + ) footprints = await connection.fetch( "SELECT footprint, count(*) AS n FROM projects GROUP BY footprint" ) - tiers = await connection.fetch("SELECT tier, count(*) AS n FROM projects GROUP BY tier") + tiers = await connection.fetch( + "SELECT tier, count(*) AS n FROM projects GROUP BY tier" + ) return Report( links_by_method={row["method"]: row["n"] for row in methods}, projects_by_footprint={row["footprint"]: row["n"] for row in footprints}, diff --git a/apps/api/src/hackproof/queries.py b/apps/api/src/hackproof/queries.py index 30b6cac..de7700d 100644 --- a/apps/api/src/hackproof/queries.py +++ b/apps/api/src/hackproof/queries.py @@ -301,7 +301,9 @@ async def list_page( placed = places() wanted = project_ids if wanted is None and (winners or awards): - wanted = [pid for pid, place in placed.items() if awards is None or place in awards] + wanted = [ + pid for pid, place in placed.items() if awards is None or place in awards + ] args = ( ( q, @@ -344,7 +346,9 @@ async def fetch_links(full_name: str) -> list[dict[str, Any]]: async def fetch_contributors(full_name: str) -> list[dict[str, Any]]: async with pool().acquire() as connection: - return [dict(row) for row in await connection.fetch(CONTRIBUTORS_SQL, full_name)] + return [ + dict(row) for row in await connection.fetch(CONTRIBUTORS_SQL, full_name) + ] async def fetch_timeline() -> list[dict[str, Any]]: @@ -357,28 +361,44 @@ async def fetch_integration_uses(full_name: str) -> list[dict[str, Any]]: async with pool().acquire() as connection: rows = await connection.fetch(INTEGRATION_USES_SQL, full_name) names = {partner.slug: partner.name for partner in get_catalog().partners} - return [{**row, "name": names[row["slug"]]} for row in map(dict, rows) if row["slug"] in names] + return [ + {**row, "name": names[row["slug"]]} + for row in map(dict, rows) + if row["slug"] in names + ] async def fetch_tool_uses(full_name: str) -> list[dict[str, Any]]: async with pool().acquire() as connection: rows = await connection.fetch(TOOL_USES_SQL, full_name) names = {tool.slug: tool.name for tool in get_catalog().tools} - return [{**row, "name": names[row["slug"]]} for row in map(dict, rows) if row["slug"] in names] + return [ + {**row, "name": names[row["slug"]]} + for row in map(dict, rows) + if row["slug"] in names + ] async def fetch_tools() -> list[dict[str, Any]]: async with pool().acquire() as connection: - counts = {row["slug"]: row["projects"] for row in await connection.fetch(TOOLS_SQL)} + counts = { + row["slug"]: row["projects"] for row in await connection.fetch(TOOLS_SQL) + } return [ - {**tool.model_dump(include={"slug", "name", "site"}), "projects": counts.get(tool.slug, 0)} + { + **tool.model_dump(include={"slug", "name", "site"}), + "projects": counts.get(tool.slug, 0), + } for tool in get_catalog().tools ] async def fetch_integrations() -> list[dict[str, Any]]: async with pool().acquire() as connection: - counts = {row["slug"]: row["projects"] for row in await connection.fetch(INTEGRATIONS_SQL)} + counts = { + row["slug"]: row["projects"] + for row in await connection.fetch(INTEGRATIONS_SQL) + } return [ { **partner.model_dump(include={"slug", "name", "category", "site"}), @@ -395,25 +415,33 @@ async def fetch_countries(limit: int = 12) -> list[dict[str, Any]]: async def fetch_wrapped() -> dict[str, Any]: start, end = window() - agents = [login.lower() for agent in get_catalog().agents for login in agent.authors] + agents = [ + login.lower() for agent in get_catalog().agents for login in agent.authors + ] async with pool().acquire() as connection: totals = dict(await connection.fetchrow(WRAPPED_TOTALS_SQL, agents, start, end)) operations = await connection.fetch(WRAPPED_OPERATIONS_SQL, start, end) teams = await connection.fetch(WRAPPED_TEAMS_SQL, agents) - multi_team = await connection.fetch(WRAPPED_BUILDERS_SQL, agents, 2, MAX_BUILDERS) + multi_team = await connection.fetch( + WRAPPED_BUILDERS_SQL, agents, 2, MAX_BUILDERS + ) top = await connection.fetch(WRAPPED_BUILDERS_SQL, agents, 1, TOP_BUILDERS) return { **totals, "operations_by_type": [dict(row) for row in operations], "team_sizes": [dict(row) for row in teams], "multi_team_builders": [dict(row) for row in multi_team], - "top_builders": [dict(row) for row in sorted(top, key=lambda row: -row["contributions"])], + "top_builders": [ + dict(row) for row in sorted(top, key=lambda row: -row["contributions"]) + ], } async def fetch_agents() -> list[dict[str, Any]]: async with pool().acquire() as connection: - counts = {row["slug"]: row["projects"] for row in await connection.fetch(AGENTS_SQL)} + counts = { + row["slug"]: row["projects"] for row in await connection.fetch(AGENTS_SQL) + } return [ { **agent.model_dump(include={"slug", "name", "site"}), diff --git a/apps/api/src/hackproof/routes.py b/apps/api/src/hackproof/routes.py index b7fb50e..4b7b178 100644 --- a/apps/api/src/hackproof/routes.py +++ b/apps/api/src/hackproof/routes.py @@ -36,7 +36,9 @@ async def stats() -> Stats: return Stats(**await queries.fetch_stats()) -async def _page(resource: queries.Resource, q: str | None, limit: int, offset: int) -> Page[Any]: +async def _page( + resource: queries.Resource, q: str | None, limit: int, offset: int +) -> Page[Any]: items, total = await queries.list_page(resource, q=q, limit=limit, offset=offset) return Page(items=items, total=total) @@ -95,21 +97,29 @@ async def projects( @router.get("/contracts") -async def contracts(q: Search = None, limit: Limit = 50, offset: Offset = 0) -> Page[Contract]: +async def contracts( + q: Search = None, limit: Limit = 50, offset: Offset = 0 +) -> Page[Contract]: return await _page("contracts", q, limit, offset) @router.get("/wasm") -async def wasm(q: Search = None, limit: Limit = 50, offset: Offset = 0) -> Page[WasmUpload]: +async def wasm( + q: Search = None, limit: Limit = 50, offset: Offset = 0 +) -> Page[WasmUpload]: return await _page("wasm", q, limit, offset) @router.get("/failed") -async def failed(q: Search = None, limit: Limit = 50, offset: Offset = 0) -> Page[FailedTx]: +async def failed( + q: Search = None, limit: Limit = 50, offset: Offset = 0 +) -> Page[FailedTx]: return await _page("failed", q, limit, offset) -async def _everything(resource: queries.Resource, **filters: str) -> list[dict[str, Any]]: +async def _everything( + resource: queries.Resource, **filters: str +) -> list[dict[str, Any]]: items, _ = await queries.list_page(resource, limit=EVERYTHING, **filters) return items @@ -121,17 +131,23 @@ async def project(project_id: str) -> ProjectDetail: raise HTTPException(status_code=404, detail="project not found") return ProjectDetail( project=found[0], - links=await queries.fetch_links(found[0]["full_name"]) if found[0]["full_name"] else [], + links=await queries.fetch_links(found[0]["full_name"]) + if found[0]["full_name"] + else [], integration_uses=( await queries.fetch_integration_uses(found[0]["full_name"]) if found[0]["full_name"] else [] ), tool_uses=( - await queries.fetch_tool_uses(found[0]["full_name"]) if found[0]["full_name"] else [] + await queries.fetch_tool_uses(found[0]["full_name"]) + if found[0]["full_name"] + else [] ), contributors=( - await queries.fetch_contributors(found[0]["full_name"]) if found[0]["full_name"] else [] + await queries.fetch_contributors(found[0]["full_name"]) + if found[0]["full_name"] + else [] ), contracts=await _everything("contracts", project_id=project_id), wasm_uploads=await _everything("wasm", project_id=project_id), @@ -143,7 +159,9 @@ async def project(project_id: str) -> ProjectDetail: async def contributor(login: str) -> Person: person = await queries.fetch_person(login) if person is None: - raise HTTPException(status_code=404, detail="no hackathon project for this contributor") + raise HTTPException( + status_code=404, detail="no hackathon project for this contributor" + ) return Person(**person) @@ -156,6 +174,8 @@ async def builder(address: str) -> Builder: wasm_uploads=await _everything("wasm", account=address), failed_txs=await _everything("failed", account=address), ) - if not (result.projects or result.contracts or result.wasm_uploads or result.failed_txs): + if not ( + result.projects or result.contracts or result.wasm_uploads or result.failed_txs + ): raise HTTPException(status_code=404, detail="no activity in the hack window") return result diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py index da86360..51d30d4 100644 --- a/apps/api/tests/conftest.py +++ b/apps/api/tests/conftest.py @@ -16,7 +16,9 @@ async def scratch_db() -> AsyncIterator[asyncpg.Connection]: name = f"hackproof_test_{uuid4().hex[:8]}" admin = await asyncpg.connect(url) await admin.execute(f'CREATE DATABASE "{name}"') - connection = await asyncpg.connect(urlunparse(urlparse(url)._replace(path=f"/{name}"))) + connection = await asyncpg.connect( + urlunparse(urlparse(url)._replace(path=f"/{name}")) + ) await connection.set_type_codec( "jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog" ) diff --git a/apps/api/tests/helpers.py b/apps/api/tests/helpers.py index 9808703..41c9cfc 100644 --- a/apps/api/tests/helpers.py +++ b/apps/api/tests/helpers.py @@ -11,26 +11,38 @@ async def add_repo(db: asyncpg.Connection, full_name: str, **fields: object) -> None: - values = {"url": f"https://github.com/{full_name}", "owner": full_name.split("/")[0], **fields} + values = { + "url": f"https://github.com/{full_name}", + "owner": full_name.split("/")[0], + **fields, + } columns = ", ".join(["full_name", *values]) marks = ", ".join(f"${n}" for n in range(1, len(values) + 2)) - await db.execute(f"INSERT INTO repos ({columns}) VALUES ({marks})", full_name, *values.values()) + await db.execute( + f"INSERT INTO repos ({columns}) VALUES ({marks})", full_name, *values.values() + ) -async def add_ref(db: asyncpg.Connection, full_name: str, kind: str, value: str) -> None: +async def add_ref( + db: asyncpg.Connection, full_name: str, kind: str, value: str +) -> None: await db.execute( "INSERT INTO repo_refs VALUES ($1, $2, $3, 'README.md')", full_name, kind, value ) -async def add_contract(db: asyncpg.Connection, contract: str, deployer: str, wasm: str) -> None: +async def add_contract( + db: asyncpg.Connection, contract: str, deployer: str, wasm: str +) -> None: await db.execute( "INSERT INTO contracts VALUES ($1, $2, $3, $4, 1, $5)", contract, deployer, wasm, f"tx-{contract}", INSIDE, ) # fmt: skip -async def add_upload(db: asyncpg.Connection, wasm: str, uploader: str, repo: str | None = None): +async def add_upload( + db: asyncpg.Connection, wasm: str, uploader: str, repo: str | None = None +): await db.execute( "INSERT INTO wasm_uploads VALUES ($1, $2, $3, 10, $4, '{}', 1, $5)", f"up-{wasm}-{uploader}", wasm, uploader, repo, INSIDE, @@ -49,7 +61,9 @@ async def add_funding(db: asyncpg.Connection, account: str) -> None: async def links(db: asyncpg.Connection) -> dict[tuple[str, str, str], str]: rows = await db.fetch("SELECT full_name, subject_kind, subject, method FROM links") - return {(r["full_name"], r["subject_kind"], r["subject"]): r["method"] for r in rows} + return { + (r["full_name"], r["subject_kind"], r["subject"]): r["method"] for r in rows + } STELLAR = {"ecosystems": ["js"], "packages": {"@stellar/stellar-sdk": "14.0.0"}} diff --git a/apps/api/tests/test_api.py b/apps/api/tests/test_api.py index b04bdae..6c0f120 100644 --- a/apps/api/tests/test_api.py +++ b/apps/api/tests/test_api.py @@ -42,7 +42,9 @@ def test_unknown_project_and_builder_are_not_found(): def test_projects_can_be_filtered_by_tier(): with TestClient(app) as client: - body = client.get("/api/projects", params={"tier": ["verified", "likely"], "limit": 100}) + body = client.get( + "/api/projects", params={"tier": ["verified", "likely"], "limit": 100} + ) assert {item["tier"] for item in body.json()["items"]} <= {"verified", "likely"} @@ -63,19 +65,26 @@ def test_countries_are_ranked_by_contributors(): def test_integrations_list_every_partner_with_a_project_count(): with TestClient(app) as client: body = client.get("/api/integrations").json() - assert {"trustless-work", "privy", "stellar-wallets-kit"} <= {row["slug"] for row in body} + assert {"trustless-work", "privy", "stellar-wallets-kit"} <= { + row["slug"] for row in body + } assert all(row["projects"] >= 0 for row in body) def test_projects_can_be_filtered_by_integration_partner(): with TestClient(app) as client: - none = client.get("/api/projects", params={"integration": "no-such-partner"}).json() + none = client.get( + "/api/projects", params={"integration": "no-such-partner"} + ).json() assert none["total"] == 0 used = next( - (row for row in client.get("/api/integrations").json() if row["projects"]), None + (row for row in client.get("/api/integrations").json() if row["projects"]), + None, ) if used: - body = client.get("/api/projects", params={"integration": used["slug"]}).json() + body = client.get( + "/api/projects", params={"integration": used["slug"]} + ).json() assert body["total"] >= used["projects"] assert all(used["slug"] in item["integrations"] for item in body["items"]) @@ -83,7 +92,8 @@ def test_projects_can_be_filtered_by_integration_partner(): def test_a_project_page_lists_where_each_integration_was_found(): with TestClient(app) as client: used = next( - (row for row in client.get("/api/integrations").json() if row["projects"]), None + (row for row in client.get("/api/integrations").json() if row["projects"]), + None, ) if used: [first, *_] = client.get( @@ -91,7 +101,10 @@ def test_a_project_page_lists_where_each_integration_was_found(): ).json()["items"] detail = client.get(f"/api/projects/{first['project_id']}").json() assert used["slug"] in {use["slug"] for use in detail["integration_uses"]} - assert {use["method"] for use in detail["integration_uses"]} <= {"package", "endpoint"} + assert {use["method"] for use in detail["integration_uses"]} <= { + "package", + "endpoint", + } def test_wrapped_team_sizes_fit_inside_the_projects(): @@ -112,7 +125,9 @@ def test_wrapped_builders_are_ranked_and_multi_team_means_two_or_more_projects() len(builder["project_names"]) == builder["projects"] for builder in body["multi_team_builders"] ) - assert not {"cursoragent", "codex"} & {b["login"] for b in body["multi_team_builders"]} + assert not {"cursoragent", "codex"} & { + b["login"] for b in body["multi_team_builders"] + } def test_agents_list_every_catalog_agent_with_a_project_count(): @@ -136,7 +151,9 @@ def test_a_contributor_lists_their_projects_with_their_commits(): return person = client.get(f"/api/contributors/{top[0]['login'].upper()}").json() assert person["login"] == top[0]["login"] - assert person["contributions"] == sum(item["contributions"] for item in person["projects"]) + assert person["contributions"] == sum( + item["contributions"] for item in person["projects"] + ) assert len(person["projects"]) == top[0]["projects"] commits = [item["contributions"] for item in person["projects"]] assert commits == sorted(commits, reverse=True) diff --git a/apps/api/tests/test_chat.py b/apps/api/tests/test_chat.py index a7106ae..81055de 100644 --- a/apps/api/tests/test_chat.py +++ b/apps/api/tests/test_chat.py @@ -9,7 +9,13 @@ from hackproof.chat.briefs import Page, describe_page, view_filters from hackproof.chat.prompts import prompt, sections from hackproof.chat.ratelimit import RateLimiter -from hackproof.chat.stream import Message, Part, stream_reply, to_anthropic, web_search_events +from hackproof.chat.stream import ( + Message, + Part, + stream_reply, + to_anthropic, + web_search_events, +) from hackproof.chat.tools import TOOL_SPECS, run_tool from hackproof.db import close_pool, open_pool from hackproof.main import app @@ -99,13 +105,17 @@ async def fake_run_tool(name, raw): ] ) - events = parse([c async for c in stream_reply(client, "m", [{"role": "user", "content": "x"}])]) + events = parse( + [c async for c in stream_reply(client, "m", [{"role": "user", "content": "x"}])] + ) types = [e["type"] if isinstance(e, dict) else e for e in events] assert types[0] == "start" and types[-2:] == ["finish", "[DONE]"] assert types.index("tool-input-start") < types.index("tool-input-available") assert types.index("tool-input-available") < types.index("tool-output-available") - deltas = [e["delta"] for e in events if isinstance(e, dict) and e["type"] == "text-delta"] + deltas = [ + e["delta"] for e in events if isinstance(e, dict) and e["type"] == "text-delta" + ] assert deltas == ["Checking.", "One project."] @@ -121,20 +131,26 @@ async def failing(name, raw): FakeStream([], SimpleNamespace(stop_reason="end_turn", content=[])), ] ) - events = parse([c async for c in stream_reply(client, "m", [{"role": "user", "content": "x"}])]) + events = parse( + [c async for c in stream_reply(client, "m", [{"role": "user", "content": "x"}])] + ) assert any(isinstance(e, dict) and e["type"] == "tool-output-error" for e in events) def post(client: TestClient): return client.post( "/api/chat", - json={"messages": [{"role": "user", "parts": [{"type": "text", "text": "hi"}]}]}, + json={ + "messages": [{"role": "user", "parts": [{"type": "text", "text": "hi"}]}] + }, ) def test_chat_is_rate_limited(monkeypatch): monkeypatch.setattr(routes, "limiter", RateLimiter()) - monkeypatch.setattr(routes, "get_settings", lambda: SimpleNamespace(anthropic_api_key=None)) + monkeypatch.setattr( + routes, "get_settings", lambda: SimpleNamespace(anthropic_api_key=None) + ) with TestClient(app) as client: statuses = [post(client).status_code for _ in range(11)] denied = post(client) @@ -159,7 +175,9 @@ async def fake_stream(client, model, messages, page=""): monkeypatch.setattr(routes, "describe_page", fake_describe) monkeypatch.setattr(routes, "stream_reply", fake_stream) monkeypatch.setattr( - routes, "get_settings", lambda: SimpleNamespace(anthropic_api_key="k", chat_model="m") + routes, + "get_settings", + lambda: SimpleNamespace(anthropic_api_key="k", chat_model="m"), ) body = { "page": {"kind": "project", "id": "repo:a/b"}, @@ -172,19 +190,31 @@ async def fake_stream(client, model, messages, page=""): def test_prompt_file_has_every_section(): - assert set(sections()) == {"system", "page-home", "page-entity", "page-empty", "page-filter"} + assert set(sections()) == { + "system", + "page-home", + "page-entity", + "page-empty", + "page-filter", + } text = prompt("system", start="A", end="B", page="ON THE HOME PAGE") assert "A to B UTC" in text and text.endswith("ON THE HOME PAGE") assert "{" not in text def test_web_search_is_reported_as_a_provider_tool(): - call = SimpleNamespace(type="server_tool_use", id="s1", name="web_search", input={"query": "q"}) + call = SimpleNamespace( + type="server_tool_use", id="s1", name="web_search", input={"query": "q"} + ) hit = SimpleNamespace(title="Meridian", url="https://meridian.stellar.org") - result = SimpleNamespace(type="web_search_tool_result", tool_use_id="s1", content=[hit]) + result = SimpleNamespace( + type="web_search_tool_result", tool_use_id="s1", content=[hit] + ) start, done = parse(list(web_search_events(call)) + list(web_search_events(result))) assert start["providerExecuted"] and start["input"] == {"query": "q"} - assert done["output"] == [{"title": "Meridian", "url": "https://meridian.stellar.org"}] + assert done["output"] == [ + {"title": "Meridian", "url": "https://meridian.stellar.org"} + ] def test_project_tools_can_filter_by_integration_partner(): @@ -201,14 +231,24 @@ def test_project_tools_filter_by_every_explorer_filter(): assert {"get_tools", "get_agents"} <= set(specs) for name in ("query_projects", "summarize_projects"): properties = specs[name]["input_schema"]["properties"] - assert {"tools", "agents", "languages", "countries", "activity", "awards"} <= set( - properties - ) + assert { + "tools", + "agents", + "languages", + "countries", + "activity", + "awards", + } <= set(properties) def test_explorer_filters_map_onto_the_project_filters(): filters = view_filters( - {"agent": ["cursor"], "partner": ["freighter"], "track": ["scale"], "tier": ["verified"]} + { + "agent": ["cursor"], + "partner": ["freighter"], + "track": ["scale"], + "tier": ["verified"], + } ) assert filters.agents == ["cursor"] @@ -216,7 +256,10 @@ def test_explorer_filters_map_onto_the_project_filters(): assert filters.track == "scale" assert filters.list_args()["tiers"] == ["verified"] assert view_filters({"track": ["genesis", "scale"]}).track is None - assert view_filters({"nonsense": ["x"]}).list_args()["tiers"] == ["verified", "likely"] + assert view_filters({"nonsense": ["x"]}).list_args()["tiers"] == [ + "verified", + "likely", + ] async def test_the_filter_page_brief_counts_the_matching_projects(): diff --git a/apps/api/tests/test_dependencies.py b/apps/api/tests/test_dependencies.py index 96e12ec..c95fda7 100644 --- a/apps/api/tests/test_dependencies.py +++ b/apps/api/tests/test_dependencies.py @@ -19,7 +19,10 @@ def test_reads_rust_and_js_manifests(): WANTED, ) assert result.ecosystems == ["rust", "js"] - assert result.packages == {"soroban-sdk": "22.0.0", "@stellar/stellar-sdk": "^14.0.0"} + assert result.packages == { + "soroban-sdk": "22.0.0", + "@stellar/stellar-sdk": "^14.0.0", + } def test_reads_table_style_cargo_dependencies_and_workspaces(): @@ -52,5 +55,8 @@ def test_broken_manifests_are_skipped(): def test_repos_without_stellar_dependencies_have_no_fingerprint(): assert ( - fingerprint({"package.json": '{"dependencies": {"react": "19"}}'}, WANTED).ecosystems == [] + fingerprint( + {"package.json": '{"dependencies": {"react": "19"}}'}, WANTED + ).ecosystems + == [] ) diff --git a/apps/api/tests/test_discovery.py b/apps/api/tests/test_discovery.py index 2e58b2e..7d26e3b 100644 --- a/apps/api/tests/test_discovery.py +++ b/apps/api/tests/test_discovery.py @@ -38,14 +38,18 @@ def test_keeps_repos_pushed_around_the_window(): def test_drops_stale_forked_and_excluded_repos(): - assert not should_keep(candidate(pushed_at=datetime(2026, 9, 1, tzinfo=UTC)), WINDOW, DISCOVERY) + assert not should_keep( + candidate(pushed_at=datetime(2026, 9, 1, tzinfo=UTC)), WINDOW, DISCOVERY + ) assert not should_keep(candidate(fork=True), WINDOW, DISCOVERY) assert not should_keep(candidate(owner="Stellar"), WINDOW, DISCOVERY) def test_repos_named_in_wasm_metadata_bypass_every_filter(): stale = candidate( - pushed_at=datetime(2025, 1, 1, tzinfo=UTC), fork=True, discovered_via={"wasm_meta"} + pushed_at=datetime(2025, 1, 1, tzinfo=UTC), + fork=True, + discovered_via={"wasm_meta"}, ) assert is_pinned(stale) assert should_keep(stale, WINDOW, DISCOVERY) @@ -53,7 +57,9 @@ def test_repos_named_in_wasm_metadata_bypass_every_filter(): def test_excluded_repos_stay_out_even_when_wasm_metadata_names_them(): assert not should_keep( - candidate(full_name="Team/Tool", discovered_via={"wasm_meta"}), WINDOW, DISCOVERY + candidate(full_name="Team/Tool", discovered_via={"wasm_meta"}), + WINDOW, + DISCOVERY, ) diff --git a/apps/api/tests/test_export.py b/apps/api/tests/test_export.py index c117b93..922ee52 100644 --- a/apps/api/tests/test_export.py +++ b/apps/api/tests/test_export.py @@ -38,10 +38,17 @@ async def test_export_carries_repo_chain_and_evidence(scratch_db): export = await populated_export(scratch_db) [project] = export.projects - assert (project.repo_full_name, project.tier, project.name) == ("team/app", "verified", "app") + assert (project.repo_full_name, project.tier, project.name) == ( + "team/app", + "verified", + "app", + ) assert project.stack == ["@stellar/stellar-sdk"] assert project.track == "genesis" - assert project.contributors[0].login == "ana" and project.contributors[0].country == "PT" + assert ( + project.contributors[0].login == "ana" + and project.contributors[0].country == "PT" + ) assert [c.contract_id for c in project.chain.contracts] == ["CXYZ"] assert [a.address for a in project.chain.accounts] == ["GDEPLOYER"] assert project.chain.accounts[0].explorer_url == f"{EXPLORER}/account/GDEPLOYER" @@ -85,7 +92,9 @@ async def test_track_is_the_declared_one_and_genesis_otherwise(scratch_db): ) await reconcile(db) - projects = (await build_export(db, get_hackathon(), ["likely", "possible"])).projects + projects = ( + await build_export(db, get_hackathon(), ["likely", "possible"]) + ).projects assert {p.repo_full_name: (p.track, p.track_declared) for p in projects} == { "team/declared": ("scale", True), @@ -112,7 +121,9 @@ async def test_export_lists_integration_partners_with_their_evidence(scratch_db) assert [p.slug for p in project.integration_partners] == ["trustless-work", "privy"] [evidence] = project.integration_partners[0].evidence assert (evidence.method, evidence.detail) == ("endpoint", "api.trustlesswork.com") - assert evidence.repo_file_url == "https://github.com/team/app/blob/HEAD/src/client.go" + assert ( + evidence.repo_file_url == "https://github.com/team/app/blob/HEAD/src/client.go" + ) async def test_export_marks_award_winners(scratch_db): diff --git a/apps/api/tests/test_export_check.py b/apps/api/tests/test_export_check.py index 02723a5..4911653 100644 --- a/apps/api/tests/test_export_check.py +++ b/apps/api/tests/test_export_check.py @@ -18,8 +18,12 @@ def test_identical_files_have_no_drift(tmp_path): def test_drift_names_missing_extra_and_changed_files(tmp_path): - database = write(tmp_path / "db", {"2026/a.yml": "x: 1\n", "2026/new.yml": "n: 1\n"}) - committed = write(tmp_path / "repo", {"2026/a.yml": "x: 2\n", "2026/old.yml": "o: 1\n"}) + database = write( + tmp_path / "db", {"2026/a.yml": "x: 1\n", "2026/new.yml": "n: 1\n"} + ) + committed = write( + tmp_path / "repo", {"2026/a.yml": "x: 2\n", "2026/old.yml": "o: 1\n"} + ) assert drift(database, committed) == [ "only in the files: 2026/old.yml", diff --git a/apps/api/tests/test_export_files.py b/apps/api/tests/test_export_files.py index 2d13938..78f0ea1 100644 --- a/apps/api/tests/test_export_files.py +++ b/apps/api/tests/test_export_files.py @@ -45,7 +45,9 @@ async def test_a_project_file_validates_and_keeps_only_public_facts(scratch_db): data = yaml.safe_load(text) jsonschema.validate(data, file_schema()) - assert text.startswith("# yaml-language-server: $schema=../schema-hackathon-project.yml\n") + assert text.startswith( + "# yaml-language-server: $schema=../schema-hackathon-project.yml\n" + ) assert data["canonical_id"] == "pkg:github/team/app" assert data["contributors"] == [{"login": "ana", "contributions": 12}] assert data["chain"]["accounts"] == ["GDEPLOYER"] @@ -88,7 +90,9 @@ async def test_a_name_without_letters_or_digits_is_an_error(scratch_db): file_names(export.projects) -async def test_writing_files_replaces_stale_ones_and_keeps_the_event_manifest(scratch_db, tmp_path): +async def test_writing_files_replaces_stale_ones_and_keeps_the_event_manifest( + scratch_db, tmp_path +): export = await likely_export(scratch_db, "team/app") year = tmp_path / str(export.hackathon.starts_at.year) year.mkdir() @@ -99,7 +103,9 @@ async def test_writing_files_replaces_stale_ones_and_keeps_the_event_manifest(sc assert count == 1 assert sorted(path.name for path in year.glob("*.yml")) == [EVENT_FILE, "app.yml"] jsonschema.validate(yaml.safe_load((year / "app.yml").read_text()), file_schema()) - assert yaml.safe_load((tmp_path / SCHEMA_FILE).read_text())["title"] == "ProjectFile" + assert ( + yaml.safe_load((tmp_path / SCHEMA_FILE).read_text())["title"] == "ProjectFile" + ) async def test_the_event_manifest_validates_and_names_the_event(scratch_db, tmp_path): diff --git a/apps/api/tests/test_files.py b/apps/api/tests/test_files.py index 192597c..9eb774a 100644 --- a/apps/api/tests/test_files.py +++ b/apps/api/tests/test_files.py @@ -14,7 +14,9 @@ def test_query_escapes_paths_and_aliases_every_file(): async def test_blank_and_missing_files_are_dropped(): class Github: async def graphql(self, query: str) -> dict: - return {"repository": {"f0": {"text": "hello"}, "f1": None, "f2": {"text": ""}}} + return { + "repository": {"f0": {"text": "hello"}, "f1": None, "f2": {"text": ""}} + } found = await fetch_texts(Github(), "a/b", "main", ["a.md", "b.md", "c.md"]) assert found == {"a.md": "hello"} diff --git a/apps/api/tests/test_hackathon.py b/apps/api/tests/test_hackathon.py index 604b6fb..7cff382 100644 --- a/apps/api/tests/test_hackathon.py +++ b/apps/api/tests/test_hackathon.py @@ -9,4 +9,6 @@ def test_awards_name_each_project_once(): def test_places_maps_every_award(): - assert places() == {award.project_id: award.place for award in get_hackathon().awards} + assert places() == { + award.project_id: award.place for award in get_hackathon().awards + } diff --git a/apps/api/tests/test_history.py b/apps/api/tests/test_history.py index d8c689f..3207f86 100644 --- a/apps/api/tests/test_history.py +++ b/apps/api/tests/test_history.py @@ -17,14 +17,18 @@ async def json(self, path: str, params: dict | None = None): if path.endswith("/commits"): return [{"sha": sha} for sha in self._commits] sha = path.rsplit("/", 1)[-1] - return {"tree": [{"type": "blob", "path": name} for name in self._repo.get(sha, {})]} + return { + "tree": [{"type": "blob", "path": name} for name in self._repo.get(sha, {})] + } async def raw(self, full_name: str, ref: str, path: str) -> bytes | None: return self._repo.get(ref, {}).get(path) async def added(base_sha: str | None, commits: list[str], repo: dict) -> bool: - return await stellar_added_in_window(FakeGithub(commits, repo), "a/b", WANTED, base_sha, END) + return await stellar_added_in_window( + FakeGithub(commits, repo), "a/b", WANTED, base_sha, END + ) async def test_stellar_missing_at_the_window_start_and_present_at_its_end_was_added_inside_it(): diff --git a/apps/api/tests/test_integrations.py b/apps/api/tests/test_integrations.py index 638e349..dd781bb 100644 --- a/apps/api/tests/test_integrations.py +++ b/apps/api/tests/test_integrations.py @@ -89,7 +89,9 @@ def test_a_detection_is_reported_once_with_the_first_file(): } assert detect(files, CATALOG, "team") == [ - Detection(slug="acme", method="package", evidence="@acme/sdk", path="package.json") + Detection( + slug="acme", method="package", evidence="@acme/sdk", path="package.json" + ) ] @@ -98,7 +100,9 @@ def test_the_shipped_catalog_is_valid_and_unique(): slugs = [partner.slug for partner in catalog.partners] assert len(slugs) == len(set(slugs)) >= 10 - assert catalog.partner("trustless-work").packages["js"][0] == "@trustless-work/escrow" + assert ( + catalog.partner("trustless-work").packages["js"][0] == "@trustless-work/escrow" + ) def test_tools_are_found_apart_from_partners(): @@ -114,7 +118,9 @@ def test_tools_are_found_apart_from_partners(): ], ) files = { - "package.json": json.dumps({"dependencies": {"@scaffold/app-lib": "1", "@acme/sdk": "1"}}) + "package.json": json.dumps( + {"dependencies": {"@scaffold/app-lib": "1", "@acme/sdk": "1"}} + ) } assert {d.slug for d in detect_tools(files, catalog, "team")} == {"scaffold"} @@ -131,13 +137,23 @@ def test_tools_are_found_apart_from_partners(): files=[".cursor/*", ".cursorrules"], authors=["cursoragent"], ), - Agent(slug="claude", name="Claude", site="https://claude.example", files=["CLAUDE.md"]), + Agent( + slug="claude", + name="Claude", + site="https://claude.example", + files=["CLAUDE.md"], + ), ], ) def test_agents_are_found_by_file_and_by_commit_author(): - paths = ["src/app.ts", "CLAUDE.md", ".cursor/rules/style.mdc", "apps/web/.cursorrules"] + paths = [ + "src/app.ts", + "CLAUDE.md", + ".cursor/rules/style.mdc", + "apps/web/.cursorrules", + ] found = detect_agents(paths, ["Someone", "CursorAgent"], AGENT_CATALOG) @@ -158,7 +174,9 @@ def test_agent_files_inside_dependencies_are_ignored(): def test_the_shipped_catalog_lists_unique_agents(): agents = get_catalog().agents - assert {"claude-code", "cursor", "codex", "github-copilot"} <= {agent.slug for agent in agents} + assert {"claude-code", "cursor", "codex", "github-copilot"} <= { + agent.slug for agent in agents + } assert len({agent.slug for agent in agents}) == len(agents) @@ -170,4 +188,7 @@ def test_stellar_raven_and_skills_are_found_in_agent_config_files(): found = {(d.slug, d.path) for d in detect_tools(files, get_catalog(), "team")} - assert found == {("stellar-raven", ".mcp.json"), ("stellar-skills", "skills-lock.json")} + assert found == { + ("stellar-raven", ".mcp.json"), + ("stellar-skills", "skills-lock.json"), + } diff --git a/apps/api/tests/test_reconcile.py b/apps/api/tests/test_reconcile.py index 065165b..aefe882 100644 --- a/apps/api/tests/test_reconcile.py +++ b/apps/api/tests/test_reconcile.py @@ -28,8 +28,14 @@ async def test_a_contract_in_a_repo_pulls_in_the_whole_deployer_footprint(scratc assert found[("team/app", "contract", "C1")] == "contract_in_repo" assert found[("team/app", "account", "GDEPLOYER")] == "footprint" assert found[("team/app", "contract", "C2")] == "footprint" - project = await db.fetchrow("SELECT * FROM projects WHERE project_id = 'repo:team/app'") - assert (project["contracts"], project["footprint"], project["builders"]) == (2, "contracts", 1) + project = await db.fetchrow( + "SELECT * FROM projects WHERE project_id = 'repo:team/app'" + ) + assert (project["contracts"], project["footprint"], project["builders"]) == ( + 2, + "contracts", + 1, + ) async def test_wasm_metadata_links_regardless_of_repo_name_case(scratch_db): @@ -67,13 +73,18 @@ async def test_hacker_accounts_in_a_repo_attach_a_contractless_project(scratch_d await add_funding(db, "GHACKER") await add_ref(db, "team/payments", "account", "GHACKER") await db.execute( - "INSERT INTO classic_ops VALUES ('t', 0, 'GHACKER', 'payment', NULL, NULL, 1, $1)", INSIDE + "INSERT INTO classic_ops VALUES ('t', 0, 'GHACKER', 'payment', NULL, NULL, 1, $1)", + INSIDE, ) await reconcile(db) - assert (await links(db))[("team/payments", "account", "GHACKER")] == "deployer_in_repo" - project = await db.fetchrow("SELECT * FROM projects WHERE project_id = 'repo:team/payments'") + assert (await links(db))[ + ("team/payments", "account", "GHACKER") + ] == "deployer_in_repo" + project = await db.fetchrow( + "SELECT * FROM projects WHERE project_id = 'repo:team/payments'" + ) assert (project["footprint"], project["classic_ops"]) == ("classic", 1) @@ -99,14 +110,20 @@ async def test_unattributed_deployers_become_account_projects(scratch_db): assert await db.fetchval("SELECT project_id FROM projects") == "acct:GLONE" -async def test_older_repos_that_added_stellar_during_the_window_become_projects(scratch_db): +async def test_older_repos_that_added_stellar_during_the_window_become_projects( + scratch_db, +): db = scratch_db await add_repo(db, "team/old", stellar_added_in_window=True, dependencies=STELLAR) - await add_repo(db, "team/stale", stellar_added_in_window=False, dependencies=STELLAR) + await add_repo( + db, "team/stale", stellar_added_in_window=False, dependencies=STELLAR + ) await reconcile(db) - ids = {row["project_id"] for row in await db.fetch("SELECT project_id FROM projects")} + ids = { + row["project_id"] for row in await db.fetch("SELECT project_id FROM projects") + } assert ids == {"repo:team/old"} @@ -119,12 +136,16 @@ async def test_repos_created_inside_the_window_become_projects(scratch_db): assert await db.fetchval("SELECT project_id FROM projects") == "repo:team/new" -async def test_projects_are_tiered_by_how_strong_the_hackmeridian_evidence_is(scratch_db): +async def test_projects_are_tiered_by_how_strong_the_hackmeridian_evidence_is( + scratch_db, +): db = scratch_db await add_repo(db, "a/chain", window_commits=1) await add_ref(db, "a/chain", "contract", "C1") await add_contract(db, "C1", "GCHAIN", "w1") - await add_repo(db, "c/likely", window_commits=5, created_at=INSIDE, dependencies=STELLAR) + await add_repo( + db, "c/likely", window_commits=5, created_at=INSIDE, dependencies=STELLAR + ) await add_repo(db, "d/possible", window_commits=5, dependencies=STELLAR) await reconcile(db) @@ -138,14 +159,18 @@ async def test_projects_are_tiered_by_how_strong_the_hackmeridian_evidence_is(sc async def test_a_new_repo_with_only_a_few_commits_is_not_likely(scratch_db): db = scratch_db - await add_repo(db, "team/scaffold", window_commits=4, created_at=INSIDE, dependencies=STELLAR) + await add_repo( + db, "team/scaffold", window_commits=4, created_at=INSIDE, dependencies=STELLAR + ) await reconcile(db) assert await tiers(db) == {"repo:team/scaffold": "possible"} -async def test_matches_that_only_touch_the_pre_start_margin_are_not_verified(scratch_db): +async def test_matches_that_only_touch_the_pre_start_margin_are_not_verified( + scratch_db, +): db = scratch_db await add_repo(db, "team/early", window_commits=1) await add_ref(db, "team/early", "contract", "C1") @@ -160,9 +185,14 @@ async def test_matches_that_only_touch_the_pre_start_margin_are_not_verified(scr assert (await tiers(db))["repo:team/early"] == "possible" -async def test_a_chain_item_matched_by_two_repos_is_credited_to_the_older_one(scratch_db): +async def test_a_chain_item_matched_by_two_repos_is_credited_to_the_older_one( + scratch_db, +): db = scratch_db - for name, created in (("team/copy", INSIDE), ("team/original", INSIDE - timedelta(days=30))): + for name, created in ( + ("team/copy", INSIDE), + ("team/original", INSIDE - timedelta(days=30)), + ): await add_repo(db, name, window_commits=3, created_at=created) await add_ref(db, name, "contract", "C1") await add_contract(db, "C1", "GDEPLOYER", "w1") diff --git a/apps/api/tests/test_repositories.py b/apps/api/tests/test_repositories.py index 182ea39..a305041 100644 --- a/apps/api/tests/test_repositories.py +++ b/apps/api/tests/test_repositories.py @@ -20,7 +20,11 @@ "before": {"nodes": [{"oid": "abc123"}]}, }, }, - "owner": {"login": "team", "avatarUrl": "https://avatars/x", "location": "Lisbon, Portugal"}, + "owner": { + "login": "team", + "avatarUrl": "https://avatars/x", + "location": "Lisbon, Portugal", + }, "repositoryTopics": {"nodes": [{"topic": {"name": "stellar"}}]}, } @@ -30,20 +34,30 @@ def test_node_carries_commit_counts_base_commit_and_owner_location(): assert (candidate.total_commits, candidate.window_commits) == (40, 31) assert candidate.base_sha == "abc123" assert candidate.owner_location == "Lisbon, Portugal" - assert (candidate.language, candidate.license, candidate.forks) == ("Rust", "MIT", 1) + assert (candidate.language, candidate.license, candidate.forks) == ( + "Rust", + "MIT", + 1, + ) assert candidate.topics == ["stellar"] assert candidate.homepage is None def test_empty_repo_has_no_branch_or_counts(): - candidate = candidate_from_node({**NODE, "defaultBranchRef": None, "primaryLanguage": None}) + candidate = candidate_from_node( + {**NODE, "defaultBranchRef": None, "primaryLanguage": None} + ) assert candidate.default_branch is None assert candidate.total_commits is None and candidate.base_sha is None assert candidate.language is None def test_repo_with_no_commit_before_the_window_has_no_base(): - target = {"total": {"totalCount": 5}, "inWindow": {"totalCount": 5}, "before": {"nodes": []}} + target = { + "total": {"totalCount": 5}, + "inWindow": {"totalCount": 5}, + "before": {"nodes": []}, + } candidate = candidate_from_node( {**NODE, "defaultBranchRef": {"name": "main", "target": target}} ) diff --git a/apps/api/tests/test_scan.py b/apps/api/tests/test_scan.py index b1cf7f3..0a6ebac 100644 --- a/apps/api/tests/test_scan.py +++ b/apps/api/tests/test_scan.py @@ -29,7 +29,9 @@ def test_select_files_prioritises_manifests_and_skips_noise(): def test_select_files_drops_oversized_files(): - texts, wasms = select_files([blob("big.json", 600_000), blob("huge.wasm", 9_000_000)]) + texts, wasms = select_files( + [blob("big.json", 600_000), blob("huge.wasm", 9_000_000)] + ) assert texts == [] and wasms == [] @@ -51,7 +53,12 @@ def test_contributors_keep_people_and_drop_bots(): "type": "User", "contributions": 12, }, - {"login": "dependabot[bot]", "html_url": "x", "type": "Bot", "contributions": 40}, + { + "login": "dependabot[bot]", + "html_url": "x", + "type": "Bot", + "contributions": 40, + }, ] ) assert [(p.login, p.contributions) for p in people] == [("ana", 12)] @@ -59,16 +66,18 @@ def test_contributors_keep_people_and_drop_bots(): def test_demo_url_pattern_only_matches_ascii_hosts(): - assert DEMO_URL.search("see https://my-app.vercel.app/docs for the demo").group() == ( - "https://my-app.vercel.app/docs" - ) + assert DEMO_URL.search( + "see https://my-app.vercel.app/docs for the demo" + ).group() == ("https://my-app.vercel.app/docs") assert DEMO_URL.search("https://你的用户名.github.io`、`SITE_BASE=") is None async def test_a_rescan_replaces_the_stored_integrations(scratch_db): db = scratch_db await add_repo(db, "team/app") - detection = Detection(slug="privy", method="package", evidence="@privy-io/node", path="a.json") + detection = Detection( + slug="privy", method="package", evidence="@privy-io/node", path="a.json" + ) def scan(*found: Detection) -> RepoScan: return RepoScan( diff --git a/apps/api/tests/test_urls.py b/apps/api/tests/test_urls.py index 0a3201d..c757a63 100644 --- a/apps/api/tests/test_urls.py +++ b/apps/api/tests/test_urls.py @@ -29,7 +29,10 @@ def test_repo_files_link_to_the_default_branch(): def test_file_paths_are_percent_encoded_so_every_link_is_a_valid_url(): url = repo_file_url("https://github.com/team/app", "board/Leo's notes #1 (v2).md") - assert url == "https://github.com/team/app/blob/HEAD/board/Leo%27s%20notes%20%231%20%28v2%29.md" + assert ( + url + == "https://github.com/team/app/blob/HEAD/board/Leo%27s%20notes%20%231%20%28v2%29.md" + ) assert " " not in url diff --git a/apps/api/tests/test_wasm.py b/apps/api/tests/test_wasm.py index 76fc0b5..2cb8f95 100644 --- a/apps/api/tests/test_wasm.py +++ b/apps/api/tests/test_wasm.py @@ -29,9 +29,18 @@ def meta_entry(key: str, value: str) -> bytes: def test_reads_every_entry_of_the_meta_section(): - payload = meta_entry("source_repo", "github:acme/vault") + meta_entry("rsver", "1.93.0") - wasm = MAGIC + custom_section("name", b"ignored") + custom_section("contractmetav0", payload) - assert contract_meta(wasm) == {"source_repo": "github:acme/vault", "rsver": "1.93.0"} + payload = meta_entry("source_repo", "github:acme/vault") + meta_entry( + "rsver", "1.93.0" + ) + wasm = ( + MAGIC + + custom_section("name", b"ignored") + + custom_section("contractmetav0", payload) + ) + assert contract_meta(wasm) == { + "source_repo": "github:acme/vault", + "rsver": "1.93.0", + } def test_wasm_without_meta_section_is_empty(): From cee8676ca8ae41270c45211776b0b2defc9c12cd Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:22:11 +0100 Subject: [PATCH 02/18] feat: add the sqlfluff plugin and config --- apps/api/.sqlfluffignore | 2 + apps/api/pyproject.toml | 17 ++ apps/api/src/hackproof/db_utils/__init__.py | 0 apps/api/src/hackproof/db_utils/sql_files.py | 11 + .../hackproof/db_utils/sqlfluff/__init__.py | 0 .../src/hackproof/db_utils/sqlfluff/plugin.py | 9 + .../src/hackproof/db_utils/sqlfluff/rules.py | 35 +++ apps/api/uv.lock | 201 ++++++++++++++++++ 8 files changed, 275 insertions(+) create mode 100644 apps/api/.sqlfluffignore create mode 100644 apps/api/src/hackproof/db_utils/__init__.py create mode 100644 apps/api/src/hackproof/db_utils/sql_files.py create mode 100644 apps/api/src/hackproof/db_utils/sqlfluff/__init__.py create mode 100644 apps/api/src/hackproof/db_utils/sqlfluff/plugin.py create mode 100644 apps/api/src/hackproof/db_utils/sqlfluff/rules.py diff --git a/apps/api/.sqlfluffignore b/apps/api/.sqlfluffignore new file mode 100644 index 0000000..6cc9839 --- /dev/null +++ b/apps/api/.sqlfluffignore @@ -0,0 +1,2 @@ +manifest.sql +migrations/ diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 9e8ccd0..d1b5c9c 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -20,6 +20,9 @@ dependencies = [ [project.scripts] hackproof = "hackproof.cli:main" +[project.entry-points."sqlfluff"] +hackproof = "hackproof.db_utils.sqlfluff.plugin" + [build-system] requires = ["uv_build>=0.12.5,<0.13.0"] build-backend = "uv_build" @@ -30,6 +33,7 @@ dev = [ "pytest>=9.1.1", "pytest-asyncio>=1.4.1", "ruff>=0.17.3", + "sqlfluff>=4.5.0", ] [tool.pytest.ini_options] @@ -57,3 +61,16 @@ ban-relative-imports = "parents" [tool.ruff.format] # Add a trailing comma to function defs with two or more args to enforce multiline formatting in ruff. skip-magic-trailing-comma = false + +[tool.sqlfluff.core] +dialect = "postgres" +# Rule codes: https://docs.sqlfluff.com/en/stable/reference/rules.html +# LT05: comments may be longer than 80 characters. +# RF04: flags legitimate column names like `version` as reserved keywords. +exclude_rules = "LT05,RF04" + +[tool.sqlfluff.rules.capitalisation.keywords] +capitalisation_policy = "upper" + +[tool.sqlfluff.rules.capitalisation.literals] +capitalisation_policy = "upper" diff --git a/apps/api/src/hackproof/db_utils/__init__.py b/apps/api/src/hackproof/db_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/src/hackproof/db_utils/sql_files.py b/apps/api/src/hackproof/db_utils/sql_files.py new file mode 100644 index 0000000..4858af5 --- /dev/null +++ b/apps/api/src/hackproof/db_utils/sql_files.py @@ -0,0 +1,11 @@ +import re +from pathlib import Path + +TABLE_NAME_RE = re.compile( + r'CREATE TABLE( IF NOT EXISTS)? "?([A-Za-z_][A-Za-z0-9_]*)"?' +) + + +def table_name(path: Path) -> str | None: + match = TABLE_NAME_RE.search(path.read_text()) + return match.group(2) if match else None diff --git a/apps/api/src/hackproof/db_utils/sqlfluff/__init__.py b/apps/api/src/hackproof/db_utils/sqlfluff/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/src/hackproof/db_utils/sqlfluff/plugin.py b/apps/api/src/hackproof/db_utils/sqlfluff/plugin.py new file mode 100644 index 0000000..8d7d26c --- /dev/null +++ b/apps/api/src/hackproof/db_utils/sqlfluff/plugin.py @@ -0,0 +1,9 @@ +from sqlfluff.core.plugin import hookimpl +from sqlfluff.core.rules import BaseRule + + +@hookimpl +def get_rules() -> list[type[BaseRule]]: + from hackproof.db_utils.sqlfluff.rules import Rule_Hackproof_CT01 + + return [Rule_Hackproof_CT01] diff --git a/apps/api/src/hackproof/db_utils/sqlfluff/rules.py b/apps/api/src/hackproof/db_utils/sqlfluff/rules.py new file mode 100644 index 0000000..c195349 --- /dev/null +++ b/apps/api/src/hackproof/db_utils/sqlfluff/rules.py @@ -0,0 +1,35 @@ +from sqlfluff.core.rules import BaseRule, LintFix, LintResult, RuleContext +from sqlfluff.core.rules.crawlers import SegmentSeekerCrawler + + +class Rule_Hackproof_CT01(BaseRule): + """Enforce upper-case CREATE TYPE names. + + Problem: sqlfluff's built-in CP05 only flags inconsistent type casing within + a single file, so a CREATE TYPE declaration living alone in its own + schema/types/.sql file never has anything to compare against and is + never checked. + Solution: this rule checks every CREATE TYPE name directly against a fixed + upper-case policy, independent of what else does or doesn't appear in the + same file. + """ # noqa: D213 -- summary must stay on this line so `sqlfluff rules` can show it as the description + + name = "custom.create_type_upper" + groups = ("all",) + crawl_behaviour = SegmentSeekerCrawler({"create_type_statement"}) + is_fix_compatible = True + + def _eval(self, context: RuleContext) -> LintResult | None: + identifier = next(context.segment.recursive_crawl("data_type_identifier"), None) + if identifier is None: + return None + + raw = identifier.raw + if raw == raw.upper(): + return None + + return LintResult( + anchor=identifier, + fixes=[LintFix.replace(identifier, [identifier.edit(raw.upper())])], + description=f"CREATE TYPE names must be upper case: '{raw}' -> '{raw.upper()}'.", + ) diff --git a/apps/api/uv.lock b/apps/api/uv.lock index 511cc9b..7d32e7c 100644 --- a/apps/api/uv.lock +++ b/apps/api/uv.lock @@ -209,6 +209,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] +[[package]] +name = "chardet" +version = "7.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/92/3d893ac5d338b9a9af34c5b517d77492dcea8a10254254eead4215a96f3f/chardet-7.7.0.tar.gz", hash = "sha256:0ba38a05201b0abf7570c12d8af45c30b2af835bd5cc52bfc7105082eda82c1e", size = 1008480, upload-time = "2026-10-18T00:06:25.352Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/73/0c434bc65adc411e49b1bbebee24ebb36a3511c80a058f1fde17f43fb537/chardet-7.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:11263887c082c3d8c32026aeddbdef37ac8feccd1faece5a22b2ff00a29b115a", size = 1202232, upload-time = "2026-10-18T00:05:45.657Z" }, + { url = "https://files.pythonhosted.org/packages/9d/be/222e5628af10d6b80750ccdda7f6d2610646dddb082c37947623d5e55bf7/chardet-7.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e8c718e6961981fe1685e1243742f326ac553f0fa0dfd0c6d3d17dd4f92cd3ff", size = 1173273, upload-time = "2026-10-18T00:05:46.984Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/889cf390db3748918789818d93ba9a7d712b326030199698432fc50bf8cd/chardet-7.7.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a76f4d1f769b4ae8888cefd46e604cf1f5769f1176b55e78b90eaeefa888ce92", size = 1616968, upload-time = "2026-10-18T00:05:48.457Z" }, + { url = "https://files.pythonhosted.org/packages/ca/04/0b5fd38375ae182cec6446dd3121aaff5ae18fd12b98b10161a58b720db9/chardet-7.7.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c519c10c315a675bcfd4b55c0e0de1859ca5a39881d8f69750500102c2fc5d2f", size = 1658830, upload-time = "2026-10-18T00:05:49.938Z" }, + { url = "https://files.pythonhosted.org/packages/99/c0/7e642d7bc6aca242ef41b4aa5f5be27dbb57912e8ce0f38043d8d1999cd1/chardet-7.7.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:46a044166fe0d99931884dc28405faac558d5eabf28316a2b3d2a1a6c7dc439a", size = 1583300, upload-time = "2026-10-18T00:05:51.567Z" }, + { url = "https://files.pythonhosted.org/packages/30/ee/53147716ced6d43837682a2acfb09dd45add8bc496107f21051c5aeae1f6/chardet-7.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:113bbd3ab85835e3e99499ae66829915ed3b4cf5c69525ab1742260f77638449", size = 1308030, upload-time = "2026-10-18T00:05:52.997Z" }, + { url = "https://files.pythonhosted.org/packages/1a/61/800a9442f352912c86f2a00de84fd6914dd9353d29091b1e223e7c08850e/chardet-7.7.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad286a0654ae7477bc454a1dee4f396cb28adffeacc6bc7d4412f607c73de0c3", size = 1200940, upload-time = "2026-10-18T00:05:54.59Z" }, + { url = "https://files.pythonhosted.org/packages/fb/92/3c6ce24dd68c626cc6aa0b5953c2aebb1d853c7b07d3535f0300b7428288/chardet-7.7.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4f67f20c9ad3f3024bf71d4aae1d25722b12da921ca9586f3cce7a4639695aa0", size = 1173087, upload-time = "2026-10-18T00:05:56.257Z" }, + { url = "https://files.pythonhosted.org/packages/4d/32/5a952b376c00dda01465b396958a8985916d9a9293a772d6bda2c37f2cc5/chardet-7.7.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a05667635c0496dab47eca8c845c044d1da6d74d34a9bf687199c9c53a38119d", size = 1624001, upload-time = "2026-10-18T00:05:57.832Z" }, + { url = "https://files.pythonhosted.org/packages/04/f7/09296ca1d4621c9c2766d074822035eee6aebb79eff80c0af3bd96283d05/chardet-7.7.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:79eec5af9d7715334695254234c8d7c29165841526c11083951d9b9f6458e372", size = 1653785, upload-time = "2026-10-18T00:05:59.379Z" }, + { url = "https://files.pythonhosted.org/packages/cb/08/80571baad57db2962d6931b3ee1828c82e3e14e9b3464f80532f536e23ac/chardet-7.7.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4167165812219e70f311083a8e06e3b627ed5b27aa8c15c3fdd56f350fd3213e", size = 1582280, upload-time = "2026-10-18T00:06:00.825Z" }, + { url = "https://files.pythonhosted.org/packages/18/82/3c07070fdcf3c4263e16680c90e9af7c2cd2e5e59cfb1d372efa60fde8cc/chardet-7.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:546180f6170da39778069d16130449a9cff49a8fe144535947d1ef26f5330b96", size = 1310108, upload-time = "2026-10-18T00:06:02.342Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b0/d703d408c7f8931b1564110078fc3dcff9825a159990fa69b80da52be160/chardet-7.7.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a4c2e83b09a540a4d33279d2bee8ed3bfd57ac6645b359ae8b8afd5b4578c3c2", size = 1237782, upload-time = "2026-10-18T00:06:03.763Z" }, + { url = "https://files.pythonhosted.org/packages/d8/3e/0eafeebd53eb096400f75138ab18614ab5be12b8794b76dd4b35ef1a8e05/chardet-7.7.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:259dd55b1ba05a66868151a42663483763046a3e6579bde6b0aa65d834635b5a", size = 1207710, upload-time = "2026-10-18T00:06:05.221Z" }, + { url = "https://files.pythonhosted.org/packages/b8/8d/3c200098027def56be1b6443a10c72efe013a42eb76444d889c809890ca5/chardet-7.7.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56e5a5c782548b301b44371de327115d13a08de391ddaf9db4ac420d8511c5d7", size = 1660832, upload-time = "2026-10-18T00:06:06.619Z" }, + { url = "https://files.pythonhosted.org/packages/08/81/02f98389fa72345f40d23d69c2d857511d11a0911779a7cf8ed1cecb5037/chardet-7.7.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d227d2aa329fcf848f4640ed9a4d99e1753840c1cfa19f7ca5a48f30c03982ad", size = 1678291, upload-time = "2026-10-18T00:06:08.196Z" }, + { url = "https://files.pythonhosted.org/packages/5e/24/0ae866b759a148d0f6087ffb30920a2b956653d0d455b4be93026edbee95/chardet-7.7.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c4a591b2dbfc0a047f8d60331c2fbfb4c3589911000406666a3085fa7d5da69f", size = 1200696, upload-time = "2026-10-18T00:06:09.701Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a6/5e0a1f56aca6c44b562d05d9e360f31b5a8f4b311d940a324fa337d277ba/chardet-7.7.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:eb625c3ab2386ca7f3be9ac2b9b822f4c5b792595c8bb8100b3b6ea68120e92c", size = 1172821, upload-time = "2026-10-18T00:06:11.04Z" }, + { url = "https://files.pythonhosted.org/packages/89/05/868f52672cde7e188a7333f3f65deaa62f5acbfcf0f81f3582013776e8ad/chardet-7.7.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddedfb252d05c86c2d6e9a3090f8bd574876bbd39679e7c588ed57d21595c097", size = 1625481, upload-time = "2026-10-18T00:06:12.499Z" }, + { url = "https://files.pythonhosted.org/packages/f9/12/2b3a006ae31a7ab4ccac11a00abb27af23b4a5d57ddb00e937375e747a2a/chardet-7.7.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c4d8a2dc8abcc067417adc13c2f64f48482e48ce48e3af769b5a44214e5a5c", size = 1660352, upload-time = "2026-10-18T00:06:13.977Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/f995a2873a78aa3ec8308f02e61f0717deb4b01c8f1a74c2da329f325925/chardet-7.7.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:78db6186c1e73d38a2106cdcbbc8f43612eaed9ef8bc4adaf0fe67b6e8a2049d", size = 1597363, upload-time = "2026-10-18T00:06:15.492Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7b/22b916b52c7bc4eb076ba1736ea9b99cf1065ebb7b56d356409144887c73/chardet-7.7.0-cp315-cp315-win_amd64.whl", hash = "sha256:8f1d611f15530e1642856d91b7aa259851466d7e961f27b888921e7e29718edb", size = 1309528, upload-time = "2026-10-18T00:06:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/02/2f/ccaf5ffa0c8adca280e2687d75524cc07113a0df70cbf542da3445f2b300/chardet-7.7.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:1db39957379a801cf1798a5e50cdb3d3fc32683e23c4166ecfa157b980d1931e", size = 1236698, upload-time = "2026-10-18T00:06:18.191Z" }, + { url = "https://files.pythonhosted.org/packages/0d/7e/32f45f7ddeb27d94feb529a1a23e5a38a6d3614d12e23f0c89a9eb89098b/chardet-7.7.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:7f3b2ca3bac2bdbe42c4dfe7a2888c4b98e2e86a83ddb631960b229d1980979e", size = 1207247, upload-time = "2026-10-18T00:06:19.751Z" }, + { url = "https://files.pythonhosted.org/packages/26/40/8c1ab360c39a0bb823786e8934984cbc3e83076dfce2a7f3760086ef778c/chardet-7.7.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:80db1c109e6e3c102ac589897449437ea2edf26dda1042a3ff669368e21d90e5", size = 1653659, upload-time = "2026-10-18T00:06:21.189Z" }, + { url = "https://files.pythonhosted.org/packages/1b/3e/bd5c25f525ff2466e6365cbfb098f928761357331b884eebc92ac50f8878/chardet-7.7.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:091f50d3567b1a066f797835ee5367dbcf3cb77c5e56993b3f61b28264b8e6f8", size = 1677031, upload-time = "2026-10-18T00:06:22.564Z" }, + { url = "https://files.pythonhosted.org/packages/cc/75/88a3eb36264375ff951bd2f9397c409727ab6aa6a18e1bd933851571d421/chardet-7.7.0-py3-none-any.whl", hash = "sha256:11cf9b719ea3675a2a7d8a3071bbe4d7ea24b26bfe39a51588d515ddd05cacad", size = 706680, upload-time = "2026-10-18T00:06:24.018Z" }, +] + [[package]] name = "charset-normalizer" version = "3.5.2" @@ -351,6 +386,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/fe/a245e1d579e7abf47208f7c1ce5684caa2cd956ede5ff646d66d0299d9e8/detect_installer-0.2.1-py3-none-any.whl", hash = "sha256:87e2ce7d05cb108b178f8e45bb80d3d88c0cb4b397983345c18ad6562e02e620", size = 5371, upload-time = "2026-09-02T14:14:38.106Z" }, ] +[[package]] +name = "diff-cover" +version = "11.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "chardet" }, + { name = "jinja2" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/fb/c1bb9a442cbfe003036032286ca340d70278c88050f158f882b33c1ed1ff/diff_cover-11.0.2.tar.gz", hash = "sha256:8e7b9cc0c5fd4cfa28c46e028a64afd0091af150bb4a460d890817271dc69902", size = 110259, upload-time = "2026-11-14T03:40:39.35Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/f7/84c6218c306481683b56d5b0146a13526822d46d9218cc5bebf560e111a7/diff_cover-11.0.2-py3-none-any.whl", hash = "sha256:27dc1101dea8bd8b69095dd09a79dea370c2d65a9145fcf48d1ae71f3aab25e6", size = 63862, upload-time = "2026-11-14T03:40:37.899Z" }, +] + [[package]] name = "dnspython" version = "2.9.0" @@ -586,6 +636,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "ruff" }, + { name = "sqlfluff" }, ] [package.metadata] @@ -606,6 +657,7 @@ dev = [ { name = "pytest", specifier = ">=9.1.1" }, { name = "pytest-asyncio", specifier = ">=1.4.1" }, { name = "ruff", specifier = ">=0.17.3" }, + { name = "sqlfluff", specifier = ">=4.5.0" }, ] [[package]] @@ -1130,6 +1182,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.14.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/9a/8475be631fb33ca1466915e70fd594c42a4e84b512db650eaa14f90fe75d/platformdirs-4.14.5.tar.gz", hash = "sha256:32e9921db7385aa8169bee4fe9501923e5715ba9fb7ea3b9c69b0d8e0639f7a1", size = 71464, upload-time = "2026-11-04T16:54:50.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/ae/ecbbbfd0e4343a6a521d49b3c8656fb4643dee2a4f6eaf941cf3280391a8/platformdirs-4.14.5-py3-none-any.whl", hash = "sha256:8e778c6fa421516f4d3369449f8df3b0e1b72c0e0005fe88e3ec772d0a14e750", size = 35402, upload-time = "2026-11-04T16:54:48.875Z" }, +] + [[package]] name = "pluggy" version = "1.7.0" @@ -1446,6 +1516,94 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "regex" +version = "2026.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/27/a5afeb952577b84ee5fe48b8b9ce47ae662dbb69cc5b76c28eff98f83b1e/regex-2026.11.9.tar.gz", hash = "sha256:316a70fbd70e999c9664681cb0eb5b4982b2cc0f623c7c86a8a112e1bd4c6277", size = 419513, upload-time = "2026-11-09T00:42:09.786Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/55/7b6278e0babda54fe4b619c13019aef42247de912180777d49afae8eefac/regex-2026.11.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:224b73a26a1e64cf87fc8da9ee7a9994f152b92a728a7786b5c002fcebb205b8", size = 498358, upload-time = "2026-11-09T00:40:22.495Z" }, + { url = "https://files.pythonhosted.org/packages/4c/96/e37e54801fcd80fe7d2929708cd0c459aef588bc887da2de58a7c3c963d8/regex-2026.11.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6852035568af30020a538f945863d2dca3d5baadfe9d9387f9d54de2f1b30a45", size = 296847, upload-time = "2026-11-09T00:40:23.623Z" }, + { url = "https://files.pythonhosted.org/packages/f6/69/8bb15a5dcc3f8c9a57b069847fd65552bb7d1de22861e283e0139e5d5c71/regex-2026.11.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:24c9479dc1215a1ebc96fedc9a0cd026240209e9b69aa223df411c443bdf1d93", size = 294046, upload-time = "2026-11-09T00:40:25.026Z" }, + { url = "https://files.pythonhosted.org/packages/19/74/b13616c437d4141e9b73ef8f44d2be36058dc83cc1316048a081b73eead7/regex-2026.11.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5e3b575067d381f59cd6896a0126aef5dfb80a16b03594e8fc86bdc9dcf1cec", size = 804872, upload-time = "2026-11-09T00:40:26.115Z" }, + { url = "https://files.pythonhosted.org/packages/45/e4/8a0597242b57ef983f50ce84a2f35ba681e72333ff34aadee2805de2deea/regex-2026.11.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:38225e2f77e094c5a118e64af55655192eb16966b1e61ed8f4674ea94bedbb04", size = 878457, upload-time = "2026-11-09T00:40:27.223Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a7/3ed73ae7e9b7ae09f80d09a59ee21fefc4a3e33a08e2b039934e57b86a76/regex-2026.11.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b4957fa9142e4850f454e7e76366245677cb917c608d7f8088b0105c9948db2f", size = 919209, upload-time = "2026-11-09T00:40:28.425Z" }, + { url = "https://files.pythonhosted.org/packages/35/3c/d6f2b69b141b3d0aa82d69fbbfe2b38dc49138e3f6843cc763774202d024/regex-2026.11.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:32183c31b090e1ee2bbb60898d236addc0029bf04a6def37175c2bb5419dcfb5", size = 808929, upload-time = "2026-11-09T00:40:29.882Z" }, + { url = "https://files.pythonhosted.org/packages/68/68/2fbbb66b68d0babc7260c1538f5d422f9edbbbdf4d6e5c916f8fe7c0fcbc/regex-2026.11.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86b0004630d82a7fa993a40c599d77180622ba356d68dc48eef8f47e8c836853", size = 783522, upload-time = "2026-11-09T00:40:31.095Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fa/50f1b4e85430cb4aba15320eb0a8204e48d175317168771b8c05a00373c5/regex-2026.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d6bf83cfc92e7f6402cc10b8542fc562d4183929bdf126a0ccea1c15b55a109f", size = 791604, upload-time = "2026-11-09T00:40:33.218Z" }, + { url = "https://files.pythonhosted.org/packages/96/6e/fbf5139423566e051cedd6f71c97bc16fb34653e921b6654e24a0c16df8a/regex-2026.11.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c88a1749323036f3b3f67c32c1df4c6f2889f1ad5508eb3b66032019e0acb14b", size = 866818, upload-time = "2026-11-09T00:40:34.861Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d7/d873103dffac3b4558437dd622edff275f88a0cd32d0bed5175ac0c389b5/regex-2026.11.9-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:0dbed59cc01d38778091845ac229dc24b909bc326b82d3043972b0acd1349bb6", size = 770466, upload-time = "2026-11-09T00:40:36.345Z" }, + { url = "https://files.pythonhosted.org/packages/6c/71/d681e7949e91de573808a0d609a84d99411fad3680d346e436f68528c4fd/regex-2026.11.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d0517aff825aaa7a242ce85938ff4a51fa73633ec14a0b6639f658b29a78e6f2", size = 861497, upload-time = "2026-11-09T00:40:37.572Z" }, + { url = "https://files.pythonhosted.org/packages/5c/17/052721465a9e96a340b73d6bc7bffacd7bd9d57b4ddc38ac95e6671b7882/regex-2026.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a1919cb74f076c93b3d7344ba8ec930e9aa6d70272801779ba0e9bb48eb12d4f", size = 795612, upload-time = "2026-11-09T00:40:38.81Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ad/f5eeae00983238972bbcd3b31122437f729299d4a7ecd661b07a61448a6f/regex-2026.11.9-cp313-cp313-win32.whl", hash = "sha256:bd66446f042de0b2fdc4870f590eed990285d7c5d677545dc5aa8fd0dcf5f0e9", size = 269754, upload-time = "2026-11-09T00:40:40.004Z" }, + { url = "https://files.pythonhosted.org/packages/f3/e5/27d16943df68676aba97db64e58b10702627eac1b983d96342b80b71f6c1/regex-2026.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:19d36876d5783f157f280e67e51038a63a30630bd069528070d83a711f170e99", size = 280845, upload-time = "2026-11-09T00:40:41.348Z" }, + { url = "https://files.pythonhosted.org/packages/6d/34/74fbef82831900af4da247486576f867e6ffdb531841adc771af5797b213/regex-2026.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:2062558aead9322ce54479b0ac356453d1455a30ec40d16b263b4cb0dcd5fd0d", size = 280234, upload-time = "2026-11-09T00:40:43.225Z" }, + { url = "https://files.pythonhosted.org/packages/ad/76/0931cbbc2faec7eb35e1dc95bb6e3256ddd20d1995dc1474886954436d33/regex-2026.11.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8d8225ae4aec6154a9c895fc5ea7a4cd9347c955432ee3cf277871518b4a78cc", size = 498414, upload-time = "2026-11-09T00:40:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/18/57/e06c598bd5935bce31faae0c20dbb27f7b3551f77c871da1fcc9d5e8806d/regex-2026.11.9-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2d8f8f23200309b13a48a43379890bcdd34db49dda203445d2c9e1ebc1919419", size = 296843, upload-time = "2026-11-09T00:40:45.828Z" }, + { url = "https://files.pythonhosted.org/packages/e4/bc/8d2e21baca091b2f2bba7b6127cb60395018579736d41958d7398f381e5f/regex-2026.11.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fb91fc79a06d276f7811b840840440c6a35178396bfab288a4814b9a05d08245", size = 294104, upload-time = "2026-11-09T00:40:47.187Z" }, + { url = "https://files.pythonhosted.org/packages/4d/14/561ec95a248d68b55598e80a8a7190fce98f8a1ab8c0aeb2fe46b153bb28/regex-2026.11.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f570129536763bce7bd06a24f555bfad989aca594459a09cd7d51e47b92e0b4", size = 805122, upload-time = "2026-11-09T00:40:48.658Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f0/6398931f37cf3c3e9a1f8af33097ada8c17c7a991791e0b2e1df60f34666/regex-2026.11.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:992f562c10b100501d17abad5307cbe5049cecc9afe6f5bb58cc0c49be964a95", size = 878892, upload-time = "2026-11-09T00:40:49.846Z" }, + { url = "https://files.pythonhosted.org/packages/ed/49/eb615960e51bf824a24ef5ad3ea100f72ac5010aab835bd72fc0fbc5b9d2/regex-2026.11.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5f571e667b3a0af4a37e231c535bec1237f531433fbeea4efe7196aae04eb734", size = 918562, upload-time = "2026-11-09T00:40:51.008Z" }, + { url = "https://files.pythonhosted.org/packages/88/4e/97c82477691afa0329b51ee801e5002cbf91ceff4864200cf2895833586a/regex-2026.11.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8d6cc5be22e9c839547d6ffc88b515845cbf47604c74d293955eea96281adbc", size = 808261, upload-time = "2026-11-09T00:40:52.586Z" }, + { url = "https://files.pythonhosted.org/packages/ab/87/c6b8dba287886b6b535b05cbb860110b84d11b864725cd39a35c7066da2b/regex-2026.11.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f026e602009c42175f5d7ab5ae0dc34884b09e5c1bb9c540e37f5f60c330173d", size = 783703, upload-time = "2026-11-09T00:40:53.631Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d5/f9707218634b435d03839d0dd4c43b7b74426c7a010ea786846b36215887/regex-2026.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8088dcf06e237da99bdf1e802716ad29e9b4103b2b04f8dee87de3d6102a6c03", size = 791919, upload-time = "2026-11-09T00:40:55.058Z" }, + { url = "https://files.pythonhosted.org/packages/68/05/352600c95571b49ed1c7a9a73e9afc76459854439c8b5c1e811b3b81b6d8/regex-2026.11.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:063cfc54e91284d0d9808befba1b9624bf230c3d22c6dd32a539f7af81b391a5", size = 867716, upload-time = "2026-11-09T00:40:56.286Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/ff96ccc1515ed2fae491fd490dab2ea22c5536ae8de59f176eea07c6f886/regex-2026.11.9-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:80c7f5a8236a751b9f0cfd160c68199db71860daa8609ec30ba79862926c6884", size = 770658, upload-time = "2026-11-09T00:40:57.546Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b2/bbbee563872d031cbe160bddac0846195aa4cc9eb2d7d6b61510598c9c3e/regex-2026.11.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:590831103234f8b044dde16352234b24a7966ed685a8f5735da48f4044f5cb55", size = 860933, upload-time = "2026-11-09T00:40:59.2Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4c/8d6060820dea2a10b3b026b462fd47986f54c0590deb13f16f1dadd7df27/regex-2026.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:961fe37a9e52575360599a4099ebb6d983b71a88e4cb44aef24e230adcb592dd", size = 795245, upload-time = "2026-11-09T00:41:00.592Z" }, + { url = "https://files.pythonhosted.org/packages/1b/32/5093383543e5c86b40e4117c0c6ee082715810b65283494aca48dc517d3f/regex-2026.11.9-cp314-cp314-win32.whl", hash = "sha256:4b9f3e8a40e00a8c46a23e780c06444fc19f048c44e0c533186bdd730a57eb92", size = 275220, upload-time = "2026-11-09T00:41:01.735Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d0/5a472f1404c2217c92ebbd98d65a273cc48adc0a1f2948743ed59a873c18/regex-2026.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:4d2bb485c68c8688201e5d56f6a7cd563ad9ee40ad2a700e77a13fceaac3a517", size = 284464, upload-time = "2026-11-09T00:41:02.734Z" }, + { url = "https://files.pythonhosted.org/packages/28/48/4addd74faf7531f713a9e765cadd1ddcb010363924fee94038e289452e5a/regex-2026.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:dd0777e5cfa8508b0c1a903bfeb3b5859c1842c0b01f3ff39064c6bce3297e6b", size = 284279, upload-time = "2026-11-09T00:41:04.021Z" }, + { url = "https://files.pythonhosted.org/packages/ed/81/c88102ab3fa7eee7badc3f1554224d9012a5cd0fc2d40e3af4a4bb10e713/regex-2026.11.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:eb7c95df50b4132f938599f39758da9ee7c73ee66a8f40974160f07567e7301c", size = 502240, upload-time = "2026-11-09T00:41:05.561Z" }, + { url = "https://files.pythonhosted.org/packages/3e/da/ba9d89af353c54727086f705fff7b60bc0994191b76f2a560d69d44c612d/regex-2026.11.9-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:18c2e3de5790cd5d1e1437f4805fda7f16b14931f86312d9542d6014434aa107", size = 298869, upload-time = "2026-11-09T00:41:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/d9079948364cbd94f693801c9673ed3f97052848380cd1b4271e6061ef4f/regex-2026.11.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8d5b856217efc33802ae7bbd1b5d98c3117dae9dd6a36b75027291e2b79f72cf", size = 296656, upload-time = "2026-11-09T00:41:07.998Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f9/dbadb579d336487bb226ab02686faa0142cc065426826d1d121cc01a59ac/regex-2026.11.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:401055fcdbef098c6054e80e52813f7c937f4209c8c727ec4135df6a6f585222", size = 819805, upload-time = "2026-11-09T00:41:09.202Z" }, + { url = "https://files.pythonhosted.org/packages/64/d7/7e433fcdd1075dfce9423accf478a636b4e9a2078311b86a25ee4973df1a/regex-2026.11.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e68a02b7ff911d611c43457b029c76d6bbccf3c1a4f7e336e086a3b6afa1d19c", size = 881836, upload-time = "2026-11-09T00:41:10.783Z" }, + { url = "https://files.pythonhosted.org/packages/15/40/89d31d421367742d2854eba4f6a3d9ae1b15630fae9c7c791415c9a9373c/regex-2026.11.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7f854a607c31bc87b64d9ca13a9d25c6fb57f59f802d2fe871c48b205d06f566", size = 924530, upload-time = "2026-11-09T00:41:12.035Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d5/5e16185a5ba2ebbf83ae24935c32e3a3fc25191347f529c06908b267b6f4/regex-2026.11.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2d6a2d52cd32a7cdfef4b3e22417eb7bde912a5624128851bb96aef5dc657067", size = 824544, upload-time = "2026-11-09T00:41:13.233Z" }, + { url = "https://files.pythonhosted.org/packages/7b/14/0e753b59a7ffed4d67c50e2674e870293fdf6f7f95ce65352407427687df/regex-2026.11.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4c1141775f79b742d9ad2d33e058cf06e7219030aadff8d4a132450cc7539dd", size = 793572, upload-time = "2026-11-09T00:41:14.551Z" }, + { url = "https://files.pythonhosted.org/packages/4d/6d/39a0e1ad06a26e04eb1fe2e00322c5d334d74c97a94a9dca01fb7e574eda/regex-2026.11.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e886209417f98c2b8b817d531e72731c3fb78f990a402759f4a4c7714f9c5415", size = 805935, upload-time = "2026-11-09T00:41:16.034Z" }, + { url = "https://files.pythonhosted.org/packages/8a/99/1667c4ce3af04e661a0eff2008126ceeca874bc52d7f2895e75b86e7d11f/regex-2026.11.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:85f327c6645bc0d924ef407d36aca97ed86bc41da28cabd77b347b1771b5eae1", size = 871258, upload-time = "2026-11-09T00:41:17.79Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2d/124b69673327013b19e6cfaf7f59bcd1e801c1ebf893a056e0a0f6e0f84a/regex-2026.11.9-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9cb9d77c5c9101f1b6aa3d22f679f7a41a5af6ab536bc7290457e9bfc40eeeac", size = 776374, upload-time = "2026-11-09T00:41:18.85Z" }, + { url = "https://files.pythonhosted.org/packages/83/4b/e4d277fe50a30bec2b98bb7c7e055efb33060c984cc41a75cc2d6b4c9e3a/regex-2026.11.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0ac46eb63c366482a268121d873a08014304ea774d349b9bd2340a622b88a3d4", size = 864755, upload-time = "2026-11-09T00:41:20.316Z" }, + { url = "https://files.pythonhosted.org/packages/25/a7/b1626d1a273b29f6104265488a7b82ef22a81f8a1066fd8d91e098887213/regex-2026.11.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9622c0306e0864fa2cce4e5de97b3f0b7cbb86054c781550fceeb9f9c9bec290", size = 811588, upload-time = "2026-11-09T00:41:21.51Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b8/2581a22a535eeaa301f8b3a71614d7bb8c30b821ba9373b991113b8b4c1a/regex-2026.11.9-cp314-cp314t-win32.whl", hash = "sha256:8cd7bda4894077adf35ec41df18d705e0f8028d40445066c870e517c754d78c2", size = 277284, upload-time = "2026-11-09T00:41:22.634Z" }, + { url = "https://files.pythonhosted.org/packages/10/21/a39d46035c96eaf530eabb6b57751c744c317c0af145240d3f1997d9ecc9/regex-2026.11.9-cp314-cp314t-win_amd64.whl", hash = "sha256:19c5affdda6dfe1bb2e5b0775cf5f4e71e664bcde2daafeec635d34d6f9704c8", size = 287062, upload-time = "2026-11-09T00:41:23.634Z" }, + { url = "https://files.pythonhosted.org/packages/20/64/1a9f6a13e365b1b23bd7ac264ceb8ecadbe962d3354a4cb4a303c88a749b/regex-2026.11.9-cp314-cp314t-win_arm64.whl", hash = "sha256:422c3dd2f8ff069c1050d4f4048bc8fa26fa92df09fcedbe13f77c922f5f7dc8", size = 286617, upload-time = "2026-11-09T00:41:25.402Z" }, + { url = "https://files.pythonhosted.org/packages/91/3e/a94fae619c5c72644732b393f241a4dc65e32a47c76a681b819390214d6f/regex-2026.11.9-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:022c80076ecb4a1e967c6ff9feeecfdfc364183281667056d1953cd345b0b9a8", size = 498649, upload-time = "2026-11-09T00:41:26.995Z" }, + { url = "https://files.pythonhosted.org/packages/41/e3/d468930952cc613f7d63ad375ebe9c1e1b850b3a4c87ed411e8db8cfdd62/regex-2026.11.9-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:639d43b0898a2fd99e10eb5736f022c0fe8ee25bf60250061f8720c47a54483d", size = 296832, upload-time = "2026-11-09T00:41:28.351Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ad/b53184107d89fc00a596d4306483a034d68353cb410cf7b19c7ee511cfa0/regex-2026.11.9-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:569ec6f0f774caeef4e0bbc8161aff2b7f07b1051ebcfc9c2c9eccebe7320b06", size = 294370, upload-time = "2026-11-09T00:41:29.497Z" }, + { url = "https://files.pythonhosted.org/packages/b6/79/0fc94e5a21468d15e649c57ac327ea307712a5a1216557f6a7508d16abfb/regex-2026.11.9-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f209033452abebfa65f0d388b1f6d4f2b4b2791b3a403bc744c03e80e996be9", size = 807071, upload-time = "2026-11-09T00:41:30.753Z" }, + { url = "https://files.pythonhosted.org/packages/83/6e/3006d97b837a876223484206e4d7700798421958de49c081ce280a38d5e4/regex-2026.11.9-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1dd7fa70f8b4956acb3e34eb9bea1c7ae3d31509abf979393181eaaa0f00e63d", size = 879313, upload-time = "2026-11-09T00:41:32.194Z" }, + { url = "https://files.pythonhosted.org/packages/cb/a9/f033d7aa75457dd3f6fa8f5701c5b4a5c685428bc7582b9c872a3e9e9c55/regex-2026.11.9-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b10639d37913075b553ba9638be170a87f70c32bd9da0655c3093e7ebc4a8065", size = 918900, upload-time = "2026-11-09T00:41:33.717Z" }, + { url = "https://files.pythonhosted.org/packages/55/7e/378e356b11dbd903d61844215bd16e63e0bdf253c4c8e66cd60ed504737e/regex-2026.11.9-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e8fd96057e04660a655f2f7fbc9a9eff6ffe2970ff01c0145a0d425f2d1bf6", size = 809955, upload-time = "2026-11-09T00:41:34.98Z" }, + { url = "https://files.pythonhosted.org/packages/28/2f/5d689c2b7b0083a54e47786699a08a2548cf317316b86385ac0646a56a61/regex-2026.11.9-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:adce69b7696dec985977bc65d025c43dd7f2fa780f4f6e1d467d939884d7f918", size = 789193, upload-time = "2026-11-09T00:41:36.85Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f4/231515fefa3051505ff3dd51d1eee987bcf637958d52dd7ae70d2daebe19/regex-2026.11.9-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:346ac3b9664ca04dd9ff7f422a33d87015657b0674a2830d804d78e987cc1f4a", size = 794166, upload-time = "2026-11-09T00:41:38.268Z" }, + { url = "https://files.pythonhosted.org/packages/9e/10/f935d85b0155d0c621113fcef32aa148ef8d078c16907fbda6950c5d3abb/regex-2026.11.9-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:0c816b00198ddca1040fa2657619b1c6b4c861d158cbad9de77d01fdc44550cd", size = 868221, upload-time = "2026-11-09T00:41:39.706Z" }, + { url = "https://files.pythonhosted.org/packages/a0/5b/9e243ee822c1680cd1468520280a8bc96fe7a8b8ca5a7ed7f213fe9da92c/regex-2026.11.9-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa62357d0dd25abd66100369828d6e28b50a5c44001a0fd20f95117c4038c648", size = 779357, upload-time = "2026-11-09T00:41:40.966Z" }, + { url = "https://files.pythonhosted.org/packages/12/67/2273b16db6d6a27bbc2a4f61ec9221d06adf3d1dacfbd6d8120f78e4c8ef/regex-2026.11.9-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:d1627410afc3f5a4b0c4d81dd1bdd1ed3de6407eda6d93524e643cf3891126d2", size = 861471, upload-time = "2026-11-09T00:41:42.048Z" }, + { url = "https://files.pythonhosted.org/packages/68/d8/58f8688bcdb9e482be40e25c13e7bb0701c2ce8c4514ce7af7c635877d81/regex-2026.11.9-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:970201018e997ed4a2626ad894b657035397fa91c1c86eff1b0c8cb9eb03d478", size = 797579, upload-time = "2026-11-09T00:41:43.758Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fd/382230164a17f509c155918771817378516a9ee8bd1b54a5ec4d3dacd7c3/regex-2026.11.9-cp315-cp315-win32.whl", hash = "sha256:1f4c9e7edad6ed655c83741413b350db98b1234a4fa8739bc1ed7167e19ce98e", size = 275213, upload-time = "2026-11-09T00:41:45.63Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1f/3b08cdef84fae97450baf8c60c0f86a85c613ddfa7b106df9186069248c0/regex-2026.11.9-cp315-cp315-win_amd64.whl", hash = "sha256:72770400851673eee68e9d6cfc45c18a20736894e3002506a987a244780df36c", size = 284471, upload-time = "2026-11-09T00:41:46.87Z" }, + { url = "https://files.pythonhosted.org/packages/24/ee/ddf3c916425e7de6774df90e9b36ea7be709233f6e617863f9eeaf855211/regex-2026.11.9-cp315-cp315-win_arm64.whl", hash = "sha256:1e8fe618fe5eeceb827db2b49750fad345be0e6145c5c41ed701ee27e081ff9d", size = 284289, upload-time = "2026-11-09T00:41:48.301Z" }, + { url = "https://files.pythonhosted.org/packages/a6/d6/0e0566737f61954b66df56cee68a7412df3fd8f5a9325f08837cc641b9f5/regex-2026.11.9-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:6dc0dd2d31bc0e2301345c3b20f64a6d864218196c7c9ea2b96788e76b964578", size = 502328, upload-time = "2026-11-09T00:41:49.337Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1e/7f95189e75cd26a634c9832b469f620bf45d02a6e2be184a1169ee7af61a/regex-2026.11.9-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:6a1ee54c1e237fee890ef7045951613147fcf77a2c898229de44f165e5650aff", size = 298773, upload-time = "2026-11-09T00:41:50.737Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cc/7e5406a25fe6cf378e770858133b044e451349ef2c1f2977b68dd061a2db/regex-2026.11.9-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:2bebc58114814ed6d78cc3d27b70d4f2dceb8a2e5d23dcc7d831809e86d3bc3c", size = 296831, upload-time = "2026-11-09T00:41:52.067Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5c/6ad943248f553ce669fd2a6ad75ec27476f90a40e0f5173337db7089c13d/regex-2026.11.9-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9bc2cb2aa025b89ebe26c8dc3c1edb1c6b22be9dadea030a89f3a477faf646e8", size = 818469, upload-time = "2026-11-09T00:41:53.192Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b6/be6a30c8bb5bc32fca59e9e6a7ace27fc5f8626b5b11a6ecca9f2d7834c6/regex-2026.11.9-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499313286c8ae346f19e6af6f78f45c356de293b37540fdea5f4309a5aa59b21", size = 882408, upload-time = "2026-11-09T00:41:54.347Z" }, + { url = "https://files.pythonhosted.org/packages/37/3d/a6fc734740e030e1479ff081c5af669c3543f6f04008c792c5566b30f07a/regex-2026.11.9-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d77014fe27bfc3565a0ef706ced0f7d5471937bce9b7ce38c46deeaa9f4b3963", size = 923235, upload-time = "2026-11-09T00:41:55.551Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7f/c1860577baf5c3acea63fa812b7bb510825b28c44b474db00900e7cb3ab0/regex-2026.11.9-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4498f31d71819a16b32e1a965bb587c7aaf153848c451d1e384ede77dc6a134", size = 822410, upload-time = "2026-11-09T00:41:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/28/af/692419879b97a63e8342ccbd138329f900af38408f0b080d72a5a5fc9160/regex-2026.11.9-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04e1c78cba5e3edd5400e9d7319a3036f652faa599bfabbf888fec584a9a4772", size = 795527, upload-time = "2026-11-09T00:41:58.169Z" }, + { url = "https://files.pythonhosted.org/packages/24/53/1ba04321ce8d15d219710e95f03feab54ca6a19ecad329a42efe7c46c873/regex-2026.11.9-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:5774579fca3431bd77518529d1ba54480d075396bdb0533daa0daa4db86e6594", size = 806153, upload-time = "2026-11-09T00:41:59.309Z" }, + { url = "https://files.pythonhosted.org/packages/e5/dc/3933476fab57134867d20d214d57393fbdb82a5a285083633bdf956372d5/regex-2026.11.9-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:34087251368439c8238cbf2bde88a8e89fd865a2a61266a5c64b360414c7fe04", size = 872784, upload-time = "2026-11-09T00:42:00.636Z" }, + { url = "https://files.pythonhosted.org/packages/e9/53/e44fc4efa4de6dd29fc0f2e237f944a392478611d54d1af43e45c7726626/regex-2026.11.9-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:4cb820ff2fdcf610278c74dcfd45857b5177429e30bd0b7d1164855a011cd58d", size = 784277, upload-time = "2026-11-09T00:42:01.72Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a5/a5e6f6b5904712646ed10023b75746d9492616f75f8ad9d8045fd28f04c5/regex-2026.11.9-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:cb152514d21e3a0440e4c30ff2323f3bbc48de3c1d1aaffe0494470cf5566a42", size = 865578, upload-time = "2026-11-09T00:42:03.197Z" }, + { url = "https://files.pythonhosted.org/packages/09/60/73045aa8b48f37e12f18eb6ee5dee9573a31f381f987f1a2d0ea27310537/regex-2026.11.9-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:38cd15cf8407a4220bf387657cbd38d961161d152c08c731fc2e05fb45a00ab9", size = 809333, upload-time = "2026-11-09T00:42:04.407Z" }, + { url = "https://files.pythonhosted.org/packages/71/de/9f668597839d8abb969846be416ed6982a4ba10b83445f241f27752456cb/regex-2026.11.9-cp315-cp315t-win32.whl", hash = "sha256:c68ed3497146b830d27642193612bb6ac51cb70dd73b5f35f07b37cec915b795", size = 277479, upload-time = "2026-11-09T00:42:05.728Z" }, + { url = "https://files.pythonhosted.org/packages/cd/af/af2858db09db8ff0b9f50e67610da2d71517defc62f6eada14a1405c8930/regex-2026.11.9-cp315-cp315t-win_amd64.whl", hash = "sha256:83ee345b9feb7973c408187778fdf2f2498b54a51bc0b92d08b9b7468ddfb5dd", size = 287021, upload-time = "2026-11-09T00:42:07.23Z" }, + { url = "https://files.pythonhosted.org/packages/05/29/f38c146b8e4b9e9d844cd5387ff82b18a35efe34795366a2a98429695584/regex-2026.11.9-cp315-cp315t-win_arm64.whl", hash = "sha256:2fd69109525129c1823765f34d650247bcd20065eb842832b99d3e512dec0ed7", size = 286551, upload-time = "2026-11-09T00:42:08.629Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1725,6 +1883,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sqlfluff" +version = "4.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "chardet" }, + { name = "click" }, + { name = "colorama" }, + { name = "diff-cover" }, + { name = "jinja2" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "tblib" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/37/6fa8831fad5ffda6e5d3c2629b4c9ed1a4e480b4d8c9f127f132463d6d75/sqlfluff-4.5.0.tar.gz", hash = "sha256:0b0d5b2cca03e7d138b8d899249fed5fa54083e407bbd161449c0b30f901b7b8", size = 1184615, upload-time = "2026-10-30T20:29:12.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/de/78c2f6d023a2dada127baf9a41a7186707d500de89947dd7259d33700bb2/sqlfluff-4.5.0-py3-none-any.whl", hash = "sha256:471a9abcad8061e36b69f711a6d89393241e30f9d1e0869f79965c0ae2c889c7", size = 1152828, upload-time = "2026-10-30T20:29:10.927Z" }, +] + [[package]] name = "starlette" version = "1.10.0" @@ -1755,6 +1935,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ad/97/630d4f115e5a98f6ea195c0e836f0e6332560d53a49ebb364a06ee3b8abe/stellar_sdk-16.1.1-py3-none-any.whl", hash = "sha256:6f6b0661cf0e5e99d1d4f994fdf048ebe4b351afc2f927739e7566426a35e38b", size = 989895, upload-time = "2026-10-22T01:42:52.7Z" }, ] +[[package]] +name = "tblib" +version = "3.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8a/14c15ae154895cc131174f858c707790d416c444fc69f93918adfd8c4c0b/tblib-3.2.2.tar.gz", hash = "sha256:e9a652692d91bf4f743d4a15bc174c0b76afc750fe8c7b6d195cc1c1d6d2ccec", size = 35046, upload-time = "2025-11-12T12:21:16.572Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/be/5d2d47b1fb58943194fb59dcf222f7c4e35122ec0ffe8c36e18b5d728f0b/tblib-3.2.2-py3-none-any.whl", hash = "sha256:26bdccf339bcce6a88b2b5432c988b266ebbe63a4e593f6b578b1d2e723d2b76", size = 12893, upload-time = "2025-11-12T12:21:14.407Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, +] + [[package]] name = "truststore" version = "0.10.5" From 5d8ec5bec00553a10057c7cb17be3d834e3fddb3 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:28:24 +0100 Subject: [PATCH 03/18] feat: describe the database schema as sql files --- apps/api/db/migrations/000001.sql | 25 ++++++++++++++ apps/api/db/schema/functions/method_rank.sql | 14 ++++++++ .../api/db/schema/indexes/accounts_funder.sql | 1 + .../db/schema/indexes/classic_ops_asset.sql | 1 + .../db/schema/indexes/classic_ops_source.sql | 1 + .../schema/indexes/contracts_deployed_at.sql | 1 + .../db/schema/indexes/contracts_deployer.sql | 1 + apps/api/db/schema/indexes/contracts_wasm.sql | 1 + .../db/schema/indexes/failed_txs_source.sql | 1 + .../db/schema/indexes/repo_agents_agent.sql | 1 + .../indexes/repo_contributors_login.sql | 1 + .../indexes/repo_integrations_partner.sql | 1 + .../api/db/schema/indexes/repo_refs_value.sql | 1 + .../api/db/schema/indexes/repo_tools_tool.sql | 1 + .../db/schema/indexes/wasm_uploads_hash.sql | 1 + .../schema/indexes/wasm_uploads_uploader.sql | 1 + apps/api/db/schema/manifest.sql | 33 +++++++++++++++++++ apps/api/db/schema/tables/accounts.sql | 8 +++++ apps/api/db/schema/tables/classic_ops.sql | 11 +++++++ apps/api/db/schema/tables/contracts.sql | 9 +++++ apps/api/db/schema/tables/failed_txs.sql | 9 +++++ apps/api/db/schema/tables/github_users.sql | 7 ++++ apps/api/db/schema/tables/links.sql | 7 ++++ apps/api/db/schema/tables/projects.sql | 15 +++++++++ apps/api/db/schema/tables/repo_agents.sql | 8 +++++ .../db/schema/tables/repo_contributors.sql | 8 +++++ .../db/schema/tables/repo_integrations.sql | 8 +++++ apps/api/db/schema/tables/repo_refs.sql | 7 ++++ apps/api/db/schema/tables/repo_tools.sql | 8 +++++ apps/api/db/schema/tables/repos.sql | 27 +++++++++++++++ apps/api/db/schema/tables/sync_state.sql | 5 +++ apps/api/db/schema/tables/wasm_repos.sql | 6 ++++ apps/api/db/schema/tables/wasm_uploads.sql | 11 +++++++ apps/api/db/schema/views/account_projects.sql | 6 ++++ apps/api/db/schema/views/attribution.sql | 9 +++++ 35 files changed, 255 insertions(+) create mode 100644 apps/api/db/migrations/000001.sql create mode 100644 apps/api/db/schema/functions/method_rank.sql create mode 100644 apps/api/db/schema/indexes/accounts_funder.sql create mode 100644 apps/api/db/schema/indexes/classic_ops_asset.sql create mode 100644 apps/api/db/schema/indexes/classic_ops_source.sql create mode 100644 apps/api/db/schema/indexes/contracts_deployed_at.sql create mode 100644 apps/api/db/schema/indexes/contracts_deployer.sql create mode 100644 apps/api/db/schema/indexes/contracts_wasm.sql create mode 100644 apps/api/db/schema/indexes/failed_txs_source.sql create mode 100644 apps/api/db/schema/indexes/repo_agents_agent.sql create mode 100644 apps/api/db/schema/indexes/repo_contributors_login.sql create mode 100644 apps/api/db/schema/indexes/repo_integrations_partner.sql create mode 100644 apps/api/db/schema/indexes/repo_refs_value.sql create mode 100644 apps/api/db/schema/indexes/repo_tools_tool.sql create mode 100644 apps/api/db/schema/indexes/wasm_uploads_hash.sql create mode 100644 apps/api/db/schema/indexes/wasm_uploads_uploader.sql create mode 100644 apps/api/db/schema/manifest.sql create mode 100644 apps/api/db/schema/tables/accounts.sql create mode 100644 apps/api/db/schema/tables/classic_ops.sql create mode 100644 apps/api/db/schema/tables/contracts.sql create mode 100644 apps/api/db/schema/tables/failed_txs.sql create mode 100644 apps/api/db/schema/tables/github_users.sql create mode 100644 apps/api/db/schema/tables/links.sql create mode 100644 apps/api/db/schema/tables/projects.sql create mode 100644 apps/api/db/schema/tables/repo_agents.sql create mode 100644 apps/api/db/schema/tables/repo_contributors.sql create mode 100644 apps/api/db/schema/tables/repo_integrations.sql create mode 100644 apps/api/db/schema/tables/repo_refs.sql create mode 100644 apps/api/db/schema/tables/repo_tools.sql create mode 100644 apps/api/db/schema/tables/repos.sql create mode 100644 apps/api/db/schema/tables/sync_state.sql create mode 100644 apps/api/db/schema/tables/wasm_repos.sql create mode 100644 apps/api/db/schema/tables/wasm_uploads.sql create mode 100644 apps/api/db/schema/views/account_projects.sql create mode 100644 apps/api/db/schema/views/attribution.sql diff --git a/apps/api/db/migrations/000001.sql b/apps/api/db/migrations/000001.sql new file mode 100644 index 0000000..ebeda57 --- /dev/null +++ b/apps/api/db/migrations/000001.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE FUNCTION method_rank( + method text +) +RETURNS integer +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT array_position(ARRAY[ + 'contract_in_repo', + 'wasm_hash_match', + 'wasm_meta', + 'asset_in_repo', + 'deployer_in_repo', + 'footprint' + ], method) +$$; + +CREATE OR REPLACE VIEW attribution AS + SELECT DISTINCT ON (l.subject_kind, l.subject) l.subject_kind, + l.subject, + l.full_name, + l.method + FROM links l + JOIN repos r ON l.full_name = r.full_name + ORDER BY l.subject_kind, l.subject, (method_rank(l.method)), r.created_at, l.full_name; diff --git a/apps/api/db/schema/functions/method_rank.sql b/apps/api/db/schema/functions/method_rank.sql new file mode 100644 index 0000000..9c65673 --- /dev/null +++ b/apps/api/db/schema/functions/method_rank.sql @@ -0,0 +1,14 @@ +CREATE OR REPLACE FUNCTION method_rank(method text) +RETURNS integer +LANGUAGE sql +IMMUTABLE +AS $$ + SELECT array_position(ARRAY[ + 'contract_in_repo', + 'wasm_hash_match', + 'wasm_meta', + 'asset_in_repo', + 'deployer_in_repo', + 'footprint' + ], method) +$$; diff --git a/apps/api/db/schema/indexes/accounts_funder.sql b/apps/api/db/schema/indexes/accounts_funder.sql new file mode 100644 index 0000000..2e5ba25 --- /dev/null +++ b/apps/api/db/schema/indexes/accounts_funder.sql @@ -0,0 +1 @@ +CREATE INDEX accounts_funder ON accounts (funder); diff --git a/apps/api/db/schema/indexes/classic_ops_asset.sql b/apps/api/db/schema/indexes/classic_ops_asset.sql new file mode 100644 index 0000000..ccf3c64 --- /dev/null +++ b/apps/api/db/schema/indexes/classic_ops_asset.sql @@ -0,0 +1 @@ +CREATE INDEX classic_ops_asset ON classic_ops (asset) WHERE (asset IS NOT NULL); diff --git a/apps/api/db/schema/indexes/classic_ops_source.sql b/apps/api/db/schema/indexes/classic_ops_source.sql new file mode 100644 index 0000000..3b187cb --- /dev/null +++ b/apps/api/db/schema/indexes/classic_ops_source.sql @@ -0,0 +1 @@ +CREATE INDEX classic_ops_source ON classic_ops (source_account); diff --git a/apps/api/db/schema/indexes/contracts_deployed_at.sql b/apps/api/db/schema/indexes/contracts_deployed_at.sql new file mode 100644 index 0000000..5608d33 --- /dev/null +++ b/apps/api/db/schema/indexes/contracts_deployed_at.sql @@ -0,0 +1 @@ +CREATE INDEX contracts_deployed_at ON contracts (deployed_at); diff --git a/apps/api/db/schema/indexes/contracts_deployer.sql b/apps/api/db/schema/indexes/contracts_deployer.sql new file mode 100644 index 0000000..2fd1506 --- /dev/null +++ b/apps/api/db/schema/indexes/contracts_deployer.sql @@ -0,0 +1 @@ +CREATE INDEX contracts_deployer ON contracts (deployer); diff --git a/apps/api/db/schema/indexes/contracts_wasm.sql b/apps/api/db/schema/indexes/contracts_wasm.sql new file mode 100644 index 0000000..55aa767 --- /dev/null +++ b/apps/api/db/schema/indexes/contracts_wasm.sql @@ -0,0 +1 @@ +CREATE INDEX contracts_wasm ON contracts (wasm_hash); diff --git a/apps/api/db/schema/indexes/failed_txs_source.sql b/apps/api/db/schema/indexes/failed_txs_source.sql new file mode 100644 index 0000000..6fb57f2 --- /dev/null +++ b/apps/api/db/schema/indexes/failed_txs_source.sql @@ -0,0 +1 @@ +CREATE INDEX failed_txs_source ON failed_txs (source_account); diff --git a/apps/api/db/schema/indexes/repo_agents_agent.sql b/apps/api/db/schema/indexes/repo_agents_agent.sql new file mode 100644 index 0000000..2fc287a --- /dev/null +++ b/apps/api/db/schema/indexes/repo_agents_agent.sql @@ -0,0 +1 @@ +CREATE INDEX repo_agents_agent ON repo_agents (agent); diff --git a/apps/api/db/schema/indexes/repo_contributors_login.sql b/apps/api/db/schema/indexes/repo_contributors_login.sql new file mode 100644 index 0000000..12953dc --- /dev/null +++ b/apps/api/db/schema/indexes/repo_contributors_login.sql @@ -0,0 +1 @@ +CREATE INDEX repo_contributors_login ON repo_contributors (login); diff --git a/apps/api/db/schema/indexes/repo_integrations_partner.sql b/apps/api/db/schema/indexes/repo_integrations_partner.sql new file mode 100644 index 0000000..625174e --- /dev/null +++ b/apps/api/db/schema/indexes/repo_integrations_partner.sql @@ -0,0 +1 @@ +CREATE INDEX repo_integrations_partner ON repo_integrations (partner); diff --git a/apps/api/db/schema/indexes/repo_refs_value.sql b/apps/api/db/schema/indexes/repo_refs_value.sql new file mode 100644 index 0000000..9efebb1 --- /dev/null +++ b/apps/api/db/schema/indexes/repo_refs_value.sql @@ -0,0 +1 @@ +CREATE INDEX repo_refs_value ON repo_refs (kind, value); diff --git a/apps/api/db/schema/indexes/repo_tools_tool.sql b/apps/api/db/schema/indexes/repo_tools_tool.sql new file mode 100644 index 0000000..4985444 --- /dev/null +++ b/apps/api/db/schema/indexes/repo_tools_tool.sql @@ -0,0 +1 @@ +CREATE INDEX repo_tools_tool ON repo_tools (tool); diff --git a/apps/api/db/schema/indexes/wasm_uploads_hash.sql b/apps/api/db/schema/indexes/wasm_uploads_hash.sql new file mode 100644 index 0000000..75f80c9 --- /dev/null +++ b/apps/api/db/schema/indexes/wasm_uploads_hash.sql @@ -0,0 +1 @@ +CREATE INDEX wasm_uploads_hash ON wasm_uploads (wasm_hash); diff --git a/apps/api/db/schema/indexes/wasm_uploads_uploader.sql b/apps/api/db/schema/indexes/wasm_uploads_uploader.sql new file mode 100644 index 0000000..497a9f1 --- /dev/null +++ b/apps/api/db/schema/indexes/wasm_uploads_uploader.sql @@ -0,0 +1 @@ +CREATE INDEX wasm_uploads_uploader ON wasm_uploads (uploader); diff --git a/apps/api/db/schema/manifest.sql b/apps/api/db/schema/manifest.sql new file mode 100644 index 0000000..a3e7f03 --- /dev/null +++ b/apps/api/db/schema/manifest.sql @@ -0,0 +1,33 @@ +\i functions/method_rank.sql +\i tables/accounts.sql +\i tables/classic_ops.sql +\i tables/contracts.sql +\i tables/failed_txs.sql +\i tables/github_users.sql +\i tables/links.sql +\i tables/projects.sql +\i tables/repo_agents.sql +\i tables/repo_contributors.sql +\i tables/repo_integrations.sql +\i tables/repo_refs.sql +\i tables/repo_tools.sql +\i tables/repos.sql +\i tables/sync_state.sql +\i tables/wasm_repos.sql +\i tables/wasm_uploads.sql +\i indexes/accounts_funder.sql +\i indexes/classic_ops_asset.sql +\i indexes/classic_ops_source.sql +\i indexes/contracts_deployed_at.sql +\i indexes/contracts_deployer.sql +\i indexes/contracts_wasm.sql +\i indexes/failed_txs_source.sql +\i indexes/repo_agents_agent.sql +\i indexes/repo_contributors_login.sql +\i indexes/repo_integrations_partner.sql +\i indexes/repo_refs_value.sql +\i indexes/repo_tools_tool.sql +\i indexes/wasm_uploads_hash.sql +\i indexes/wasm_uploads_uploader.sql +\i views/attribution.sql +\i views/account_projects.sql diff --git a/apps/api/db/schema/tables/accounts.sql b/apps/api/db/schema/tables/accounts.sql new file mode 100644 index 0000000..7ca8efa --- /dev/null +++ b/apps/api/db/schema/tables/accounts.sql @@ -0,0 +1,8 @@ +CREATE TABLE accounts ( + account text, + funder text NOT NULL, + tx_hash text NOT NULL, + ledger bigint NOT NULL, + funded_at timestamptz NOT NULL, + CONSTRAINT accounts_pkey PRIMARY KEY (account) +); diff --git a/apps/api/db/schema/tables/classic_ops.sql b/apps/api/db/schema/tables/classic_ops.sql new file mode 100644 index 0000000..68e6bec --- /dev/null +++ b/apps/api/db/schema/tables/classic_ops.sql @@ -0,0 +1,11 @@ +CREATE TABLE classic_ops ( + tx_hash text, + op_index integer, + source_account text NOT NULL, + op_type text NOT NULL, + asset text, + counterparty text, + ledger bigint NOT NULL, + occurred_at timestamptz NOT NULL, + CONSTRAINT classic_ops_pkey PRIMARY KEY (tx_hash, op_index) +); diff --git a/apps/api/db/schema/tables/contracts.sql b/apps/api/db/schema/tables/contracts.sql new file mode 100644 index 0000000..e943601 --- /dev/null +++ b/apps/api/db/schema/tables/contracts.sql @@ -0,0 +1,9 @@ +CREATE TABLE contracts ( + contract_id text, + deployer text NOT NULL, + wasm_hash text NOT NULL, + tx_hash text NOT NULL, + ledger bigint NOT NULL, + deployed_at timestamptz NOT NULL, + CONSTRAINT contracts_pkey PRIMARY KEY (contract_id) +); diff --git a/apps/api/db/schema/tables/failed_txs.sql b/apps/api/db/schema/tables/failed_txs.sql new file mode 100644 index 0000000..1b1ee38 --- /dev/null +++ b/apps/api/db/schema/tables/failed_txs.sql @@ -0,0 +1,9 @@ +CREATE TABLE failed_txs ( + tx_hash text, + source_account text NOT NULL, + kind text NOT NULL, + result_code text NOT NULL, + ledger bigint NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT failed_txs_pkey PRIMARY KEY (tx_hash) +); diff --git a/apps/api/db/schema/tables/github_users.sql b/apps/api/db/schema/tables/github_users.sql new file mode 100644 index 0000000..7cc791d --- /dev/null +++ b/apps/api/db/schema/tables/github_users.sql @@ -0,0 +1,7 @@ +CREATE TABLE github_users ( + login text, + location text, + country text, + fetched_at timestamptz DEFAULT now() NOT NULL, + CONSTRAINT github_users_pkey PRIMARY KEY (login) +); diff --git a/apps/api/db/schema/tables/links.sql b/apps/api/db/schema/tables/links.sql new file mode 100644 index 0000000..5f58408 --- /dev/null +++ b/apps/api/db/schema/tables/links.sql @@ -0,0 +1,7 @@ +CREATE TABLE links ( + full_name text, + subject_kind text, + subject text, + method text NOT NULL, + CONSTRAINT links_pkey PRIMARY KEY (subject_kind, subject, full_name) +); diff --git a/apps/api/db/schema/tables/projects.sql b/apps/api/db/schema/tables/projects.sql new file mode 100644 index 0000000..3ed0e02 --- /dev/null +++ b/apps/api/db/schema/tables/projects.sql @@ -0,0 +1,15 @@ +CREATE TABLE projects ( + project_id text, + full_name text, + footprint text NOT NULL, + tier text NOT NULL, + contracts integer NOT NULL, + classic_ops integer NOT NULL, + assets integer NOT NULL, + wasm_uploads integer NOT NULL, + failed_txs integer NOT NULL, + builders integer NOT NULL, + first_activity timestamptz, + last_activity timestamptz, + CONSTRAINT projects_pkey PRIMARY KEY (project_id) +); diff --git a/apps/api/db/schema/tables/repo_agents.sql b/apps/api/db/schema/tables/repo_agents.sql new file mode 100644 index 0000000..c1ebfd9 --- /dev/null +++ b/apps/api/db/schema/tables/repo_agents.sql @@ -0,0 +1,8 @@ +CREATE TABLE repo_agents ( + full_name text, + agent text, + method text, + evidence text, + path text NOT NULL, + CONSTRAINT repo_agents_pkey PRIMARY KEY (full_name, agent, method, evidence) +); diff --git a/apps/api/db/schema/tables/repo_contributors.sql b/apps/api/db/schema/tables/repo_contributors.sql new file mode 100644 index 0000000..24dfa75 --- /dev/null +++ b/apps/api/db/schema/tables/repo_contributors.sql @@ -0,0 +1,8 @@ +CREATE TABLE repo_contributors ( + full_name text, + login text, + url text NOT NULL, + avatar_url text, + contributions integer NOT NULL, + CONSTRAINT repo_contributors_pkey PRIMARY KEY (full_name, login) +); diff --git a/apps/api/db/schema/tables/repo_integrations.sql b/apps/api/db/schema/tables/repo_integrations.sql new file mode 100644 index 0000000..941cd14 --- /dev/null +++ b/apps/api/db/schema/tables/repo_integrations.sql @@ -0,0 +1,8 @@ +CREATE TABLE repo_integrations ( + full_name text, + partner text, + method text, + evidence text, + path text NOT NULL, + CONSTRAINT repo_integrations_pkey PRIMARY KEY (full_name, partner, method, evidence) +); diff --git a/apps/api/db/schema/tables/repo_refs.sql b/apps/api/db/schema/tables/repo_refs.sql new file mode 100644 index 0000000..15ffb73 --- /dev/null +++ b/apps/api/db/schema/tables/repo_refs.sql @@ -0,0 +1,7 @@ +CREATE TABLE repo_refs ( + full_name text, + kind text, + value text, + path text NOT NULL, + CONSTRAINT repo_refs_pkey PRIMARY KEY (full_name, kind, value) +); diff --git a/apps/api/db/schema/tables/repo_tools.sql b/apps/api/db/schema/tables/repo_tools.sql new file mode 100644 index 0000000..7eec3cb --- /dev/null +++ b/apps/api/db/schema/tables/repo_tools.sql @@ -0,0 +1,8 @@ +CREATE TABLE repo_tools ( + full_name text, + tool text, + method text, + evidence text, + path text NOT NULL, + CONSTRAINT repo_tools_pkey PRIMARY KEY (full_name, tool, method, evidence) +); diff --git a/apps/api/db/schema/tables/repos.sql b/apps/api/db/schema/tables/repos.sql new file mode 100644 index 0000000..20012e8 --- /dev/null +++ b/apps/api/db/schema/tables/repos.sql @@ -0,0 +1,27 @@ +CREATE TABLE repos ( + full_name text, + url text NOT NULL, + description text, + stars integer DEFAULT 0 NOT NULL, + owner text NOT NULL, + owner_avatar text, + owner_location text, + country text, + default_branch text, + homepage text, + created_at timestamptz, + pushed_at timestamptz, + discovered_via text[] DEFAULT '{}' NOT NULL, + dependencies jsonb DEFAULT '{}' NOT NULL, + window_commits integer, + total_commits integer, + scanned_at timestamptz, + topics text[] DEFAULT '{}' NOT NULL, + stellar_added_in_window boolean, + language text, + license text, + forks integer DEFAULT 0 NOT NULL, + base_sha text, + declared_track text, + CONSTRAINT repos_pkey PRIMARY KEY (full_name) +); diff --git a/apps/api/db/schema/tables/sync_state.sql b/apps/api/db/schema/tables/sync_state.sql new file mode 100644 index 0000000..8098902 --- /dev/null +++ b/apps/api/db/schema/tables/sync_state.sql @@ -0,0 +1,5 @@ +CREATE TABLE sync_state ( + key text, + value text NOT NULL, + CONSTRAINT sync_state_pkey PRIMARY KEY (key) +); diff --git a/apps/api/db/schema/tables/wasm_repos.sql b/apps/api/db/schema/tables/wasm_repos.sql new file mode 100644 index 0000000..b49fc4f --- /dev/null +++ b/apps/api/db/schema/tables/wasm_repos.sql @@ -0,0 +1,6 @@ +CREATE TABLE wasm_repos ( + wasm_hash text, + full_name text NOT NULL, + source text NOT NULL, + CONSTRAINT wasm_repos_pkey PRIMARY KEY (wasm_hash) +); diff --git a/apps/api/db/schema/tables/wasm_uploads.sql b/apps/api/db/schema/tables/wasm_uploads.sql new file mode 100644 index 0000000..08db759 --- /dev/null +++ b/apps/api/db/schema/tables/wasm_uploads.sql @@ -0,0 +1,11 @@ +CREATE TABLE wasm_uploads ( + tx_hash text, + wasm_hash text NOT NULL, + uploader text NOT NULL, + size bigint NOT NULL, + source_repo text, + meta jsonb NOT NULL, + ledger bigint NOT NULL, + uploaded_at timestamptz NOT NULL, + CONSTRAINT wasm_uploads_pkey PRIMARY KEY (tx_hash) +); diff --git a/apps/api/db/schema/views/account_projects.sql b/apps/api/db/schema/views/account_projects.sql new file mode 100644 index 0000000..57a6bfc --- /dev/null +++ b/apps/api/db/schema/views/account_projects.sql @@ -0,0 +1,6 @@ +CREATE OR REPLACE VIEW account_projects AS +SELECT + subject AS account, + 'repo:'::text || full_name AS project_id +FROM attribution +WHERE subject_kind = 'account'::text; diff --git a/apps/api/db/schema/views/attribution.sql b/apps/api/db/schema/views/attribution.sql new file mode 100644 index 0000000..1a95745 --- /dev/null +++ b/apps/api/db/schema/views/attribution.sql @@ -0,0 +1,9 @@ +CREATE OR REPLACE VIEW attribution AS +SELECT DISTINCT ON (l.subject_kind, l.subject) + l.subject_kind, + l.subject, + l.full_name, + l.method +FROM links AS l +INNER JOIN repos AS r ON l.full_name = r.full_name +ORDER BY l.subject_kind, l.subject, method_rank(l.method), r.created_at, l.full_name; From 1674d73f5cc2f5b9193484b05bc946a2ac0e3dfa Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:28:24 +0100 Subject: [PATCH 04/18] refactor: build the database from the schema files --- apps/api/src/hackproof/db.py | 191 ----------------------------------- apps/api/tests/conftest.py | 5 +- 2 files changed, 3 insertions(+), 193 deletions(-) diff --git a/apps/api/src/hackproof/db.py b/apps/api/src/hackproof/db.py index 51ffc87..310e0d2 100644 --- a/apps/api/src/hackproof/db.py +++ b/apps/api/src/hackproof/db.py @@ -3,199 +3,10 @@ import asyncpg from hackproof.config import get_settings -from hackproof.linking.methods import METHOD_RANK_SQL - -TABLES = """ -CREATE TABLE IF NOT EXISTS contracts ( - contract_id TEXT PRIMARY KEY, - deployer TEXT NOT NULL, - wasm_hash TEXT NOT NULL, - tx_hash TEXT NOT NULL, - ledger BIGINT NOT NULL, - deployed_at TIMESTAMPTZ NOT NULL -); -CREATE INDEX IF NOT EXISTS contracts_deployer ON contracts (deployer); -CREATE INDEX IF NOT EXISTS contracts_wasm ON contracts (wasm_hash); -CREATE INDEX IF NOT EXISTS contracts_deployed_at ON contracts (deployed_at); - -CREATE TABLE IF NOT EXISTS wasm_uploads ( - tx_hash TEXT PRIMARY KEY, - wasm_hash TEXT NOT NULL, - uploader TEXT NOT NULL, - size BIGINT NOT NULL, - source_repo TEXT, - meta JSONB NOT NULL, - ledger BIGINT NOT NULL, - uploaded_at TIMESTAMPTZ NOT NULL -); -CREATE INDEX IF NOT EXISTS wasm_uploads_uploader ON wasm_uploads (uploader); -CREATE INDEX IF NOT EXISTS wasm_uploads_hash ON wasm_uploads (wasm_hash); - -CREATE TABLE IF NOT EXISTS failed_txs ( - tx_hash TEXT PRIMARY KEY, - source_account TEXT NOT NULL, - kind TEXT NOT NULL, - result_code TEXT NOT NULL, - ledger BIGINT NOT NULL, - created_at TIMESTAMPTZ NOT NULL -); -CREATE INDEX IF NOT EXISTS failed_txs_source ON failed_txs (source_account); - -CREATE TABLE IF NOT EXISTS accounts ( - account TEXT PRIMARY KEY, - funder TEXT NOT NULL, - tx_hash TEXT NOT NULL, - ledger BIGINT NOT NULL, - funded_at TIMESTAMPTZ NOT NULL -); -CREATE INDEX IF NOT EXISTS accounts_funder ON accounts (funder); - -CREATE TABLE IF NOT EXISTS classic_ops ( - tx_hash TEXT NOT NULL, - op_index INTEGER NOT NULL, - source_account TEXT NOT NULL, - op_type TEXT NOT NULL, - asset TEXT, - counterparty TEXT, - ledger BIGINT NOT NULL, - occurred_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (tx_hash, op_index) -); -CREATE INDEX IF NOT EXISTS classic_ops_source ON classic_ops (source_account); -CREATE INDEX IF NOT EXISTS classic_ops_asset ON classic_ops (asset) WHERE asset IS NOT NULL; - -CREATE TABLE IF NOT EXISTS repos ( - full_name TEXT PRIMARY KEY, - url TEXT NOT NULL, - description TEXT, - stars INTEGER NOT NULL DEFAULT 0, - owner TEXT NOT NULL, - owner_avatar TEXT, - owner_location TEXT, - country TEXT, - default_branch TEXT, - homepage TEXT, - language TEXT, - license TEXT, - forks INTEGER NOT NULL DEFAULT 0, - created_at TIMESTAMPTZ, - pushed_at TIMESTAMPTZ, - topics TEXT[] NOT NULL DEFAULT '{}', - discovered_via TEXT[] NOT NULL DEFAULT '{}', - dependencies JSONB NOT NULL DEFAULT '{}', - window_commits INTEGER, - base_sha TEXT, - total_commits INTEGER, - stellar_added_in_window BOOLEAN, - scanned_at TIMESTAMPTZ, - declared_track TEXT -); -ALTER TABLE repos ADD COLUMN IF NOT EXISTS declared_track TEXT; - -CREATE TABLE IF NOT EXISTS repo_contributors ( - full_name TEXT NOT NULL, - login TEXT NOT NULL, - url TEXT NOT NULL, - avatar_url TEXT, - contributions INTEGER NOT NULL, - PRIMARY KEY (full_name, login) -); -CREATE INDEX IF NOT EXISTS repo_contributors_login ON repo_contributors (login); - -CREATE TABLE IF NOT EXISTS github_users ( - login TEXT PRIMARY KEY, - location TEXT, - country TEXT, - fetched_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE TABLE IF NOT EXISTS repo_refs ( - full_name TEXT NOT NULL, - kind TEXT NOT NULL, - value TEXT NOT NULL, - path TEXT NOT NULL, - PRIMARY KEY (full_name, kind, value) -); -CREATE INDEX IF NOT EXISTS repo_refs_value ON repo_refs (kind, value); - -CREATE TABLE IF NOT EXISTS repo_integrations ( - full_name TEXT NOT NULL, - partner TEXT NOT NULL, - method TEXT NOT NULL, - evidence TEXT NOT NULL, - path TEXT NOT NULL, - PRIMARY KEY (full_name, partner, method, evidence) -); -CREATE INDEX IF NOT EXISTS repo_integrations_partner ON repo_integrations (partner); - -CREATE TABLE IF NOT EXISTS repo_tools ( - full_name TEXT NOT NULL, - tool TEXT NOT NULL, - method TEXT NOT NULL, - evidence TEXT NOT NULL, - path TEXT NOT NULL, - PRIMARY KEY (full_name, tool, method, evidence) -); -CREATE INDEX IF NOT EXISTS repo_tools_tool ON repo_tools (tool); - -CREATE TABLE IF NOT EXISTS repo_agents ( - full_name TEXT NOT NULL, - agent TEXT NOT NULL, - method TEXT NOT NULL, - evidence TEXT NOT NULL, - path TEXT NOT NULL, - PRIMARY KEY (full_name, agent, method, evidence) -); -CREATE INDEX IF NOT EXISTS repo_agents_agent ON repo_agents (agent); - -CREATE TABLE IF NOT EXISTS links ( - full_name TEXT NOT NULL, - subject_kind TEXT NOT NULL, - subject TEXT NOT NULL, - method TEXT NOT NULL, - PRIMARY KEY (subject_kind, subject, full_name) -); - -CREATE TABLE IF NOT EXISTS projects ( - project_id TEXT PRIMARY KEY, - full_name TEXT, - footprint TEXT NOT NULL, - tier TEXT NOT NULL, - contracts INTEGER NOT NULL, - classic_ops INTEGER NOT NULL, - assets INTEGER NOT NULL, - wasm_uploads INTEGER NOT NULL, - failed_txs INTEGER NOT NULL, - builders INTEGER NOT NULL, - first_activity TIMESTAMPTZ, - last_activity TIMESTAMPTZ -); - -CREATE TABLE IF NOT EXISTS sync_state ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); -""" _pool: asyncpg.Pool | None = None -VIEWS = f""" -CREATE OR REPLACE VIEW attribution AS -SELECT DISTINCT ON (l.subject_kind, l.subject) - l.subject_kind, l.subject, l.full_name, l.method -FROM links l -JOIN repos r ON r.full_name = l.full_name -ORDER BY l.subject_kind, l.subject, {METHOD_RANK_SQL}, r.created_at NULLS LAST, l.full_name; - -CREATE OR REPLACE VIEW account_projects AS -SELECT subject AS account, 'repo:' || full_name AS project_id -FROM attribution WHERE subject_kind = 'account'; -""" - -SCHEMA = TABLES + VIEWS - - async def _init_connection(connection: asyncpg.Connection) -> None: await connection.set_type_codec( "jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog" @@ -207,8 +18,6 @@ async def open_pool() -> asyncpg.Pool: _pool = await asyncpg.create_pool( get_settings().database_url, min_size=1, max_size=10, init=_init_connection ) - async with _pool.acquire() as connection: - await connection.execute(SCHEMA) return _pool diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py index 51d30d4..fdbf25a 100644 --- a/apps/api/tests/conftest.py +++ b/apps/api/tests/conftest.py @@ -7,7 +7,8 @@ import pytest from hackproof.config import get_settings -from hackproof.db import SCHEMA +from hackproof.db_utils.manifest import schema_sql +from hackproof.db_utils.pgschema import SCHEMA_DIR @pytest.fixture @@ -22,7 +23,7 @@ async def scratch_db() -> AsyncIterator[asyncpg.Connection]: await connection.set_type_codec( "jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog" ) - await connection.execute(SCHEMA) + await connection.execute(schema_sql(SCHEMA_DIR)) yield connection await connection.close() await admin.execute(f'DROP DATABASE "{name}"') From 16852c2be0aa8d33d956cb58d85caacad38ab09b Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:28:38 +0100 Subject: [PATCH 05/18] feat: add the pgschema commands with a guard against data loss --- apps/api/pyproject.toml | 8 - apps/api/src/hackproof/cli.py | 53 +++++ apps/api/src/hackproof/config.py | 1 + apps/api/src/hackproof/db_utils/connection.py | 50 +++++ apps/api/src/hackproof/db_utils/guard.py | 43 ++++ apps/api/src/hackproof/db_utils/manifest.py | 89 +++++++++ apps/api/src/hackproof/db_utils/pgschema.py | 184 ++++++++++++++++++ apps/api/src/hackproof/db_utils/sql_files.py | 11 -- .../src/hackproof/db_utils/sqlfluff/rules.py | 11 +- apps/api/tests/test_pgschema.py | 148 ++++++++++++++ 10 files changed, 569 insertions(+), 29 deletions(-) create mode 100644 apps/api/src/hackproof/db_utils/connection.py create mode 100644 apps/api/src/hackproof/db_utils/guard.py create mode 100644 apps/api/src/hackproof/db_utils/manifest.py create mode 100644 apps/api/src/hackproof/db_utils/pgschema.py delete mode 100644 apps/api/src/hackproof/db_utils/sql_files.py create mode 100644 apps/api/tests/test_pgschema.py diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index d1b5c9c..fbed6d1 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -45,28 +45,20 @@ line-length = 88 [tool.ruff.lint] select = ["E", "F", "I", "UP", "B"] -# E501: the formatter wraps what it can, long SQL and message strings may stay long. ignore = ["E501"] -# D209: closing """ goes on its own line (https://docs.astral.sh/ruff/rules/new-line-after-last-paragraph/) -# D213: docstring summary starts on the line after the opening """ (https://docs.astral.sh/ruff/rules/multi-line-summary-second-line/) extend-select = ["TID", "D209", "D213"] [tool.ruff.lint.isort] -# Keep `from x import a, b as c` on one line instead of splitting the alias out. combine-as-imports = true [tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "parents" [tool.ruff.format] -# Add a trailing comma to function defs with two or more args to enforce multiline formatting in ruff. skip-magic-trailing-comma = false [tool.sqlfluff.core] dialect = "postgres" -# Rule codes: https://docs.sqlfluff.com/en/stable/reference/rules.html -# LT05: comments may be longer than 80 characters. -# RF04: flags legitimate column names like `version` as reserved keywords. exclude_rules = "LT05,RF04" [tool.sqlfluff.rules.capitalisation.keywords] diff --git a/apps/api/src/hackproof/cli.py b/apps/api/src/hackproof/cli.py index 62c9715..8425d4e 100644 --- a/apps/api/src/hackproof/cli.py +++ b/apps/api/src/hackproof/cli.py @@ -1,13 +1,40 @@ import argparse import asyncio import logging +import os +from pathlib import Path +from hackproof.config import get_settings +from hackproof.db_utils import pgschema +from hackproof.db_utils.connection import Connection from hackproof.export.jobs import run_check, run_export from hackproof.github.jobs import run_discover, run_scan from hackproof.indexer.sync import run_sync from hackproof.linking.reconcile import run_reconcile +def run_schema(args: argparse.Namespace) -> int: + connection = Connection.from_url(get_settings().database_url) + match args.action: + case "plan": + return pgschema.plan(connection) + case "apply": + if not (connection.is_local or args.remote or os.environ.get("CI")): + print( + f"{connection.label} is not on this machine. Pass --remote to apply " + "to it, or let CI do it on merge." + ) + return 1 + return pgschema.apply( + connection, + allow_destructive=args.allow_destructive, + out_dir=args.out_dir, + ) + case "verify": + return pgschema.verify(connection) + return 1 + + def main() -> None: parser = argparse.ArgumentParser(prog="hackproof") commands = parser.add_subparsers(dest="command", required=True) @@ -39,6 +66,30 @@ def main() -> None: "check", help="check, read-only, that the database matches the committed data files", ) + schema = commands.add_parser( + "schema", help="manage the database schema with pgschema" + ) + actions = schema.add_subparsers(dest="action", required=True) + actions.add_parser( + "plan", help="write the pending changes to the next numbered migration file" + ) + apply = actions.add_parser( + "apply", help="apply the schema files, refusing any change that could lose data" + ) + apply.add_argument( + "--allow-destructive", + action="store_true", + help="apply a plan that drops or rewrites data", + ) + apply.add_argument( + "--remote", + action="store_true", + help="allow a database that is not on this machine", + ) + apply.add_argument("--out-dir", type=Path, help="keep plan.json and plan.sql here") + actions.add_parser( + "verify", help="fail unless the manifest is current and nothing is pending" + ) args = parser.parse_args() logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") @@ -56,3 +107,5 @@ def main() -> None: asyncio.run(run_export()) case "check": raise SystemExit(asyncio.run(run_check())) + case "schema": + raise SystemExit(run_schema(args)) diff --git a/apps/api/src/hackproof/config.py b/apps/api/src/hackproof/config.py index 616c222..ad0b1ca 100644 --- a/apps/api/src/hackproof/config.py +++ b/apps/api/src/hackproof/config.py @@ -25,6 +25,7 @@ class Settings(BaseSettings): integrations_file: Path = ROOT / "data" / "integrations.yaml" friendbot_account: str = "GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3QJZNSR" rpc_concurrency: int = 6 + pgschema_image: str = "pgplex/pgschema:1.14.1" anthropic_api_key: str | None = Field( default=None, validation_alias=AliasChoices( diff --git a/apps/api/src/hackproof/db_utils/connection.py b/apps/api/src/hackproof/db_utils/connection.py new file mode 100644 index 0000000..453702d --- /dev/null +++ b/apps/api/src/hackproof/db_utils/connection.py @@ -0,0 +1,50 @@ +import sys +from typing import Self +from urllib.parse import parse_qs, unquote, urlparse + +from pydantic import BaseModel + +LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1", "host.docker.internal"} + + +class Connection(BaseModel): + """One database, as pgschema's container needs to reach it.""" + + host: str + port: int = 5432 + user: str + password: str + database: str + sslmode: str = "prefer" + + @classmethod + def from_url(cls, url: str) -> Self: + parsed = urlparse(url) + query = parse_qs(parsed.query) + return cls( + host=parsed.hostname or "localhost", + port=parsed.port or 5432, + user=unquote(parsed.username or ""), + password=unquote(parsed.password or ""), + database=parsed.path.lstrip("/"), + sslmode=query.get("sslmode", ["prefer"])[0], + ) + + @property + def is_local(self) -> bool: + return self.host in LOCAL_HOSTS + + @property + def label(self) -> str: + return f"{self.user}@{self.host}:{self.port}/{self.database}" + + def container_route(self) -> tuple[list[str], str]: + """Docker network flags and the host name the container should dial.""" + if not self.is_local: + return [], self.host + if sys.platform == "linux": + return ["--network", "host"], "localhost" + return [ + "--add-host", + "host.docker.internal:host-gateway", + ], "host.docker.internal" diff --git a/apps/api/src/hackproof/db_utils/guard.py b/apps/api/src/hackproof/db_utils/guard.py new file mode 100644 index 0000000..f386f49 --- /dev/null +++ b/apps/api/src/hackproof/db_utils/guard.py @@ -0,0 +1,43 @@ +import re + +from pydantic import BaseModel + +RULES = [ + (r"\bDROP\s+TABLE\b", "drops a table and every row in it"), + (r"\bDROP\s+COLUMN\b", "drops a column and its data"), + (r"\bDROP\s+TYPE\b", "drops a type"), + (r"\bDROP\s+SCHEMA\b", "drops a schema"), + (r"\bDROP\s+SEQUENCE\b", "drops a sequence"), + (r"\bDROP\s+MATERIALIZED\s+VIEW\b", "drops stored view data"), + ( + r"\bALTER\s+COLUMN\b[^;]*\bTYPE\b", + "changes a column type and can rewrite or lose data", + ), + (r"\bTRUNCATE\b", "empties a table"), + (r"\bDELETE\s+FROM\b", "deletes rows"), + (r"\bUPDATE\s+\S+\s+SET\b", "rewrites rows"), +] + + +class Finding(BaseModel): + reason: str + statement: str + + +def strip_comments(sql: str) -> str: + return re.sub(r"--[^\n]*", "", sql) + + +def destructive_statements(sql: str) -> list[Finding]: + """ + Every statement in a plan that could lose data. Dropping an index, view, + trigger, function or constraint loses none, so those pass. + """ + findings = [] + for statement in strip_comments(sql).split(";"): + for pattern, reason in RULES: + if re.search(pattern, statement, re.IGNORECASE): + findings.append( + Finding(reason=reason, statement=" ".join(statement.split())[:200]) + ) + return findings diff --git a/apps/api/src/hackproof/db_utils/manifest.py b/apps/api/src/hackproof/db_utils/manifest.py new file mode 100644 index 0000000..267d6fe --- /dev/null +++ b/apps/api/src/hackproof/db_utils/manifest.py @@ -0,0 +1,89 @@ +import re +from collections.abc import Callable +from pathlib import Path + +LAYERS = [ + "types", + "functions", + "tables", + "constraints", + "indexes", + "views", + "triggers", +] + +OBJECT_RE = { + "tables": re.compile(r'CREATE TABLE( IF NOT EXISTS)? "?([A-Za-z_][A-Za-z0-9_]*)"?'), + "views": re.compile( + r'CREATE (?:OR REPLACE )?VIEW "?([A-Za-z_][A-Za-z0-9_]*)"?', re.IGNORECASE + ), +} +REFERENCE_RE = { + "tables": re.compile(r'REFERENCES "?([A-Za-z_][A-Za-z0-9_]*)"?'), + "views": re.compile(r'(?:FROM|JOIN)\s+"?([A-Za-z_][A-Za-z0-9_]*)"?', re.IGNORECASE), +} + + +def object_name(layer: str, path: Path) -> str | None: + match = OBJECT_RE[layer].search(path.read_text()) + return match.group(match.lastindex or 1) if match else None + + +def order_by_dependency(layer: str, directory: Path) -> list[Path]: + """ + Objects another object points at must load first, and file names alone do not + guarantee that. Tables point at each other through REFERENCES, views through + FROM and JOIN. + """ + files = sorted(directory.glob("*.sql")) + by_name = {name: f for f in files if (name := object_name(layer, f))} + ordered: list[Path] = [] + visited: set[str] = set() + + def visit(name: str) -> None: + if name in visited: + return + visited.add(name) + path = by_name[name] + for dependency in REFERENCE_RE[layer].findall(path.read_text()): + if dependency != name and dependency in by_name: + visit(dependency) + ordered.append(path) + + for name in by_name: + visit(name) + return ordered + + +def layer_files(schema_dir: Path, layer: str) -> list[Path]: + directory = schema_dir / layer + if not directory.is_dir(): + return [] + ordering: Callable[[], list[Path]] = ( + (lambda: order_by_dependency(layer, directory)) + if layer in OBJECT_RE + else (lambda: sorted(directory.glob("*.sql"))) + ) + return ordering() + + +def manifest_text(schema_dir: Path) -> str: + lines = [ + f"\\i {layer}/{path.name}" + for layer in LAYERS + for path in layer_files(schema_dir, layer) + ] + return "\n".join(lines) + "\n" if lines else "" + + +def generate_manifest(schema_dir: Path) -> None: + (schema_dir / "manifest.sql").write_text(manifest_text(schema_dir)) + + +def schema_sql(schema_dir: Path) -> str: + paths = [ + schema_dir / line.removeprefix("\\i ").strip() + for line in (schema_dir / "manifest.sql").read_text().splitlines() + if line.startswith("\\i ") + ] + return "\n".join(path.read_text() for path in paths) diff --git a/apps/api/src/hackproof/db_utils/pgschema.py b/apps/api/src/hackproof/db_utils/pgschema.py new file mode 100644 index 0000000..c55eeb1 --- /dev/null +++ b/apps/api/src/hackproof/db_utils/pgschema.py @@ -0,0 +1,184 @@ +import os +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +from hackproof.config import ROOT, get_settings +from hackproof.db_utils.connection import Connection +from hackproof.db_utils.guard import destructive_statements +from hackproof.db_utils.manifest import generate_manifest, manifest_text + +API_DIR = ROOT / "apps" / "api" +DB_DIR = API_DIR / "db" +SCHEMA_DIR = DB_DIR / "schema" +MIGRATIONS_DIR = DB_DIR / "migrations" +MANIFEST = "/db/schema/manifest.sql" + + +def docker_command( + command: str, + *args: str, + connection: Connection, + mounts: dict[Path, str], + image: str, +) -> list[str]: + network, host = connection.container_route() + mount_flags = [ + flag + for source, target in mounts.items() + for flag in ("-v", f"{source}:{target}") + ] + return [ + "docker", + "run", + "--rm", + "-i", + *network, + "-e", + "PGPASSWORD", + *mount_flags, + image, + command, + "--host", + host, + "--port", + str(connection.port), + "--db", + connection.database, + "--user", + connection.user, + "--schema", + "public", + "--sslmode", + connection.sslmode, + *args, + ] + + +def run_pgschema( + command: str, + *args: str, + connection: Connection, + mounts: dict[Path, str] | None = None, +) -> int: + cmd = docker_command( + command, + *args, + connection=connection, + mounts={DB_DIR: "/db", **(mounts or {})}, + image=get_settings().pgschema_image, + ) + env = {**os.environ, "PGPASSWORD": connection.password} + return subprocess.run(cmd, env=env, check=False).returncode + + +def format_schema() -> None: + """Fix what sqlfluff can, then lint so what is left shows its file and rule.""" + for action in ("fix", "lint"): + subprocess.run( + [sys.executable, "-m", "sqlfluff", action, str(SCHEMA_DIR)], + cwd=API_DIR, + check=False, + ) + + +def next_migration_path() -> Path: + MIGRATIONS_DIR.mkdir(parents=True, exist_ok=True) + numbers = [ + int(f.stem) for f in MIGRATIONS_DIR.glob("[0-9][0-9][0-9][0-9][0-9][0-9].sql") + ] + return MIGRATIONS_DIR / f"{max(numbers, default=0) + 1:06d}.sql" + + +def plan_sql(connection: Connection, out_dir: Path) -> tuple[int, str]: + returncode = run_pgschema( + "plan", + "--file", + MANIFEST, + "--output-json", + "/out/plan.json", + "--output-sql", + "/out/plan.sql", + connection=connection, + mounts={out_dir: "/out"}, + ) + sql_file = out_dir / "plan.sql" + return returncode, sql_file.read_text() if sql_file.is_file() else "" + + +def plan(connection: Connection, *, format: bool = True) -> int: + if format: + format_schema() + generate_manifest(SCHEMA_DIR) + path = next_migration_path() + returncode = run_pgschema( + "plan", + "--file", + MANIFEST, + "--output-sql", + f"/db/migrations/{path.name}", + connection=connection, + ) + if returncode != 0: + return returncode + if not path.is_file() or path.stat().st_size == 0: + path.unlink(missing_ok=True) + print("No changes detected.") + return 0 + print(f"Wrote the migration plan to {path.relative_to(ROOT)}") + return 0 + + +def apply( + connection: Connection, + *, + allow_destructive: bool = False, + out_dir: Path | None = None, +) -> int: + generate_manifest(SCHEMA_DIR) + with TemporaryDirectory() as scratch: + target = out_dir or Path(scratch) + target.mkdir(parents=True, exist_ok=True) + returncode, sql = plan_sql(connection, target) + if returncode != 0: + return returncode + if not sql.strip(): + print(f"No changes: {connection.label} already matches the schema files.") + return 0 + print(f"Pending changes for {connection.label}:\n\n{sql}") + findings = destructive_statements(sql) + if findings and not allow_destructive: + print("Refusing to apply: the plan could lose data.", file=sys.stderr) + for finding in findings: + print(f" {finding.reason}: {finding.statement}", file=sys.stderr) + return 2 + return run_pgschema( + "apply", + "--plan", + "/out/plan.json", + "--auto-approve", + connection=connection, + mounts={target: "/out"}, + ) + + +def verify(connection: Connection) -> int: + manifest = SCHEMA_DIR / "manifest.sql" + if not manifest.is_file() or manifest.read_text() != manifest_text(SCHEMA_DIR): + print( + "manifest.sql is out of date: run `hackproof schema plan`.", file=sys.stderr + ) + return 1 + with TemporaryDirectory() as scratch: + returncode, sql = plan_sql(connection, Path(scratch)) + if returncode != 0: + return returncode + if sql.strip(): + print( + f"{connection.label} differs from the schema files:\n\n{sql}", + file=sys.stderr, + ) + return 1 + print(f"In sync: {connection.label} matches the schema files.") + return 0 diff --git a/apps/api/src/hackproof/db_utils/sql_files.py b/apps/api/src/hackproof/db_utils/sql_files.py deleted file mode 100644 index 4858af5..0000000 --- a/apps/api/src/hackproof/db_utils/sql_files.py +++ /dev/null @@ -1,11 +0,0 @@ -import re -from pathlib import Path - -TABLE_NAME_RE = re.compile( - r'CREATE TABLE( IF NOT EXISTS)? "?([A-Za-z_][A-Za-z0-9_]*)"?' -) - - -def table_name(path: Path) -> str | None: - match = TABLE_NAME_RE.search(path.read_text()) - return match.group(2) if match else None diff --git a/apps/api/src/hackproof/db_utils/sqlfluff/rules.py b/apps/api/src/hackproof/db_utils/sqlfluff/rules.py index c195349..3a3b513 100644 --- a/apps/api/src/hackproof/db_utils/sqlfluff/rules.py +++ b/apps/api/src/hackproof/db_utils/sqlfluff/rules.py @@ -3,16 +3,7 @@ class Rule_Hackproof_CT01(BaseRule): - """Enforce upper-case CREATE TYPE names. - - Problem: sqlfluff's built-in CP05 only flags inconsistent type casing within - a single file, so a CREATE TYPE declaration living alone in its own - schema/types/.sql file never has anything to compare against and is - never checked. - Solution: this rule checks every CREATE TYPE name directly against a fixed - upper-case policy, independent of what else does or doesn't appear in the - same file. - """ # noqa: D213 -- summary must stay on this line so `sqlfluff rules` can show it as the description + """Enforce upper-case CREATE TYPE names.""" name = "custom.create_type_upper" groups = ("all",) diff --git a/apps/api/tests/test_pgschema.py b/apps/api/tests/test_pgschema.py new file mode 100644 index 0000000..4cfe4d5 --- /dev/null +++ b/apps/api/tests/test_pgschema.py @@ -0,0 +1,148 @@ +import sys +from pathlib import Path + +import pytest + +from hackproof.db_utils import pgschema +from hackproof.db_utils.connection import Connection +from hackproof.db_utils.guard import destructive_statements +from hackproof.db_utils.manifest import manifest_text, order_by_dependency +from hackproof.linking.methods import LinkMethod + +NEON = "postgresql://owner:p%40ss@ep-cool-1234.us-east-2.aws.neon.tech/hackproof?sslmode=require&channel_binding=require" + + +def write(root: Path, files: dict[str, str]) -> Path: + for name, text in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return root + + +def test_a_connection_is_read_from_a_url(): + connection = Connection.from_url(NEON) + + assert connection.host == "ep-cool-1234.us-east-2.aws.neon.tech" + assert connection.port == 5432 + assert connection.user == "owner" + assert connection.password == "p@ss" + assert connection.database == "hackproof" + assert connection.sslmode == "require" + assert not connection.is_local + + +def test_a_local_database_is_reached_through_the_host_the_container_can_see( + monkeypatch, +): + local = Connection.from_url("postgresql://u:p@localhost:5440/hackproof") + + monkeypatch.setattr(sys, "platform", "darwin") + assert local.container_route() == ( + ["--add-host", "host.docker.internal:host-gateway"], + "host.docker.internal", + ) + monkeypatch.setattr(sys, "platform", "linux") + assert local.container_route() == (["--network", "host"], "localhost") + assert Connection.from_url(NEON).container_route() == ( + [], + "ep-cool-1234.us-east-2.aws.neon.tech", + ) + + +def test_the_password_never_appears_in_the_docker_command(tmp_path): + connection = Connection.from_url(NEON) + + command = pgschema.docker_command( + "plan", + "--file", + "/db/schema/manifest.sql", + connection=connection, + mounts={tmp_path: "/db"}, + image="pgplex/pgschema:1.14.1", + ) + + assert not any("p@ss" in part or "p%40ss" in part for part in command) + assert command[command.index("-e") + 1] == "PGPASSWORD" + assert command[command.index("--sslmode") + 1] == "require" + assert f"{tmp_path}:/db" in command + + +@pytest.mark.parametrize( + "sql", + [ + "DROP TABLE IF EXISTS wasm_repos CASCADE;", + "ALTER TABLE repos DROP COLUMN stars;", + "ALTER TABLE repos ALTER COLUMN stars TYPE bigint;", + "alter table repos drop column stars;", + "TRUNCATE contracts;", + "DELETE FROM projects;", + ], +) +def test_the_guard_flags_statements_that_can_lose_data(sql): + assert destructive_statements(sql) + + +def test_the_guard_lets_additive_and_data_safe_changes_through(): + sql = """ + CREATE TABLE notes (id text PRIMARY KEY); + ALTER TABLE repos ADD COLUMN notes text; + CREATE INDEX notes_id ON notes (id); + DROP INDEX IF EXISTS contracts_wasm; + DROP VIEW IF EXISTS attribution; + DROP FUNCTION IF EXISTS method_rank(text); + ALTER TABLE repos DROP CONSTRAINT repos_pkey; + -- DROP TABLE contracts; + """ + + assert destructive_statements(sql) == [] + + +def test_tables_load_after_the_tables_they_reference(tmp_path): + root = write( + tmp_path, + { + "a_child.sql": "CREATE TABLE a_child (p text REFERENCES z_parent (id));", + "z_parent.sql": "CREATE TABLE z_parent (id text PRIMARY KEY);", + }, + ) + + assert [f.name for f in order_by_dependency("tables", root)] == [ + "z_parent.sql", + "a_child.sql", + ] + + +def test_views_load_after_the_views_they_select_from(tmp_path): + root = write( + tmp_path, + { + "account_projects.sql": "CREATE OR REPLACE VIEW account_projects AS SELECT 1 FROM attribution;", + "attribution.sql": "CREATE OR REPLACE VIEW attribution AS SELECT 1 FROM links;", + }, + ) + + assert [f.name for f in order_by_dependency("views", root)] == [ + "attribution.sql", + "account_projects.sql", + ] + + +def test_the_committed_manifest_matches_the_schema_files(): + committed = (pgschema.SCHEMA_DIR / "manifest.sql").read_text() + + assert committed == manifest_text(pgschema.SCHEMA_DIR) + + +def test_migrations_are_numbered_one_after_another(tmp_path, monkeypatch): + monkeypatch.setattr(pgschema, "MIGRATIONS_DIR", tmp_path) + assert pgschema.next_migration_path().name == "000001.sql" + + write(tmp_path, {"000001.sql": "x", "000002.sql": "y", "notes.md": "z"}) + assert pgschema.next_migration_path().name == "000003.sql" + + +async def test_the_sql_method_rank_follows_the_python_order(scratch_db): + for position, method in enumerate(LinkMethod, start=1): + assert await scratch_db.fetchval("SELECT method_rank($1)", method) == position + assert await scratch_db.fetchval("SELECT method_rank('unknown')") is None From 987d0cc9da08374a78ae0c45ec21a27307166d31 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:30:00 +0100 Subject: [PATCH 06/18] feat: mask the database in ci logs and time out migration locks --- apps/api/src/hackproof/cli.py | 2 + apps/api/src/hackproof/db_utils/connection.py | 7 ++- apps/api/src/hackproof/db_utils/pgschema.py | 3 ++ apps/api/tests/test_pgschema.py | 48 +++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/apps/api/src/hackproof/cli.py b/apps/api/src/hackproof/cli.py index 8425d4e..1eadecb 100644 --- a/apps/api/src/hackproof/cli.py +++ b/apps/api/src/hackproof/cli.py @@ -15,6 +15,8 @@ def run_schema(args: argparse.Namespace) -> int: connection = Connection.from_url(get_settings().database_url) + if os.environ.get("GITHUB_ACTIONS") == "true": + print("\n".join(connection.mask_lines()), flush=True) match args.action: case "plan": return pgschema.plan(connection) diff --git a/apps/api/src/hackproof/db_utils/connection.py b/apps/api/src/hackproof/db_utils/connection.py index 453702d..6f38ac7 100644 --- a/apps/api/src/hackproof/db_utils/connection.py +++ b/apps/api/src/hackproof/db_utils/connection.py @@ -1,6 +1,6 @@ import sys from typing import Self -from urllib.parse import parse_qs, unquote, urlparse +from urllib.parse import parse_qs, quote, unquote, urlparse from pydantic import BaseModel @@ -38,6 +38,11 @@ def is_local(self) -> bool: def label(self) -> str: return f"{self.user}@{self.host}:{self.port}/{self.database}" + def mask_lines(self) -> list[str]: + """Workflow commands that keep this database out of public CI logs.""" + secrets = {self.host, self.user, self.password, quote(self.password, safe="")} + return [f"::add-mask::{value}" for value in sorted(secrets) if value] + def container_route(self) -> tuple[list[str], str]: """Docker network flags and the host name the container should dial.""" if not self.is_local: diff --git a/apps/api/src/hackproof/db_utils/pgschema.py b/apps/api/src/hackproof/db_utils/pgschema.py index c55eeb1..56ed422 100644 --- a/apps/api/src/hackproof/db_utils/pgschema.py +++ b/apps/api/src/hackproof/db_utils/pgschema.py @@ -14,6 +14,7 @@ SCHEMA_DIR = DB_DIR / "schema" MIGRATIONS_DIR = DB_DIR / "migrations" MANIFEST = "/db/schema/manifest.sql" +LOCK_TIMEOUT = "30s" def docker_command( @@ -158,6 +159,8 @@ def apply( "--plan", "/out/plan.json", "--auto-approve", + "--lock-timeout", + LOCK_TIMEOUT, connection=connection, mounts={target: "/out"}, ) diff --git a/apps/api/tests/test_pgschema.py b/apps/api/tests/test_pgschema.py index 4cfe4d5..2657665 100644 --- a/apps/api/tests/test_pgschema.py +++ b/apps/api/tests/test_pgschema.py @@ -32,6 +32,15 @@ def test_a_connection_is_read_from_a_url(): assert not connection.is_local +def test_every_part_of_a_remote_url_is_masked_in_ci_logs(): + lines = Connection.from_url(NEON).mask_lines() + + assert "::add-mask::ep-cool-1234.us-east-2.aws.neon.tech" in lines + assert "::add-mask::owner" in lines + assert "::add-mask::p@ss" in lines + assert "::add-mask::p%40ss" in lines + + def test_a_local_database_is_reached_through_the_host_the_container_can_see( monkeypatch, ): @@ -146,3 +155,42 @@ async def test_the_sql_method_rank_follows_the_python_order(scratch_db): for position, method in enumerate(LinkMethod, start=1): assert await scratch_db.fetchval("SELECT method_rank($1)", method) == position assert await scratch_db.fetchval("SELECT method_rank('unknown')") is None + + +def test_apply_runs_the_reviewed_plan_with_a_lock_timeout(monkeypatch, tmp_path): + seen = [] + monkeypatch.setattr( + pgschema, + "run_pgschema", + lambda command, *args, **kwargs: seen.append((command, args)) or 0, + ) + monkeypatch.setattr( + pgschema, "plan_sql", lambda *_: (0, "CREATE TABLE x (id text);") + ) + monkeypatch.setattr(pgschema, "generate_manifest", lambda *_: None) + + code = pgschema.apply(Connection.from_url(NEON), out_dir=tmp_path) + + assert code == 0 + command, args = seen[0] + assert command == "apply" + assert args[:2] == ("--plan", "/out/plan.json") + assert args[args.index("--lock-timeout") + 1] == "30s" + + +def test_apply_refuses_a_plan_that_drops_data_and_never_runs_it(monkeypatch, tmp_path): + ran = [] + monkeypatch.setattr(pgschema, "run_pgschema", lambda *a, **k: ran.append(a) or 0) + monkeypatch.setattr(pgschema, "plan_sql", lambda *_: (0, "DROP TABLE contracts;")) + monkeypatch.setattr(pgschema, "generate_manifest", lambda *_: None) + + code = pgschema.apply(Connection.from_url(NEON), out_dir=tmp_path) + + assert code == 2 + assert ran == [] + assert ( + pgschema.apply( + Connection.from_url(NEON), allow_destructive=True, out_dir=tmp_path + ) + == 0 + ) From 0968275e33c5cacf68d7f641ca4c6f289b7f3d10 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:30:51 +0100 Subject: [PATCH 07/18] feat: check the schema on every pull request and migrate production on merge --- .github/workflows/ci.yml | 56 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8fc21b..acf47a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,8 @@ jobs: DATABASE_URL: postgresql://hackproof:hackproof@localhost:5432/hackproof steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: astral-sh/setup-uv@v6 with: enable-cache: true @@ -40,6 +42,22 @@ jobs: - run: uv sync --frozen - run: uv run ruff check . - run: uv run ruff format --check . + - run: uv run sqlfluff lint db/schema + - name: A schema change needs a migration file + if: github.event_name == 'pull_request' + working-directory: . + run: | + base="origin/${{ github.base_ref }}" + changed=$(git diff --name-only "$base...HEAD" -- apps/api/db/schema ':!apps/api/db/schema/manifest.sql') + added=$(git diff --name-only --diff-filter=A "$base...HEAD" -- apps/api/db/migrations) + if [ -n "$changed" ] && [ -z "$added" ]; then + echo "::error::The schema files changed but no migration file was added. Run 'hackproof schema plan' and commit the file it writes." + exit 1 + fi + - name: Build the schema on an empty database + run: uv run hackproof schema apply + - name: The schema files and the database agree + run: uv run hackproof schema verify - run: uv run pytest web: @@ -56,10 +74,46 @@ jobs: - run: pnpm format:check - run: pnpm build - deploy-api: + migrate: needs: api if: github.event_name == 'push' runs-on: ubuntu-latest + environment: production + concurrency: + group: migrate-production + cancel-in-progress: false + defaults: + run: + working-directory: apps/api + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: apps/api/uv.lock + - run: uv sync --frozen + - name: Apply the schema to production + env: + DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }} + run: | + if [ -z "$DATABASE_URL" ]; then + echo "::error::Add the PRODUCTION_DATABASE_URL secret. The schema must be migrated before the API deploys." + exit 1 + fi + mkdir -p "$RUNNER_TEMP/plan" + uv run hackproof schema apply --remote --out-dir "$RUNNER_TEMP/plan" + - name: Keep the plan that ran + if: always() + uses: actions/upload-artifact@v4 + with: + name: production-plan + path: ${{ runner.temp }}/plan + if-no-files-found: ignore + + deploy-api: + needs: [api, migrate] + if: github.event_name == 'push' + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 From 6f979eca0db8d75a692e352ef6ca610dce08f5be Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:33:04 +0100 Subject: [PATCH 08/18] refactor: move the api queries into sql files --- apps/api/src/hackproof/queries.py | 321 +++--------------- apps/api/src/hackproof/sql/queries/agents.sql | 6 + .../src/hackproof/sql/queries/contracts.sql | 23 ++ .../hackproof/sql/queries/contributors.sql | 10 + .../src/hackproof/sql/queries/countries.sql | 10 + apps/api/src/hackproof/sql/queries/failed.sql | 22 ++ .../sql/queries/integration_uses.sql | 8 + .../hackproof/sql/queries/integrations.sql | 6 + apps/api/src/hackproof/sql/queries/links.sql | 10 + apps/api/src/hackproof/sql/queries/person.sql | 11 + .../src/hackproof/sql/queries/projects.sql | 93 +++++ apps/api/src/hackproof/sql/queries/stats.sql | 56 +++ .../src/hackproof/sql/queries/timeline.sql | 9 + .../src/hackproof/sql/queries/tool_uses.sql | 8 + apps/api/src/hackproof/sql/queries/tools.sql | 6 + apps/api/src/hackproof/sql/queries/wasm.sql | 31 ++ .../src/hackproof/sql/wrapped/builders.sql | 12 + .../src/hackproof/sql/wrapped/operations.sql | 8 + apps/api/src/hackproof/sql/wrapped/teams.sql | 13 + apps/api/src/hackproof/sql/wrapped/totals.sql | 23 ++ apps/api/src/hackproof/sql_loader.py | 9 + 21 files changed, 418 insertions(+), 277 deletions(-) create mode 100644 apps/api/src/hackproof/sql/queries/agents.sql create mode 100644 apps/api/src/hackproof/sql/queries/contracts.sql create mode 100644 apps/api/src/hackproof/sql/queries/contributors.sql create mode 100644 apps/api/src/hackproof/sql/queries/countries.sql create mode 100644 apps/api/src/hackproof/sql/queries/failed.sql create mode 100644 apps/api/src/hackproof/sql/queries/integration_uses.sql create mode 100644 apps/api/src/hackproof/sql/queries/integrations.sql create mode 100644 apps/api/src/hackproof/sql/queries/links.sql create mode 100644 apps/api/src/hackproof/sql/queries/person.sql create mode 100644 apps/api/src/hackproof/sql/queries/projects.sql create mode 100644 apps/api/src/hackproof/sql/queries/stats.sql create mode 100644 apps/api/src/hackproof/sql/queries/timeline.sql create mode 100644 apps/api/src/hackproof/sql/queries/tool_uses.sql create mode 100644 apps/api/src/hackproof/sql/queries/tools.sql create mode 100644 apps/api/src/hackproof/sql/queries/wasm.sql create mode 100644 apps/api/src/hackproof/sql/wrapped/builders.sql create mode 100644 apps/api/src/hackproof/sql/wrapped/operations.sql create mode 100644 apps/api/src/hackproof/sql/wrapped/teams.sql create mode 100644 apps/api/src/hackproof/sql/wrapped/totals.sql create mode 100644 apps/api/src/hackproof/sql_loader.py diff --git a/apps/api/src/hackproof/queries.py b/apps/api/src/hackproof/queries.py index de7700d..75cc405 100644 --- a/apps/api/src/hackproof/queries.py +++ b/apps/api/src/hackproof/queries.py @@ -4,270 +4,16 @@ from hackproof.db import pool from hackproof.hackathon import get_hackathon, places from hackproof.integrations.catalog import get_catalog -from hackproof.linking.methods import METHOD_RANK_SQL - -PROJECTS_SQL = """ -SELECT p.project_id, p.full_name, r.url, p.footprint, p.tier, p.contracts, p.classic_ops, p.assets, - p.builders, p.wasm_uploads, p.failed_txs, p.first_activity, p.last_activity, - r.description, r.homepage, r.stars, r.forks, r.language, r.license, r.owner, r.owner_avatar, - r.country, r.dependencies, - r.window_commits, r.total_commits, r.stellar_added_in_window, - r.created_at, r.topics, COALESCE(r.declared_track, 'genesis') AS track, - ARRAY(SELECT i.partner FROM repo_integrations i WHERE i.full_name = p.full_name - GROUP BY i.partner ORDER BY i.partner) AS integrations, - ARRAY(SELECT t.tool FROM repo_tools t WHERE t.full_name = p.full_name - GROUP BY t.tool ORDER BY t.tool) AS tools, - ARRAY(SELECT a.agent FROM repo_agents a WHERE a.full_name = p.full_name - GROUP BY a.agent ORDER BY a.agent) AS agents, - count(*) OVER () AS total -FROM projects p -LEFT JOIN repos r ON r.full_name = p.full_name -WHERE ($1::text IS NULL - OR p.project_id ILIKE '%' || $1 || '%' - OR r.description ILIKE '%' || $1 || '%') - AND ($2::text IS NULL - OR p.project_id = 'acct:' || $2 - OR p.project_id IN (SELECT project_id FROM account_projects WHERE account = $2)) - AND ($3::text IS NULL OR p.project_id = $3) - AND ($4::text[] IS NULL OR p.tier = ANY($4)) - AND ($7::text IS NULL OR COALESCE(r.declared_track, 'genesis') = $7) - AND ($8::text[] IS NULL OR EXISTS ( - SELECT 1 FROM repo_integrations i - WHERE i.full_name = p.full_name AND i.partner = ANY($8))) - AND ($9::text[] IS NULL OR p.project_id = ANY($9)) - AND ($10::text[] IS NULL OR EXISTS ( - SELECT 1 FROM repo_tools t - WHERE t.full_name = p.full_name AND t.tool = ANY($10))) - AND ($11::text[] IS NULL OR EXISTS ( - SELECT 1 FROM repo_agents a - WHERE a.full_name = p.full_name AND a.agent = ANY($11))) - AND ($12::text[] IS NULL OR r.language = ANY($12)) - AND ($13::text[] IS NULL OR r.country = ANY($13)) - AND ($14::text[] IS NULL - OR ('contracts' = ANY($14) AND p.contracts > 0) - OR ('failed' = ANY($14) AND p.failed_txs > 0)) -ORDER BY array_position( - ARRAY['verified', 'likely', 'possible', 'unattributed'], p.tier), - p.contracts DESC, p.classic_ops DESC, r.stars DESC NULLS LAST, p.last_activity DESC -LIMIT $5 OFFSET $6 -""" - -CONTRACTS_SQL = """ -SELECT c.contract_id, c.deployer, c.wasm_hash, c.tx_hash, c.ledger, c.deployed_at, - COALESCE('repo:' || a.full_name, 'acct:' || c.deployer) AS project_id, - count(*) OVER () AS total -FROM contracts c -LEFT JOIN attribution a ON a.subject_kind = 'contract' AND a.subject = c.contract_id -WHERE c.deployed_at BETWEEN $1 AND $2 - AND ($3::text IS NULL - OR c.contract_id ILIKE '%' || $3 || '%' - OR c.deployer ILIKE '%' || $3 || '%' - OR c.wasm_hash ILIKE '%' || $3 || '%') - AND ($4::text IS NULL OR c.deployer = $4) - AND ($5::text IS NULL OR COALESCE('repo:' || a.full_name, 'acct:' || c.deployer) = $5) -ORDER BY c.deployed_at DESC -LIMIT $6 OFFSET $7 -""" - -WASM_SQL = """ -SELECT w.wasm_hash, w.uploader, w.tx_hash, w.size, w.source_repo, w.uploaded_at, - COALESCE(aa.full_name, aw.full_name) AS full_name, - (SELECT count(*) FROM contracts c WHERE c.wasm_hash = w.wasm_hash) AS contracts, - count(*) OVER () AS total -FROM wasm_uploads w -LEFT JOIN attribution aa ON aa.subject_kind = 'account' AND aa.subject = w.uploader -LEFT JOIN attribution aw ON aw.subject_kind = 'wasm' AND aw.subject = w.wasm_hash -WHERE w.uploaded_at BETWEEN $1 AND $2 - AND ($3::text IS NULL - OR w.wasm_hash ILIKE '%' || $3 || '%' - OR w.uploader ILIKE '%' || $3 || '%' - OR w.source_repo ILIKE '%' || $3 || '%') - AND ($4::text IS NULL OR w.uploader = $4) - AND ($5::text IS NULL - OR COALESCE('repo:' || COALESCE(aa.full_name, aw.full_name), 'acct:' || w.uploader) = $5) -ORDER BY w.uploaded_at DESC -LIMIT $6 OFFSET $7 -""" - -FAILED_SQL = """ -SELECT f.tx_hash, f.source_account, f.kind, f.result_code, f.ledger, f.created_at, - count(*) OVER () AS total -FROM failed_txs f -LEFT JOIN account_projects ap ON ap.account = f.source_account -WHERE f.created_at BETWEEN $1 AND $2 - AND ($3::text IS NULL - OR f.tx_hash ILIKE '%' || $3 || '%' - OR f.source_account ILIKE '%' || $3 || '%' - OR f.result_code ILIKE '%' || $3 || '%') - AND ($4::text IS NULL OR f.source_account = $4) - AND ($5::text IS NULL OR COALESCE(ap.project_id, 'acct:' || f.source_account) = $5) -ORDER BY f.created_at DESC -LIMIT $6 OFFSET $7 -""" - -STATS_SQL = """ -SELECT - (SELECT count(*) FROM projects WHERE tier IN ('verified', 'likely')) AS projects, - (SELECT count(*) FROM projects WHERE tier = 'verified') AS verified_projects, - (SELECT count(*) FROM projects WHERE tier = 'likely') AS likely_projects, - (SELECT count(*) FROM projects WHERE tier = 'possible') AS possible_projects, - (SELECT count(*) FROM projects WHERE tier = 'unattributed') AS unattributed_accounts, - (SELECT count(*) FROM contracts WHERE deployed_at BETWEEN $1 AND $2) AS contracts, - (SELECT count(*) FROM ( - SELECT deployer FROM contracts WHERE deployed_at BETWEEN $1 AND $2 - UNION SELECT uploader FROM wasm_uploads WHERE uploaded_at BETWEEN $1 AND $2) builders) - AS builders, - (SELECT count(DISTINCT c.login) FROM repo_contributors c - JOIN projects p ON p.full_name = c.full_name - WHERE p.tier IN ('verified', 'likely')) AS contributors, - (SELECT count(DISTINCT country) FROM ( - SELECT r.country FROM projects p JOIN repos r ON r.full_name = p.full_name - WHERE p.tier IN ('verified', 'likely') - UNION SELECT u.country FROM projects p - JOIN repo_contributors c ON c.full_name = p.full_name - JOIN github_users u ON u.login = c.login - WHERE p.tier IN ('verified', 'likely')) found - WHERE country IS NOT NULL) AS countries, - (SELECT count(*) FROM wasm_uploads WHERE uploaded_at BETWEEN $1 AND $2) AS wasm_uploads, - (SELECT count(*) FROM failed_txs WHERE created_at BETWEEN $1 AND $2) AS failed_txs -""" - -LINKS_SQL = f""" -SELECT l.subject_kind, l.subject, l.method, r.path -FROM links l -LEFT JOIN repo_refs r - ON r.full_name = l.full_name AND r.kind = l.subject_kind AND r.value = l.subject -WHERE l.full_name = $1 -ORDER BY {METHOD_RANK_SQL}, l.subject_kind, l.subject -""" - -PERSON_SQL = """ -SELECT c.login, c.url, c.avatar_url, c.contributions, c.full_name, u.country -FROM repo_contributors c -LEFT JOIN github_users u ON u.login = c.login -WHERE lower(c.login) = lower($1) -ORDER BY c.contributions DESC, c.full_name -""" - -CONTRIBUTORS_SQL = """ -SELECT c.login, c.url, c.avatar_url, c.contributions, u.country -FROM repo_contributors c -LEFT JOIN github_users u ON u.login = c.login -WHERE c.full_name = $1 -ORDER BY c.contributions DESC, c.login -""" - -TIMELINE_SQL = """ -SELECT g.hour, count(c.contract_id) AS contracts -FROM generate_series($1::timestamptz, $2::timestamptz - interval '1 hour', interval '1 hour') - AS g(hour) -LEFT JOIN contracts c ON c.deployed_at >= g.hour AND c.deployed_at < g.hour + interval '1 hour' -GROUP BY g.hour -ORDER BY g.hour -""" - -INTEGRATIONS_SQL = """ -SELECT i.partner AS slug, count(DISTINCT i.full_name) AS projects -FROM repo_integrations i -JOIN projects p ON p.full_name = i.full_name AND p.tier IN ('verified', 'likely') -GROUP BY i.partner -""" - -INTEGRATION_USES_SQL = """ -SELECT partner AS slug, method, evidence, path -FROM repo_integrations -WHERE full_name = $1 -ORDER BY partner, method, evidence -""" - -TOOLS_SQL = """ -SELECT t.tool AS slug, count(DISTINCT t.full_name) AS projects -FROM repo_tools t -JOIN projects p ON p.full_name = t.full_name AND p.tier IN ('verified', 'likely') -GROUP BY t.tool -""" - -AGENTS_SQL = """ -SELECT a.agent AS slug, count(DISTINCT a.full_name) AS projects -FROM repo_agents a -JOIN projects p ON p.full_name = a.full_name AND p.tier IN ('verified', 'likely') -GROUP BY a.agent -""" - -TOOL_USES_SQL = """ -SELECT tool AS slug, method, evidence, path -FROM repo_tools -WHERE full_name = $1 -ORDER BY tool, method, evidence -""" - -COUNTRIES_SQL = """ -SELECT u.country, count(DISTINCT c.login) AS contributors -FROM repo_contributors c -JOIN projects p ON p.full_name = c.full_name AND p.tier IN ('verified', 'likely') -JOIN github_users u ON u.login = c.login -WHERE u.country IS NOT NULL -GROUP BY u.country -ORDER BY contributors DESC, u.country -LIMIT $1 -""" - -SHIPPED = "p.tier IN ('verified', 'likely')" -HUMAN = "lower(c.login) <> ALL($1::text[]) AND c.login NOT LIKE '%[bot]'" +from hackproof.sql_loader import load_sql + TOP_BUILDERS = 12 MAX_BUILDERS = 1_000 -WRAPPED_TOTALS_SQL = f""" -SELECT - (SELECT count(DISTINCT c.login) FROM repo_contributors c - JOIN projects p ON p.full_name = c.full_name - WHERE {SHIPPED} AND {HUMAN}) AS builders, - (SELECT count(*) FROM accounts WHERE funded_at BETWEEN $2 AND $3) AS accounts_funded, - (SELECT count(DISTINCT deployer) FROM contracts - WHERE deployed_at BETWEEN $2 AND $3) AS deployers, - (SELECT count(*) FROM classic_ops WHERE occurred_at BETWEEN $2 AND $3) AS operations, - (SELECT coalesce(max(n), 0) FROM ( - SELECT count(*) AS n FROM contracts WHERE deployed_at BETWEEN $2 AND $3 - GROUP BY wasm_hash) shared) AS largest_wasm_contracts -""" - -WRAPPED_OPERATIONS_SQL = """ -SELECT op_type AS label, count(*) AS value -FROM classic_ops -WHERE occurred_at BETWEEN $1 AND $2 -GROUP BY op_type -ORDER BY value DESC, op_type -LIMIT 5 -""" - -WRAPPED_TEAMS_SQL = f""" -SELECT least(n, 5)::text AS label, count(*) AS value -FROM ( - SELECT c.full_name, count(*) AS n - FROM repo_contributors c JOIN projects p ON p.full_name = c.full_name - WHERE {SHIPPED} AND {HUMAN} - GROUP BY c.full_name) teams -GROUP BY 1 -ORDER BY 1 -""" - -WRAPPED_BUILDERS_SQL = f""" -SELECT c.login, max(c.avatar_url) AS avatar_url, - count(DISTINCT c.full_name) AS projects, sum(c.contributions)::int AS contributions, - array_agg(DISTINCT c.full_name ORDER BY c.full_name) AS project_names -FROM repo_contributors c JOIN projects p ON p.full_name = c.full_name -WHERE {SHIPPED} AND {HUMAN} -GROUP BY c.login -HAVING count(DISTINCT c.full_name) >= $2 -ORDER BY projects DESC, contributions DESC, c.login -LIMIT $3 -""" - RESOURCE_SQL = { - "projects": PROJECTS_SQL, - "contracts": CONTRACTS_SQL, - "wasm": WASM_SQL, - "failed": FAILED_SQL, + "projects": "queries/projects", + "contracts": "queries/contracts", + "wasm": "queries/wasm", + "failed": "queries/failed", } Resource = Literal["projects", "contracts", "wasm", "failed"] @@ -325,7 +71,7 @@ async def list_page( else (*window(), q, account, project_id, limit, offset) ) async with pool().acquire() as connection: - rows = await connection.fetch(RESOURCE_SQL[resource], *args) + rows = await connection.fetch(load_sql(RESOURCE_SQL[resource]), *args) total = rows[0]["total"] if rows else 0 items = [{k: v for k, v in dict(row).items() if k != "total"} for row in rows] if resource == "projects": @@ -336,30 +82,39 @@ async def list_page( async def fetch_stats() -> dict[str, int]: start, end = window() async with pool().acquire() as connection: - return dict(await connection.fetchrow(STATS_SQL, start, end)) + return dict(await connection.fetchrow(load_sql("queries/stats"), start, end)) async def fetch_links(full_name: str) -> list[dict[str, Any]]: async with pool().acquire() as connection: - return [dict(row) for row in await connection.fetch(LINKS_SQL, full_name)] + return [ + dict(row) + for row in await connection.fetch(load_sql("queries/links"), full_name) + ] async def fetch_contributors(full_name: str) -> list[dict[str, Any]]: async with pool().acquire() as connection: return [ - dict(row) for row in await connection.fetch(CONTRIBUTORS_SQL, full_name) + dict(row) + for row in await connection.fetch( + load_sql("queries/contributors"), full_name + ) ] async def fetch_timeline() -> list[dict[str, Any]]: start, end = window() async with pool().acquire() as connection: - return [dict(row) for row in await connection.fetch(TIMELINE_SQL, start, end)] + return [ + dict(row) + for row in await connection.fetch(load_sql("queries/timeline"), start, end) + ] async def fetch_integration_uses(full_name: str) -> list[dict[str, Any]]: async with pool().acquire() as connection: - rows = await connection.fetch(INTEGRATION_USES_SQL, full_name) + rows = await connection.fetch(load_sql("queries/integration_uses"), full_name) names = {partner.slug: partner.name for partner in get_catalog().partners} return [ {**row, "name": names[row["slug"]]} @@ -370,7 +125,7 @@ async def fetch_integration_uses(full_name: str) -> list[dict[str, Any]]: async def fetch_tool_uses(full_name: str) -> list[dict[str, Any]]: async with pool().acquire() as connection: - rows = await connection.fetch(TOOL_USES_SQL, full_name) + rows = await connection.fetch(load_sql("queries/tool_uses"), full_name) names = {tool.slug: tool.name for tool in get_catalog().tools} return [ {**row, "name": names[row["slug"]]} @@ -382,7 +137,8 @@ async def fetch_tool_uses(full_name: str) -> list[dict[str, Any]]: async def fetch_tools() -> list[dict[str, Any]]: async with pool().acquire() as connection: counts = { - row["slug"]: row["projects"] for row in await connection.fetch(TOOLS_SQL) + row["slug"]: row["projects"] + for row in await connection.fetch(load_sql("queries/tools")) } return [ { @@ -397,7 +153,7 @@ async def fetch_integrations() -> list[dict[str, Any]]: async with pool().acquire() as connection: counts = { row["slug"]: row["projects"] - for row in await connection.fetch(INTEGRATIONS_SQL) + for row in await connection.fetch(load_sql("queries/integrations")) } return [ { @@ -410,7 +166,10 @@ async def fetch_integrations() -> list[dict[str, Any]]: async def fetch_countries(limit: int = 12) -> list[dict[str, Any]]: async with pool().acquire() as connection: - return [dict(row) for row in await connection.fetch(COUNTRIES_SQL, limit)] + return [ + dict(row) + for row in await connection.fetch(load_sql("queries/countries"), limit) + ] async def fetch_wrapped() -> dict[str, Any]: @@ -419,13 +178,17 @@ async def fetch_wrapped() -> dict[str, Any]: login.lower() for agent in get_catalog().agents for login in agent.authors ] async with pool().acquire() as connection: - totals = dict(await connection.fetchrow(WRAPPED_TOTALS_SQL, agents, start, end)) - operations = await connection.fetch(WRAPPED_OPERATIONS_SQL, start, end) - teams = await connection.fetch(WRAPPED_TEAMS_SQL, agents) + totals = dict( + await connection.fetchrow(load_sql("wrapped/totals"), agents, start, end) + ) + operations = await connection.fetch(load_sql("wrapped/operations"), start, end) + teams = await connection.fetch(load_sql("wrapped/teams"), agents) multi_team = await connection.fetch( - WRAPPED_BUILDERS_SQL, agents, 2, MAX_BUILDERS + load_sql("wrapped/builders"), agents, 2, MAX_BUILDERS + ) + top = await connection.fetch( + load_sql("wrapped/builders"), agents, 1, TOP_BUILDERS ) - top = await connection.fetch(WRAPPED_BUILDERS_SQL, agents, 1, TOP_BUILDERS) return { **totals, "operations_by_type": [dict(row) for row in operations], @@ -440,7 +203,8 @@ async def fetch_wrapped() -> dict[str, Any]: async def fetch_agents() -> list[dict[str, Any]]: async with pool().acquire() as connection: counts = { - row["slug"]: row["projects"] for row in await connection.fetch(AGENTS_SQL) + row["slug"]: row["projects"] + for row in await connection.fetch(load_sql("queries/agents")) } return [ { @@ -453,7 +217,10 @@ async def fetch_agents() -> list[dict[str, Any]]: async def fetch_person(login: str) -> dict[str, Any] | None: async with pool().acquire() as connection: - rows = [dict(row) for row in await connection.fetch(PERSON_SQL, login)] + rows = [ + dict(row) + for row in await connection.fetch(load_sql("queries/person"), login) + ] if not rows: return None commits = {f"repo:{row['full_name']}": row["contributions"] for row in rows} diff --git a/apps/api/src/hackproof/sql/queries/agents.sql b/apps/api/src/hackproof/sql/queries/agents.sql new file mode 100644 index 0000000..1f3e14b --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/agents.sql @@ -0,0 +1,6 @@ +SELECT + a.agent AS slug, + count(DISTINCT a.full_name) AS projects +FROM repo_agents AS a +INNER JOIN projects AS p ON a.full_name = p.full_name AND p.tier IN ('verified', 'likely') +GROUP BY a.agent diff --git a/apps/api/src/hackproof/sql/queries/contracts.sql b/apps/api/src/hackproof/sql/queries/contracts.sql new file mode 100644 index 0000000..a3ae5ac --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/contracts.sql @@ -0,0 +1,23 @@ +SELECT + c.contract_id, + c.deployer, + c.wasm_hash, + c.tx_hash, + c.ledger, + c.deployed_at, + COALESCE('repo:' || a.full_name, 'acct:' || c.deployer) AS project_id, + COUNT(*) OVER () AS total +FROM contracts AS c +LEFT JOIN attribution AS a ON a.subject_kind = 'contract' AND c.contract_id = a.subject +WHERE + c.deployed_at BETWEEN $1 AND $2 + AND ( + $3::text IS NULL + OR c.contract_id ILIKE '%' || $3 || '%' + OR c.deployer ILIKE '%' || $3 || '%' + OR c.wasm_hash ILIKE '%' || $3 || '%' + ) + AND ($4::text IS NULL OR c.deployer = $4) + AND ($5::text IS NULL OR COALESCE('repo:' || a.full_name, 'acct:' || c.deployer) = $5) +ORDER BY c.deployed_at DESC +LIMIT $6 OFFSET $7 diff --git a/apps/api/src/hackproof/sql/queries/contributors.sql b/apps/api/src/hackproof/sql/queries/contributors.sql new file mode 100644 index 0000000..7a8ae50 --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/contributors.sql @@ -0,0 +1,10 @@ +SELECT + c.login, + c.url, + c.avatar_url, + c.contributions, + u.country +FROM repo_contributors AS c +LEFT JOIN github_users AS u ON c.login = u.login +WHERE c.full_name = $1 +ORDER BY c.contributions DESC, c.login ASC diff --git a/apps/api/src/hackproof/sql/queries/countries.sql b/apps/api/src/hackproof/sql/queries/countries.sql new file mode 100644 index 0000000..2b62a7d --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/countries.sql @@ -0,0 +1,10 @@ +SELECT + u.country, + count(DISTINCT c.login) AS contributors +FROM repo_contributors AS c +INNER JOIN projects AS p ON c.full_name = p.full_name AND p.tier IN ('verified', 'likely') +INNER JOIN github_users AS u ON c.login = u.login +WHERE u.country IS NOT NULL +GROUP BY u.country +ORDER BY contributors DESC, u.country ASC +LIMIT $1 diff --git a/apps/api/src/hackproof/sql/queries/failed.sql b/apps/api/src/hackproof/sql/queries/failed.sql new file mode 100644 index 0000000..864b513 --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/failed.sql @@ -0,0 +1,22 @@ +SELECT + f.tx_hash, + f.source_account, + f.kind, + f.result_code, + f.ledger, + f.created_at, + count(*) OVER () AS total +FROM failed_txs AS f +LEFT JOIN account_projects AS ap ON f.source_account = ap.account +WHERE + f.created_at BETWEEN $1 AND $2 + AND ( + $3::text IS NULL + OR f.tx_hash ILIKE '%' || $3 || '%' + OR f.source_account ILIKE '%' || $3 || '%' + OR f.result_code ILIKE '%' || $3 || '%' + ) + AND ($4::text IS NULL OR f.source_account = $4) + AND ($5::text IS NULL OR coalesce(ap.project_id, 'acct:' || f.source_account) = $5) +ORDER BY f.created_at DESC +LIMIT $6 OFFSET $7 diff --git a/apps/api/src/hackproof/sql/queries/integration_uses.sql b/apps/api/src/hackproof/sql/queries/integration_uses.sql new file mode 100644 index 0000000..09deb4f --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/integration_uses.sql @@ -0,0 +1,8 @@ +SELECT + partner AS slug, + method, + evidence, + path +FROM repo_integrations +WHERE full_name = $1 +ORDER BY partner, method, evidence diff --git a/apps/api/src/hackproof/sql/queries/integrations.sql b/apps/api/src/hackproof/sql/queries/integrations.sql new file mode 100644 index 0000000..641c0f3 --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/integrations.sql @@ -0,0 +1,6 @@ +SELECT + i.partner AS slug, + count(DISTINCT i.full_name) AS projects +FROM repo_integrations AS i +INNER JOIN projects AS p ON i.full_name = p.full_name AND p.tier IN ('verified', 'likely') +GROUP BY i.partner diff --git a/apps/api/src/hackproof/sql/queries/links.sql b/apps/api/src/hackproof/sql/queries/links.sql new file mode 100644 index 0000000..9834e38 --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/links.sql @@ -0,0 +1,10 @@ +SELECT + l.subject_kind, + l.subject, + l.method, + r.path +FROM links AS l +LEFT JOIN repo_refs AS r + ON l.full_name = r.full_name AND l.subject_kind = r.kind AND l.subject = r.value +WHERE l.full_name = $1 +ORDER BY method_rank(l.method), l.subject_kind, l.subject diff --git a/apps/api/src/hackproof/sql/queries/person.sql b/apps/api/src/hackproof/sql/queries/person.sql new file mode 100644 index 0000000..16c12ad --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/person.sql @@ -0,0 +1,11 @@ +SELECT + c.login, + c.url, + c.avatar_url, + c.contributions, + c.full_name, + u.country +FROM repo_contributors AS c +LEFT JOIN github_users AS u ON c.login = u.login +WHERE lower(c.login) = lower($1) +ORDER BY c.contributions DESC, c.full_name ASC diff --git a/apps/api/src/hackproof/sql/queries/projects.sql b/apps/api/src/hackproof/sql/queries/projects.sql new file mode 100644 index 0000000..ffa6043 --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/projects.sql @@ -0,0 +1,93 @@ +SELECT + p.project_id, + p.full_name, + r.url, + p.footprint, + p.tier, + p.contracts, + p.classic_ops, + p.assets, + p.builders, + p.wasm_uploads, + p.failed_txs, + p.first_activity, + p.last_activity, + r.description, + r.homepage, + r.stars, + r.forks, + r.language, + r.license, + r.owner, + r.owner_avatar, + r.country, + r.dependencies, + r.window_commits, + r.total_commits, + r.stellar_added_in_window, + r.created_at, + r.topics, + COALESCE(r.declared_track, 'genesis') AS track, + ARRAY( + SELECT i.partner FROM repo_integrations AS i + WHERE i.full_name = p.full_name + GROUP BY i.partner + ORDER BY i.partner + ) AS integrations, + ARRAY( + SELECT t.tool FROM repo_tools AS t + WHERE t.full_name = p.full_name + GROUP BY t.tool + ORDER BY t.tool + ) AS tools, + ARRAY( + SELECT a.agent FROM repo_agents AS a + WHERE a.full_name = p.full_name + GROUP BY a.agent + ORDER BY a.agent + ) AS agents, + COUNT(*) OVER () AS total +FROM projects AS p +LEFT JOIN repos AS r ON p.full_name = r.full_name +WHERE ( + $1::text IS NULL + OR p.project_id ILIKE '%' || $1 || '%' + OR r.description ILIKE '%' || $1 || '%' +) +AND ( + $2::text IS NULL + OR p.project_id = 'acct:' || $2 + OR p.project_id IN ( + SELECT ap.project_id FROM account_projects AS ap + WHERE ap.account = $2 + ) +) +AND ($3::text IS NULL OR p.project_id = $3) +AND ($4::text[] IS NULL OR p.tier = ANY($4)) +AND ($7::text IS NULL OR COALESCE(r.declared_track, 'genesis') = $7) +AND ($8::text[] IS NULL OR EXISTS ( + SELECT 1 FROM repo_integrations AS i + WHERE i.full_name = p.full_name AND i.partner = ANY($8) +)) +AND ($9::text[] IS NULL OR p.project_id = ANY($9)) +AND ($10::text[] IS NULL OR EXISTS ( + SELECT 1 FROM repo_tools AS t + WHERE t.full_name = p.full_name AND t.tool = ANY($10) +)) +AND ($11::text[] IS NULL OR EXISTS ( + SELECT 1 FROM repo_agents AS a + WHERE a.full_name = p.full_name AND a.agent = ANY($11) +)) +AND ($12::text[] IS NULL OR r.language = ANY($12)) +AND ($13::text[] IS NULL OR r.country = ANY($13)) +AND ( + $14::text[] IS NULL + OR ('contracts' = ANY($14) AND p.contracts > 0) + OR ('failed' = ANY($14) AND p.failed_txs > 0) +) +ORDER BY + ARRAY_POSITION( + ARRAY['verified', 'likely', 'possible', 'unattributed'], p.tier + ), + p.contracts DESC, p.classic_ops DESC, r.stars DESC NULLS LAST, p.last_activity DESC +LIMIT $5 OFFSET $6 diff --git a/apps/api/src/hackproof/sql/queries/stats.sql b/apps/api/src/hackproof/sql/queries/stats.sql new file mode 100644 index 0000000..d07167a --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/stats.sql @@ -0,0 +1,56 @@ +SELECT + ( + SELECT count(*) FROM projects + WHERE tier IN ('verified', 'likely') + ) AS projects, + ( + SELECT count(*) FROM projects + WHERE tier = 'verified' + ) AS verified_projects, + ( + SELECT count(*) FROM projects + WHERE tier = 'likely' + ) AS likely_projects, + ( + SELECT count(*) FROM projects + WHERE tier = 'possible' + ) AS possible_projects, + ( + SELECT count(*) FROM projects + WHERE tier = 'unattributed' + ) AS unattributed_accounts, + ( + SELECT count(*) FROM contracts + WHERE deployed_at BETWEEN $1 AND $2 + ) AS contracts, + (SELECT count(*) FROM ( + SELECT deployer FROM contracts + WHERE deployed_at BETWEEN $1 AND $2 + UNION + SELECT uploader FROM wasm_uploads + WHERE uploaded_at BETWEEN $1 AND $2 + ) AS builders) + AS builders, + ( + SELECT count(DISTINCT c.login) FROM repo_contributors AS c + INNER JOIN projects AS p ON c.full_name = p.full_name + WHERE p.tier IN ('verified', 'likely') + ) AS contributors, + (SELECT count(DISTINCT country) FROM ( + SELECT r.country FROM projects AS p INNER JOIN repos AS r ON p.full_name = r.full_name + WHERE p.tier IN ('verified', 'likely') + UNION + SELECT u.country FROM projects AS p + INNER JOIN repo_contributors AS c ON p.full_name = c.full_name + INNER JOIN github_users AS u ON c.login = u.login + WHERE p.tier IN ('verified', 'likely') + ) AS found + WHERE country IS NOT NULL) AS countries, + ( + SELECT count(*) FROM wasm_uploads + WHERE uploaded_at BETWEEN $1 AND $2 + ) AS wasm_uploads, + ( + SELECT count(*) FROM failed_txs + WHERE created_at BETWEEN $1 AND $2 + ) AS failed_txs diff --git a/apps/api/src/hackproof/sql/queries/timeline.sql b/apps/api/src/hackproof/sql/queries/timeline.sql new file mode 100644 index 0000000..a6434a3 --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/timeline.sql @@ -0,0 +1,9 @@ +SELECT + g.hour, + count(c.contract_id) AS contracts +FROM + generate_series($1::timestamptz, $2::timestamptz - interval '1 hour', interval '1 hour') + AS g (hour) +LEFT JOIN contracts AS c ON g.hour <= c.deployed_at AND c.deployed_at < g.hour + interval '1 hour' +GROUP BY g.hour +ORDER BY g.hour diff --git a/apps/api/src/hackproof/sql/queries/tool_uses.sql b/apps/api/src/hackproof/sql/queries/tool_uses.sql new file mode 100644 index 0000000..43c211d --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/tool_uses.sql @@ -0,0 +1,8 @@ +SELECT + tool AS slug, + method, + evidence, + path +FROM repo_tools +WHERE full_name = $1 +ORDER BY tool, method, evidence diff --git a/apps/api/src/hackproof/sql/queries/tools.sql b/apps/api/src/hackproof/sql/queries/tools.sql new file mode 100644 index 0000000..dba99ed --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/tools.sql @@ -0,0 +1,6 @@ +SELECT + t.tool AS slug, + count(DISTINCT t.full_name) AS projects +FROM repo_tools AS t +INNER JOIN projects AS p ON t.full_name = p.full_name AND p.tier IN ('verified', 'likely') +GROUP BY t.tool diff --git a/apps/api/src/hackproof/sql/queries/wasm.sql b/apps/api/src/hackproof/sql/queries/wasm.sql new file mode 100644 index 0000000..4553593 --- /dev/null +++ b/apps/api/src/hackproof/sql/queries/wasm.sql @@ -0,0 +1,31 @@ +SELECT + w.wasm_hash, + w.uploader, + w.tx_hash, + w.size, + w.source_repo, + w.uploaded_at, + COALESCE(aa.full_name, aw.full_name) AS full_name, + ( + SELECT COUNT(*) FROM contracts AS c + WHERE c.wasm_hash = w.wasm_hash + ) AS contracts, + COUNT(*) OVER () AS total +FROM wasm_uploads AS w +LEFT JOIN attribution AS aa ON aa.subject_kind = 'account' AND w.uploader = aa.subject +LEFT JOIN attribution AS aw ON aw.subject_kind = 'wasm' AND w.wasm_hash = aw.subject +WHERE + w.uploaded_at BETWEEN $1 AND $2 + AND ( + $3::text IS NULL + OR w.wasm_hash ILIKE '%' || $3 || '%' + OR w.uploader ILIKE '%' || $3 || '%' + OR w.source_repo ILIKE '%' || $3 || '%' + ) + AND ($4::text IS NULL OR w.uploader = $4) + AND ( + $5::text IS NULL + OR COALESCE('repo:' || COALESCE(aa.full_name, aw.full_name), 'acct:' || w.uploader) = $5 + ) +ORDER BY w.uploaded_at DESC +LIMIT $6 OFFSET $7 diff --git a/apps/api/src/hackproof/sql/wrapped/builders.sql b/apps/api/src/hackproof/sql/wrapped/builders.sql new file mode 100644 index 0000000..fd9a98e --- /dev/null +++ b/apps/api/src/hackproof/sql/wrapped/builders.sql @@ -0,0 +1,12 @@ +SELECT + c.login, + max(c.avatar_url) AS avatar_url, + count(DISTINCT c.full_name) AS projects, + sum(c.contributions)::int AS contributions, + array_agg(DISTINCT c.full_name ORDER BY c.full_name) AS project_names +FROM repo_contributors AS c INNER JOIN projects AS p ON c.full_name = p.full_name +WHERE p.tier IN ('verified', 'likely') AND lower(c.login) <> all($1::text[]) AND c.login NOT LIKE '%[bot]' +GROUP BY c.login +HAVING count(DISTINCT c.full_name) >= $2 +ORDER BY projects DESC, contributions DESC, c.login ASC +LIMIT $3 diff --git a/apps/api/src/hackproof/sql/wrapped/operations.sql b/apps/api/src/hackproof/sql/wrapped/operations.sql new file mode 100644 index 0000000..43df789 --- /dev/null +++ b/apps/api/src/hackproof/sql/wrapped/operations.sql @@ -0,0 +1,8 @@ +SELECT + op_type AS label, + count(*) AS value +FROM classic_ops +WHERE occurred_at BETWEEN $1 AND $2 +GROUP BY op_type +ORDER BY value DESC, op_type ASC +LIMIT 5 diff --git a/apps/api/src/hackproof/sql/wrapped/teams.sql b/apps/api/src/hackproof/sql/wrapped/teams.sql new file mode 100644 index 0000000..384ea8a --- /dev/null +++ b/apps/api/src/hackproof/sql/wrapped/teams.sql @@ -0,0 +1,13 @@ +SELECT + team_size AS label, + count(*) AS value +FROM ( + SELECT + c.full_name, + least(count(*), 5)::text AS team_size + FROM repo_contributors AS c INNER JOIN projects AS p ON c.full_name = p.full_name + WHERE p.tier IN ('verified', 'likely') AND lower(c.login) <> all($1::text[]) AND c.login NOT LIKE '%[bot]' + GROUP BY c.full_name +) AS teams +GROUP BY team_size +ORDER BY team_size diff --git a/apps/api/src/hackproof/sql/wrapped/totals.sql b/apps/api/src/hackproof/sql/wrapped/totals.sql new file mode 100644 index 0000000..865ff95 --- /dev/null +++ b/apps/api/src/hackproof/sql/wrapped/totals.sql @@ -0,0 +1,23 @@ +SELECT + ( + SELECT count(DISTINCT c.login) FROM repo_contributors AS c + INNER JOIN projects AS p ON c.full_name = p.full_name + WHERE p.tier IN ('verified', 'likely') AND lower(c.login) <> all($1::text[]) AND c.login NOT LIKE '%[bot]' + ) AS builders, + ( + SELECT count(*) FROM accounts + WHERE funded_at BETWEEN $2 AND $3 + ) AS accounts_funded, + ( + SELECT count(DISTINCT deployer) FROM contracts + WHERE deployed_at BETWEEN $2 AND $3 + ) AS deployers, + ( + SELECT count(*) FROM classic_ops + WHERE occurred_at BETWEEN $2 AND $3 + ) AS operations, + (SELECT coalesce(max(n), 0) FROM ( + SELECT count(*) AS n FROM contracts + WHERE deployed_at BETWEEN $2 AND $3 + GROUP BY wasm_hash + ) AS shared) AS largest_wasm_contracts diff --git a/apps/api/src/hackproof/sql_loader.py b/apps/api/src/hackproof/sql_loader.py new file mode 100644 index 0000000..88cb0d8 --- /dev/null +++ b/apps/api/src/hackproof/sql_loader.py @@ -0,0 +1,9 @@ +from functools import cache +from pathlib import Path + +SQL_DIR = Path(__file__).with_name("sql") + + +@cache +def load_sql(name: str) -> str: + return (SQL_DIR / f"{name}.sql").read_text() From 5bcda408308f1de6703a667e6c1fa2bd4d794fbb Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:38:46 +0100 Subject: [PATCH 09/18] feat: add the hackers function --- apps/api/db/migrations/000002.sql | 14 ++++++++++++++ apps/api/db/schema/functions/hackers.sql | 14 ++++++++++++++ apps/api/db/schema/manifest.sql | 1 + apps/api/src/hackproof/db_utils/manifest.py | 3 ++- 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 apps/api/db/migrations/000002.sql create mode 100644 apps/api/db/schema/functions/hackers.sql diff --git a/apps/api/db/migrations/000002.sql b/apps/api/db/migrations/000002.sql new file mode 100644 index 0000000..1b18389 --- /dev/null +++ b/apps/api/db/migrations/000002.sql @@ -0,0 +1,14 @@ +CREATE OR REPLACE FUNCTION hackers( + window_start timestamptz, + window_end timestamptz, + friendbot text +) +RETURNS SETOF text +LANGUAGE sql +STABLE +AS $$ + SELECT account FROM accounts + WHERE funder = friendbot AND funded_at BETWEEN window_start AND window_end + UNION SELECT deployer FROM contracts + UNION SELECT uploader FROM wasm_uploads +$$; diff --git a/apps/api/db/schema/functions/hackers.sql b/apps/api/db/schema/functions/hackers.sql new file mode 100644 index 0000000..1b18389 --- /dev/null +++ b/apps/api/db/schema/functions/hackers.sql @@ -0,0 +1,14 @@ +CREATE OR REPLACE FUNCTION hackers( + window_start timestamptz, + window_end timestamptz, + friendbot text +) +RETURNS SETOF text +LANGUAGE sql +STABLE +AS $$ + SELECT account FROM accounts + WHERE funder = friendbot AND funded_at BETWEEN window_start AND window_end + UNION SELECT deployer FROM contracts + UNION SELECT uploader FROM wasm_uploads +$$; diff --git a/apps/api/db/schema/manifest.sql b/apps/api/db/schema/manifest.sql index a3e7f03..7ed0beb 100644 --- a/apps/api/db/schema/manifest.sql +++ b/apps/api/db/schema/manifest.sql @@ -1,3 +1,4 @@ +\i functions/hackers.sql \i functions/method_rank.sql \i tables/accounts.sql \i tables/classic_ops.sql diff --git a/apps/api/src/hackproof/db_utils/manifest.py b/apps/api/src/hackproof/db_utils/manifest.py index 267d6fe..7bd684d 100644 --- a/apps/api/src/hackproof/db_utils/manifest.py +++ b/apps/api/src/hackproof/db_utils/manifest.py @@ -86,4 +86,5 @@ def schema_sql(schema_dir: Path) -> str: for line in (schema_dir / "manifest.sql").read_text().splitlines() if line.startswith("\\i ") ] - return "\n".join(path.read_text() for path in paths) + body = "\n".join(path.read_text() for path in paths) + return f"SET check_function_bodies = off;\n{body}" From 5acb12133123ecc0b9b9712258998e2aa39088b7 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:38:46 +0100 Subject: [PATCH 10/18] refactor: move the reconcile queries into sql files --- apps/api/src/hackproof/linking/reconcile.py | 218 +++--------------- .../sql/reconcile/account_projects.sql | 79 +++++++ .../sql/reconcile/exclude_projects.sql | 2 + .../hackproof/sql/reconcile/insert_link.sql | 1 + .../sql/reconcile/link_asset_in_repo.sql | 15 ++ .../sql/reconcile/link_contract_in_repo.sql | 11 + .../sql/reconcile/link_deployer_in_repo.sql | 18 ++ .../sql/reconcile/link_footprint_assets.sql | 14 ++ .../reconcile/link_footprint_contracts.sql | 9 + .../reconcile/link_footprint_deployers.sql | 9 + .../reconcile/link_footprint_uploaders.sql | 15 ++ .../sql/reconcile/link_wasm_hash_match.sql | 14 ++ .../hackproof/sql/reconcile/repo_names.sql | 1 + .../hackproof/sql/reconcile/repo_projects.sql | 108 +++++++++ .../sql/reconcile/report_footprints.sql | 5 + .../sql/reconcile/report_methods.sql | 5 + .../hackproof/sql/reconcile/report_tiers.sql | 5 + .../hackproof/sql/reconcile/reset_links.sql | 1 + .../sql/reconcile/reset_projects.sql | 1 + .../hackproof/sql/reconcile/wasm_sources.sql | 5 + 20 files changed, 349 insertions(+), 187 deletions(-) create mode 100644 apps/api/src/hackproof/sql/reconcile/account_projects.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/exclude_projects.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/insert_link.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_asset_in_repo.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_contract_in_repo.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_deployer_in_repo.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_footprint_assets.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_footprint_contracts.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_footprint_deployers.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_footprint_uploaders.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/link_wasm_hash_match.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/repo_names.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/repo_projects.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/report_footprints.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/report_methods.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/report_tiers.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/reset_links.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/reset_projects.sql create mode 100644 apps/api/src/hackproof/sql/reconcile/wasm_sources.sql diff --git a/apps/api/src/hackproof/linking/reconcile.py b/apps/api/src/hackproof/linking/reconcile.py index 9a4be19..d4db4c8 100644 --- a/apps/api/src/hackproof/linking/reconcile.py +++ b/apps/api/src/hackproof/linking/reconcile.py @@ -8,6 +8,7 @@ from hackproof.hackathon import get_hackathon from hackproof.linking.methods import LinkMethod from hackproof.linking.repo_ref import normalize_repo +from hackproof.sql_loader import load_sql log = logging.getLogger("hackproof.reconcile") @@ -15,164 +16,20 @@ MIN_WINDOW_COMMITS = 3 MIN_LIKELY_COMMITS = 5 -HACKERS = """ -SELECT account FROM accounts WHERE funder = $3 AND funded_at BETWEEN $1 AND $2 -UNION SELECT deployer FROM contracts -UNION SELECT uploader FROM wasm_uploads -""" - EVIDENCE = { - LinkMethod.CONTRACT_IN_REPO: """ - SELECT r.full_name, 'contract', r.value FROM repo_refs r - WHERE r.kind = 'contract' AND r.value IN (SELECT contract_id FROM contracts) - """, - LinkMethod.WASM_HASH_MATCH: """ - SELECT r.full_name, 'wasm', r.value FROM repo_refs r - WHERE r.kind = 'wasm' AND (r.value IN (SELECT wasm_hash FROM wasm_uploads) - OR r.value IN (SELECT wasm_hash FROM contracts)) - """, - LinkMethod.DEPLOYER_IN_REPO: f""" - SELECT r.full_name, 'account', r.value FROM repo_refs r - WHERE r.kind = 'account' - AND r.value IN ({HACKERS}) - AND r.value <> ALL($4::text[]) - AND r.value NOT IN ( - SELECT value FROM repo_refs WHERE kind = 'account' - GROUP BY value HAVING count(*) > $5) - """, - LinkMethod.ASSET_IN_REPO: f""" - SELECT r.full_name, 'asset', r.value FROM repo_refs r - WHERE r.kind = 'asset' - AND split_part(r.value, ':', 2) IN ({HACKERS}) - AND r.value IN (SELECT asset FROM classic_ops WHERE asset IS NOT NULL - AND occurred_at BETWEEN $1 AND $2) - """, + LinkMethod.CONTRACT_IN_REPO: "reconcile/link_contract_in_repo", + LinkMethod.WASM_HASH_MATCH: "reconcile/link_wasm_hash_match", + LinkMethod.DEPLOYER_IN_REPO: "reconcile/link_deployer_in_repo", + LinkMethod.ASSET_IN_REPO: "reconcile/link_asset_in_repo", } -SINGLE_OWNER_WASM = """ - SELECT wasm_hash FROM wasm_uploads GROUP BY wasm_hash HAVING count(DISTINCT uploader) = 1 -""" - FOOTPRINT = [ - """ - SELECT l.full_name, 'account', c.deployer FROM links l - JOIN contracts c ON l.subject_kind = 'contract' AND c.contract_id = l.subject - """, - f""" - SELECT l.full_name, 'account', w.uploader FROM links l - JOIN wasm_uploads w ON l.subject_kind = 'wasm' AND w.wasm_hash = l.subject - WHERE l.subject IN ({SINGLE_OWNER_WASM}) - """, - """ - SELECT l.full_name, 'contract', c.contract_id FROM links l - JOIN contracts c ON l.subject_kind = 'account' AND c.deployer = l.subject - """, - """ - SELECT l.full_name, 'asset', a.asset FROM links l - JOIN (SELECT DISTINCT asset FROM classic_ops WHERE asset IS NOT NULL) a - ON l.subject_kind = 'account' AND split_part(a.asset, ':', 2) = l.subject - """, + "reconcile/link_footprint_deployers", + "reconcile/link_footprint_uploaders", + "reconcile/link_footprint_contracts", + "reconcile/link_footprint_assets", ] -REPO_PROJECTS = """ -INSERT INTO projects -WITH contract_stats AS ( - SELECT a.full_name, count(*) AS contracts, count(DISTINCT c.deployer) AS builders, - min(c.deployed_at) AS first_at, max(c.deployed_at) AS last_at - FROM contracts c - JOIN attribution a ON a.subject_kind = 'contract' AND a.subject = c.contract_id - WHERE c.deployed_at BETWEEN $1 AND $2 - GROUP BY a.full_name -), op_stats AS ( - SELECT a.full_name, count(*) AS classic_ops FROM classic_ops o - JOIN attribution a ON a.subject_kind = 'account' AND a.subject = o.source_account - WHERE o.occurred_at BETWEEN $1 AND $2 - GROUP BY a.full_name -), asset_stats AS ( - SELECT full_name, count(*) AS assets FROM attribution - WHERE subject_kind = 'asset' GROUP BY full_name -), wasm_stats AS ( - SELECT COALESCE(aa.full_name, aw.full_name) AS full_name, count(DISTINCT w.wasm_hash) AS wasm - FROM wasm_uploads w - LEFT JOIN attribution aa ON aa.subject_kind = 'account' AND aa.subject = w.uploader - LEFT JOIN attribution aw ON aw.subject_kind = 'wasm' AND aw.subject = w.wasm_hash - WHERE w.uploaded_at BETWEEN $1 AND $2 - GROUP BY 1 -), failed_stats AS ( - SELECT a.full_name, count(*) AS failed FROM failed_txs f - JOIN attribution a ON a.subject_kind = 'account' AND a.subject = f.source_account - WHERE f.created_at BETWEEN $1 AND $2 - GROUP BY a.full_name -) -SELECT 'repo:' || r.full_name, r.full_name, - CASE WHEN COALESCE(cs.contracts, 0) > 0 THEN 'contracts' - WHEN COALESCE(os.classic_ops, 0) + COALESCE(ast.assets, 0) > 0 THEN 'classic' - ELSE 'github' END, - CASE - WHEN EXISTS (SELECT 1 FROM links l WHERE l.full_name = r.full_name) - AND COALESCE(cs.contracts, 0) + COALESCE(os.classic_ops, 0) - + COALESCE(ast.assets, 0) + COALESCE(ws.wasm, 0) + COALESCE(fs.failed, 0) > 0 - THEN 'verified' - WHEN jsonb_array_length(r.dependencies -> 'ecosystems') > 0 - AND COALESCE(r.window_commits, 0) >= $4 - AND (r.created_at BETWEEN $1 AND $2 - OR COALESCE(r.stellar_added_in_window, FALSE)) THEN 'likely' - ELSE 'possible' END, - COALESCE(cs.contracts, 0), COALESCE(os.classic_ops, 0), COALESCE(ast.assets, 0), - COALESCE(ws.wasm, 0), COALESCE(fs.failed, 0), COALESCE(cs.builders, 0), - COALESCE(cs.first_at, r.pushed_at), COALESCE(cs.last_at, r.pushed_at) -FROM repos r -LEFT JOIN contract_stats cs USING (full_name) -LEFT JOIN op_stats os USING (full_name) -LEFT JOIN asset_stats ast USING (full_name) -LEFT JOIN wasm_stats ws USING (full_name) -LEFT JOIN failed_stats fs USING (full_name) -WHERE r.full_name IN (SELECT full_name FROM links) - OR 'wasm_meta' = ANY(r.discovered_via) - OR (jsonb_array_length(r.dependencies -> 'ecosystems') > 0 - AND (COALESCE(r.window_commits, 0) >= $3 - OR COALESCE(r.stellar_added_in_window, FALSE) - OR r.created_at BETWEEN $1 AND $2)) -""" - -ACCOUNT_PROJECTS = f""" -INSERT INTO projects -WITH builders AS ( - SELECT deployer AS account FROM contracts WHERE deployed_at BETWEEN $1 AND $2 - UNION SELECT uploader FROM wasm_uploads WHERE uploaded_at BETWEEN $1 AND $2 - UNION SELECT split_part(asset, ':', 2) FROM classic_ops - WHERE asset IS NOT NULL AND occurred_at BETWEEN $1 AND $2 - AND split_part(asset, ':', 2) IN ({HACKERS}) -), orphans AS ( - SELECT account FROM builders - WHERE account NOT IN (SELECT subject FROM links WHERE subject_kind = 'account') -) -SELECT 'acct:' || o.account, NULL, - CASE WHEN EXISTS (SELECT 1 FROM contracts c WHERE c.deployer = o.account - AND c.deployed_at BETWEEN $1 AND $2) THEN 'contracts' ELSE 'classic' END, - 'unattributed', - (SELECT count(*) FROM contracts c - WHERE c.deployer = o.account AND c.deployed_at BETWEEN $1 AND $2), - (SELECT count(*) FROM classic_ops x - WHERE x.source_account = o.account AND x.occurred_at BETWEEN $1 AND $2), - (SELECT count(DISTINCT asset) FROM classic_ops x - WHERE split_part(x.asset, ':', 2) = o.account AND x.occurred_at BETWEEN $1 AND $2), - (SELECT count(DISTINCT wasm_hash) FROM wasm_uploads w - WHERE w.uploader = o.account AND w.uploaded_at BETWEEN $1 AND $2), - (SELECT count(*) FROM failed_txs f - WHERE f.source_account = o.account AND f.created_at BETWEEN $1 AND $2), - 1, - (SELECT min(at) FROM ( - SELECT deployed_at AS at FROM contracts WHERE deployer = o.account - UNION ALL SELECT uploaded_at FROM wasm_uploads WHERE uploader = o.account - UNION ALL SELECT occurred_at FROM classic_ops WHERE source_account = o.account) a), - (SELECT max(at) FROM ( - SELECT deployed_at AS at FROM contracts WHERE deployer = o.account - UNION ALL SELECT uploaded_at FROM wasm_uploads WHERE uploader = o.account - UNION ALL SELECT occurred_at FROM classic_ops WHERE source_account = o.account) a) -FROM orphans o -""" - class Report(BaseModel): links_by_method: dict[str, int] @@ -181,27 +38,17 @@ class Report(BaseModel): async def _insert_links( - connection: asyncpg.Connection, method: LinkMethod, select: str, *params: object + connection: asyncpg.Connection, method: LinkMethod, query: str, *params: object ) -> None: - await connection.execute( - f""" - INSERT INTO links (full_name, subject_kind, subject, method) - SELECT s.full_name, s.subject_kind, s.subject, '{method}' - FROM ({select}) AS s(full_name, subject_kind, subject) - ON CONFLICT DO NOTHING - """, - *params, - ) + await connection.execute(load_sql(query), method.value, *params) async def _wasm_meta_links(connection: asyncpg.Connection) -> None: canonical = { row["full_name"].lower(): row["full_name"] - for row in await connection.fetch("SELECT full_name FROM repos") + for row in await connection.fetch(load_sql("reconcile/repo_names")) } - rows = await connection.fetch( - "SELECT DISTINCT wasm_hash, source_repo FROM wasm_uploads WHERE source_repo IS NOT NULL" - ) + rows = await connection.fetch(load_sql("reconcile/wasm_sources")) links = [] for row in rows: name = normalize_repo(row["source_repo"]) @@ -214,9 +61,7 @@ async def _wasm_meta_links(connection: asyncpg.Connection) -> None: LinkMethod.WASM_META, ) ) - await connection.executemany( - "INSERT INTO links VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", links - ) + await connection.executemany(load_sql("reconcile/insert_link"), links) async def reconcile(connection: asyncpg.Connection) -> Report: @@ -237,34 +82,33 @@ async def reconcile(connection: asyncpg.Connection) -> Report: } async with connection.transaction(): - await connection.execute("DELETE FROM links") - await connection.execute("DELETE FROM projects") + await connection.execute(load_sql("reconcile/reset_links")) + await connection.execute(load_sql("reconcile/reset_projects")) await _wasm_meta_links(connection) - for method, select in EVIDENCE.items(): - await _insert_links(connection, method, select, *evidence_params[method]) - for select in FOOTPRINT: - await _insert_links(connection, LinkMethod.FOOTPRINT, select) + for method, query in EVIDENCE.items(): + await _insert_links(connection, method, query, *evidence_params[method]) + for query in FOOTPRINT: + await _insert_links(connection, LinkMethod.FOOTPRINT, query) official = (window.start, window.end) await connection.execute( - REPO_PROJECTS, *official, MIN_WINDOW_COMMITS, MIN_LIKELY_COMMITS + load_sql("reconcile/repo_projects"), + *official, + MIN_WINDOW_COMMITS, + MIN_LIKELY_COMMITS, ) await connection.execute( - ACCOUNT_PROJECTS, *official, settings.friendbot_account + load_sql("reconcile/account_projects"), + *official, + settings.friendbot_account, ) await connection.execute( - "DELETE FROM projects WHERE lower(project_id) = ANY($1)", + load_sql("reconcile/exclude_projects"), [f"repo:{name.lower()}" for name in hackathon.discovery.exclude_repos], ) - methods = await connection.fetch( - "SELECT method, count(*) AS n FROM links GROUP BY method" - ) - footprints = await connection.fetch( - "SELECT footprint, count(*) AS n FROM projects GROUP BY footprint" - ) - tiers = await connection.fetch( - "SELECT tier, count(*) AS n FROM projects GROUP BY tier" - ) + methods = await connection.fetch(load_sql("reconcile/report_methods")) + footprints = await connection.fetch(load_sql("reconcile/report_footprints")) + tiers = await connection.fetch(load_sql("reconcile/report_tiers")) return Report( links_by_method={row["method"]: row["n"] for row in methods}, projects_by_footprint={row["footprint"]: row["n"] for row in footprints}, diff --git a/apps/api/src/hackproof/sql/reconcile/account_projects.sql b/apps/api/src/hackproof/sql/reconcile/account_projects.sql new file mode 100644 index 0000000..ccab25a --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/account_projects.sql @@ -0,0 +1,79 @@ +INSERT INTO projects ( + project_id, full_name, footprint, tier, contracts, classic_ops, assets, + wasm_uploads, failed_txs, builders, first_activity, last_activity +) +WITH builders AS ( + SELECT c.deployer AS account FROM contracts AS c + WHERE c.deployed_at BETWEEN $1 AND $2 + UNION + SELECT w.uploader FROM wasm_uploads AS w + WHERE w.uploaded_at BETWEEN $1 AND $2 + UNION + SELECT split_part(x.asset, ':', 2) FROM classic_ops AS x + WHERE + x.asset IS NOT NULL AND x.occurred_at BETWEEN $1 AND $2 + AND split_part(x.asset, ':', 2) IN (SELECT hackers($1, $2, $3) AS account) +), + +orphans AS ( + SELECT b.account FROM builders AS b + WHERE b.account NOT IN ( + SELECT l.subject FROM links AS l + WHERE l.subject_kind = 'account' + ) +) + +SELECT + 'acct:' || o.account AS project_id, + NULL AS full_name, + CASE WHEN EXISTS ( + SELECT 1 FROM contracts AS c + WHERE c.deployer = o.account AND c.deployed_at BETWEEN $1 AND $2 + ) THEN 'contracts' ELSE 'classic' END AS footprint, + 'unattributed' AS tier, + ( + SELECT count(*) FROM contracts AS c + WHERE c.deployer = o.account AND c.deployed_at BETWEEN $1 AND $2 + ) AS contracts, + ( + SELECT count(*) FROM classic_ops AS x + WHERE x.source_account = o.account AND x.occurred_at BETWEEN $1 AND $2 + ) AS classic_ops, + ( + SELECT count(DISTINCT x.asset) FROM classic_ops AS x + WHERE split_part(x.asset, ':', 2) = o.account AND x.occurred_at BETWEEN $1 AND $2 + ) AS assets, + ( + SELECT count(DISTINCT w.wasm_hash) FROM wasm_uploads AS w + WHERE w.uploader = o.account AND w.uploaded_at BETWEEN $1 AND $2 + ) AS wasm_uploads, + ( + SELECT count(*) FROM failed_txs AS f + WHERE f.source_account = o.account AND f.created_at BETWEEN $1 AND $2 + ) AS failed_txs, + 1 AS builders, + ( + SELECT min(t.at) FROM ( + SELECT c.deployed_at AS at FROM contracts AS c + WHERE c.deployer = o.account + UNION ALL + SELECT w.uploaded_at AS at FROM wasm_uploads AS w + WHERE w.uploader = o.account + UNION ALL + SELECT x.occurred_at AS at FROM classic_ops AS x + WHERE x.source_account = o.account + ) AS t + ) AS first_activity, + ( + SELECT max(t.at) FROM ( + SELECT c.deployed_at AS at FROM contracts AS c + WHERE c.deployer = o.account + UNION ALL + SELECT w.uploaded_at AS at FROM wasm_uploads AS w + WHERE w.uploader = o.account + UNION ALL + SELECT x.occurred_at AS at FROM classic_ops AS x + WHERE x.source_account = o.account + ) AS t + ) AS last_activity +FROM orphans AS o diff --git a/apps/api/src/hackproof/sql/reconcile/exclude_projects.sql b/apps/api/src/hackproof/sql/reconcile/exclude_projects.sql new file mode 100644 index 0000000..4750bde --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/exclude_projects.sql @@ -0,0 +1,2 @@ +DELETE FROM projects +WHERE lower(project_id) = any($1) diff --git a/apps/api/src/hackproof/sql/reconcile/insert_link.sql b/apps/api/src/hackproof/sql/reconcile/insert_link.sql new file mode 100644 index 0000000..86f9141 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/insert_link.sql @@ -0,0 +1 @@ +INSERT INTO links VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_asset_in_repo.sql b/apps/api/src/hackproof/sql/reconcile/link_asset_in_repo.sql new file mode 100644 index 0000000..2f69aea --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_asset_in_repo.sql @@ -0,0 +1,15 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +SELECT + r.full_name, + 'asset' AS subject_kind, + r.value AS subject, + $1::text AS method +FROM repo_refs AS r +WHERE + r.kind = 'asset' + AND split_part(r.value, ':', 2) IN (SELECT hackers($2, $3, $4) AS account) + AND r.value IN ( + SELECT o.asset FROM classic_ops AS o + WHERE o.asset IS NOT NULL AND o.occurred_at BETWEEN $2 AND $3 + ) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_contract_in_repo.sql b/apps/api/src/hackproof/sql/reconcile/link_contract_in_repo.sql new file mode 100644 index 0000000..2c7bee3 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_contract_in_repo.sql @@ -0,0 +1,11 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +SELECT + r.full_name, + 'contract' AS subject_kind, + r.value AS subject, + $1::text AS method +FROM repo_refs AS r +WHERE + r.kind = 'contract' + AND r.value IN (SELECT c.contract_id FROM contracts AS c) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_deployer_in_repo.sql b/apps/api/src/hackproof/sql/reconcile/link_deployer_in_repo.sql new file mode 100644 index 0000000..2aa4969 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_deployer_in_repo.sql @@ -0,0 +1,18 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +SELECT + r.full_name, + 'account' AS subject_kind, + r.value AS subject, + $1::text AS method +FROM repo_refs AS r +WHERE + r.kind = 'account' + AND r.value IN (SELECT hackers($2, $3, $4) AS account) + AND r.value <> all($5::text[]) + AND r.value NOT IN ( + SELECT x.value FROM repo_refs AS x + WHERE x.kind = 'account' + GROUP BY x.value + HAVING count(*) > $6 + ) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_footprint_assets.sql b/apps/api/src/hackproof/sql/reconcile/link_footprint_assets.sql new file mode 100644 index 0000000..214e5d1 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_footprint_assets.sql @@ -0,0 +1,14 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +WITH a AS ( + SELECT DISTINCT o.asset FROM classic_ops AS o + WHERE o.asset IS NOT NULL +) + +SELECT + l.full_name, + 'asset' AS subject_kind, + a.asset AS subject, + $1::text AS method +FROM links AS l +INNER JOIN a ON l.subject_kind = 'account' AND split_part(a.asset, ':', 2) = l.subject +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_footprint_contracts.sql b/apps/api/src/hackproof/sql/reconcile/link_footprint_contracts.sql new file mode 100644 index 0000000..17d121f --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_footprint_contracts.sql @@ -0,0 +1,9 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +SELECT + l.full_name, + 'contract' AS subject_kind, + c.contract_id AS subject, + $1::text AS method +FROM links AS l +INNER JOIN contracts AS c ON l.subject_kind = 'account' AND l.subject = c.deployer +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_footprint_deployers.sql b/apps/api/src/hackproof/sql/reconcile/link_footprint_deployers.sql new file mode 100644 index 0000000..c673f3a --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_footprint_deployers.sql @@ -0,0 +1,9 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +SELECT + l.full_name, + 'account' AS subject_kind, + c.deployer AS subject, + $1::text AS method +FROM links AS l +INNER JOIN contracts AS c ON l.subject_kind = 'contract' AND l.subject = c.contract_id +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_footprint_uploaders.sql b/apps/api/src/hackproof/sql/reconcile/link_footprint_uploaders.sql new file mode 100644 index 0000000..db6fc35 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_footprint_uploaders.sql @@ -0,0 +1,15 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +SELECT + l.full_name, + 'account' AS subject_kind, + w.uploader AS subject, + $1::text AS method +FROM links AS l +INNER JOIN wasm_uploads AS w ON l.subject_kind = 'wasm' AND l.subject = w.wasm_hash +WHERE + l.subject IN ( + SELECT s.wasm_hash FROM wasm_uploads AS s + GROUP BY s.wasm_hash + HAVING count(DISTINCT s.uploader) = 1 + ) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/link_wasm_hash_match.sql b/apps/api/src/hackproof/sql/reconcile/link_wasm_hash_match.sql new file mode 100644 index 0000000..951a570 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/link_wasm_hash_match.sql @@ -0,0 +1,14 @@ +INSERT INTO links (full_name, subject_kind, subject, method) +SELECT + r.full_name, + 'wasm' AS subject_kind, + r.value AS subject, + $1::text AS method +FROM repo_refs AS r +WHERE + r.kind = 'wasm' + AND ( + r.value IN (SELECT w.wasm_hash FROM wasm_uploads AS w) + OR r.value IN (SELECT c.wasm_hash FROM contracts AS c) + ) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/reconcile/repo_names.sql b/apps/api/src/hackproof/sql/reconcile/repo_names.sql new file mode 100644 index 0000000..8699b43 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/repo_names.sql @@ -0,0 +1 @@ +SELECT full_name FROM repos diff --git a/apps/api/src/hackproof/sql/reconcile/repo_projects.sql b/apps/api/src/hackproof/sql/reconcile/repo_projects.sql new file mode 100644 index 0000000..b2cbb17 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/repo_projects.sql @@ -0,0 +1,108 @@ +INSERT INTO projects ( + project_id, full_name, footprint, tier, contracts, classic_ops, assets, + wasm_uploads, failed_txs, builders, first_activity, last_activity +) +WITH contract_stats AS ( + SELECT + a.full_name, + count(*) AS contracts, + count(DISTINCT c.deployer) AS builders, + min(c.deployed_at) AS first_at, + max(c.deployed_at) AS last_at + FROM contracts AS c + INNER JOIN attribution AS a ON a.subject_kind = 'contract' AND c.contract_id = a.subject + WHERE c.deployed_at BETWEEN $1 AND $2 + GROUP BY a.full_name +), + +op_stats AS ( + SELECT + a.full_name, + count(*) AS classic_ops + FROM classic_ops AS o + INNER JOIN attribution AS a ON a.subject_kind = 'account' AND o.source_account = a.subject + WHERE o.occurred_at BETWEEN $1 AND $2 + GROUP BY a.full_name +), + +asset_stats AS ( + SELECT + t.full_name, + count(*) AS assets + FROM attribution AS t + WHERE t.subject_kind = 'asset' + GROUP BY t.full_name +), + +wasm_stats AS ( + SELECT + coalesce(aa.full_name, aw.full_name) AS full_name, + count(DISTINCT w.wasm_hash) AS wasm + FROM wasm_uploads AS w + LEFT JOIN attribution AS aa ON aa.subject_kind = 'account' AND w.uploader = aa.subject + LEFT JOIN attribution AS aw ON aw.subject_kind = 'wasm' AND w.wasm_hash = aw.subject + WHERE w.uploaded_at BETWEEN $1 AND $2 + GROUP BY coalesce(aa.full_name, aw.full_name) +), + +failed_stats AS ( + SELECT + a.full_name, + count(*) AS failed + FROM failed_txs AS f + INNER JOIN attribution AS a ON a.subject_kind = 'account' AND f.source_account = a.subject + WHERE f.created_at BETWEEN $1 AND $2 + GROUP BY a.full_name +) + +SELECT + 'repo:' || r.full_name AS project_id, + r.full_name, + CASE + WHEN coalesce(cs.contracts, 0) > 0 THEN 'contracts' + WHEN coalesce(os.classic_ops, 0) + coalesce(ast.assets, 0) > 0 THEN 'classic' + ELSE 'github' + END AS footprint, + CASE + WHEN + EXISTS ( + SELECT 1 FROM links AS l + WHERE l.full_name = r.full_name + ) + AND coalesce(cs.contracts, 0) + coalesce(os.classic_ops, 0) + + coalesce(ast.assets, 0) + coalesce(ws.wasm, 0) + coalesce(fs.failed, 0) > 0 + THEN 'verified' + WHEN + jsonb_array_length(r.dependencies -> 'ecosystems') > 0 + AND coalesce(r.window_commits, 0) >= $4 + AND ( + r.created_at BETWEEN $1 AND $2 + OR coalesce(r.stellar_added_in_window, FALSE) + ) THEN 'likely' + ELSE 'possible' + END AS tier, + coalesce(cs.contracts, 0) AS contracts, + coalesce(os.classic_ops, 0) AS classic_ops, + coalesce(ast.assets, 0) AS assets, + coalesce(ws.wasm, 0) AS wasm_uploads, + coalesce(fs.failed, 0) AS failed_txs, + coalesce(cs.builders, 0) AS builders, + coalesce(cs.first_at, r.pushed_at) AS first_activity, + coalesce(cs.last_at, r.pushed_at) AS last_activity +FROM repos AS r +LEFT JOIN contract_stats AS cs ON r.full_name = cs.full_name +LEFT JOIN op_stats AS os ON r.full_name = os.full_name +LEFT JOIN asset_stats AS ast ON r.full_name = ast.full_name +LEFT JOIN wasm_stats AS ws ON r.full_name = ws.full_name +LEFT JOIN failed_stats AS fs ON r.full_name = fs.full_name +WHERE + r.full_name IN (SELECT l.full_name FROM links AS l) + OR 'wasm_meta' = any(r.discovered_via) + OR ( + jsonb_array_length(r.dependencies -> 'ecosystems') > 0 + AND ( + coalesce(r.window_commits, 0) >= $3 + OR coalesce(r.stellar_added_in_window, FALSE) + OR r.created_at BETWEEN $1 AND $2 + ) + ) diff --git a/apps/api/src/hackproof/sql/reconcile/report_footprints.sql b/apps/api/src/hackproof/sql/reconcile/report_footprints.sql new file mode 100644 index 0000000..e3197b7 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/report_footprints.sql @@ -0,0 +1,5 @@ +SELECT + footprint, + count(*) AS n +FROM projects +GROUP BY footprint diff --git a/apps/api/src/hackproof/sql/reconcile/report_methods.sql b/apps/api/src/hackproof/sql/reconcile/report_methods.sql new file mode 100644 index 0000000..3a6ccfa --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/report_methods.sql @@ -0,0 +1,5 @@ +SELECT + method, + count(*) AS n +FROM links +GROUP BY method diff --git a/apps/api/src/hackproof/sql/reconcile/report_tiers.sql b/apps/api/src/hackproof/sql/reconcile/report_tiers.sql new file mode 100644 index 0000000..68cd0cc --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/report_tiers.sql @@ -0,0 +1,5 @@ +SELECT + tier, + count(*) AS n +FROM projects +GROUP BY tier diff --git a/apps/api/src/hackproof/sql/reconcile/reset_links.sql b/apps/api/src/hackproof/sql/reconcile/reset_links.sql new file mode 100644 index 0000000..4b2627f --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/reset_links.sql @@ -0,0 +1 @@ +DELETE FROM links diff --git a/apps/api/src/hackproof/sql/reconcile/reset_projects.sql b/apps/api/src/hackproof/sql/reconcile/reset_projects.sql new file mode 100644 index 0000000..cdbb886 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/reset_projects.sql @@ -0,0 +1 @@ +DELETE FROM projects diff --git a/apps/api/src/hackproof/sql/reconcile/wasm_sources.sql b/apps/api/src/hackproof/sql/reconcile/wasm_sources.sql new file mode 100644 index 0000000..89b9e69 --- /dev/null +++ b/apps/api/src/hackproof/sql/reconcile/wasm_sources.sql @@ -0,0 +1,5 @@ +SELECT DISTINCT + wasm_hash, + source_repo +FROM wasm_uploads +WHERE source_repo IS NOT NULL From 7c61400426a858b1848cf15243d1ac5b40ab3a01 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:39:22 +0100 Subject: [PATCH 11/18] refactor: move the export queries into sql files --- apps/api/src/hackproof/export/build.py | 100 ++++-------------- .../api/src/hackproof/sql/export/accounts.sql | 6 ++ apps/api/src/hackproof/sql/export/agents.sql | 9 ++ .../src/hackproof/sql/export/contracts.sql | 12 +++ .../src/hackproof/sql/export/contributors.sql | 11 ++ .../api/src/hackproof/sql/export/evidence.sql | 11 ++ .../src/hackproof/sql/export/integrations.sql | 9 ++ .../api/src/hackproof/sql/export/projects.sql | 29 +++++ apps/api/src/hackproof/sql/export/tools.sql | 9 ++ apps/api/src/hackproof/sql/export/wasm.sql | 8 ++ 10 files changed, 122 insertions(+), 82 deletions(-) create mode 100644 apps/api/src/hackproof/sql/export/accounts.sql create mode 100644 apps/api/src/hackproof/sql/export/agents.sql create mode 100644 apps/api/src/hackproof/sql/export/contracts.sql create mode 100644 apps/api/src/hackproof/sql/export/contributors.sql create mode 100644 apps/api/src/hackproof/sql/export/evidence.sql create mode 100644 apps/api/src/hackproof/sql/export/integrations.sql create mode 100644 apps/api/src/hackproof/sql/export/projects.sql create mode 100644 apps/api/src/hackproof/sql/export/tools.sql create mode 100644 apps/api/src/hackproof/sql/export/wasm.sql diff --git a/apps/api/src/hackproof/export/build.py b/apps/api/src/hackproof/export/build.py index 1517423..2372320 100644 --- a/apps/api/src/hackproof/export/build.py +++ b/apps/api/src/hackproof/export/build.py @@ -30,83 +30,11 @@ ) from hackproof.hackathon import Hackathon from hackproof.integrations.catalog import Catalog, get_catalog -from hackproof.linking.methods import METHOD_RANK_SQL +from hackproof.sql_loader import load_sql STRICT_TIERS = ("verified", "likely") TIER_ORDER = ("verified", "likely", "possible") -PROJECTS_SQL = """ -SELECT p.full_name, p.tier, p.contracts, p.classic_ops, p.assets, p.wasm_uploads, p.failed_txs, - r.url, r.description, r.homepage, r.language, r.license, r.stars, r.forks, r.topics, - r.dependencies, r.created_at, r.pushed_at, r.window_commits, r.total_commits, - r.owner, r.owner_avatar, r.country, r.declared_track -FROM projects p -JOIN repos r ON r.full_name = p.full_name -WHERE p.tier = ANY($1) -ORDER BY array_position($2::text[], p.tier), p.contracts DESC, r.stars DESC, p.full_name -""" - -CONTRIBUTORS_SQL = """ -SELECT c.full_name, c.login, c.url, c.avatar_url, c.contributions, u.country -FROM repo_contributors c -LEFT JOIN github_users u ON u.login = c.login -WHERE c.full_name = ANY($1) -ORDER BY c.contributions DESC, c.login -""" - -CONTRACTS_SQL = """ -SELECT a.full_name, c.contract_id, c.deployer, c.wasm_hash, c.deployed_at, c.ledger, c.tx_hash -FROM contracts c -JOIN attribution a ON a.subject_kind = 'contract' AND a.subject = c.contract_id -WHERE a.full_name = ANY($1) AND c.deployed_at BETWEEN $2 AND $3 -ORDER BY c.deployed_at -""" - -ACCOUNTS_SQL = """ -SELECT full_name, subject FROM links -WHERE subject_kind = 'account' AND full_name = ANY($1) -ORDER BY subject -""" - -WASM_SQL = """ -SELECT DISTINCT ON (a.full_name, w.wasm_hash) a.full_name, w.wasm_hash, w.tx_hash -FROM wasm_uploads w -JOIN attribution a ON a.subject_kind = 'account' AND a.subject = w.uploader -WHERE a.full_name = ANY($1) AND w.uploaded_at BETWEEN $2 AND $3 -ORDER BY a.full_name, w.wasm_hash, w.uploaded_at -""" - -EVIDENCE_SQL = f""" -SELECT l.full_name, l.subject_kind, l.subject, l.method, r.path -FROM links l -LEFT JOIN repo_refs r - ON r.full_name = l.full_name AND r.kind = l.subject_kind AND r.value = l.subject -WHERE l.full_name = ANY($1) AND l.method <> 'footprint' -ORDER BY {METHOD_RANK_SQL}, l.subject_kind, l.subject -""" - -INTEGRATIONS_SQL = """ -SELECT full_name, partner, method, evidence, path -FROM repo_integrations -WHERE full_name = ANY($1) -ORDER BY partner, method, evidence -""" - -TOOLS_SQL = """ -SELECT full_name, tool, method, evidence, path -FROM repo_tools -WHERE full_name = ANY($1) -ORDER BY tool, method, evidence -""" - - -AGENTS_SQL = """ -SELECT full_name, agent, method, evidence, path -FROM repo_agents -WHERE full_name = ANY($1) -ORDER BY agent, method, evidence -""" - def _evidence_url(kind: str, subject: str) -> str | None: match kind: @@ -206,18 +134,26 @@ async def build_export( tiers: Iterable[str] = STRICT_TIERS, ) -> HackathonExport: window = hackathon.window - projects = await connection.fetch(PROJECTS_SQL, list(tiers), list(TIER_ORDER)) + projects = await connection.fetch( + load_sql("export/projects"), list(tiers), list(TIER_ORDER) + ) names = [row["full_name"] for row in projects] span = (window.start, window.end) - contributors = _group(await connection.fetch(CONTRIBUTORS_SQL, names)) - contracts = _group(await connection.fetch(CONTRACTS_SQL, names, *span)) - accounts = _group(await connection.fetch(ACCOUNTS_SQL, names)) - wasm = _group(await connection.fetch(WASM_SQL, names, *span)) - evidence = _group(await connection.fetch(EVIDENCE_SQL, names)) - integrations = _group(await connection.fetch(INTEGRATIONS_SQL, names)) - tools = _group(await connection.fetch(TOOLS_SQL, names)) - agents = _group(await connection.fetch(AGENTS_SQL, names)) + contributors = _group( + await connection.fetch(load_sql("export/contributors"), names) + ) + contracts = _group( + await connection.fetch(load_sql("export/contracts"), names, *span) + ) + accounts = _group(await connection.fetch(load_sql("export/accounts"), names)) + wasm = _group(await connection.fetch(load_sql("export/wasm"), names, *span)) + evidence = _group(await connection.fetch(load_sql("export/evidence"), names)) + integrations = _group( + await connection.fetch(load_sql("export/integrations"), names) + ) + tools = _group(await connection.fetch(load_sql("export/tools"), names)) + agents = _group(await connection.fetch(load_sql("export/agents"), names)) catalog = get_catalog() awards = {award.project_id: award.place for award in hackathon.awards} diff --git a/apps/api/src/hackproof/sql/export/accounts.sql b/apps/api/src/hackproof/sql/export/accounts.sql new file mode 100644 index 0000000..db4ce0d --- /dev/null +++ b/apps/api/src/hackproof/sql/export/accounts.sql @@ -0,0 +1,6 @@ +SELECT + full_name, + subject +FROM links +WHERE subject_kind = 'account' AND full_name = ANY($1) +ORDER BY subject diff --git a/apps/api/src/hackproof/sql/export/agents.sql b/apps/api/src/hackproof/sql/export/agents.sql new file mode 100644 index 0000000..1818086 --- /dev/null +++ b/apps/api/src/hackproof/sql/export/agents.sql @@ -0,0 +1,9 @@ +SELECT + full_name, + agent, + method, + evidence, + path +FROM repo_agents +WHERE full_name = ANY($1) +ORDER BY agent, method, evidence diff --git a/apps/api/src/hackproof/sql/export/contracts.sql b/apps/api/src/hackproof/sql/export/contracts.sql new file mode 100644 index 0000000..7dc83f5 --- /dev/null +++ b/apps/api/src/hackproof/sql/export/contracts.sql @@ -0,0 +1,12 @@ +SELECT + a.full_name, + c.contract_id, + c.deployer, + c.wasm_hash, + c.deployed_at, + c.ledger, + c.tx_hash +FROM contracts AS c +INNER JOIN attribution AS a ON a.subject_kind = 'contract' AND c.contract_id = a.subject +WHERE a.full_name = ANY($1) AND c.deployed_at BETWEEN $2 AND $3 +ORDER BY c.deployed_at diff --git a/apps/api/src/hackproof/sql/export/contributors.sql b/apps/api/src/hackproof/sql/export/contributors.sql new file mode 100644 index 0000000..b56cb22 --- /dev/null +++ b/apps/api/src/hackproof/sql/export/contributors.sql @@ -0,0 +1,11 @@ +SELECT + c.full_name, + c.login, + c.url, + c.avatar_url, + c.contributions, + u.country +FROM repo_contributors AS c +LEFT JOIN github_users AS u ON c.login = u.login +WHERE c.full_name = ANY($1) +ORDER BY c.contributions DESC, c.login ASC diff --git a/apps/api/src/hackproof/sql/export/evidence.sql b/apps/api/src/hackproof/sql/export/evidence.sql new file mode 100644 index 0000000..9cdc824 --- /dev/null +++ b/apps/api/src/hackproof/sql/export/evidence.sql @@ -0,0 +1,11 @@ +SELECT + l.full_name, + l.subject_kind, + l.subject, + l.method, + r.path +FROM links AS l +LEFT JOIN repo_refs AS r + ON l.full_name = r.full_name AND l.subject_kind = r.kind AND l.subject = r.value +WHERE l.full_name = ANY($1) AND l.method <> 'footprint' +ORDER BY METHOD_RANK(l.method), l.subject_kind, l.subject diff --git a/apps/api/src/hackproof/sql/export/integrations.sql b/apps/api/src/hackproof/sql/export/integrations.sql new file mode 100644 index 0000000..711fe83 --- /dev/null +++ b/apps/api/src/hackproof/sql/export/integrations.sql @@ -0,0 +1,9 @@ +SELECT + full_name, + partner, + method, + evidence, + path +FROM repo_integrations +WHERE full_name = ANY($1) +ORDER BY partner, method, evidence diff --git a/apps/api/src/hackproof/sql/export/projects.sql b/apps/api/src/hackproof/sql/export/projects.sql new file mode 100644 index 0000000..6722acb --- /dev/null +++ b/apps/api/src/hackproof/sql/export/projects.sql @@ -0,0 +1,29 @@ +SELECT + p.full_name, + p.tier, + p.contracts, + p.classic_ops, + p.assets, + p.wasm_uploads, + p.failed_txs, + r.url, + r.description, + r.homepage, + r.language, + r.license, + r.stars, + r.forks, + r.topics, + r.dependencies, + r.created_at, + r.pushed_at, + r.window_commits, + r.total_commits, + r.owner, + r.owner_avatar, + r.country, + r.declared_track +FROM projects AS p +INNER JOIN repos AS r ON p.full_name = r.full_name +WHERE p.tier = ANY($1) +ORDER BY ARRAY_POSITION($2::text[], p.tier), p.contracts DESC, r.stars DESC, p.full_name ASC diff --git a/apps/api/src/hackproof/sql/export/tools.sql b/apps/api/src/hackproof/sql/export/tools.sql new file mode 100644 index 0000000..b52ee2d --- /dev/null +++ b/apps/api/src/hackproof/sql/export/tools.sql @@ -0,0 +1,9 @@ +SELECT + full_name, + tool, + method, + evidence, + path +FROM repo_tools +WHERE full_name = ANY($1) +ORDER BY tool, method, evidence diff --git a/apps/api/src/hackproof/sql/export/wasm.sql b/apps/api/src/hackproof/sql/export/wasm.sql new file mode 100644 index 0000000..2845cf0 --- /dev/null +++ b/apps/api/src/hackproof/sql/export/wasm.sql @@ -0,0 +1,8 @@ +SELECT DISTINCT ON (a.full_name, w.wasm_hash) + a.full_name, + w.wasm_hash, + w.tx_hash +FROM wasm_uploads AS w +INNER JOIN attribution AS a ON a.subject_kind = 'account' AND w.uploader = a.subject +WHERE a.full_name = ANY($1) AND w.uploaded_at BETWEEN $2 AND $3 +ORDER BY a.full_name, w.wasm_hash, w.uploaded_at From ef944bce85699d6ed61d4c1520f6b41b0f510bf9 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:42:38 +0100 Subject: [PATCH 12/18] refactor: move the github queries into sql files --- apps/api/src/hackproof/github/discovery.py | 25 ++------------ apps/api/src/hackproof/github/jobs.py | 21 ++---------- apps/api/src/hackproof/github/profiles.py | 11 ++----- apps/api/src/hackproof/github/scan.py | 33 ++++++------------- apps/api/src/hackproof/linking/methods.py | 5 --- .../hackproof/sql/github/delete_agents.sql | 2 ++ .../sql/github/delete_integrations.sql | 2 ++ .../src/hackproof/sql/github/delete_tools.sql | 2 ++ .../src/hackproof/sql/github/insert_agent.sql | 2 ++ .../sql/github/insert_integration.sql | 2 ++ .../src/hackproof/sql/github/insert_ref.sql | 2 ++ .../src/hackproof/sql/github/insert_tool.sql | 2 ++ .../src/hackproof/sql/github/insert_user.sql | 2 ++ .../hackproof/sql/github/scan_candidates.sql | 16 +++++++++ .../src/hackproof/sql/github/scan_names.sql | 7 ++++ .../sql/github/unprofiled_logins.sql | 2 ++ .../src/hackproof/sql/github/update_scan.sql | 6 ++++ .../sql/github/upsert_contributor.sql | 5 +++ .../src/hackproof/sql/github/upsert_repo.sql | 21 ++++++++++++ .../sql/github/wasm_source_repos.sql | 2 ++ 20 files changed, 94 insertions(+), 76 deletions(-) create mode 100644 apps/api/src/hackproof/sql/github/delete_agents.sql create mode 100644 apps/api/src/hackproof/sql/github/delete_integrations.sql create mode 100644 apps/api/src/hackproof/sql/github/delete_tools.sql create mode 100644 apps/api/src/hackproof/sql/github/insert_agent.sql create mode 100644 apps/api/src/hackproof/sql/github/insert_integration.sql create mode 100644 apps/api/src/hackproof/sql/github/insert_ref.sql create mode 100644 apps/api/src/hackproof/sql/github/insert_tool.sql create mode 100644 apps/api/src/hackproof/sql/github/insert_user.sql create mode 100644 apps/api/src/hackproof/sql/github/scan_candidates.sql create mode 100644 apps/api/src/hackproof/sql/github/scan_names.sql create mode 100644 apps/api/src/hackproof/sql/github/unprofiled_logins.sql create mode 100644 apps/api/src/hackproof/sql/github/update_scan.sql create mode 100644 apps/api/src/hackproof/sql/github/upsert_contributor.sql create mode 100644 apps/api/src/hackproof/sql/github/upsert_repo.sql create mode 100644 apps/api/src/hackproof/sql/github/wasm_source_repos.sql diff --git a/apps/api/src/hackproof/github/discovery.py b/apps/api/src/hackproof/github/discovery.py index edb06bb..cc7f850 100644 --- a/apps/api/src/hackproof/github/discovery.py +++ b/apps/api/src/hackproof/github/discovery.py @@ -12,6 +12,7 @@ from hackproof.hackathon import Discovery, Window from hackproof.linking.country import resolve_country from hackproof.linking.repo_ref import normalize_repo +from hackproof.sql_loader import load_sql log = logging.getLogger("hackproof.discovery") @@ -196,25 +197,7 @@ async def store_candidates( connection: asyncpg.Connection, candidates: list[Candidate] ) -> None: await connection.executemany( - """ - INSERT INTO repos (full_name, url, description, stars, owner, owner_avatar, - default_branch, homepage, language, license, forks, created_at, - pushed_at, topics, window_commits, total_commits, base_sha, - owner_location, country, discovered_via) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, - $17, $18, $19, $20) - ON CONFLICT (full_name) DO UPDATE SET - description = EXCLUDED.description, stars = EXCLUDED.stars, - default_branch = EXCLUDED.default_branch, homepage = EXCLUDED.homepage, - language = EXCLUDED.language, license = EXCLUDED.license, forks = EXCLUDED.forks, - pushed_at = EXCLUDED.pushed_at, topics = EXCLUDED.topics, - window_commits = EXCLUDED.window_commits, total_commits = EXCLUDED.total_commits, - base_sha = EXCLUDED.base_sha, owner_location = EXCLUDED.owner_location, - country = EXCLUDED.country, - discovered_via = ARRAY( - SELECT DISTINCT via FROM unnest(repos.discovered_via || EXCLUDED.discovered_via) via - ) - """, + load_sql("github/upsert_repo"), [ ( c.full_name, @@ -244,9 +227,7 @@ async def store_candidates( async def wasm_meta_repos(connection: asyncpg.Connection) -> list[str]: - rows = await connection.fetch( - "SELECT DISTINCT source_repo FROM wasm_uploads WHERE source_repo IS NOT NULL" - ) + rows = await connection.fetch(load_sql("github/wasm_source_repos")) return sorted( {name for row in rows if (name := normalize_repo(row["source_repo"]))} ) diff --git a/apps/api/src/hackproof/github/jobs.py b/apps/api/src/hackproof/github/jobs.py index df920e7..86810e5 100644 --- a/apps/api/src/hackproof/github/jobs.py +++ b/apps/api/src/hackproof/github/jobs.py @@ -12,6 +12,7 @@ from hackproof.github.repositories import fetch_repositories from hackproof.github.scan import Scanner, store_scan from hackproof.hackathon import get_hackathon +from hackproof.sql_loader import load_sql log = logging.getLogger("hackproof.github") @@ -38,11 +39,6 @@ async def run_discover(include_strings: bool) -> None: log.info("stored %s candidate repos", len(candidates)) -PROJECT_REPOS = ( - "full_name IN (SELECT full_name FROM projects WHERE full_name IS NOT NULL)" -) - - async def run_scan(force: bool, projects_only: bool = False) -> None: settings, hackathon = get_settings(), get_hackathon() await open_pool() @@ -54,10 +50,7 @@ async def run_scan(force: bool, projects_only: bool = False) -> None: names = [ row["full_name"] for row in await connection.fetch( - f""" - SELECT full_name FROM repos - WHERE ($1 OR scanned_at IS NULL) AND ($2 IS FALSE OR {PROJECT_REPOS}) - """, + load_sql("github/scan_names"), force, projects_only, ) @@ -65,15 +58,7 @@ async def run_scan(force: bool, projects_only: bool = False) -> None: refreshed = await fetch_repositories(github, names, hackathon.window) await store_candidates(connection, refreshed) rows = await connection.fetch( - f""" - SELECT full_name, url, owner, default_branch, homepage, base_sha - FROM repos - WHERE ($1 OR scanned_at IS NULL) - AND ($2 IS FALSE OR {PROJECT_REPOS}) - AND (COALESCE(window_commits, 0) > 0 - OR 'wasm_meta' = ANY(discovered_via)) - ORDER BY pushed_at DESC NULLS LAST - """, + load_sql("github/scan_candidates"), force, projects_only, ) diff --git a/apps/api/src/hackproof/github/profiles.py b/apps/api/src/hackproof/github/profiles.py index c8abdad..612b4bb 100644 --- a/apps/api/src/hackproof/github/profiles.py +++ b/apps/api/src/hackproof/github/profiles.py @@ -4,6 +4,7 @@ from hackproof.github.client import GithubClient from hackproof.linking.country import resolve_country +from hackproof.sql_loader import load_sql BATCH = 100 @@ -17,12 +18,7 @@ def _query(logins: list[str]) -> str: async def resolve_profiles(github: GithubClient, connection: asyncpg.Connection) -> int: - rows = await connection.fetch( - """ - SELECT DISTINCT login FROM repo_contributors - WHERE login NOT IN (SELECT login FROM github_users) - """ - ) + rows = await connection.fetch(load_sql("github/unprofiled_logins")) logins = [row["login"] for row in rows] for start in range(0, len(logins), BATCH): batch = logins[start : start + BATCH] @@ -31,8 +27,7 @@ async def resolve_profiles(github: GithubClient, connection: asyncpg.Connection) node["login"].lower(): node["location"] for node in data.values() if node } await connection.executemany( - "INSERT INTO github_users (login, location, country) VALUES ($1, $2, $3)" - " ON CONFLICT (login) DO NOTHING", + load_sql("github/insert_user"), [ ( login, diff --git a/apps/api/src/hackproof/github/scan.py b/apps/api/src/hackproof/github/scan.py index 13b7734..4d7b98e 100644 --- a/apps/api/src/hackproof/github/scan.py +++ b/apps/api/src/hackproof/github/scan.py @@ -17,6 +17,7 @@ from hackproof.hackathon import Discovery, Window from hackproof.integrations.catalog import Catalog, get_catalog from hackproof.integrations.detect import detect, detect_agents, detect_tools +from hackproof.sql_loader import load_sql TEXT_SUFFIXES = { ".rs", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".toml", ".json", @@ -182,56 +183,42 @@ def _is_readme(path: str) -> bool: async def store_scan(connection: asyncpg.Connection, scan: RepoScan) -> None: async with connection.transaction(): await connection.executemany( - "INSERT INTO repo_refs VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", + load_sql("github/insert_ref"), [(scan.full_name, ref.kind, ref.value, ref.path) for ref in scan.refs], ) await connection.executemany( - """ - INSERT INTO repo_contributors VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (full_name, login) DO UPDATE SET contributions = EXCLUDED.contributions, - avatar_url = EXCLUDED.avatar_url - """, + load_sql("github/upsert_contributor"), [ (scan.full_name, c.login, c.url, c.avatar_url, c.contributions) for c in scan.contributors ], ) - await connection.execute( - "DELETE FROM repo_integrations WHERE full_name = $1", scan.full_name - ) + await connection.execute(load_sql("github/delete_integrations"), scan.full_name) await connection.executemany( - "INSERT INTO repo_integrations VALUES ($1, $2, $3, $4, $5)", + load_sql("github/insert_integration"), [ (scan.full_name, i.slug, i.method, i.evidence, i.path) for i in scan.integrations ], ) - await connection.execute( - "DELETE FROM repo_tools WHERE full_name = $1", scan.full_name - ) + await connection.execute(load_sql("github/delete_tools"), scan.full_name) await connection.executemany( - "INSERT INTO repo_tools VALUES ($1, $2, $3, $4, $5)", + load_sql("github/insert_tool"), [ (scan.full_name, t.slug, t.method, t.evidence, t.path) for t in scan.tools ], ) - await connection.execute( - "DELETE FROM repo_agents WHERE full_name = $1", scan.full_name - ) + await connection.execute(load_sql("github/delete_agents"), scan.full_name) await connection.executemany( - "INSERT INTO repo_agents VALUES ($1, $2, $3, $4, $5)", + load_sql("github/insert_agent"), [ (scan.full_name, a.slug, a.method, a.evidence, a.path) for a in scan.agents ], ) await connection.execute( - """ - UPDATE repos SET dependencies = $2, stellar_added_in_window = $3, - declared_track = $4, scanned_at = now() - WHERE full_name = $1 - """, + load_sql("github/update_scan"), scan.full_name, scan.dependencies.model_dump(), scan.stellar_added_in_window, diff --git a/apps/api/src/hackproof/linking/methods.py b/apps/api/src/hackproof/linking/methods.py index 3532aa9..4e527e4 100644 --- a/apps/api/src/hackproof/linking/methods.py +++ b/apps/api/src/hackproof/linking/methods.py @@ -10,8 +10,3 @@ class LinkMethod(StrEnum): ASSET_IN_REPO = "asset_in_repo" DEPLOYER_IN_REPO = "deployer_in_repo" FOOTPRINT = "footprint" - - -METHOD_RANK_SQL = "array_position(ARRAY[{}], l.method)".format( - ", ".join(f"'{method}'" for method in LinkMethod) -) diff --git a/apps/api/src/hackproof/sql/github/delete_agents.sql b/apps/api/src/hackproof/sql/github/delete_agents.sql new file mode 100644 index 0000000..1ffaf9d --- /dev/null +++ b/apps/api/src/hackproof/sql/github/delete_agents.sql @@ -0,0 +1,2 @@ +DELETE FROM repo_agents +WHERE full_name = $1 diff --git a/apps/api/src/hackproof/sql/github/delete_integrations.sql b/apps/api/src/hackproof/sql/github/delete_integrations.sql new file mode 100644 index 0000000..41e158a --- /dev/null +++ b/apps/api/src/hackproof/sql/github/delete_integrations.sql @@ -0,0 +1,2 @@ +DELETE FROM repo_integrations +WHERE full_name = $1 diff --git a/apps/api/src/hackproof/sql/github/delete_tools.sql b/apps/api/src/hackproof/sql/github/delete_tools.sql new file mode 100644 index 0000000..5f140e2 --- /dev/null +++ b/apps/api/src/hackproof/sql/github/delete_tools.sql @@ -0,0 +1,2 @@ +DELETE FROM repo_tools +WHERE full_name = $1 diff --git a/apps/api/src/hackproof/sql/github/insert_agent.sql b/apps/api/src/hackproof/sql/github/insert_agent.sql new file mode 100644 index 0000000..f97e28a --- /dev/null +++ b/apps/api/src/hackproof/sql/github/insert_agent.sql @@ -0,0 +1,2 @@ +INSERT INTO repo_agents (full_name, agent, method, evidence, path) +VALUES ($1, $2, $3, $4, $5) diff --git a/apps/api/src/hackproof/sql/github/insert_integration.sql b/apps/api/src/hackproof/sql/github/insert_integration.sql new file mode 100644 index 0000000..fce2af3 --- /dev/null +++ b/apps/api/src/hackproof/sql/github/insert_integration.sql @@ -0,0 +1,2 @@ +INSERT INTO repo_integrations (full_name, partner, method, evidence, path) +VALUES ($1, $2, $3, $4, $5) diff --git a/apps/api/src/hackproof/sql/github/insert_ref.sql b/apps/api/src/hackproof/sql/github/insert_ref.sql new file mode 100644 index 0000000..b62eb91 --- /dev/null +++ b/apps/api/src/hackproof/sql/github/insert_ref.sql @@ -0,0 +1,2 @@ +INSERT INTO repo_refs (full_name, kind, value, path) VALUES ($1, $2, $3, $4) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/github/insert_tool.sql b/apps/api/src/hackproof/sql/github/insert_tool.sql new file mode 100644 index 0000000..73fb24d --- /dev/null +++ b/apps/api/src/hackproof/sql/github/insert_tool.sql @@ -0,0 +1,2 @@ +INSERT INTO repo_tools (full_name, tool, method, evidence, path) +VALUES ($1, $2, $3, $4, $5) diff --git a/apps/api/src/hackproof/sql/github/insert_user.sql b/apps/api/src/hackproof/sql/github/insert_user.sql new file mode 100644 index 0000000..ff6711b --- /dev/null +++ b/apps/api/src/hackproof/sql/github/insert_user.sql @@ -0,0 +1,2 @@ +INSERT INTO github_users (login, location, country) VALUES ($1, $2, $3) +ON CONFLICT (login) DO NOTHING diff --git a/apps/api/src/hackproof/sql/github/scan_candidates.sql b/apps/api/src/hackproof/sql/github/scan_candidates.sql new file mode 100644 index 0000000..9b754b9 --- /dev/null +++ b/apps/api/src/hackproof/sql/github/scan_candidates.sql @@ -0,0 +1,16 @@ +SELECT + r.full_name, + r.url, + r.owner, + r.default_branch, + r.homepage, + r.base_sha +FROM repos AS r +WHERE + ($1 OR r.scanned_at IS NULL) + AND ($2 IS FALSE OR r.full_name IN ( + SELECT p.full_name FROM projects AS p + WHERE p.full_name IS NOT NULL + )) + AND (COALESCE(r.window_commits, 0) > 0 OR 'wasm_meta' = ANY(r.discovered_via)) +ORDER BY r.pushed_at DESC NULLS LAST diff --git a/apps/api/src/hackproof/sql/github/scan_names.sql b/apps/api/src/hackproof/sql/github/scan_names.sql new file mode 100644 index 0000000..ef6b797 --- /dev/null +++ b/apps/api/src/hackproof/sql/github/scan_names.sql @@ -0,0 +1,7 @@ +SELECT r.full_name FROM repos AS r +WHERE + ($1 OR r.scanned_at IS NULL) + AND ($2 IS FALSE OR r.full_name IN ( + SELECT p.full_name FROM projects AS p + WHERE p.full_name IS NOT NULL + )) diff --git a/apps/api/src/hackproof/sql/github/unprofiled_logins.sql b/apps/api/src/hackproof/sql/github/unprofiled_logins.sql new file mode 100644 index 0000000..c53d62a --- /dev/null +++ b/apps/api/src/hackproof/sql/github/unprofiled_logins.sql @@ -0,0 +1,2 @@ +SELECT DISTINCT c.login FROM repo_contributors AS c +WHERE c.login NOT IN (SELECT u.login FROM github_users AS u) diff --git a/apps/api/src/hackproof/sql/github/update_scan.sql b/apps/api/src/hackproof/sql/github/update_scan.sql new file mode 100644 index 0000000..39029a7 --- /dev/null +++ b/apps/api/src/hackproof/sql/github/update_scan.sql @@ -0,0 +1,6 @@ +UPDATE repos SET + dependencies = $2, + stellar_added_in_window = $3, + declared_track = $4, + scanned_at = now() +WHERE full_name = $1 diff --git a/apps/api/src/hackproof/sql/github/upsert_contributor.sql b/apps/api/src/hackproof/sql/github/upsert_contributor.sql new file mode 100644 index 0000000..d15e4f8 --- /dev/null +++ b/apps/api/src/hackproof/sql/github/upsert_contributor.sql @@ -0,0 +1,5 @@ +INSERT INTO repo_contributors (full_name, login, url, avatar_url, contributions) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (full_name, login) DO UPDATE SET + contributions = excluded.contributions, + avatar_url = excluded.avatar_url diff --git a/apps/api/src/hackproof/sql/github/upsert_repo.sql b/apps/api/src/hackproof/sql/github/upsert_repo.sql new file mode 100644 index 0000000..0ab76af --- /dev/null +++ b/apps/api/src/hackproof/sql/github/upsert_repo.sql @@ -0,0 +1,21 @@ +INSERT INTO repos ( + full_name, url, description, stars, owner, owner_avatar, + default_branch, homepage, language, license, forks, created_at, + pushed_at, topics, window_commits, total_commits, base_sha, + owner_location, country, discovered_via +) +VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, + $17, $18, $19, $20 +) +ON CONFLICT (full_name) DO UPDATE SET + description = excluded.description, stars = excluded.stars, + default_branch = excluded.default_branch, homepage = excluded.homepage, + language = excluded.language, license = excluded.license, forks = excluded.forks, + pushed_at = excluded.pushed_at, topics = excluded.topics, + window_commits = excluded.window_commits, total_commits = excluded.total_commits, + base_sha = excluded.base_sha, owner_location = excluded.owner_location, + country = excluded.country, + discovered_via = ARRAY( + SELECT DISTINCT via FROM UNNEST(repos.discovered_via || excluded.discovered_via) AS via -- noqa: RF01 + ) diff --git a/apps/api/src/hackproof/sql/github/wasm_source_repos.sql b/apps/api/src/hackproof/sql/github/wasm_source_repos.sql new file mode 100644 index 0000000..9474b1e --- /dev/null +++ b/apps/api/src/hackproof/sql/github/wasm_source_repos.sql @@ -0,0 +1,2 @@ +SELECT DISTINCT source_repo FROM wasm_uploads +WHERE source_repo IS NOT NULL From c11051d44108026bdcfcba8555b331b34ca38baf Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:42:38 +0100 Subject: [PATCH 13/18] refactor: move the indexer queries into sql files --- apps/api/src/hackproof/indexer/sync.py | 8 ++++---- apps/api/src/hackproof/sql/indexer/done_chunks.sql | 2 ++ apps/api/src/hackproof/sql/indexer/insert_accounts.sql | 3 +++ apps/api/src/hackproof/sql/indexer/insert_classic_ops.sql | 3 +++ apps/api/src/hackproof/sql/indexer/insert_contracts.sql | 3 +++ apps/api/src/hackproof/sql/indexer/insert_failed_txs.sql | 3 +++ apps/api/src/hackproof/sql/indexer/insert_sync_state.sql | 1 + .../api/src/hackproof/sql/indexer/insert_wasm_uploads.sql | 3 +++ apps/api/tests/test_sql_files.py | 6 ++++++ 9 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 apps/api/src/hackproof/sql/indexer/done_chunks.sql create mode 100644 apps/api/src/hackproof/sql/indexer/insert_accounts.sql create mode 100644 apps/api/src/hackproof/sql/indexer/insert_classic_ops.sql create mode 100644 apps/api/src/hackproof/sql/indexer/insert_contracts.sql create mode 100644 apps/api/src/hackproof/sql/indexer/insert_failed_txs.sql create mode 100644 apps/api/src/hackproof/sql/indexer/insert_sync_state.sql create mode 100644 apps/api/src/hackproof/sql/indexer/insert_wasm_uploads.sql create mode 100644 apps/api/tests/test_sql_files.py diff --git a/apps/api/src/hackproof/indexer/sync.py b/apps/api/src/hackproof/indexer/sync.py index aee5312..442d514 100644 --- a/apps/api/src/hackproof/indexer/sync.py +++ b/apps/api/src/hackproof/indexer/sync.py @@ -14,6 +14,7 @@ from hackproof.indexer.horizon import Horizon from hackproof.indexer.parser import parse_transaction from hackproof.indexer.rpc import SorobanRpc +from hackproof.sql_loader import load_sql log = logging.getLogger("hackproof.sync") @@ -64,13 +65,12 @@ async def _insert( by_table[event.table].append(event) async with connection.transaction(): for table, rows in sorted(by_table.items()): - placeholders = ", ".join(f"${n}" for n in range(1, len(rows[0].row()) + 1)) await connection.executemany( - f"INSERT INTO {table} VALUES ({placeholders}) ON CONFLICT DO NOTHING", + load_sql(f"indexer/insert_{table}"), [event.row() for event in rows], ) await connection.execute( - "INSERT INTO sync_state VALUES ($1, 'done') ON CONFLICT DO NOTHING", + load_sql("indexer/insert_sync_state"), chunk_key, ) @@ -102,7 +102,7 @@ async def run_sync() -> None: done = { row["key"] for row in await connection.fetch( - "SELECT key FROM sync_state WHERE key LIKE $1", f"{CHUNK_PREFIX}%" + load_sql("indexer/done_chunks"), f"{CHUNK_PREFIX}%" ) } pending = [s for s in starts if f"{CHUNK_PREFIX}{s}" not in done] diff --git a/apps/api/src/hackproof/sql/indexer/done_chunks.sql b/apps/api/src/hackproof/sql/indexer/done_chunks.sql new file mode 100644 index 0000000..8a63c8e --- /dev/null +++ b/apps/api/src/hackproof/sql/indexer/done_chunks.sql @@ -0,0 +1,2 @@ +SELECT key FROM sync_state +WHERE key LIKE $1 diff --git a/apps/api/src/hackproof/sql/indexer/insert_accounts.sql b/apps/api/src/hackproof/sql/indexer/insert_accounts.sql new file mode 100644 index 0000000..05ba64a --- /dev/null +++ b/apps/api/src/hackproof/sql/indexer/insert_accounts.sql @@ -0,0 +1,3 @@ +INSERT INTO accounts (account, funder, tx_hash, ledger, funded_at) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/indexer/insert_classic_ops.sql b/apps/api/src/hackproof/sql/indexer/insert_classic_ops.sql new file mode 100644 index 0000000..10743c4 --- /dev/null +++ b/apps/api/src/hackproof/sql/indexer/insert_classic_ops.sql @@ -0,0 +1,3 @@ +INSERT INTO classic_ops (tx_hash, op_index, source_account, op_type, asset, counterparty, ledger, occurred_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/indexer/insert_contracts.sql b/apps/api/src/hackproof/sql/indexer/insert_contracts.sql new file mode 100644 index 0000000..6f88a53 --- /dev/null +++ b/apps/api/src/hackproof/sql/indexer/insert_contracts.sql @@ -0,0 +1,3 @@ +INSERT INTO contracts (contract_id, deployer, wasm_hash, tx_hash, ledger, deployed_at) +VALUES ($1, $2, $3, $4, $5, $6) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/indexer/insert_failed_txs.sql b/apps/api/src/hackproof/sql/indexer/insert_failed_txs.sql new file mode 100644 index 0000000..d781ec4 --- /dev/null +++ b/apps/api/src/hackproof/sql/indexer/insert_failed_txs.sql @@ -0,0 +1,3 @@ +INSERT INTO failed_txs (tx_hash, source_account, kind, result_code, ledger, created_at) +VALUES ($1, $2, $3, $4, $5, $6) +ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/indexer/insert_sync_state.sql b/apps/api/src/hackproof/sql/indexer/insert_sync_state.sql new file mode 100644 index 0000000..1daed69 --- /dev/null +++ b/apps/api/src/hackproof/sql/indexer/insert_sync_state.sql @@ -0,0 +1 @@ +INSERT INTO sync_state VALUES ($1, 'done') ON CONFLICT DO NOTHING diff --git a/apps/api/src/hackproof/sql/indexer/insert_wasm_uploads.sql b/apps/api/src/hackproof/sql/indexer/insert_wasm_uploads.sql new file mode 100644 index 0000000..6d5ed23 --- /dev/null +++ b/apps/api/src/hackproof/sql/indexer/insert_wasm_uploads.sql @@ -0,0 +1,3 @@ +INSERT INTO wasm_uploads (tx_hash, wasm_hash, uploader, size, source_repo, meta, ledger, uploaded_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +ON CONFLICT DO NOTHING diff --git a/apps/api/tests/test_sql_files.py b/apps/api/tests/test_sql_files.py new file mode 100644 index 0000000..f563eb6 --- /dev/null +++ b/apps/api/tests/test_sql_files.py @@ -0,0 +1,6 @@ +from hackproof.sql_loader import SQL_DIR + + +async def test_every_statement_prepares(scratch_db): + for path in sorted(SQL_DIR.rglob("*.sql")): + await scratch_db.prepare(path.read_text()) From cc8368d4d5130b2d652e3b4126b4524b37c01e78 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:42:47 +0100 Subject: [PATCH 14/18] ci: lint the query sql files --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acf47a6..7d13f85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: - run: uv sync --frozen - run: uv run ruff check . - run: uv run ruff format --check . - - run: uv run sqlfluff lint db/schema + - run: uv run sqlfluff lint db/schema src/hackproof/sql - name: A schema change needs a migration file if: github.event_name == 'pull_request' working-directory: . From a0aaf9c2700fe3f3a88487c4a288c2efbc1e990b Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:44:38 +0100 Subject: [PATCH 15/18] ci: open a pull request when the data files drift from the database --- .github/workflows/data-sync.yml | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/data-sync.yml diff --git a/.github/workflows/data-sync.yml b/.github/workflows/data-sync.yml new file mode 100644 index 0000000..ddc2a36 --- /dev/null +++ b/.github/workflows/data-sync.yml @@ -0,0 +1,57 @@ +name: data-sync + +on: + schedule: + - cron: "17 */6 * * *" + workflow_dispatch: + +concurrency: + group: data-sync + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + sync: + runs-on: ubuntu-latest + environment: production + defaults: + run: + working-directory: apps/api + env: + DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }} + steps: + - name: Require the database secret + run: | + if [ -z "$DATABASE_URL" ]; then + echo "::error::Add the PRODUCTION_DATABASE_URL secret" + exit 1 + fi + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - run: uv sync --frozen + - name: Mask the connection details + run: > + uv run python -c "import os; + from hackproof.db_utils.connection import Connection; + print('\n'.join(Connection.from_url(os.environ['DATABASE_URL']).mask_lines()))" + - name: Regenerate the data files from the database + run: uv run hackproof export + - uses: peter-evans/create-pull-request@v7 + with: + branch: chore/sync-data + delete-branch: true + commit-message: "fix: sync the data files with the database" + title: "fix: sync the data files with the database" + body: | + The database and the committed data files had drifted apart. This regenerates the files from the database so they match again. + + - the files are rewritten by `hackproof export`, the same command that produced them + - review the diff, then merge to bring the repo back in sync + - this branch is refreshed on every scheduled run until it is merged + labels: data + add-paths: data/** From 6bf71b91740269a5190e591640df9100e7670d58 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:44:49 +0100 Subject: [PATCH 16/18] docs: describe the database workflow --- CONTRIBUTING.md | 2 +- README.md | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4df977..4b5fe3d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,7 +22,7 @@ The README has the commands to run everything locally. - Pydantic models for anything that crosses a module boundary. No dataclasses, NamedTuple or TypedDict. - Models for upstream responses use `extra="allow"` so new fields are not lost. Models we derive use `extra="forbid"`. - Tests run against the scratch Postgres database from `tests/conftest.py` and never touch the live network. Anything that needs an upstream response uses a recorded fixture in `apps/api/tests/fixtures/`. -- New pipeline steps follow the existing shape: a `run_*` job in the relevant package, a subcommand in `cli.py`, and tables added to `db.py`. +- New pipeline steps follow the existing shape: a `run_*` job in the relevant package, a subcommand in `cli.py`, and any schema change as files in `db/schema` with a migration from `hackproof schema plan`. SQL goes in `src/hackproof/sql`, loaded with `load_sql`, never in Python strings. ## React diff --git a/README.md b/README.md index fbb52e6..3b223be 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,21 @@ uv run hackproof reconcile uv run hackproof export ``` +## Database + +The schema lives in `apps/api/db/schema`, one file per table, index, view and function, and is applied with [pgschema](https://github.com/pgplex/pgschema). Queries live in `apps/api/src/hackproof/sql`. Run from `apps/api`. + +```sh +uv run hackproof schema apply +uv run hackproof schema plan +uv run hackproof schema verify +``` + +To change the schema, edit the files in `db/schema`, run `schema plan`, and commit the numbered file it writes to `db/migrations` with your change. `apply` refuses a plan that drops data. CI checks every pull request against an empty database and migrates production when the pull request merges to `main`. + ## Checks ```sh -cd apps/api && uv run ruff check . && uv run ruff format --check . && uv run pytest +cd apps/api && uv run ruff check . && uv run ruff format --check . && uv run sqlfluff lint db/schema src/hackproof/sql && uv run pytest pnpm lint && pnpm format:check && pnpm build ``` From a4091d1c1f8fac91c1bf629b17686730e7b4c081 Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 16:44:49 +0100 Subject: [PATCH 17/18] feat: add a taskfile for the database and lint commands --- taskfile.yml | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 taskfile.yml diff --git a/taskfile.yml b/taskfile.yml new file mode 100644 index 0000000..dab937c --- /dev/null +++ b/taskfile.yml @@ -0,0 +1,64 @@ +version: "3" + +tasks: + db:up: + desc: Start the local Postgres and apply the schema + cmds: + - docker compose up -d --wait db + - task: schema:apply + + db:down: + desc: Stop the local Postgres + cmds: + - docker compose down + + schema:plan: + desc: Write the pending changes to the next file in db/migrations + dir: apps/api + cmds: + - uv run hackproof schema plan + + schema:apply: + desc: Apply the schema files to the local database + dir: apps/api + cmds: + - uv run hackproof schema apply + + schema:verify: + desc: Check the database matches the schema files + dir: apps/api + cmds: + - uv run hackproof schema verify + + sql:fmt: + desc: Format the schema and query files + dir: apps/api + cmds: + - uv run sqlfluff fix db/schema src/hackproof/sql + + sql:lint: + desc: Lint the schema and query files + dir: apps/api + cmds: + - uv run sqlfluff lint db/schema src/hackproof/sql + + py:fmt: + desc: Format the Python code + dir: apps/api + cmds: + - uv run ruff check --fix . + - uv run ruff format . + + py:check: + desc: Lint and test the Python code + dir: apps/api + cmds: + - uv run ruff check . + - uv run ruff format --check . + - uv run pytest + + data:check: + desc: Check the database matches the committed data files + dir: apps/api + cmds: + - uv run hackproof check From 472794dbde99aa4bd283bb3bcc4698f2635e1ccb Mon Sep 17 00:00:00 2001 From: Emmanuel Odii Date: Sun, 15 Nov 2026 17:34:57 +0100 Subject: [PATCH 18/18] refactor: drop the unused wasm_repos table from the schema --- apps/api/db/schema/manifest.sql | 1 - apps/api/db/schema/tables/wasm_repos.sql | 6 ------ 2 files changed, 7 deletions(-) delete mode 100644 apps/api/db/schema/tables/wasm_repos.sql diff --git a/apps/api/db/schema/manifest.sql b/apps/api/db/schema/manifest.sql index 7ed0beb..7e7a44e 100644 --- a/apps/api/db/schema/manifest.sql +++ b/apps/api/db/schema/manifest.sql @@ -14,7 +14,6 @@ \i tables/repo_tools.sql \i tables/repos.sql \i tables/sync_state.sql -\i tables/wasm_repos.sql \i tables/wasm_uploads.sql \i indexes/accounts_funder.sql \i indexes/classic_ops_asset.sql diff --git a/apps/api/db/schema/tables/wasm_repos.sql b/apps/api/db/schema/tables/wasm_repos.sql deleted file mode 100644 index b49fc4f..0000000 --- a/apps/api/db/schema/tables/wasm_repos.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE wasm_repos ( - wasm_hash text, - full_name text NOT NULL, - source text NOT NULL, - CONSTRAINT wasm_repos_pkey PRIMARY KEY (wasm_hash) -);