From ceea3b6c0a63d48a28bc188fba29f55f2a4f7156 Mon Sep 17 00:00:00 2001 From: Yan Wang Date: Wed, 23 Sep 2026 16:54:18 -0400 Subject: [PATCH] feat(task3): open the development and test phases Both hidden datasets exist now: 680 cases each, split so that no company appears on both sides, with the gold in the private workspaces and the questions published without answers or rule labels. The two rows derive their state rather than carrying a hard-coded true. Development runs on the same scoring workspace as practice, so it shares that flag; test has its own. A literal `true` is how a build with no Task 3 configuration ends up announcing an open phase -- the defect this table was rewritten to prevent. The test that pinned both rows to pending asserted a fact that has changed. It now asserts what still holds: no row may claim to be open while the site has no verified workspace, and development and practice, sharing a workspace, may never report different states. A new test carries the other half. "Development: Live" is an empty claim if the questions are nowhere to be found, so an open ranked phase must say where to get them, on the hub and in the guide both. It asserts only that direction -- a pending phase may still say where its data will live. Verified by mutation in final mode. Note: `npm test` reports one pre-existing failure, "renders direct web upload routes without a GitHub Issue intake", which expects /TASK 1 LIVE/ that the homepage redesign in a98fd52 removed. It fails identically on a clean main and is untouched here. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NTMENQHcMNFRj6XC3NqxiD --- app/task3/page.tsx | 23 ++++++++++++---- app/task3/submission-guide.tsx | 9 ++++++- tests/rendered-html.test.mjs | 48 +++++++++++++++++++++++++++++++--- 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/app/task3/page.tsx b/app/task3/page.tsx index a2e303f..97a8b95 100644 --- a/app/task3/page.tsx +++ b/app/task3/page.tsx @@ -13,6 +13,7 @@ export const metadata: Metadata = { const finmrDataset = "https://huggingface.co/datasets/TheFinAI/FinMR"; const starterKit = "https://github.com/The-FinAI/IEEE-bigdata-cup/tree/main/finreason_task3"; +const hiddenDataset = "https://huggingface.co/datasets/YanAdjeNole/FinReason-Task3"; type PhaseCard = { name: string; @@ -31,6 +32,8 @@ export default function Task3HubPage() { const config = getTask3PublicConfig(); const scoringIsLive = config.siteMode === "final" && config.scoringSpace.state === "ready"; + const testIsLive = + config.siteMode === "final" && config.testSpace.state === "ready"; const phases: PhaseCard[] = [ { @@ -48,16 +51,19 @@ export default function Task3HubPage() { { name: "Development", slug: "development", - live: false, - gold: "Held by the organizers", + // Development runs on the same scoring workspace as practice, so it is + // live under exactly the same condition. Hard-coding this would let a + // build with no Task 3 configuration announce an open phase. + live: scoringIsLive, + gold: "680 held-out cases, answers withheld", feedback: "Score, rank, and a public leaderboard", ranked: "Ranked", }, { name: "Test", slug: "test", - live: false, - gold: "Held by the organizers", + live: testIsLive, + gold: "680 held-out cases, answers withheld", feedback: "An acceptance receipt only", ranked: "Decides the final result", }, @@ -145,7 +151,7 @@ export default function Task3HubPage() {

DATA

-

Where the practice data comes from

+

Where the data comes from

This site hosts no Task 3 files. The 332 public practice cases are the Financial @@ -154,6 +160,13 @@ export default function Task3HubPage() { Task 3 starter kit downloads them for you, and ships the validator, the scorer, and one baseline per way of running a model.

+

+ The development and test questions are published separately, without their + answers, as{" "} + YanAdjeNole/FinReason-Task3 — 680 cases each. They + carry no rule label: for one of the three rules the label alone gives away the + answer’s shape. +

Every one of the 332 practice cases is built around one flagged Data Quality diff --git a/app/task3/submission-guide.tsx b/app/task3/submission-guide.tsx index 8b3d0c7..c072c97 100644 --- a/app/task3/submission-guide.tsx +++ b/app/task3/submission-guide.tsx @@ -1,5 +1,6 @@ const basePath = process.env.NEXT_PUBLIC_BASE_PATH ?? ""; const starterKit = "https://github.com/The-FinAI/IEEE-bigdata-cup/tree/main/finreason_task3"; +const hiddenDataset = "https://huggingface.co/datasets/YanAdjeNole/FinReason-Task3"; type SubmissionGuideProps = { scoringSpaceUrl: string | null; @@ -51,12 +52,18 @@ export function SubmissionGuide({

  • -

    Step 1 of 5: Get the practice data.

    +

    Step 1 of 5: Get the data.

    Clone the Task 3 starter kit and run its prepare script. It downloads the 332 public cases and writes both the answered and unanswered copies.

    +

    + For the two ranked phases, download{" "} + development_inputs.jsonl and test_inputs.jsonl from{" "} + YanAdjeNole/FinReason-Task3. Each holds 680 + questions with the answers withheld, and no rule label. + 

    {prepareCommand}
    • diff --git a/tests/rendered-html.test.mjs b/tests/rendered-html.test.mjs index 958f635..8cbd540 100644 --- a/tests/rendered-html.test.mjs +++ b/tests/rendered-html.test.mjs @@ -279,10 +279,20 @@ test("publishes the Task 3 participant hub with honest phase status", async () = assert.notEqual(rowOf(slug), "", `${slug} row is missing`); } - // Development and test have no datasets, in any build. - for (const slug of ["development", "test"]) { - assert.match(rowOf(slug), /data-state="pending"/, `${slug} must be pending`); - assert.doesNotMatch(rowOf(slug), /data-state="ready"/, `${slug} must not claim to be live`); + // The hidden datasets exist now, so these rows are no longer pinned to + // pending. What must still hold is that no row claims to be open in a build + // that has no workspace to send people to -- the original defect. Each row + // tracks its own workspace: practice and development share the scoring one, + // test has its own. + const panelPending = /
    Link under verification<\/dd>/.test(hub); + for (const slug of ["practice", "development", "test"]) { + if (panelPending) { + assert.match( + rowOf(slug), + /data-state="pending"/, + `${slug} must be pending while the site has no verified Space`, + ); + } } // Practice tracks the configuration, and this assertion works in BOTH site @@ -295,6 +305,15 @@ test("publishes the Task 3 participant hub with honest phase status", async () = "the practice row disagrees with the quick-facts panel", ); + // Development runs on the same workspace as practice, so the two rows must + // never disagree: one of them being open while the other is not would mean + // the page is reporting a workspace state that does not exist. + assert.equal( + /data-state="ready"/.test(rowOf("development")), + /data-state="ready"/.test(rowOf("practice")), + "development and practice share a workspace but report different states", + ); + // The page must never contradict itself: if the panel says the link is still // being verified, no phase row may simultaneously claim to be open. This holds // in both site modes, which is why it is the assertion that would have caught @@ -443,6 +462,27 @@ test("a Task 3 phase the hub calls open has a working upload link", async () => ); }); +test("an open ranked phase says where to get its questions", async () => { + // "Development: Live" is an empty claim if the questions are nowhere to be + // found. The gold stays private, but the inputs must be reachable from both + // the hub and the guide before either page calls those phases open. + const [hub, submit] = await Promise.all([ + text("out/task3/index.html"), + text("out/task3/submit/index.html"), + ]); + const devRow = hub.match(/]*data-phase="development"[\s\S]*?<\/tr>/)?.[0] ?? ""; + const ranked = /data-state="ready"/.test(devRow); + const dataset = /huggingface\.co\/datasets\/[\w.-]+\/FinReason-Task3/; + if (!ranked) return; + for (const [name, page] of [["hub", hub], ["submit guide", submit]]) { + assert.match( + page, + dataset, + `${name} calls development open but never says where its questions are`, + ); + } +}); + test("the home page links to all three task hubs", async () => { // A participant site nobody can navigate to does not do its job. The Task 3 // route existed in the sitemap but no page linked to it.