diff --git a/app/task3/page.tsx b/app/task3/page.tsx index a2e303f..97a8b95 100644 --- a/app/task3/page.tsx +++ b/app/task3/page.tsx @@ -13,6 +13,7 @@ export const metadata: Metadata = { const finmrDataset = "https://huggingface.co/datasets/TheFinAI/FinMR"; const starterKit = "https://github.com/The-FinAI/IEEE-bigdata-cup/tree/main/finreason_task3"; +const hiddenDataset = "https://huggingface.co/datasets/YanAdjeNole/FinReason-Task3"; type PhaseCard = { name: string; @@ -31,6 +32,8 @@ export default function Task3HubPage() { const config = getTask3PublicConfig(); const scoringIsLive = config.siteMode === "final" && config.scoringSpace.state === "ready"; + const testIsLive = + config.siteMode === "final" && config.testSpace.state === "ready"; const phases: PhaseCard[] = [ { @@ -48,16 +51,19 @@ export default function Task3HubPage() { { name: "Development", slug: "development", - live: false, - gold: "Held by the organizers", + // Development runs on the same scoring workspace as practice, so it is + // live under exactly the same condition. Hard-coding this would let a + // build with no Task 3 configuration announce an open phase. + live: scoringIsLive, + gold: "680 held-out cases, answers withheld", feedback: "Score, rank, and a public leaderboard", ranked: "Ranked", }, { name: "Test", slug: "test", - live: false, - gold: "Held by the organizers", + live: testIsLive, + gold: "680 held-out cases, answers withheld", feedback: "An acceptance receipt only", ranked: "Decides the final result", }, @@ -145,7 +151,7 @@ export default function Task3HubPage() {

DATA

-

Where the practice data comes from

+

Where the data comes from

This site hosts no Task 3 files. The 332 public practice cases are the Financial @@ -154,6 +160,13 @@ export default function Task3HubPage() { Task 3 starter kit downloads them for you, and ships the validator, the scorer, and one baseline per way of running a model.

+

+ The development and test questions are published separately, without their + answers, as{" "} + YanAdjeNole/FinReason-Task3 — 680 cases each. They + carry no rule label: for one of the three rules the label alone gives away the + answer’s shape. +

Every one of the 332 practice cases is built around one flagged Data Quality diff --git a/app/task3/submission-guide.tsx b/app/task3/submission-guide.tsx index 8b3d0c7..c072c97 100644 --- a/app/task3/submission-guide.tsx +++ b/app/task3/submission-guide.tsx @@ -1,5 +1,6 @@ const basePath = process.env.NEXT_PUBLIC_BASE_PATH ?? ""; const starterKit = "https://github.com/The-FinAI/IEEE-bigdata-cup/tree/main/finreason_task3"; +const hiddenDataset = "https://huggingface.co/datasets/YanAdjeNole/FinReason-Task3"; type SubmissionGuideProps = { scoringSpaceUrl: string | null; @@ -51,12 +52,18 @@ export function SubmissionGuide({

  • -

    Step 1 of 5: Get the practice data.

    +

    Step 1 of 5: Get the data.

    Clone the Task 3 starter kit and run its prepare script. It downloads the 332 public cases and writes both the answered and unanswered copies.

    +

    + For the two ranked phases, download{" "} + development_inputs.jsonl and test_inputs.jsonl from{" "} + YanAdjeNole/FinReason-Task3. Each holds 680 + questions with the answers withheld, and no rule label. +

    {prepareCommand}
  • diff --git a/tests/rendered-html.test.mjs b/tests/rendered-html.test.mjs index 958f635..8cbd540 100644 --- a/tests/rendered-html.test.mjs +++ b/tests/rendered-html.test.mjs @@ -279,10 +279,20 @@ test("publishes the Task 3 participant hub with honest phase status", async () = assert.notEqual(rowOf(slug), "", `${slug} row is missing`); } - // Development and test have no datasets, in any build. - for (const slug of ["development", "test"]) { - assert.match(rowOf(slug), /data-state="pending"/, `${slug} must be pending`); - assert.doesNotMatch(rowOf(slug), /data-state="ready"/, `${slug} must not claim to be live`); + // The hidden datasets exist now, so these rows are no longer pinned to + // pending. What must still hold is that no row claims to be open in a build + // that has no workspace to send people to -- the original defect. Each row + // tracks its own workspace: practice and development share the scoring one, + // test has its own. + const panelPending = /
    Link under verification<\/dd>/.test(hub); + for (const slug of ["practice", "development", "test"]) { + if (panelPending) { + assert.match( + rowOf(slug), + /data-state="pending"/, + `${slug} must be pending while the site has no verified Space`, + ); + } } // Practice tracks the configuration, and this assertion works in BOTH site @@ -295,6 +305,15 @@ test("publishes the Task 3 participant hub with honest phase status", async () = "the practice row disagrees with the quick-facts panel", ); + // Development runs on the same workspace as practice, so the two rows must + // never disagree: one of them being open while the other is not would mean + // the page is reporting a workspace state that does not exist. + assert.equal( + /data-state="ready"/.test(rowOf("development")), + /data-state="ready"/.test(rowOf("practice")), + "development and practice share a workspace but report different states", + ); + // The page must never contradict itself: if the panel says the link is still // being verified, no phase row may simultaneously claim to be open. This holds // in both site modes, which is why it is the assertion that would have caught @@ -443,6 +462,27 @@ test("a Task 3 phase the hub calls open has a working upload link", async () => ); }); +test("an open ranked phase says where to get its questions", async () => { + // "Development: Live" is an empty claim if the questions are nowhere to be + // found. The gold stays private, but the inputs must be reachable from both + // the hub and the guide before either page calls those phases open. + const [hub, submit] = await Promise.all([ + text("out/task3/index.html"), + text("out/task3/submit/index.html"), + ]); + const devRow = hub.match(/]*data-phase="development"[\s\S]*?<\/tr>/)?.[0] ?? ""; + const ranked = /data-state="ready"/.test(devRow); + const dataset = /huggingface\.co\/datasets\/[\w.-]+\/FinReason-Task3/; + if (!ranked) return; + for (const [name, page] of [["hub", hub], ["submit guide", submit]]) { + assert.match( + page, + dataset, + `${name} calls development open but never says where its questions are`, + ); + } +}); + test("the home page links to all three task hubs", async () => { // A participant site nobody can navigate to does not do its job. The Task 3 // route existed in the sitemap but no page linked to it.