diff --git a/app/task3/page.tsx b/app/task3/page.tsx
index a2e303f..97a8b95 100644
--- a/app/task3/page.tsx
+++ b/app/task3/page.tsx
@@ -13,6 +13,7 @@ export const metadata: Metadata = {
const finmrDataset = "https://huggingface.co/datasets/TheFinAI/FinMR";
const starterKit = "https://github.com/The-FinAI/IEEE-bigdata-cup/tree/main/finreason_task3";
+const hiddenDataset = "https://huggingface.co/datasets/YanAdjeNole/FinReason-Task3";
type PhaseCard = {
name: string;
@@ -31,6 +32,8 @@ export default function Task3HubPage() {
const config = getTask3PublicConfig();
const scoringIsLive =
config.siteMode === "final" && config.scoringSpace.state === "ready";
+ const testIsLive =
+ config.siteMode === "final" && config.testSpace.state === "ready";
const phases: PhaseCard[] = [
{
@@ -48,16 +51,19 @@ export default function Task3HubPage() {
{
name: "Development",
slug: "development",
- live: false,
- gold: "Held by the organizers",
+ // Development runs on the same scoring workspace as practice, so it is
+ // live under exactly the same condition. Hard-coding this would let a
+ // build with no Task 3 configuration announce an open phase.
+ live: scoringIsLive,
+ gold: "680 held-out cases, answers withheld",
feedback: "Score, rank, and a public leaderboard",
ranked: "Ranked",
},
{
name: "Test",
slug: "test",
- live: false,
- gold: "Held by the organizers",
+ live: testIsLive,
+ gold: "680 held-out cases, answers withheld",
feedback: "An acceptance receipt only",
ranked: "Decides the final result",
},
@@ -145,7 +151,7 @@ export default function Task3HubPage() {
DATA
-
Where the practice data comes from
+
Where the data comes from
This site hosts no Task 3 files. The 332 public practice cases are the Financial
@@ -154,6 +160,13 @@ export default function Task3HubPage() {
Task 3 starter kit downloads them for you, and ships the
validator, the scorer, and one baseline per way of running a model.
+
+ The development and test questions are published separately, without their
+ answers, as{" "}
+ YanAdjeNole/FinReason-Task3 — 680 cases each. They
+ carry no rule label: for one of the three rules the label alone gives away the
+ answer’s shape.
+
Every one of the 332 practice cases is built around one flagged Data Quality
diff --git a/app/task3/submission-guide.tsx b/app/task3/submission-guide.tsx
index 8b3d0c7..c072c97 100644
--- a/app/task3/submission-guide.tsx
+++ b/app/task3/submission-guide.tsx
@@ -1,5 +1,6 @@
const basePath = process.env.NEXT_PUBLIC_BASE_PATH ?? "";
const starterKit = "https://github.com/The-FinAI/IEEE-bigdata-cup/tree/main/finreason_task3";
+const hiddenDataset = "https://huggingface.co/datasets/YanAdjeNole/FinReason-Task3";
type SubmissionGuideProps = {
scoringSpaceUrl: string | null;
@@ -51,12 +52,18 @@ export function SubmissionGuide({
01
-
Step 1 of 5: Get the practice data.
+
Step 1 of 5: Get the data.
Clone the Task 3 starter kit and run its prepare script.
It downloads the 332 public cases and writes both the answered and unanswered
copies.
+
+ For the two ranked phases, download{" "}
+ development_inputs.jsonl and test_inputs.jsonl from{" "}
+ YanAdjeNole/FinReason-Task3. Each holds 680
+ questions with the answers withheld, and no rule label.
+
{prepareCommand}
diff --git a/tests/rendered-html.test.mjs b/tests/rendered-html.test.mjs
index 958f635..8cbd540 100644
--- a/tests/rendered-html.test.mjs
+++ b/tests/rendered-html.test.mjs
@@ -279,10 +279,20 @@ test("publishes the Task 3 participant hub with honest phase status", async () =
assert.notEqual(rowOf(slug), "", `${slug} row is missing`);
}
- // Development and test have no datasets, in any build.
- for (const slug of ["development", "test"]) {
- assert.match(rowOf(slug), /data-state="pending"/, `${slug} must be pending`);
- assert.doesNotMatch(rowOf(slug), /data-state="ready"/, `${slug} must not claim to be live`);
+ // The hidden datasets exist now, so these rows are no longer pinned to
+ // pending. What must still hold is that no row claims to be open in a build
+ // that has no workspace to send people to -- the original defect. Each row
+ // tracks its own workspace: practice and development share the scoring one,
+ // test has its own.
+ const panelPending = /Link under verification<\/dd>/.test(hub);
+ for (const slug of ["practice", "development", "test"]) {
+ if (panelPending) {
+ assert.match(
+ rowOf(slug),
+ /data-state="pending"/,
+ `${slug} must be pending while the site has no verified Space`,
+ );
+ }
}
// Practice tracks the configuration, and this assertion works in BOTH site
@@ -295,6 +305,15 @@ test("publishes the Task 3 participant hub with honest phase status", async () =
"the practice row disagrees with the quick-facts panel",
);
+ // Development runs on the same workspace as practice, so the two rows must
+ // never disagree: one of them being open while the other is not would mean
+ // the page is reporting a workspace state that does not exist.
+ assert.equal(
+ /data-state="ready"/.test(rowOf("development")),
+ /data-state="ready"/.test(rowOf("practice")),
+ "development and practice share a workspace but report different states",
+ );
+
// The page must never contradict itself: if the panel says the link is still
// being verified, no phase row may simultaneously claim to be open. This holds
// in both site modes, which is why it is the assertion that would have caught
@@ -443,6 +462,27 @@ test("a Task 3 phase the hub calls open has a working upload link", async () =>
);
});
+test("an open ranked phase says where to get its questions", async () => {
+ // "Development: Live" is an empty claim if the questions are nowhere to be
+ // found. The gold stays private, but the inputs must be reachable from both
+ // the hub and the guide before either page calls those phases open.
+ const [hub, submit] = await Promise.all([
+ text("out/task3/index.html"),
+ text("out/task3/submit/index.html"),
+ ]);
+ const devRow = hub.match(/]*data-phase="development"[\s\S]*?<\/tr>/)?.[0] ?? "";
+ const ranked = /data-state="ready"/.test(devRow);
+ const dataset = /huggingface\.co\/datasets\/[\w.-]+\/FinReason-Task3/;
+ if (!ranked) return;
+ for (const [name, page] of [["hub", hub], ["submit guide", submit]]) {
+ assert.match(
+ page,
+ dataset,
+ `${name} calls development open but never says where its questions are`,
+ );
+ }
+});
+
test("the home page links to all three task hubs", async () => {
// A participant site nobody can navigate to does not do its job. The Task 3
// route existed in the sitemap but no page linked to it.