From 09724eceb544e631cfcb8b0e5e162593d1327be9 Mon Sep 17 00:00:00 2001 From: yl231 <111810457+yl231@users.noreply.github.com> Date: Mon, 1 Jun 2026 05:15:00 +0000 Subject: [PATCH] Sync leaderboard data from RouterArena@78fb83a966fd99da589812e4efb2f3ab0b8971dd --- .../flip_labels/flip_labels_agentforge.json | 1682 +++++ src/data/flip_labels/flip_labels_azure.json | 462 +- src/data/flip_labels/flip_labels_nadir.json | 1682 +++++ .../flip_labels/flip_labels_orcarouter.json | 1682 +++++ .../flip_labels/flip_labels_weave-router.json | 176 +- src/data/routerMetrics/category_scores.json | 6196 ++++++++++++----- src/data/routerMetrics/leaderboard.json | 89 +- 7 files changed, 9976 insertions(+), 1993 deletions(-) create mode 100644 src/data/flip_labels/flip_labels_agentforge.json create mode 100644 src/data/flip_labels/flip_labels_nadir.json create mode 100644 src/data/flip_labels/flip_labels_orcarouter.json diff --git a/src/data/flip_labels/flip_labels_agentforge.json b/src/data/flip_labels/flip_labels_agentforge.json new file mode 100644 index 0000000..2bc24b5 --- /dev/null +++ b/src/data/flip_labels/flip_labels_agentforge.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 0 + }, + { + "global index": "ArcMMLU_12", + "flip": 0 + }, + { + "global index": "ArcMMLU_123", + "flip": 0 + }, + { + "global index": "ArcMMLU_16", + "flip": 1 + }, + { + "global index": "ArcMMLU_182", + "flip": 0 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 1 + }, + { + "global index": "ArcMMLU_496", + "flip": 0 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 1 + }, + { + "global index": "ArcMMLU_659", + "flip": 0 + }, + { + "global index": "ArcMMLU_676", + "flip": 1 + }, + { + "global index": "ArcMMLU_685", + "flip": 1 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 1 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 0 + }, + { + "global index": "AsDiv_1165", + "flip": 0 + }, + { + "global index": "AsDiv_1347", + "flip": 1 + }, + { + "global index": "AsDiv_472", + "flip": 1 + }, + { + "global index": "AsDiv_733", + "flip": 1 + }, + { + "global index": "ChessInstruct_0", + "flip": 0 + }, + { + "global index": "ChessInstruct_107", + "flip": 1 + }, + { + "global index": "ChessInstruct_144", + "flip": 1 + }, + { + "global index": "ChessInstruct_42", + "flip": 1 + }, + { + "global index": "ChessInstruct_58", + "flip": 1 + }, + { + "global index": "ChessInstruct_71", + "flip": 1 + }, + { + "global index": "ChessInstruct_84", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 1 + }, + { + "global index": "Ethics_deontology_0", + "flip": 1 + }, + { + "global index": "Ethics_deontology_2", + "flip": 1 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 1 + }, + { + "global index": "Ethics_deontology_56", + "flip": 1 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 1 + }, + { + "global index": "Ethics_virtue_26", + "flip": 1 + }, + { + "global index": "Ethics_virtue_30", + "flip": 1 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 1 + }, + { + "global index": "FinQA_208", + "flip": 0 + }, + { + "global index": "FinQA_56", + "flip": 0 + }, + { + "global index": "FinQA_60", + "flip": 1 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 0 + }, + { + "global index": "GeoBench_1094", + "flip": 1 + }, + { + "global index": "GeoBench_1102", + "flip": 0 + }, + { + "global index": "GeoBench_1113", + "flip": 1 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 0 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 0 + }, + { + "global index": "GeoBench_526", + "flip": 0 + }, + { + "global index": "GeoBench_591", + "flip": 1 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 0 + }, + { + "global index": "GeoBench_915", + "flip": 0 + }, + { + "global index": "GeoBench_944", + "flip": 0 + }, + { + "global index": "GeoBench_968", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 0 + }, + { + "global index": "LiveCodeBench_105", + "flip": 1 + }, + { + "global index": "LiveCodeBench_114", + "flip": 1 + }, + { + "global index": "LiveCodeBench_118", + "flip": 1 + }, + { + "global index": "LiveCodeBench_131", + "flip": 1 + }, + { + "global index": "LiveCodeBench_136", + "flip": 1 + }, + { + "global index": "LiveCodeBench_181", + "flip": 1 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 1 + }, + { + "global index": "LiveCodeBench_386", + "flip": 0 + }, + { + "global index": "LiveCodeBench_405", + "flip": 0 + }, + { + "global index": "LiveCodeBench_43", + "flip": 1 + }, + { + "global index": "LiveCodeBench_431", + "flip": 1 + }, + { + "global index": "LiveCodeBench_437", + "flip": 0 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 1 + }, + { + "global index": "LiveCodeBench_49", + "flip": 1 + }, + { + "global index": "LiveCodeBench_491", + "flip": 1 + }, + { + "global index": "LiveCodeBench_499", + "flip": 1 + }, + { + "global index": "MATH_108", + "flip": 1 + }, + { + "global index": "MATH_442", + "flip": 0 + }, + { + "global index": "MATH_53", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 1 + }, + { + "global index": "MMLUPro_business_294", + "flip": 1 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 1 + }, + { + "global index": "MMLUPro_business_503", + "flip": 1 + }, + { + "global index": "MMLUPro_business_507", + "flip": 0 + }, + { + "global index": "MMLUPro_business_6", + "flip": 1 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 1 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 1 + }, + { + "global index": "MMLU_management_93", + "flip": 1 + }, + { + "global index": "MathQA_158", + "flip": 1 + }, + { + "global index": "MathQA_1742", + "flip": 1 + }, + { + "global index": "MathQA_202", + "flip": 1 + }, + { + "global index": "MathQA_2092", + "flip": 1 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 1 + }, + { + "global index": "MathQA_827", + "flip": 0 + }, + { + "global index": "MathQA_84", + "flip": 1 + }, + { + "global index": "MedMCQA_1005", + "flip": 1 + }, + { + "global index": "MedMCQA_1054", + "flip": 0 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 1 + }, + { + "global index": "MedMCQA_1362", + "flip": 1 + }, + { + "global index": "MedMCQA_145", + "flip": 1 + }, + { + "global index": "MedMCQA_2010", + "flip": 0 + }, + { + "global index": "MedMCQA_2323", + "flip": 1 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 1 + }, + { + "global index": "MedMCQA_511", + "flip": 1 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 1 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 0 + }, + { + "global index": "NarrativeQA_131", + "flip": 1 + }, + { + "global index": "NarrativeQA_1683", + "flip": 1 + }, + { + "global index": "NarrativeQA_2474", + "flip": 0 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 1 + }, + { + "global index": "NarrativeQA_4102", + "flip": 0 + }, + { + "global index": "NarrativeQA_4128", + "flip": 1 + }, + { + "global index": "NarrativeQA_4347", + "flip": 1 + }, + { + "global index": "NarrativeQA_4540", + "flip": 1 + }, + { + "global index": "NarrativeQA_5022", + "flip": 1 + }, + { + "global index": "NarrativeQA_5259", + "flip": 1 + }, + { + "global index": "NarrativeQA_533", + "flip": 1 + }, + { + "global index": "NarrativeQA_5894", + "flip": 1 + }, + { + "global index": "NarrativeQA_6829", + "flip": 1 + }, + { + "global index": "NarrativeQA_7678", + "flip": 1 + }, + { + "global index": "NarrativeQA_7964", + "flip": 1 + }, + { + "global index": "NarrativeQA_8215", + "flip": 1 + }, + { + "global index": "NarrativeQA_8598", + "flip": 0 + }, + { + "global index": "NarrativeQA_927", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 0 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 1 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 0 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 0 + }, + { + "global index": "PubMedQA_238", + "flip": 0 + }, + { + "global index": "PubMedQA_250", + "flip": 1 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 0 + }, + { + "global index": "PubMedQA_437", + "flip": 0 + }, + { + "global index": "PubMedQA_510", + "flip": 0 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 1 + }, + { + "global index": "PubMedQA_582", + "flip": 0 + }, + { + "global index": "PubMedQA_588", + "flip": 0 + }, + { + "global index": "PubMedQA_610", + "flip": 1 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 0 + }, + { + "global index": "PubMedQA_687", + "flip": 1 + }, + { + "global index": "PubMedQA_722", + "flip": 1 + }, + { + "global index": "PubMedQA_73", + "flip": 0 + }, + { + "global index": "PubMedQA_755", + "flip": 1 + }, + { + "global index": "PubMedQA_8", + "flip": 1 + }, + { + "global index": "PubMedQA_81", + "flip": 0 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 0 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 0 + }, + { + "global index": "QANTA_Geography_304", + "flip": 0 + }, + { + "global index": "QANTA_History_1084", + "flip": 0 + }, + { + "global index": "QANTA_History_1154", + "flip": 1 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 1 + }, + { + "global index": "QANTA_Literature_386", + "flip": 0 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 0 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 1 + }, + { + "global index": "QANTA_Science_1360", + "flip": 0 + }, + { + "global index": "QANTA_Science_1473", + "flip": 0 + }, + { + "global index": "QANTA_Science_308", + "flip": 0 + }, + { + "global index": "QANTA_Science_619", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 0 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 1 + }, + { + "global index": "SocialiQA_26846", + "flip": 1 + }, + { + "global index": "SocialiQA_7839", + "flip": 1 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 0 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 0 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 0 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 1 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 0 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 1 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 1 + } +] diff --git a/src/data/flip_labels/flip_labels_azure.json b/src/data/flip_labels/flip_labels_azure.json index abd8a07..3b7abc8 100644 --- a/src/data/flip_labels/flip_labels_azure.json +++ b/src/data/flip_labels/flip_labels_azure.json @@ -13,11 +13,11 @@ }, { "global index": "ArcMMLU_123", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_16", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_182", @@ -29,7 +29,7 @@ }, { "global index": "ArcMMLU_293", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_349", @@ -37,11 +37,11 @@ }, { "global index": "ArcMMLU_378", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_443", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_496", @@ -49,15 +49,15 @@ }, { "global index": "ArcMMLU_631", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_646", - "flip": 0 + "flip": 1 }, { "global index": "ArcMMLU_659", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_676", @@ -65,7 +65,7 @@ }, { "global index": "ArcMMLU_685", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_689", @@ -77,7 +77,7 @@ }, { "global index": "ArcMMLU_713", - "flip": 1 + "flip": 0 }, { "global index": "ArcMMLU_98", @@ -85,7 +85,7 @@ }, { "global index": "AsDiv_1165", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_1347", @@ -93,19 +93,19 @@ }, { "global index": "AsDiv_472", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_733", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_0", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_107", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_144", @@ -113,35 +113,35 @@ }, { "global index": "ChessInstruct_42", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_58", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_71", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_84", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_28", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_51", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_6", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_62", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_commonsense_70", @@ -149,7 +149,7 @@ }, { "global index": "Ethics_commonsense_85", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_90", @@ -157,7 +157,7 @@ }, { "global index": "Ethics_deontology_0", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_deontology_2", @@ -169,11 +169,11 @@ }, { "global index": "Ethics_deontology_32", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_deontology_56", - "flip": 0 + "flip": 1 }, { "global index": "Ethics_justice_1", @@ -189,11 +189,11 @@ }, { "global index": "Ethics_justice_84", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_virtue_14", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_virtue_26", @@ -209,11 +209,11 @@ }, { "global index": "Ethics_virtue_51", - "flip": 1 + "flip": 0 }, { "global index": "FinQA_149", - "flip": 0 + "flip": 1 }, { "global index": "FinQA_208", @@ -233,7 +233,7 @@ }, { "global index": "GeoBench_1002", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1094", @@ -249,15 +249,15 @@ }, { "global index": "GeoBench_124", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_1243", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_30", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_502", @@ -265,23 +265,23 @@ }, { "global index": "GeoBench_526", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_591", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_766", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_87", - "flip": 0 + "flip": 1 }, { "global index": "GeoBench_915", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_944", @@ -289,12 +289,20 @@ }, { "global index": "GeoBench_968", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_27", "flip": 1 }, { - "global index": "LiveCodeBench_105", + "global index": "GeoGraphyData_100k_42", "flip": 1 }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, { "global index": "LiveCodeBench_114", "flip": 0 @@ -305,7 +313,7 @@ }, { "global index": "LiveCodeBench_131", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_136", @@ -349,7 +357,7 @@ }, { "global index": "LiveCodeBench_476", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_485", @@ -357,7 +365,7 @@ }, { "global index": "LiveCodeBench_49", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_491", @@ -365,7 +373,7 @@ }, { "global index": "LiveCodeBench_499", - "flip": 1 + "flip": 0 }, { "global index": "MATH_108", @@ -381,15 +389,15 @@ }, { "global index": "MMLUPro_biology_2808", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_biology_2912", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_2980", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_2985", @@ -401,11 +409,11 @@ }, { "global index": "MMLUPro_biology_3215", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_3225", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_business_226", @@ -413,11 +421,11 @@ }, { "global index": "MMLUPro_business_294", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_378", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_430", @@ -433,15 +441,15 @@ }, { "global index": "MMLUPro_business_6", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_business_784", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_3796", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_chemistry_3837", @@ -449,15 +457,15 @@ }, { "global index": "MMLUPro_chemistry_3974", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_4067", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_4407", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9086", @@ -469,11 +477,11 @@ }, { "global index": "MMLUPro_computer science_9136", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_computer science_9138", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9149", @@ -481,7 +489,7 @@ }, { "global index": "MMLUPro_computer science_9200", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9212", @@ -493,7 +501,7 @@ }, { "global index": "MMLUPro_computer science_9264", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9285", @@ -505,7 +513,7 @@ }, { "global index": "MMLUPro_computer science_9414", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9415", @@ -533,7 +541,7 @@ }, { "global index": "MMLUPro_economics_5907", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5931", @@ -541,15 +549,15 @@ }, { "global index": "MMLUPro_economics_5965", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_6114", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_6122", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_6135", @@ -561,11 +569,11 @@ }, { "global index": "MMLUPro_economics_6353", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10076", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_engineering_10125", @@ -573,7 +581,7 @@ }, { "global index": "MMLUPro_engineering_10179", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10195", @@ -597,11 +605,11 @@ }, { "global index": "MMLUPro_engineering_10428", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10432", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10473", @@ -625,15 +633,15 @@ }, { "global index": "MMLUPro_health_4885", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_health_4973", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5093", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5144", @@ -641,11 +649,11 @@ }, { "global index": "MMLUPro_health_5214", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5215", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5261", @@ -657,7 +665,7 @@ }, { "global index": "MMLUPro_health_5514", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4486", @@ -665,7 +673,7 @@ }, { "global index": "MMLUPro_history_4490", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4497", @@ -673,15 +681,15 @@ }, { "global index": "MMLUPro_history_4509", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4517", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_history_4523", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4605", @@ -689,15 +697,15 @@ }, { "global index": "MMLUPro_history_4629", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4638", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4717", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4749", @@ -709,7 +717,7 @@ }, { "global index": "MMLUPro_history_4774", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4810", @@ -717,7 +725,7 @@ }, { "global index": "MMLUPro_history_4833", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4836", @@ -733,11 +741,11 @@ }, { "global index": "MMLUPro_law_1031", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1386", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1462", @@ -745,11 +753,11 @@ }, { "global index": "MMLUPro_law_1484", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1518", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1818", @@ -757,7 +765,7 @@ }, { "global index": "MMLUPro_law_806", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_899", @@ -769,19 +777,19 @@ }, { "global index": "MMLUPro_math_6526", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_math_6623", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6848", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7101", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7249", @@ -789,7 +797,7 @@ }, { "global index": "MMLUPro_math_7284", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7451", @@ -805,11 +813,11 @@ }, { "global index": "MMLUPro_philosophy_9536", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_philosophy_9663", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_philosophy_9672", @@ -837,15 +845,15 @@ }, { "global index": "MMLUPro_physics_9017", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2005", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2186", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2329", @@ -861,20 +869,68 @@ }, { "global index": "MMLUPro_psychology_2450", - "flip": 0 + "flip": 1 }, { "global index": "MMLUPro_psychology_2457", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", "flip": 0 }, { - "global index": "MathQA_158", + "global index": "MMLU_formal_logic_16", "flip": 1 }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 0 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 0 + }, + { + "global index": "MMLU_management_93", + "flip": 1 + }, + { + "global index": "MathQA_158", + "flip": 0 + }, { "global index": "MathQA_1742", "flip": 0 @@ -885,7 +941,7 @@ }, { "global index": "MathQA_2092", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_2102", @@ -909,7 +965,7 @@ }, { "global index": "MedMCQA_1054", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1298", @@ -917,27 +973,31 @@ }, { "global index": "MedMCQA_1309", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_1362", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_145", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_2010", "flip": 0 }, + { + "global index": "MedMCQA_2323", + "flip": 0 + }, { "global index": "MedMCQA_2366", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_2581", - "flip": 0 + "flip": 1 }, { "global index": "MedMCQA_511", @@ -949,7 +1009,7 @@ }, { "global index": "MedMCQA_643", - "flip": 1 + "flip": 0 }, { "global index": "MedMCQA_853", @@ -957,7 +1017,7 @@ }, { "global index": "MusicTheoryBench_126", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_14", @@ -973,7 +1033,7 @@ }, { "global index": "MusicTheoryBench_188", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_189", @@ -993,31 +1053,31 @@ }, { "global index": "MusicTheoryBench_340", - "flip": 0 + "flip": 1 }, { "global index": "MusicTheoryBench_70", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_131", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_1683", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_2474", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_2820", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_3282", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_4102", @@ -1029,11 +1089,11 @@ }, { "global index": "NarrativeQA_4347", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_4540", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_5022", @@ -1045,7 +1105,7 @@ }, { "global index": "NarrativeQA_533", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_5894", @@ -1053,7 +1113,7 @@ }, { "global index": "NarrativeQA_6829", - "flip": 0 + "flip": 1 }, { "global index": "NarrativeQA_7678", @@ -1069,11 +1129,11 @@ }, { "global index": "NarrativeQA_8598", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_927", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Animals_2545", @@ -1081,15 +1141,15 @@ }, { "global index": "OpenTDB_Animals_262", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Art_1429", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Celebrities_1078", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Celebrities_1456", @@ -1105,11 +1165,11 @@ }, { "global index": "OpenTDB_General Knowledge_1178", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_1807", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_1957", @@ -1117,7 +1177,7 @@ }, { "global index": "OpenTDB_General Knowledge_2181", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_General Knowledge_3404", @@ -1125,11 +1185,11 @@ }, { "global index": "OpenTDB_General Knowledge_3727", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_General Knowledge_4019", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Geography_2099", @@ -1145,11 +1205,11 @@ }, { "global index": "OpenTDB_Geography_792", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_History_1162", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_2026", @@ -1157,11 +1217,11 @@ }, { "global index": "OpenTDB_History_3712", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_History_3902", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Science & Nature_1175", @@ -1169,7 +1229,7 @@ }, { "global index": "OpenTDB_Science & Nature_1560", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Science & Nature_3716", @@ -1177,27 +1237,27 @@ }, { "global index": "OpenTDB_Science & Nature_476", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Sports_2289", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Vehicles_1173", - "flip": 0 + "flip": 1 }, { "global index": "OpenTDB_Vehicles_1419", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Vehicles_2519", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Vehicles_3234", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_0", @@ -1213,7 +1273,7 @@ }, { "global index": "PubMedQA_238", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_250", @@ -1221,7 +1281,7 @@ }, { "global index": "PubMedQA_337", - "flip": 0 + "flip": 1 }, { "global index": "PubMedQA_362", @@ -1241,15 +1301,15 @@ }, { "global index": "PubMedQA_575", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_582", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_588", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_610", @@ -1261,7 +1321,7 @@ }, { "global index": "PubMedQA_643", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_687", @@ -1269,15 +1329,15 @@ }, { "global index": "PubMedQA_722", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_73", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_755", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_8", @@ -1293,15 +1353,15 @@ }, { "global index": "PubMedQA_905", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Fine Arts_1212", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Fine Arts_1702", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Fine Arts_828", @@ -1313,7 +1373,7 @@ }, { "global index": "QANTA_Geography_1023", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Geography_1555", @@ -1325,11 +1385,11 @@ }, { "global index": "QANTA_History_1084", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_History_1154", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_History_433", @@ -1341,23 +1401,23 @@ }, { "global index": "QANTA_History_926", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_1045", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1073", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1239", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_1326", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_1727", @@ -1365,7 +1425,7 @@ }, { "global index": "QANTA_Literature_1843", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_386", @@ -1373,11 +1433,11 @@ }, { "global index": "QANTA_Literature_408", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_475", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Literature_833", @@ -1401,7 +1461,7 @@ }, { "global index": "QANTA_Science_1473", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Science_308", @@ -1413,11 +1473,11 @@ }, { "global index": "QANTA_Social Science_1847", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_2", - "flip": 0 + "flip": 1 }, { "global index": "QANTA_Social Science_77", @@ -1425,7 +1485,7 @@ }, { "global index": "SocialiQA_13810", - "flip": 1 + "flip": 0 }, { "global index": "SocialiQA_22095", @@ -1437,7 +1497,7 @@ }, { "global index": "SocialiQA_7839", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-CausalReasoning_4526", @@ -1465,7 +1525,7 @@ }, { "global index": "SuperGLUE-Entailment_522", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Entailment_767", @@ -1473,15 +1533,15 @@ }, { "global index": "SuperGLUE-QA_1408", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_3137", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_4046", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_4102", @@ -1497,15 +1557,15 @@ }, { "global index": "SuperGLUE-RC_7738", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-RC_8531", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_19695", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_19738", @@ -1513,15 +1573,15 @@ }, { "global index": "SuperGLUE-Wic_20079", - "flip": 0 + "flip": 1 }, { "global index": "SuperGLUE-Wic_20189", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_20253", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wsc_20368", @@ -1529,27 +1589,27 @@ }, { "global index": "SuperGLUE-Wsc_20370", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-cs-en_156", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-cs-en_246", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-cs-en_568", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-de-en_46", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-de-en_715", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-de-en_883", @@ -1557,7 +1617,7 @@ }, { "global index": "WMT19-fi-en_222", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-fi-en_610", @@ -1573,7 +1633,7 @@ }, { "global index": "WMT19-gu-en_191", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-gu-en_491", @@ -1581,7 +1641,7 @@ }, { "global index": "WMT19-gu-en_968", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-kk-en_528", @@ -1589,23 +1649,23 @@ }, { "global index": "WMT19-kk-en_826", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-lt-en_135", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-lt-en_269", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-lt-en_636", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-ru-en_222", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-zh-en_218", @@ -1613,10 +1673,10 @@ }, { "global index": "WMT19-zh-en_252", - "flip": 0 + "flip": 1 }, { "global index": "WMT19-zh-en_59", - "flip": 1 + "flip": 0 } ] diff --git a/src/data/flip_labels/flip_labels_nadir.json b/src/data/flip_labels/flip_labels_nadir.json new file mode 100644 index 0000000..687d086 --- /dev/null +++ b/src/data/flip_labels/flip_labels_nadir.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 1 + }, + { + "global index": "AIME_58", + "flip": 1 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 1 + }, + { + "global index": "ArcMMLU_16", + "flip": 1 + }, + { + "global index": "ArcMMLU_182", + "flip": 1 + }, + { + "global index": "ArcMMLU_230", + "flip": 1 + }, + { + "global index": "ArcMMLU_293", + "flip": 1 + }, + { + "global index": "ArcMMLU_349", + "flip": 1 + }, + { + "global index": "ArcMMLU_378", + "flip": 0 + }, + { + "global index": "ArcMMLU_443", + "flip": 1 + }, + { + "global index": "ArcMMLU_496", + "flip": 1 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 1 + }, + { + "global index": "ArcMMLU_659", + "flip": 1 + }, + { + "global index": "ArcMMLU_676", + "flip": 1 + }, + { + "global index": "ArcMMLU_685", + "flip": 1 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 1 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 1 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 1 + }, + { + "global index": "AsDiv_733", + "flip": 0 + }, + { + "global index": "ChessInstruct_0", + "flip": 1 + }, + { + "global index": "ChessInstruct_107", + "flip": 1 + }, + { + "global index": "ChessInstruct_144", + "flip": 0 + }, + { + "global index": "ChessInstruct_42", + "flip": 1 + }, + { + "global index": "ChessInstruct_58", + "flip": 1 + }, + { + "global index": "ChessInstruct_71", + "flip": 1 + }, + { + "global index": "ChessInstruct_84", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 0 + }, + { + "global index": "Ethics_deontology_0", + "flip": 1 + }, + { + "global index": "Ethics_deontology_2", + "flip": 1 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 1 + }, + { + "global index": "Ethics_deontology_56", + "flip": 1 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 0 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 1 + }, + { + "global index": "Ethics_virtue_26", + "flip": 1 + }, + { + "global index": "Ethics_virtue_30", + "flip": 0 + }, + { + "global index": "Ethics_virtue_48", + "flip": 1 + }, + { + "global index": "Ethics_virtue_51", + "flip": 1 + }, + { + "global index": "FinQA_149", + "flip": 1 + }, + { + "global index": "FinQA_208", + "flip": 1 + }, + { + "global index": "FinQA_56", + "flip": 1 + }, + { + "global index": "FinQA_60", + "flip": 1 + }, + { + "global index": "GSM8K_43", + "flip": 1 + }, + { + "global index": "GeoBench_1002", + "flip": 1 + }, + { + "global index": "GeoBench_1094", + "flip": 0 + }, + { + "global index": "GeoBench_1102", + "flip": 1 + }, + { + "global index": "GeoBench_1113", + "flip": 1 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 1 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 1 + }, + { + "global index": "GeoBench_526", + "flip": 1 + }, + { + "global index": "GeoBench_591", + "flip": 1 + }, + { + "global index": "GeoBench_766", + "flip": 1 + }, + { + "global index": "GeoBench_87", + "flip": 1 + }, + { + "global index": "GeoBench_915", + "flip": 1 + }, + { + "global index": "GeoBench_944", + "flip": 1 + }, + { + "global index": "GeoBench_968", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 1 + }, + { + "global index": "LiveCodeBench_105", + "flip": 1 + }, + { + "global index": "LiveCodeBench_114", + "flip": 1 + }, + { + "global index": "LiveCodeBench_118", + "flip": 1 + }, + { + "global index": "LiveCodeBench_131", + "flip": 1 + }, + { + "global index": "LiveCodeBench_136", + "flip": 1 + }, + { + "global index": "LiveCodeBench_181", + "flip": 0 + }, + { + "global index": "LiveCodeBench_237", + "flip": 1 + }, + { + "global index": "LiveCodeBench_271", + "flip": 0 + }, + { + "global index": "LiveCodeBench_350", + "flip": 1 + }, + { + "global index": "LiveCodeBench_386", + "flip": 1 + }, + { + "global index": "LiveCodeBench_405", + "flip": 1 + }, + { + "global index": "LiveCodeBench_43", + "flip": 1 + }, + { + "global index": "LiveCodeBench_431", + "flip": 1 + }, + { + "global index": "LiveCodeBench_437", + "flip": 1 + }, + { + "global index": "LiveCodeBench_476", + "flip": 0 + }, + { + "global index": "LiveCodeBench_485", + "flip": 1 + }, + { + "global index": "LiveCodeBench_49", + "flip": 1 + }, + { + "global index": "LiveCodeBench_491", + "flip": 1 + }, + { + "global index": "LiveCodeBench_499", + "flip": 1 + }, + { + "global index": "MATH_108", + "flip": 1 + }, + { + "global index": "MATH_442", + "flip": 1 + }, + { + "global index": "MATH_53", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 0 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 1 + }, + { + "global index": "MMLUPro_business_226", + "flip": 0 + }, + { + "global index": "MMLUPro_business_294", + "flip": 0 + }, + { + "global index": "MMLUPro_business_378", + "flip": 1 + }, + { + "global index": "MMLUPro_business_430", + "flip": 1 + }, + { + "global index": "MMLUPro_business_503", + "flip": 1 + }, + { + "global index": "MMLUPro_business_507", + "flip": 1 + }, + { + "global index": "MMLUPro_business_6", + "flip": 1 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 0 + }, + { + "global index": "MMLUPro_law_806", + "flip": 0 + }, + { + "global index": "MMLUPro_law_899", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 0 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 0 + }, + { + "global index": "MMLU_management_4", + "flip": 1 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 1 + }, + { + "global index": "MMLU_management_93", + "flip": 1 + }, + { + "global index": "MathQA_158", + "flip": 1 + }, + { + "global index": "MathQA_1742", + "flip": 1 + }, + { + "global index": "MathQA_202", + "flip": 1 + }, + { + "global index": "MathQA_2092", + "flip": 0 + }, + { + "global index": "MathQA_2102", + "flip": 0 + }, + { + "global index": "MathQA_2851", + "flip": 0 + }, + { + "global index": "MathQA_827", + "flip": 1 + }, + { + "global index": "MathQA_84", + "flip": 0 + }, + { + "global index": "MedMCQA_1005", + "flip": 1 + }, + { + "global index": "MedMCQA_1054", + "flip": 1 + }, + { + "global index": "MedMCQA_1298", + "flip": 1 + }, + { + "global index": "MedMCQA_1309", + "flip": 1 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 0 + }, + { + "global index": "MedMCQA_2010", + "flip": 1 + }, + { + "global index": "MedMCQA_2323", + "flip": 1 + }, + { + "global index": "MedMCQA_2366", + "flip": 0 + }, + { + "global index": "MedMCQA_2581", + "flip": 1 + }, + { + "global index": "MedMCQA_511", + "flip": 1 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 1 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 1 + }, + { + "global index": "NarrativeQA_131", + "flip": 1 + }, + { + "global index": "NarrativeQA_1683", + "flip": 1 + }, + { + "global index": "NarrativeQA_2474", + "flip": 1 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 1 + }, + { + "global index": "NarrativeQA_4102", + "flip": 1 + }, + { + "global index": "NarrativeQA_4128", + "flip": 1 + }, + { + "global index": "NarrativeQA_4347", + "flip": 1 + }, + { + "global index": "NarrativeQA_4540", + "flip": 0 + }, + { + "global index": "NarrativeQA_5022", + "flip": 0 + }, + { + "global index": "NarrativeQA_5259", + "flip": 0 + }, + { + "global index": "NarrativeQA_533", + "flip": 0 + }, + { + "global index": "NarrativeQA_5894", + "flip": 1 + }, + { + "global index": "NarrativeQA_6829", + "flip": 1 + }, + { + "global index": "NarrativeQA_7678", + "flip": 1 + }, + { + "global index": "NarrativeQA_7964", + "flip": 1 + }, + { + "global index": "NarrativeQA_8215", + "flip": 1 + }, + { + "global index": "NarrativeQA_8598", + "flip": 1 + }, + { + "global index": "NarrativeQA_927", + "flip": 0 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 0 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 0 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 1 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 1 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 1 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 0 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 1 + }, + { + "global index": "PubMedQA_0", + "flip": 1 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 1 + }, + { + "global index": "PubMedQA_238", + "flip": 1 + }, + { + "global index": "PubMedQA_250", + "flip": 1 + }, + { + "global index": "PubMedQA_337", + "flip": 1 + }, + { + "global index": "PubMedQA_362", + "flip": 1 + }, + { + "global index": "PubMedQA_437", + "flip": 1 + }, + { + "global index": "PubMedQA_510", + "flip": 1 + }, + { + "global index": "PubMedQA_520", + "flip": 1 + }, + { + "global index": "PubMedQA_575", + "flip": 1 + }, + { + "global index": "PubMedQA_582", + "flip": 1 + }, + { + "global index": "PubMedQA_588", + "flip": 1 + }, + { + "global index": "PubMedQA_610", + "flip": 1 + }, + { + "global index": "PubMedQA_63", + "flip": 0 + }, + { + "global index": "PubMedQA_643", + "flip": 1 + }, + { + "global index": "PubMedQA_687", + "flip": 1 + }, + { + "global index": "PubMedQA_722", + "flip": 1 + }, + { + "global index": "PubMedQA_73", + "flip": 1 + }, + { + "global index": "PubMedQA_755", + "flip": 1 + }, + { + "global index": "PubMedQA_8", + "flip": 1 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 0 + }, + { + "global index": "PubMedQA_905", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 1 + }, + { + "global index": "QANTA_Geography_304", + "flip": 1 + }, + { + "global index": "QANTA_History_1084", + "flip": 1 + }, + { + "global index": "QANTA_History_1154", + "flip": 0 + }, + { + "global index": "QANTA_History_433", + "flip": 0 + }, + { + "global index": "QANTA_History_473", + "flip": 0 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 0 + }, + { + "global index": "QANTA_Literature_386", + "flip": 1 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 1 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 1 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 1 + }, + { + "global index": "QANTA_Science_308", + "flip": 1 + }, + { + "global index": "QANTA_Science_619", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 1 + }, + { + "global index": "SocialiQA_13810", + "flip": 0 + }, + { + "global index": "SocialiQA_22095", + "flip": 1 + }, + { + "global index": "SocialiQA_26846", + "flip": 0 + }, + { + "global index": "SocialiQA_7839", + "flip": 1 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 0 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 1 + }, + { + "global index": "WMT19-de-en_46", + "flip": 0 + }, + { + "global index": "WMT19-de-en_715", + "flip": 1 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 1 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 0 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 1 + } +] diff --git a/src/data/flip_labels/flip_labels_orcarouter.json b/src/data/flip_labels/flip_labels_orcarouter.json new file mode 100644 index 0000000..0c03174 --- /dev/null +++ b/src/data/flip_labels/flip_labels_orcarouter.json @@ -0,0 +1,1682 @@ +[ + { + "global index": "AIME_112", + "flip": 0 + }, + { + "global index": "AIME_58", + "flip": 1 + }, + { + "global index": "ArcMMLU_12", + "flip": 1 + }, + { + "global index": "ArcMMLU_123", + "flip": 1 + }, + { + "global index": "ArcMMLU_16", + "flip": 1 + }, + { + "global index": "ArcMMLU_182", + "flip": 1 + }, + { + "global index": "ArcMMLU_230", + "flip": 0 + }, + { + "global index": "ArcMMLU_293", + "flip": 0 + }, + { + "global index": "ArcMMLU_349", + "flip": 0 + }, + { + "global index": "ArcMMLU_378", + "flip": 1 + }, + { + "global index": "ArcMMLU_443", + "flip": 1 + }, + { + "global index": "ArcMMLU_496", + "flip": 1 + }, + { + "global index": "ArcMMLU_631", + "flip": 1 + }, + { + "global index": "ArcMMLU_646", + "flip": 1 + }, + { + "global index": "ArcMMLU_659", + "flip": 1 + }, + { + "global index": "ArcMMLU_676", + "flip": 1 + }, + { + "global index": "ArcMMLU_685", + "flip": 0 + }, + { + "global index": "ArcMMLU_689", + "flip": 1 + }, + { + "global index": "ArcMMLU_702", + "flip": 1 + }, + { + "global index": "ArcMMLU_713", + "flip": 1 + }, + { + "global index": "ArcMMLU_98", + "flip": 1 + }, + { + "global index": "AsDiv_1165", + "flip": 1 + }, + { + "global index": "AsDiv_1347", + "flip": 0 + }, + { + "global index": "AsDiv_472", + "flip": 0 + }, + { + "global index": "AsDiv_733", + "flip": 1 + }, + { + "global index": "ChessInstruct_0", + "flip": 1 + }, + { + "global index": "ChessInstruct_107", + "flip": 0 + }, + { + "global index": "ChessInstruct_144", + "flip": 1 + }, + { + "global index": "ChessInstruct_42", + "flip": 0 + }, + { + "global index": "ChessInstruct_58", + "flip": 1 + }, + { + "global index": "ChessInstruct_71", + "flip": 1 + }, + { + "global index": "ChessInstruct_84", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_28", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_51", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_6", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_62", + "flip": 0 + }, + { + "global index": "Ethics_commonsense_70", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_85", + "flip": 1 + }, + { + "global index": "Ethics_commonsense_90", + "flip": 1 + }, + { + "global index": "Ethics_deontology_0", + "flip": 1 + }, + { + "global index": "Ethics_deontology_2", + "flip": 1 + }, + { + "global index": "Ethics_deontology_31", + "flip": 1 + }, + { + "global index": "Ethics_deontology_32", + "flip": 1 + }, + { + "global index": "Ethics_deontology_56", + "flip": 1 + }, + { + "global index": "Ethics_justice_1", + "flip": 1 + }, + { + "global index": "Ethics_justice_45", + "flip": 1 + }, + { + "global index": "Ethics_justice_76", + "flip": 1 + }, + { + "global index": "Ethics_justice_84", + "flip": 0 + }, + { + "global index": "Ethics_virtue_14", + "flip": 1 + }, + { + "global index": "Ethics_virtue_26", + "flip": 1 + }, + { + "global index": "Ethics_virtue_30", + "flip": 1 + }, + { + "global index": "Ethics_virtue_48", + "flip": 0 + }, + { + "global index": "Ethics_virtue_51", + "flip": 0 + }, + { + "global index": "FinQA_149", + "flip": 1 + }, + { + "global index": "FinQA_208", + "flip": 1 + }, + { + "global index": "FinQA_56", + "flip": 1 + }, + { + "global index": "FinQA_60", + "flip": 1 + }, + { + "global index": "GSM8K_43", + "flip": 0 + }, + { + "global index": "GeoBench_1002", + "flip": 1 + }, + { + "global index": "GeoBench_1094", + "flip": 1 + }, + { + "global index": "GeoBench_1102", + "flip": 1 + }, + { + "global index": "GeoBench_1113", + "flip": 1 + }, + { + "global index": "GeoBench_124", + "flip": 1 + }, + { + "global index": "GeoBench_1243", + "flip": 1 + }, + { + "global index": "GeoBench_30", + "flip": 1 + }, + { + "global index": "GeoBench_502", + "flip": 1 + }, + { + "global index": "GeoBench_526", + "flip": 1 + }, + { + "global index": "GeoBench_591", + "flip": 1 + }, + { + "global index": "GeoBench_766", + "flip": 0 + }, + { + "global index": "GeoBench_87", + "flip": 1 + }, + { + "global index": "GeoBench_915", + "flip": 1 + }, + { + "global index": "GeoBench_944", + "flip": 1 + }, + { + "global index": "GeoBench_968", + "flip": 1 + }, + { + "global index": "GeoGraphyData_100k_27", + "flip": 0 + }, + { + "global index": "GeoGraphyData_100k_42", + "flip": 1 + }, + { + "global index": "LiveCodeBench_105", + "flip": 0 + }, + { + "global index": "LiveCodeBench_114", + "flip": 0 + }, + { + "global index": "LiveCodeBench_118", + "flip": 1 + }, + { + "global index": "LiveCodeBench_131", + "flip": 1 + }, + { + "global index": "LiveCodeBench_136", + "flip": 0 + }, + { + "global index": "LiveCodeBench_181", + "flip": 1 + }, + { + "global index": "LiveCodeBench_237", + "flip": 0 + }, + { + "global index": "LiveCodeBench_271", + "flip": 1 + }, + { + "global index": "LiveCodeBench_350", + "flip": 0 + }, + { + "global index": "LiveCodeBench_386", + "flip": 1 + }, + { + "global index": "LiveCodeBench_405", + "flip": 1 + }, + { + "global index": "LiveCodeBench_43", + "flip": 1 + }, + { + "global index": "LiveCodeBench_431", + "flip": 1 + }, + { + "global index": "LiveCodeBench_437", + "flip": 1 + }, + { + "global index": "LiveCodeBench_476", + "flip": 1 + }, + { + "global index": "LiveCodeBench_485", + "flip": 0 + }, + { + "global index": "LiveCodeBench_49", + "flip": 1 + }, + { + "global index": "LiveCodeBench_491", + "flip": 0 + }, + { + "global index": "LiveCodeBench_499", + "flip": 0 + }, + { + "global index": "MATH_108", + "flip": 1 + }, + { + "global index": "MATH_442", + "flip": 1 + }, + { + "global index": "MATH_53", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2808", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2912", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2980", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_2985", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3188", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3215", + "flip": 1 + }, + { + "global index": "MMLUPro_biology_3225", + "flip": 0 + }, + { + "global index": "MMLUPro_business_226", + "flip": 1 + }, + { + "global index": "MMLUPro_business_294", + "flip": 1 + }, + { + "global index": "MMLUPro_business_378", + "flip": 0 + }, + { + "global index": "MMLUPro_business_430", + "flip": 0 + }, + { + "global index": "MMLUPro_business_503", + "flip": 1 + }, + { + "global index": "MMLUPro_business_507", + "flip": 1 + }, + { + "global index": "MMLUPro_business_6", + "flip": 1 + }, + { + "global index": "MMLUPro_business_784", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3796", + "flip": 0 + }, + { + "global index": "MMLUPro_chemistry_3837", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_3974", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4067", + "flip": 1 + }, + { + "global index": "MMLUPro_chemistry_4407", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9086", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9110", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9136", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9138", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9149", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9200", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9212", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9239", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9264", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9285", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9289", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9414", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9415", + "flip": 0 + }, + { + "global index": "MMLUPro_computer science_9430", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9452", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9471", + "flip": 1 + }, + { + "global index": "MMLUPro_computer science_9475", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5769", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5907", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5931", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_5965", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6114", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6122", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6135", + "flip": 1 + }, + { + "global index": "MMLUPro_economics_6325", + "flip": 0 + }, + { + "global index": "MMLUPro_economics_6353", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10076", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10125", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10179", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10195", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10199", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10298", + "flip": 0 + }, + { + "global index": "MMLUPro_engineering_10342", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10395", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10428", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10432", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10473", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10537", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10701", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10823", + "flip": 1 + }, + { + "global index": "MMLUPro_engineering_10864", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4885", + "flip": 1 + }, + { + "global index": "MMLUPro_health_4973", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5093", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5144", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5214", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5215", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5261", + "flip": 0 + }, + { + "global index": "MMLUPro_health_5473", + "flip": 1 + }, + { + "global index": "MMLUPro_health_5514", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4486", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4490", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4497", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4509", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4517", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4523", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4605", + "flip": 0 + }, + { + "global index": "MMLUPro_history_4629", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4638", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4717", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4749", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4752", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4774", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4810", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4833", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4836", + "flip": 1 + }, + { + "global index": "MMLUPro_history_4841", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1007", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1031", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1386", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1462", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1484", + "flip": 0 + }, + { + "global index": "MMLUPro_law_1518", + "flip": 1 + }, + { + "global index": "MMLUPro_law_1818", + "flip": 1 + }, + { + "global index": "MMLUPro_law_806", + "flip": 1 + }, + { + "global index": "MMLUPro_law_899", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6429", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6526", + "flip": 1 + }, + { + "global index": "MMLUPro_math_6623", + "flip": 0 + }, + { + "global index": "MMLUPro_math_6848", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7101", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7249", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7284", + "flip": 1 + }, + { + "global index": "MMLUPro_math_7451", + "flip": 0 + }, + { + "global index": "MMLUPro_math_7577", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9510", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9536", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9663", + "flip": 1 + }, + { + "global index": "MMLUPro_philosophy_9672", + "flip": 0 + }, + { + "global index": "MMLUPro_philosophy_9943", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7773", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7887", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_7893", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_8888", + "flip": 1 + }, + { + "global index": "MMLUPro_physics_9017", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2005", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2186", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2329", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2406", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2420", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2450", + "flip": 1 + }, + { + "global index": "MMLUPro_psychology_2457", + "flip": 0 + }, + { + "global index": "MMLUPro_psychology_2524", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_121", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_16", + "flip": 0 + }, + { + "global index": "MMLU_formal_logic_32", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_63", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_7", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_70", + "flip": 1 + }, + { + "global index": "MMLU_formal_logic_85", + "flip": 1 + }, + { + "global index": "MMLU_management_4", + "flip": 1 + }, + { + "global index": "MMLU_management_41", + "flip": 1 + }, + { + "global index": "MMLU_management_77", + "flip": 0 + }, + { + "global index": "MMLU_management_91", + "flip": 1 + }, + { + "global index": "MMLU_management_93", + "flip": 1 + }, + { + "global index": "MathQA_158", + "flip": 1 + }, + { + "global index": "MathQA_1742", + "flip": 0 + }, + { + "global index": "MathQA_202", + "flip": 1 + }, + { + "global index": "MathQA_2092", + "flip": 1 + }, + { + "global index": "MathQA_2102", + "flip": 1 + }, + { + "global index": "MathQA_2851", + "flip": 1 + }, + { + "global index": "MathQA_827", + "flip": 1 + }, + { + "global index": "MathQA_84", + "flip": 1 + }, + { + "global index": "MedMCQA_1005", + "flip": 0 + }, + { + "global index": "MedMCQA_1054", + "flip": 1 + }, + { + "global index": "MedMCQA_1298", + "flip": 0 + }, + { + "global index": "MedMCQA_1309", + "flip": 1 + }, + { + "global index": "MedMCQA_1362", + "flip": 0 + }, + { + "global index": "MedMCQA_145", + "flip": 1 + }, + { + "global index": "MedMCQA_2010", + "flip": 1 + }, + { + "global index": "MedMCQA_2323", + "flip": 1 + }, + { + "global index": "MedMCQA_2366", + "flip": 1 + }, + { + "global index": "MedMCQA_2581", + "flip": 1 + }, + { + "global index": "MedMCQA_511", + "flip": 1 + }, + { + "global index": "MedMCQA_59", + "flip": 1 + }, + { + "global index": "MedMCQA_643", + "flip": 1 + }, + { + "global index": "MedMCQA_853", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_126", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_14", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_147", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_152", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_188", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_189", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_240", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_33", + "flip": 1 + }, + { + "global index": "MusicTheoryBench_337", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_340", + "flip": 0 + }, + { + "global index": "MusicTheoryBench_70", + "flip": 1 + }, + { + "global index": "NarrativeQA_131", + "flip": 1 + }, + { + "global index": "NarrativeQA_1683", + "flip": 0 + }, + { + "global index": "NarrativeQA_2474", + "flip": 1 + }, + { + "global index": "NarrativeQA_2820", + "flip": 1 + }, + { + "global index": "NarrativeQA_3282", + "flip": 0 + }, + { + "global index": "NarrativeQA_4102", + "flip": 1 + }, + { + "global index": "NarrativeQA_4128", + "flip": 0 + }, + { + "global index": "NarrativeQA_4347", + "flip": 1 + }, + { + "global index": "NarrativeQA_4540", + "flip": 1 + }, + { + "global index": "NarrativeQA_5022", + "flip": 1 + }, + { + "global index": "NarrativeQA_5259", + "flip": 1 + }, + { + "global index": "NarrativeQA_533", + "flip": 1 + }, + { + "global index": "NarrativeQA_5894", + "flip": 1 + }, + { + "global index": "NarrativeQA_6829", + "flip": 1 + }, + { + "global index": "NarrativeQA_7678", + "flip": 1 + }, + { + "global index": "NarrativeQA_7964", + "flip": 0 + }, + { + "global index": "NarrativeQA_8215", + "flip": 1 + }, + { + "global index": "NarrativeQA_8598", + "flip": 1 + }, + { + "global index": "NarrativeQA_927", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_2545", + "flip": 1 + }, + { + "global index": "OpenTDB_Animals_262", + "flip": 1 + }, + { + "global index": "OpenTDB_Art_1429", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1078", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_1456", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3577", + "flip": 1 + }, + { + "global index": "OpenTDB_Celebrities_3968", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_1178", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1807", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_1957", + "flip": 0 + }, + { + "global index": "OpenTDB_General Knowledge_2181", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3404", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_3727", + "flip": 1 + }, + { + "global index": "OpenTDB_General Knowledge_4019", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_2099", + "flip": 0 + }, + { + "global index": "OpenTDB_Geography_2346", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_3880", + "flip": 1 + }, + { + "global index": "OpenTDB_Geography_792", + "flip": 1 + }, + { + "global index": "OpenTDB_History_1162", + "flip": 1 + }, + { + "global index": "OpenTDB_History_2026", + "flip": 0 + }, + { + "global index": "OpenTDB_History_3712", + "flip": 1 + }, + { + "global index": "OpenTDB_History_3902", + "flip": 0 + }, + { + "global index": "OpenTDB_Science & Nature_1175", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_1560", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_3716", + "flip": 1 + }, + { + "global index": "OpenTDB_Science & Nature_476", + "flip": 1 + }, + { + "global index": "OpenTDB_Sports_2289", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1173", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_1419", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_2519", + "flip": 1 + }, + { + "global index": "OpenTDB_Vehicles_3234", + "flip": 1 + }, + { + "global index": "PubMedQA_0", + "flip": 0 + }, + { + "global index": "PubMedQA_154", + "flip": 1 + }, + { + "global index": "PubMedQA_18", + "flip": 1 + }, + { + "global index": "PubMedQA_238", + "flip": 1 + }, + { + "global index": "PubMedQA_250", + "flip": 0 + }, + { + "global index": "PubMedQA_337", + "flip": 0 + }, + { + "global index": "PubMedQA_362", + "flip": 1 + }, + { + "global index": "PubMedQA_437", + "flip": 1 + }, + { + "global index": "PubMedQA_510", + "flip": 1 + }, + { + "global index": "PubMedQA_520", + "flip": 0 + }, + { + "global index": "PubMedQA_575", + "flip": 0 + }, + { + "global index": "PubMedQA_582", + "flip": 1 + }, + { + "global index": "PubMedQA_588", + "flip": 1 + }, + { + "global index": "PubMedQA_610", + "flip": 1 + }, + { + "global index": "PubMedQA_63", + "flip": 1 + }, + { + "global index": "PubMedQA_643", + "flip": 1 + }, + { + "global index": "PubMedQA_687", + "flip": 0 + }, + { + "global index": "PubMedQA_722", + "flip": 1 + }, + { + "global index": "PubMedQA_73", + "flip": 1 + }, + { + "global index": "PubMedQA_755", + "flip": 1 + }, + { + "global index": "PubMedQA_8", + "flip": 1 + }, + { + "global index": "PubMedQA_81", + "flip": 1 + }, + { + "global index": "PubMedQA_854", + "flip": 1 + }, + { + "global index": "PubMedQA_905", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_1212", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_1702", + "flip": 0 + }, + { + "global index": "QANTA_Fine Arts_828", + "flip": 1 + }, + { + "global index": "QANTA_Fine Arts_865", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1023", + "flip": 1 + }, + { + "global index": "QANTA_Geography_1555", + "flip": 1 + }, + { + "global index": "QANTA_Geography_304", + "flip": 1 + }, + { + "global index": "QANTA_History_1084", + "flip": 1 + }, + { + "global index": "QANTA_History_1154", + "flip": 1 + }, + { + "global index": "QANTA_History_433", + "flip": 1 + }, + { + "global index": "QANTA_History_473", + "flip": 1 + }, + { + "global index": "QANTA_History_926", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1045", + "flip": 0 + }, + { + "global index": "QANTA_Literature_1073", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1239", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1326", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1727", + "flip": 1 + }, + { + "global index": "QANTA_Literature_1843", + "flip": 1 + }, + { + "global index": "QANTA_Literature_386", + "flip": 1 + }, + { + "global index": "QANTA_Literature_408", + "flip": 1 + }, + { + "global index": "QANTA_Literature_475", + "flip": 1 + }, + { + "global index": "QANTA_Literature_833", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_1270", + "flip": 0 + }, + { + "global index": "QANTA_Philosophy_499", + "flip": 1 + }, + { + "global index": "QANTA_Philosophy_91", + "flip": 0 + }, + { + "global index": "QANTA_Science_1360", + "flip": 1 + }, + { + "global index": "QANTA_Science_1473", + "flip": 1 + }, + { + "global index": "QANTA_Science_308", + "flip": 1 + }, + { + "global index": "QANTA_Science_619", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_1847", + "flip": 0 + }, + { + "global index": "QANTA_Social Science_2", + "flip": 1 + }, + { + "global index": "QANTA_Social Science_77", + "flip": 1 + }, + { + "global index": "SocialiQA_13810", + "flip": 1 + }, + { + "global index": "SocialiQA_22095", + "flip": 0 + }, + { + "global index": "SocialiQA_26846", + "flip": 1 + }, + { + "global index": "SocialiQA_7839", + "flip": 1 + }, + { + "global index": "SuperGLUE-CausalReasoning_4526", + "flip": 1 + }, + { + "global index": "SuperGLUE-ClozeTest_12894", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_17965", + "flip": 0 + }, + { + "global index": "SuperGLUE-ClozeTest_18766", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19410", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_19567", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_522", + "flip": 1 + }, + { + "global index": "SuperGLUE-Entailment_767", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_1408", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_3137", + "flip": 0 + }, + { + "global index": "SuperGLUE-QA_4046", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4102", + "flip": 1 + }, + { + "global index": "SuperGLUE-QA_4160", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7725", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_7738", + "flip": 1 + }, + { + "global index": "SuperGLUE-RC_8531", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_19695", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wic_19738", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20079", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20189", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wic_20253", + "flip": 0 + }, + { + "global index": "SuperGLUE-Wsc_20368", + "flip": 1 + }, + { + "global index": "SuperGLUE-Wsc_20370", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_156", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_246", + "flip": 1 + }, + { + "global index": "WMT19-cs-en_568", + "flip": 1 + }, + { + "global index": "WMT19-de-en_46", + "flip": 1 + }, + { + "global index": "WMT19-de-en_715", + "flip": 0 + }, + { + "global index": "WMT19-de-en_883", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_222", + "flip": 1 + }, + { + "global index": "WMT19-fi-en_610", + "flip": 0 + }, + { + "global index": "WMT19-gu-en_116", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_123", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_191", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_491", + "flip": 1 + }, + { + "global index": "WMT19-gu-en_968", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_528", + "flip": 1 + }, + { + "global index": "WMT19-kk-en_826", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_135", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_269", + "flip": 1 + }, + { + "global index": "WMT19-lt-en_636", + "flip": 1 + }, + { + "global index": "WMT19-ru-en_222", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_218", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_252", + "flip": 1 + }, + { + "global index": "WMT19-zh-en_59", + "flip": 1 + } +] diff --git a/src/data/flip_labels/flip_labels_weave-router.json b/src/data/flip_labels/flip_labels_weave-router.json index 699ecfc..9450917 100644 --- a/src/data/flip_labels/flip_labels_weave-router.json +++ b/src/data/flip_labels/flip_labels_weave-router.json @@ -81,7 +81,7 @@ }, { "global index": "ArcMMLU_98", - "flip": 1 + "flip": 0 }, { "global index": "AsDiv_1165", @@ -101,11 +101,11 @@ }, { "global index": "ChessInstruct_0", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_107", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_144", @@ -121,11 +121,11 @@ }, { "global index": "ChessInstruct_71", - "flip": 1 + "flip": 0 }, { "global index": "ChessInstruct_84", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_commonsense_28", @@ -153,7 +153,7 @@ }, { "global index": "Ethics_commonsense_90", - "flip": 1 + "flip": 0 }, { "global index": "Ethics_deontology_0", @@ -241,7 +241,7 @@ }, { "global index": "GeoBench_1102", - "flip": 1 + "flip": 0 }, { "global index": "GeoBench_1113", @@ -293,7 +293,7 @@ }, { "global index": "GeoGraphyData_100k_27", - "flip": 1 + "flip": 0 }, { "global index": "GeoGraphyData_100k_42", @@ -309,7 +309,7 @@ }, { "global index": "LiveCodeBench_118", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_131", @@ -329,7 +329,7 @@ }, { "global index": "LiveCodeBench_271", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_350", @@ -341,7 +341,7 @@ }, { "global index": "LiveCodeBench_405", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_43", @@ -349,11 +349,11 @@ }, { "global index": "LiveCodeBench_431", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_437", - "flip": 1 + "flip": 0 }, { "global index": "LiveCodeBench_476", @@ -401,7 +401,7 @@ }, { "global index": "MMLUPro_biology_2985", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_biology_3188", @@ -453,7 +453,7 @@ }, { "global index": "MMLUPro_chemistry_3837", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_chemistry_3974", @@ -477,7 +477,7 @@ }, { "global index": "MMLUPro_computer science_9136", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9138", @@ -509,11 +509,11 @@ }, { "global index": "MMLUPro_computer science_9289", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9414", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_computer science_9415", @@ -533,7 +533,7 @@ }, { "global index": "MMLUPro_computer science_9475", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5769", @@ -545,7 +545,7 @@ }, { "global index": "MMLUPro_economics_5931", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_economics_5965", @@ -593,7 +593,7 @@ }, { "global index": "MMLUPro_engineering_10298", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_engineering_10342", @@ -633,7 +633,7 @@ }, { "global index": "MMLUPro_health_4885", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_4973", @@ -657,11 +657,11 @@ }, { "global index": "MMLUPro_health_5261", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5473", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_health_5514", @@ -673,23 +673,23 @@ }, { "global index": "MMLUPro_history_4490", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4497", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4509", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4517", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4523", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4605", @@ -717,7 +717,7 @@ }, { "global index": "MMLUPro_history_4774", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_history_4810", @@ -749,11 +749,11 @@ }, { "global index": "MMLUPro_law_1462", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1484", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_law_1518", @@ -773,7 +773,7 @@ }, { "global index": "MMLUPro_math_6429", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_6526", @@ -785,7 +785,7 @@ }, { "global index": "MMLUPro_math_6848", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_math_7101", @@ -825,11 +825,11 @@ }, { "global index": "MMLUPro_philosophy_9943", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_7773", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_physics_7887", @@ -845,7 +845,7 @@ }, { "global index": "MMLUPro_physics_9017", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2005", @@ -869,15 +869,15 @@ }, { "global index": "MMLUPro_psychology_2450", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2457", - "flip": 1 + "flip": 0 }, { "global index": "MMLUPro_psychology_2524", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_121", @@ -901,7 +901,7 @@ }, { "global index": "MMLU_formal_logic_70", - "flip": 1 + "flip": 0 }, { "global index": "MMLU_formal_logic_85", @@ -933,7 +933,7 @@ }, { "global index": "MathQA_1742", - "flip": 1 + "flip": 0 }, { "global index": "MathQA_202", @@ -1029,7 +1029,7 @@ }, { "global index": "MusicTheoryBench_152", - "flip": 1 + "flip": 0 }, { "global index": "MusicTheoryBench_188", @@ -1105,7 +1105,7 @@ }, { "global index": "NarrativeQA_533", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_5894", @@ -1129,7 +1129,7 @@ }, { "global index": "NarrativeQA_8598", - "flip": 1 + "flip": 0 }, { "global index": "NarrativeQA_927", @@ -1141,7 +1141,7 @@ }, { "global index": "OpenTDB_Animals_262", - "flip": 1 + "flip": 0 }, { "global index": "OpenTDB_Art_1429", @@ -1265,7 +1265,7 @@ }, { "global index": "PubMedQA_154", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_18", @@ -1273,15 +1273,15 @@ }, { "global index": "PubMedQA_238", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_250", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_337", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_362", @@ -1289,11 +1289,11 @@ }, { "global index": "PubMedQA_437", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_510", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_520", @@ -1321,11 +1321,11 @@ }, { "global index": "PubMedQA_643", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_687", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_722", @@ -1337,7 +1337,7 @@ }, { "global index": "PubMedQA_755", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_8", @@ -1349,11 +1349,11 @@ }, { "global index": "PubMedQA_854", - "flip": 1 + "flip": 0 }, { "global index": "PubMedQA_905", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_1212", @@ -1365,11 +1365,11 @@ }, { "global index": "QANTA_Fine Arts_828", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Fine Arts_865", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Geography_1023", @@ -1401,7 +1401,7 @@ }, { "global index": "QANTA_History_926", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1045", @@ -1413,7 +1413,7 @@ }, { "global index": "QANTA_Literature_1239", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1326", @@ -1421,11 +1421,11 @@ }, { "global index": "QANTA_Literature_1727", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_1843", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_386", @@ -1433,19 +1433,19 @@ }, { "global index": "QANTA_Literature_408", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_475", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Literature_833", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Philosophy_1270", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Philosophy_499", @@ -1457,11 +1457,11 @@ }, { "global index": "QANTA_Science_1360", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_1473", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Science_308", @@ -1473,7 +1473,7 @@ }, { "global index": "QANTA_Social Science_1847", - "flip": 1 + "flip": 0 }, { "global index": "QANTA_Social Science_2", @@ -1481,7 +1481,7 @@ }, { "global index": "QANTA_Social Science_77", - "flip": 1 + "flip": 0 }, { "global index": "SocialiQA_13810", @@ -1505,7 +1505,7 @@ }, { "global index": "SuperGLUE-ClozeTest_12894", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-ClozeTest_17965", @@ -1513,7 +1513,7 @@ }, { "global index": "SuperGLUE-ClozeTest_18766", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Entailment_19410", @@ -1521,23 +1521,23 @@ }, { "global index": "SuperGLUE-Entailment_19567", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Entailment_522", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Entailment_767", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_1408", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_3137", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-QA_4046", @@ -1549,7 +1549,7 @@ }, { "global index": "SuperGLUE-QA_4160", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-RC_7725", @@ -1561,19 +1561,19 @@ }, { "global index": "SuperGLUE-RC_8531", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_19695", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_19738", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_20079", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wic_20189", @@ -1581,15 +1581,15 @@ }, { "global index": "SuperGLUE-Wic_20253", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wsc_20368", - "flip": 1 + "flip": 0 }, { "global index": "SuperGLUE-Wsc_20370", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-cs-en_156", @@ -1649,7 +1649,7 @@ }, { "global index": "WMT19-kk-en_826", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-lt-en_135", @@ -1661,7 +1661,7 @@ }, { "global index": "WMT19-lt-en_636", - "flip": 1 + "flip": 0 }, { "global index": "WMT19-ru-en_222", diff --git a/src/data/routerMetrics/category_scores.json b/src/data/routerMetrics/category_scores.json index 3c47759..5a73a6f 100644 --- a/src/data/routerMetrics/category_scores.json +++ b/src/data/routerMetrics/category_scores.json @@ -3682,475 +3682,553 @@ "azure": { "metrics": { "easy": { - "accuracy": 93.5, - "cost": 0.0003, - "robustness": 55.9 + "accuracy": 94.3, + "cost": 0.0002, + "robustness": 69.2 }, "medium": { - "accuracy": 61.3, - "cost": 0.0007, - "robustness": 58.3 + "accuracy": 66.0, + "cost": 0.0002, + "robustness": 78.8 }, "hard": { - "accuracy": 26.5, - "cost": 0.0009, - "robustness": 46.8 + "accuracy": 34.2, + "cost": 0.0004, + "robustness": 67.9 }, "all": { - "accuracy": 68.1, - "cost": 0.0005, - "robustness": 54.1 + "accuracy": 72.0, + "cost": 0.0002, + "robustness": 71.4 } }, "categories": { - "Technology": { + "Arts & recreation": { "metrics": { "easy": { - "accuracy": 93.7, - "cost": 0.0003, - "robustness": 61.5 + "accuracy": 93.3, + "cost": 0.0001, + "robustness": 75.0 }, "medium": { - "accuracy": 70.3, - "cost": 0.0007, - "robustness": 68.4 + "accuracy": 54.9, + "cost": 0.0002, + "robustness": 81.8 }, "hard": { - "accuracy": 23.9, - "cost": 0.0007, - "robustness": 57.1 + "accuracy": 15.6, + "cost": 0.0004, + "robustness": 40.0 }, "all": { - "accuracy": 78.3, - "cost": 0.0005, - "robustness": 63.1 + "accuracy": 63.0, + "cost": 0.0002, + "robustness": 70.8 } }, "subcategories": { - "Medicine and health": { + "Arts": { "metrics": { "easy": { - "accuracy": 92.8, - "cost": 0.0003, - "robustness": 64.7 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 0.0 }, "medium": { - "accuracy": 61.0, - "cost": 0.0004, + "accuracy": 69.2, + "cost": 0.0002, "robustness": 50.0 }, "hard": { - "accuracy": 15.0, - "cost": 0.0004, - "robustness": 50.0 + "accuracy": 14.3, + "cost": 0.0003, + "robustness": 0.0 }, "all": { - "accuracy": 76.1, - "cost": 0.0003, - "robustness": 60.9 + "accuracy": 61.0, + "cost": 0.0002, + "robustness": 20.0 } } }, - "Engineering": { + "Music": { "metrics": { "easy": { - "accuracy": 98.4, - "cost": 0.0005, - "robustness": 40.0 + "accuracy": 93.9, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 83.8, - "cost": 0.001, - "robustness": 76.9 + "accuracy": 46.6, + "cost": 0.0002, + "robustness": 80.0 }, "hard": { - "accuracy": 47.6, - "cost": 0.0015, + "accuracy": 24.0, + "cost": 0.0002, "robustness": 100.0 }, "all": { - "accuracy": 84.8, - "cost": 0.0009, - "robustness": 68.4 + "accuracy": 63.7, + "cost": 0.0002, + "robustness": 90.9 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 90.0, + "cost": 0.0002, + "robustness": 50.0 + }, + "medium": { + "accuracy": 59.0, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 13.6, + "cost": 0.0006, + "robustness": 50.0 + }, + "all": { + "accuracy": 63.4, + "cost": 0.0003, + "robustness": 75.0 } } } } }, - "Philosophy and psychology": { + "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 92.4, - "cost": 0.0003, - "robustness": 60.9 + "accuracy": 95.4, + "cost": 0.0002, + "robustness": 69.2 }, "medium": { - "accuracy": 51.4, - "cost": 0.0004, - "robustness": 14.3 + "accuracy": 81.8, + "cost": 0.0002, + "robustness": 85.7 }, "hard": { - "accuracy": 7.9, - "cost": 0.0005, - "robustness": 42.9 + "accuracy": 63.7, + "cost": 0.0003, + "robustness": 93.3 }, "all": { - "accuracy": 69.5, - "cost": 0.0003, - "robustness": 48.6 + "accuracy": 85.4, + "cost": 0.0002, + "robustness": 80.6 } }, "subcategories": { - "Philosophy": { + "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 86.5, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 95.7, + "cost": 0.0001, + "robustness": 69.2 }, "medium": { - "accuracy": 62.5, - "cost": 0.0005, - "robustness": 50.0 - }, - "hard": { - "accuracy": 11.1, - "cost": 0.0007, - "robustness": 33.3 - }, - "all": { - "accuracy": 50.8, - "cost": 0.0005, - "robustness": 62.5 - } - } - }, - "Psychology": { - "metrics": { - "easy": { - "accuracy": 95.0, + "accuracy": 88.3, "cost": 0.0002, - "robustness": 83.3 - }, - "medium": { - "accuracy": 77.1, - "cost": 0.0004, - "robustness": 0 + "robustness": 88.2 }, "hard": { - "accuracy": 5.6, + "accuracy": 73.7, "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 81.8, - "cost": 0.0003, - "robustness": 87.5 + "accuracy": 88.3, + "cost": 0.0002, + "robustness": 86.0 } } }, - "Ethics": { + "Library and information sciences": { "metrics": { "easy": { - "accuracy": 91.8, - "cost": 0.0003, - "robustness": 42.9 + "accuracy": 94.9, + "cost": 0.0002, + "robustness": 69.2 }, "medium": { - "accuracy": 31.8, - "cost": 0.0004, - "robustness": 0.0 + "accuracy": 52.9, + "cost": 0.0002, + "robustness": 75.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0003, - "robustness": 0.0 + "accuracy": 5.6, + "cost": 0.0002, + "robustness": 50.0 }, "all": { - "accuracy": 69.4, - "cost": 0.0003, - "robustness": 28.6 + "accuracy": 77.8, + "cost": 0.0002, + "robustness": 68.4 } } } } }, - "Social Science": { + "History": { "metrics": { "easy": { - "accuracy": 93.3, - "cost": 0.0003, - "robustness": 33.3 + "accuracy": 97.6, + "cost": 0.0002, + "robustness": 65.2 }, "medium": { - "accuracy": 65.1, - "cost": 0.0007, - "robustness": 75.0 + "accuracy": 65.5, + "cost": 0.0002, + "robustness": 50.0 }, "hard": { - "accuracy": 6.7, - "cost": 0.0006, - "robustness": 14.3 + "accuracy": 12.8, + "cost": 0.0003, + "robustness": 80.0 }, "all": { - "accuracy": 64.7, - "cost": 0.0005, - "robustness": 43.2 + "accuracy": 71.4, + "cost": 0.0002, + "robustness": 66.7 } }, "subcategories": { - "Social sciences, sociology, and anthropology": { + "Biography and genealogy": { "metrics": { "easy": { "accuracy": 100.0, "cost": 0.0001, - "robustness": 0 + "robustness": 100.0 }, "medium": { - "accuracy": 33.3, - "cost": 0.0006, - "robustness": 50.0 + "accuracy": 88.9, + "cost": 0.0001, + "robustness": 0 }, "hard": { - "accuracy": 2.9, - "cost": 0.0006, - "robustness": 0.0 + "accuracy": 100.0, + "cost": 0.0004, + "robustness": 0 }, "all": { - "accuracy": 18.2, - "cost": 0.0006, - "robustness": 33.3 + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 100.0 } } }, - "Economics": { + "Geography": { "metrics": { "easy": { - "accuracy": 93.4, - "cost": 0.0003, - "robustness": 25.0 + "accuracy": 99.1, + "cost": 0.0002, + "robustness": 37.5 }, "medium": { - "accuracy": 70.6, - "cost": 0.0007, - "robustness": 83.3 + "accuracy": 55.2, + "cost": 0.0002, + "robustness": 0.0 }, "hard": { - "accuracy": 7.5, - "cost": 0.0006, - "robustness": 33.3 + "accuracy": 41.7, + "cost": 0.0003, + "robustness": 0 }, "all": { - "accuracy": 68.5, - "cost": 0.0005, - "robustness": 42.9 + "accuracy": 86.0, + "cost": 0.0002, + "robustness": 33.3 } } }, - "Social problems": { + "History": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 96.4, "cost": 0.0002, - "robustness": 66.7 + "robustness": 72.7 }, "medium": { - "accuracy": 44.4, - "cost": 0.0003, - "robustness": 0 - }, - "hard": { - "accuracy": 0.0, + "accuracy": 66.2, "cost": 0.0002, - "robustness": 0.0 + "robustness": 60.0 + }, + "hard": { + "accuracy": 9.6, + "cost": 0.0003, + "robustness": 80.0 }, "all": { - "accuracy": 83.6, + "accuracy": 64.3, "cost": 0.0002, - "robustness": 50.0 + "robustness": 73.1 } } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 80.9, + "cost": 0.0002, + "robustness": 72.7 }, - "Law": { + "medium": { + "accuracy": 31.1, + "cost": 0.0002, + "robustness": 55.6 + }, + "hard": { + "accuracy": 54.6, + "cost": 0.0009, + "robustness": 80.0 + }, + "all": { + "accuracy": 56.8, + "cost": 0.0005, + "robustness": 73.3 + } + }, + "subcategories": { + "Language": { "metrics": { "easy": { - "accuracy": 87.7, - "cost": 0.0004, - "robustness": 33.3 + "accuracy": 80.9, + "cost": 0.0002, + "robustness": 72.7 }, "medium": { - "accuracy": 62.9, - "cost": 0.0006, - "robustness": 75.0 + "accuracy": 31.1, + "cost": 0.0002, + "robustness": 55.6 }, "hard": { - "accuracy": 10.0, - "cost": 0.0005, - "robustness": 0.0 + "accuracy": 54.6, + "cost": 0.0009, + "robustness": 80.0 }, "all": { - "accuracy": 63.1, + "accuracy": 56.8, "cost": 0.0005, - "robustness": 44.4 + "robustness": 73.3 } } } } }, - "Science": { + "Literature": { "metrics": { "easy": { - "accuracy": 97.0, + "accuracy": 97.3, "cost": 0.0002, - "robustness": 59.0 + "robustness": 66.7 }, "medium": { - "accuracy": 75.0, - "cost": 0.0006, - "robustness": 61.9 + "accuracy": 47.9, + "cost": 0.0004, + "robustness": 0.0 }, "hard": { - "accuracy": 27.8, - "cost": 0.0012, - "robustness": 55.6 + "accuracy": 32.9, + "cost": 0.0003, + "robustness": 41.7 }, "all": { - "accuracy": 80.6, - "cost": 0.0005, - "robustness": 59.4 + "accuracy": 44.7, + "cost": 0.0003, + "robustness": 41.4 } }, "subcategories": { - "Science": { + "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 96.7, + "accuracy": 97.3, "cost": 0.0002, - "robustness": 100.0 + "robustness": 66.7 }, "medium": { - "accuracy": 20.0, + "accuracy": 47.9, "cost": 0.0004, - "robustness": 0 + "robustness": 0.0 }, "hard": { - "accuracy": 3.4, - "cost": 0.0006, - "robustness": 33.3 + "accuracy": 32.9, + "cost": 0.0003, + "robustness": 41.7 }, "all": { - "accuracy": 31.6, - "cost": 0.0005, - "robustness": 75.0 + "accuracy": 44.7, + "cost": 0.0003, + "robustness": 41.4 } } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0002, + "robustness": 55.6 }, - "Mathematics": { + "medium": { + "accuracy": 63.5, + "cost": 0.0003, + "robustness": 70.0 + }, + "hard": { + "accuracy": 14.8, + "cost": 0.0004, + "robustness": 28.6 + }, + "all": { + "accuracy": 75.0, + "cost": 0.0002, + "robustness": 54.5 + } + }, + "subcategories": { + "Ethics": { "metrics": { "easy": { - "accuracy": 96.9, + "accuracy": 87.6, "cost": 0.0003, - "robustness": 36.4 + "robustness": 42.9 }, "medium": { - "accuracy": 80.4, - "cost": 0.0008, - "robustness": 81.8 + "accuracy": 35.3, + "cost": 0.0004, + "robustness": 60.0 }, "hard": { - "accuracy": 40.4, - "cost": 0.0019, - "robustness": 80.0 + "accuracy": 0.0, + "cost": 0.0003, + "robustness": 0.0 }, "all": { - "accuracy": 81.7, - "cost": 0.0007, - "robustness": 63.0 + "accuracy": 67.8, + "cost": 0.0003, + "robustness": 42.9 } } }, - "Biology": { + "Philosophical logic": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0003, - "robustness": 57.1 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 64.7, - "cost": 0.0004, - "robustness": 0 + "accuracy": 89.5, + "cost": 0.0001, + "robustness": 66.7 }, "hard": { - "accuracy": 22.2, - "cost": 0.0004, + "accuracy": 80.0, + "cost": 0.0001, "robustness": 0 }, "all": { - "accuracy": 89.5, - "cost": 0.0003, - "robustness": 57.1 + "accuracy": 95.4, + "cost": 0.0001, + "robustness": 85.7 } } }, - "Chemistry": { + "Philosophy": { "metrics": { "easy": { - "accuracy": 93.0, - "cost": 0.0004, + "accuracy": 97.3, + "cost": 0.0002, "robustness": 66.7 }, "medium": { - "accuracy": 91.2, - "cost": 0.0006, - "robustness": 0.0 + "accuracy": 77.5, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 30.8, + "accuracy": 11.1, "cost": 0.0005, - "robustness": 0 + "robustness": 33.3 }, "all": { - "accuracy": 83.3, - "cost": 0.0005, - "robustness": 40.0 + "accuracy": 59.0, + "cost": 0.0003, + "robustness": 62.5 } } }, - "Physics": { + "Psychology": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0003, - "robustness": 66.7 + "accuracy": 95.9, + "cost": 0.0002, + "robustness": 50.0 }, "medium": { - "accuracy": 73.8, - "cost": 0.0006, - "robustness": 50.0 + "accuracy": 81.2, + "cost": 0.0002, + "robustness": 0 }, "hard": { - "accuracy": 54.5, - "cost": 0.0006, - "robustness": 0 + "accuracy": 16.7, + "cost": 0.0002, + "robustness": 50.0 }, "all": { - "accuracy": 82.5, - "cost": 0.0005, - "robustness": 60.0 + "accuracy": 84.5, + "cost": 0.0002, + "robustness": 50.0 } } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0002, + "robustness": 64.1 + }, + "medium": { + "accuracy": 72.7, + "cost": 0.0003, + "robustness": 81.0 + }, + "hard": { + "accuracy": 28.9, + "cost": 0.0004, + "robustness": 77.8 }, + "all": { + "accuracy": 80.0, + "cost": 0.0002, + "robustness": 71.0 + } + }, + "subcategories": { "Animals (Zoology)": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 95.0, + "cost": 0.0001, + "robustness": 0.0 }, "medium": { "accuracy": 60.0, - "cost": 0.0003, + "cost": 0.0001, "robustness": 0 }, "hard": { @@ -4159,403 +4237,373 @@ "robustness": 0 }, "all": { - "accuracy": 92.0, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 88.0, + "cost": 0.0001, + "robustness": 0.0 } } }, - "Earth sciences and geology": { + "Biology": { "metrics": { "easy": { - "accuracy": 96.9, + "accuracy": 99.1, "cost": 0.0002, - "robustness": 62.5 + "robustness": 71.4 }, "medium": { - "accuracy": 71.2, - "cost": 0.0004, - "robustness": 50.0 + "accuracy": 52.9, + "cost": 0.0002, + "robustness": 0 }, "hard": { - "accuracy": 30.8, + "accuracy": 22.2, "cost": 0.0004, - "robustness": 0.0 + "robustness": 0 }, "all": { - "accuracy": 89.7, - "cost": 0.0003, - "robustness": 53.3 + "accuracy": 88.8, + "cost": 0.0002, + "robustness": 71.4 } } - } - } - }, - "Arts & recreation": { - "metrics": { - "easy": { - "accuracy": 90.9, - "cost": 0.0006, - "robustness": 75.0 - }, - "medium": { - "accuracy": 52.9, - "cost": 0.0012, - "robustness": 45.5 - }, - "hard": { - "accuracy": 10.9, - "cost": 0.0017, - "robustness": 40.0 }, - "all": { - "accuracy": 60.3, - "cost": 0.001, - "robustness": 54.2 - } - }, - "subcategories": { - "Arts": { + "Chemistry": { "metrics": { "easy": { - "accuracy": 94.9, + "accuracy": 95.3, "cost": 0.0002, - "robustness": 100.0 + "robustness": 33.3 }, "medium": { - "accuracy": 51.3, - "cost": 0.0006, + "accuracy": 79.4, + "cost": 0.0002, "robustness": 50.0 }, "hard": { - "accuracy": 5.4, - "cost": 0.0009, + "accuracy": 15.4, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 77.8, + "cost": 0.0003, + "robustness": 40.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0001, + "robustness": 75.0 + }, + "medium": { + "accuracy": 67.8, + "cost": 0.0002, + "robustness": 83.3 + }, + "hard": { + "accuracy": 30.8, + "cost": 0.0002, "robustness": 100.0 }, "all": { - "accuracy": 51.3, - "cost": 0.0006, + "accuracy": 88.2, + "cost": 0.0001, "robustness": 80.0 } } }, - "Sports, games and entertainment": { + "Mathematics": { "metrics": { "easy": { - "accuracy": 87.9, - "cost": 0.0009, - "robustness": 50.0 + "accuracy": 97.7, + "cost": 0.0002, + "robustness": 90.9 }, "medium": { - "accuracy": 52.0, - "cost": 0.0016, - "robustness": 0.0 + "accuracy": 78.4, + "cost": 0.0003, + "robustness": 90.9 }, "hard": { - "accuracy": 15.2, - "cost": 0.0025, - "robustness": 0.0 + "accuracy": 49.4, + "cost": 0.0004, + "robustness": 100.0 }, "all": { - "accuracy": 60.5, - "cost": 0.0015, - "robustness": 12.5 + "accuracy": 82.5, + "cost": 0.0003, + "robustness": 92.6 } } }, - "Music": { + "Physics": { "metrics": { "easy": { - "accuracy": 92.9, - "cost": 0.0004, - "robustness": 80.0 + "accuracy": 95.5, + "cost": 0.0002, + "robustness": 33.3 }, "medium": { - "accuracy": 54.3, - "cost": 0.0011, - "robustness": 80.0 + "accuracy": 73.8, + "cost": 0.0003, + "robustness": 50.0 }, "hard": { - "accuracy": 12.0, - "cost": 0.0012, - "robustness": 0.0 + "accuracy": 27.3, + "cost": 0.0002, + "robustness": 0 }, "all": { - "accuracy": 65.8, - "cost": 0.0008, - "robustness": 72.7 + "accuracy": 78.4, + "cost": 0.0003, + "robustness": 40.0 } } - } - } - }, - "Literature": { - "metrics": { - "easy": { - "accuracy": 93.1, - "cost": 0.0002, - "robustness": 66.7 - }, - "medium": { - "accuracy": 45.1, - "cost": 0.0006, - "robustness": 100.0 - }, - "hard": { - "accuracy": 24.8, - "cost": 0.0006, - "robustness": 58.3 }, - "all": { - "accuracy": 38.0, - "cost": 0.0006, - "robustness": 62.1 - } - }, - "subcategories": { - "Literature, rhetoric and criticism": { + "Science": { "metrics": { "easy": { - "accuracy": 93.1, + "accuracy": 96.7, "cost": 0.0002, - "robustness": 66.7 + "robustness": 40.0 }, "medium": { - "accuracy": 45.1, - "cost": 0.0006, - "robustness": 100.0 + "accuracy": 32.0, + "cost": 0.0004, + "robustness": 0 }, "hard": { - "accuracy": 24.8, - "cost": 0.0006, - "robustness": 58.3 + "accuracy": 1.7, + "cost": 0.0005, + "robustness": 33.3 }, "all": { - "accuracy": 38.0, - "cost": 0.0006, - "robustness": 62.1 + "accuracy": 33.3, + "cost": 0.0004, + "robustness": 37.5 } } } } }, - "History": { + "Social Science": { "metrics": { "easy": { - "accuracy": 97.0, - "cost": 0.0003, - "robustness": 57.1 + "accuracy": 95.3, + "cost": 0.0001, + "robustness": 88.9 }, "medium": { - "accuracy": 55.5, - "cost": 0.0004, - "robustness": 66.7 + "accuracy": 63.2, + "cost": 0.0002, + "robustness": 83.3 }, "hard": { - "accuracy": 4.0, - "cost": 0.0006, - "robustness": 40.0 + "accuracy": 11.4, + "cost": 0.0004, + "robustness": 71.4 }, "all": { - "accuracy": 65.3, - "cost": 0.0004, - "robustness": 54.1 + "accuracy": 65.9, + "cost": 0.0002, + "robustness": 83.8 } }, "subcategories": { - "Geography": { + "Economics": { "metrics": { "easy": { - "accuracy": 97.3, + "accuracy": 95.6, "cost": 0.0002, - "robustness": 66.7 + "robustness": 83.3 }, "medium": { - "accuracy": 42.1, - "cost": 0.0004, + "accuracy": 70.6, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 8.3, - "cost": 0.0007, - "robustness": 0 + "accuracy": 13.8, + "cost": 0.0004, + "robustness": 33.3 }, "all": { - "accuracy": 76.9, - "cost": 0.0003, - "robustness": 71.4 + "accuracy": 70.7, + "cost": 0.0002, + "robustness": 81.0 } } }, - "History": { + "Law": { "metrics": { "easy": { - "accuracy": 96.4, - "cost": 0.0003, - "robustness": 45.5 + "accuracy": 93.8, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 56.6, - "cost": 0.0005, - "robustness": 60.0 + "accuracy": 60.3, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 3.7, - "cost": 0.0006, - "robustness": 40.0 + "accuracy": 13.3, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 60.0, - "cost": 0.0004, - "robustness": 46.2 + "accuracy": 65.2, + "cost": 0.0001, + "robustness": 100.0 } } }, - "Biography and genealogy": { + "Social problems": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0002, - "robustness": 75.0 + "accuracy": 97.9, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 66.7, - "cost": 0.0003, + "accuracy": 11.1, + "cost": 0.0001, "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0003, - "robustness": 0 + "cost": 0.0001, + "robustness": 100.0 }, "all": { - "accuracy": 92.3, - "cost": 0.0002, - "robustness": 75.0 + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 100.0 } } - } - } - }, - "Language": { - "metrics": { - "easy": { - "accuracy": 84.7, - "cost": 0.0003, - "robustness": 45.5 - }, - "medium": { - "accuracy": 41.2, - "cost": 0.0003, - "robustness": 55.6 - }, - "hard": { - "accuracy": 56.4, - "cost": 0.0006, - "robustness": 28.0 }, - "all": { - "accuracy": 61.3, - "cost": 0.0004, - "robustness": 37.8 - } - }, - "subcategories": { - "Language": { + "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 84.7, - "cost": 0.0003, - "robustness": 45.5 + "accuracy": 66.7, + "cost": 0.0002, + "robustness": 0 }, "medium": { - "accuracy": 41.2, - "cost": 0.0003, - "robustness": 55.6 + "accuracy": 33.3, + "cost": 0.0006, + "robustness": 0.0 }, "hard": { - "accuracy": 56.4, - "cost": 0.0006, - "robustness": 28.0 + "accuracy": 5.9, + "cost": 0.0005, + "robustness": 100.0 }, "all": { - "accuracy": 61.3, - "cost": 0.0004, - "robustness": 37.8 + "accuracy": 18.2, + "cost": 0.0005, + "robustness": 33.3 } } } } }, - "Computer science, information, and general works": { + "Technology": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 93.3, + "cost": 0.0001, + "robustness": 74.4 }, "medium": { - "accuracy": 58.2, - "cost": 0.0006, - "robustness": 52.4 + "accuracy": 68.3, + "cost": 0.0002, + "robustness": 95.2 }, "hard": { - "accuracy": 31.8, - "cost": 0.0018, - "robustness": 73.3 + "accuracy": 26.8, + "cost": 0.0002, + "robustness": 85.7 }, "all": { - "accuracy": 70.3, - "cost": 0.0007, - "robustness": 56.5 + "accuracy": 78.4, + "cost": 0.0002, + "robustness": 81.7 } }, "subcategories": { - "Computer science, knowledge, and systems": { + "Engineering": { "metrics": { "easy": { - "accuracy": 89.7, - "cost": 0.0003, - "robustness": 53.8 + "accuracy": 97.6, + "cost": 0.0001, + "robustness": 60.0 }, "medium": { - "accuracy": 58.6, - "cost": 0.0007, - "robustness": 64.7 + "accuracy": 86.8, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 36.4, - "cost": 0.0021, - "robustness": 76.9 + "accuracy": 50.0, + "cost": 0.0003, + "robustness": 100.0 }, "all": { - "accuracy": 67.0, - "cost": 0.0008, - "robustness": 65.1 + "accuracy": 86.3, + "cost": 0.0002, + "robustness": 89.5 } } }, - "Library and information sciences": { + "Management and public relations": { "metrics": { "easy": { - "accuracy": 95.3, + "accuracy": 96.7, "cost": 0.0002, - "robustness": 46.2 + "robustness": 75.0 }, "medium": { - "accuracy": 56.5, - "cost": 0.0004, + "accuracy": 45.5, + "cost": 0.0002, "robustness": 0.0 }, "hard": { - "accuracy": 5.6, - "cost": 0.0003, - "robustness": 50.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 0 }, "all": { - "accuracy": 78.8, - "cost": 0.0003, - "robustness": 36.8 + "accuracy": 86.3, + "cost": 0.0002, + "robustness": 60.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 92.2, + "cost": 0.0001, + "robustness": 76.5 + }, + "medium": { + "accuracy": 56.6, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 18.6, + "cost": 0.0002, + "robustness": 83.3 + }, + "all": { + "accuracy": 75.1, + "cost": 0.0002, + "robustness": 80.9 } } } @@ -12043,319 +12091,271 @@ } }, "categories": { - "Computer science, information, and general works": { + "Arts & recreation": { "metrics": { "easy": { - "accuracy": 97.3, + "accuracy": 94.6, "cost": 0.0001, - "robustness": 42.3 + "robustness": 50.0 }, "medium": { - "accuracy": 80.7, - "cost": 0.0001, - "robustness": 42.9 + "accuracy": 53.7, + "cost": 0.0002, + "robustness": 72.7 }, "hard": { - "accuracy": 32.2, - "cost": 0.0003, - "robustness": 53.3 + "accuracy": 6.1, + "cost": 0.0001, + "robustness": 0.0 }, "all": { - "accuracy": 80.4, + "accuracy": 61.1, "cost": 0.0001, - "robustness": 45.2 + "robustness": 50.0 } }, "subcategories": { - "Library and information sciences": { + "Arts": { "metrics": { "easy": { - "accuracy": 96.0, + "accuracy": 94.9, "cost": 0.0, - "robustness": 38.5 + "robustness": 0.0 }, "medium": { - "accuracy": 62.4, + "accuracy": 84.6, "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 }, "hard": { - "accuracy": 2.8, + "accuracy": 5.4, "cost": 0.0, - "robustness": 50.0 + "robustness": 0.0 }, "all": { - "accuracy": 80.3, + "accuracy": 59.7, "cost": 0.0, - "robustness": 31.6 + "robustness": 40.0 } } }, - "Computer science, knowledge, and systems": { + "Music": { "metrics": { "easy": { - "accuracy": 98.1, + "accuracy": 98.0, "cost": 0.0001, - "robustness": 46.2 + "robustness": 40.0 }, "medium": { - "accuracy": 84.9, - "cost": 0.0001, - "robustness": 52.9 + "accuracy": 43.1, + "cost": 0.0002, + "robustness": 60.0 }, "hard": { - "accuracy": 37.3, - "cost": 0.0004, - "robustness": 53.8 + "accuracy": 8.0, + "cost": 0.0003, + "robustness": 0.0 }, "all": { - "accuracy": 80.5, + "accuracy": 62.1, "cost": 0.0002, - "robustness": 51.2 + "robustness": 45.5 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 92.1, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 54.0, + "cost": 0.0001, + "robustness": 75.0 + }, + "hard": { + "accuracy": 6.1, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 61.1, + "cost": 0.0001, + "robustness": 62.5 } } } } }, - "Philosophy and psychology": { + "Computer science, information, and general works": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0, - "robustness": 22.2 + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 42.3 }, "medium": { - "accuracy": 62.6, + "accuracy": 80.7, "cost": 0.0001, - "robustness": 40.0 + "robustness": 42.9 }, "hard": { - "accuracy": 13.6, - "cost": 0.0001, - "robustness": 42.9 + "accuracy": 32.2, + "cost": 0.0003, + "robustness": 53.3 }, "all": { - "accuracy": 74.0, + "accuracy": 80.4, "cost": 0.0001, - "robustness": 29.5 + "robustness": 45.2 } }, "subcategories": { - "Ethics": { + "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 85.9, - "cost": 0.0, - "robustness": 21.4 + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 46.2 }, "medium": { - "accuracy": 47.1, + "accuracy": 84.9, "cost": 0.0001, - "robustness": 20.0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0001, - "robustness": 100.0 - }, - "all": { - "accuracy": 70.3, - "cost": 0.0001, - "robustness": 28.6 - } - } - }, - "Philosophy": { - "metrics": { - "easy": { - "accuracy": 94.6, - "cost": 0.0, - "robustness": 0.0 - }, - "medium": { - "accuracy": 52.5, - "cost": 0.0001, - "robustness": 50.0 + "robustness": 52.9 }, "hard": { - "accuracy": 13.3, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 37.3, + "cost": 0.0004, + "robustness": 53.8 }, "all": { - "accuracy": 50.8, - "cost": 0.0001, - "robustness": 12.5 + "accuracy": 80.5, + "cost": 0.0002, + "robustness": 51.2 } } }, - "Psychology": { + "Library and information sciences": { "metrics": { "easy": { - "accuracy": 97.5, + "accuracy": 96.0, "cost": 0.0, - "robustness": 33.3 + "robustness": 38.5 }, "medium": { - "accuracy": 81.2, + "accuracy": 62.4, "cost": 0.0, - "robustness": 0 + "robustness": 0.0 }, "hard": { - "accuracy": 11.1, + "accuracy": 2.8, "cost": 0.0, "robustness": 50.0 }, "all": { - "accuracy": 85.0, + "accuracy": 80.3, "cost": 0.0, - "robustness": 37.5 - } - } - }, - "Philosophical logic": { - "metrics": { - "easy": { - "accuracy": 96.9, - "cost": 0.0001, - "robustness": 25.0 - }, - "medium": { - "accuracy": 84.2, - "cost": 0.0001, - "robustness": 66.7 - }, - "hard": { - "accuracy": 60.0, - "cost": 0.0003, - "robustness": 0 - }, - "all": { - "accuracy": 90.7, - "cost": 0.0001, - "robustness": 42.9 + "robustness": 31.6 } } } } }, - "Social Science": { + "History": { "metrics": { "easy": { - "accuracy": 95.3, + "accuracy": 96.8, "cost": 0.0, - "robustness": 55.6 + "robustness": 52.2 }, "medium": { - "accuracy": 60.5, + "accuracy": 62.1, "cost": 0.0001, - "robustness": 83.3 + "robustness": 16.7 }, "hard": { - "accuracy": 8.7, + "accuracy": 10.1, "cost": 0.0001, - "robustness": 28.6 + "robustness": 30.0 }, "all": { - "accuracy": 64.3, + "accuracy": 69.6, "cost": 0.0001, - "robustness": 59.5 + "robustness": 41.0 } }, "subcategories": { - "Economics": { + "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 97.8, + "accuracy": 92.9, "cost": 0.0, - "robustness": 66.7 + "robustness": 75.0 }, "medium": { - "accuracy": 66.3, - "cost": 0.0001, - "robustness": 83.3 + "accuracy": 77.8, + "cost": 0.0, + "robustness": 0 }, "hard": { - "accuracy": 12.5, - "cost": 0.0001, - "robustness": 66.7 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0 }, "all": { - "accuracy": 69.7, - "cost": 0.0001, - "robustness": 71.4 - } - } - }, - "Law": { - "metrics": { - "easy": { - "accuracy": 87.7, - "cost": 0.0001, - "robustness": 66.7 - }, - "medium": { - "accuracy": 54.0, - "cost": 0.0001, + "accuracy": 88.5, + "cost": 0.0, "robustness": 75.0 - }, - "hard": { - "accuracy": 6.7, - "cost": 0.0001, - "robustness": 0.0 - }, - "all": { - "accuracy": 58.9, - "cost": 0.0001, - "robustness": 55.6 } } }, - "Social sciences, sociology, and anthropology": { + "Geography": { "metrics": { "easy": { "accuracy": 100.0, "cost": 0.0, - "robustness": 0 + "robustness": 37.5 }, "medium": { - "accuracy": 61.1, + "accuracy": 51.7, "cost": 0.0, - "robustness": 100.0 + "robustness": 0.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 8.3, + "cost": 0.0001, + "robustness": 0 }, "all": { - "accuracy": 25.5, + "accuracy": 83.3, "cost": 0.0, - "robustness": 66.7 + "robustness": 33.3 } } }, - "Social problems": { + "History": { "metrics": { "easy": { - "accuracy": 95.7, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 96.0, + "cost": 0.0001, + "robustness": 54.5 }, "medium": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0 + "accuracy": 63.3, + "cost": 0.0001, + "robustness": 20.0 }, "hard": { - "accuracy": 20.0, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 10.3, + "cost": 0.0001, + "robustness": 30.0 }, "all": { - "accuracy": 75.4, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 63.5, + "cost": 0.0001, + "robustness": 38.5 } } } @@ -12411,75 +12411,223 @@ } } }, - "Science": { + "Literature": { "metrics": { "easy": { - "accuracy": 98.6, + "accuracy": 100.0, "cost": 0.0, - "robustness": 61.5 + "robustness": 66.7 }, "medium": { - "accuracy": 76.2, + "accuracy": 56.2, "cost": 0.0001, - "robustness": 57.1 + "robustness": 50.0 }, "hard": { - "accuracy": 11.3, + "accuracy": 30.1, "cost": 0.0001, - "robustness": 44.4 + "robustness": 50.0 }, "all": { - "accuracy": 79.6, + "accuracy": 44.0, "cost": 0.0001, - "robustness": 58.0 + "robustness": 51.7 } }, "subcategories": { - "Mathematics": { + "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 98.1, - "cost": 0.0001, - "robustness": 72.7 + "accuracy": 100.0, + "cost": 0.0, + "robustness": 66.7 }, "medium": { - "accuracy": 79.2, + "accuracy": 56.2, "cost": 0.0001, - "robustness": 63.6 + "robustness": 50.0 }, "hard": { - "accuracy": 12.4, - "cost": 0.0002, - "robustness": 80.0 + "accuracy": 30.1, + "cost": 0.0001, + "robustness": 50.0 }, "all": { - "accuracy": 77.5, + "accuracy": 44.0, "cost": 0.0001, - "robustness": 70.4 + "robustness": 51.7 } } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0, + "robustness": 22.2 }, - "Earth sciences and geology": { + "medium": { + "accuracy": 62.6, + "cost": 0.0001, + "robustness": 40.0 + }, + "hard": { + "accuracy": 13.6, + "cost": 0.0001, + "robustness": 42.9 + }, + "all": { + "accuracy": 74.0, + "cost": 0.0001, + "robustness": 29.5 + } + }, + "subcategories": { + "Ethics": { "metrics": { "easy": { - "accuracy": 98.1, + "accuracy": 85.9, "cost": 0.0, - "robustness": 50.0 + "robustness": 21.4 }, "medium": { - "accuracy": 72.9, + "accuracy": 47.1, + "cost": 0.0001, + "robustness": 20.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 70.3, + "cost": 0.0001, + "robustness": 28.6 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0001, + "robustness": 25.0 + }, + "medium": { + "accuracy": 84.2, + "cost": 0.0001, + "robustness": 66.7 + }, + "hard": { + "accuracy": 60.0, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 90.7, + "cost": 0.0001, + "robustness": 42.9 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 94.6, + "cost": 0.0, + "robustness": 0.0 + }, + "medium": { + "accuracy": 52.5, "cost": 0.0001, "robustness": 50.0 }, "hard": { - "accuracy": 7.7, + "accuracy": 13.3, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 50.8, + "cost": 0.0001, + "robustness": 12.5 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0, + "robustness": 33.3 + }, + "medium": { + "accuracy": 81.2, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0, + "robustness": 50.0 + }, + "all": { + "accuracy": 85.0, + "cost": 0.0, + "robustness": 37.5 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.6, + "cost": 0.0, + "robustness": 61.5 + }, + "medium": { + "accuracy": 76.2, + "cost": 0.0001, + "robustness": 57.1 + }, + "hard": { + "accuracy": 11.3, + "cost": 0.0001, + "robustness": 44.4 + }, + "all": { + "accuracy": 79.6, + "cost": 0.0001, + "robustness": 58.0 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, "cost": 0.0, "robustness": 0.0 }, + "medium": { + "accuracy": 80.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, "all": { - "accuracy": 90.0, + "accuracy": 96.0, "cost": 0.0, - "robustness": 46.7 + "robustness": 0.0 } } }, @@ -12531,51 +12679,75 @@ } } }, - "Physics": { + "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 98.1, "cost": 0.0, - "robustness": 100.0 + "robustness": 50.0 }, "medium": { - "accuracy": 83.3, + "accuracy": 72.9, "cost": 0.0001, "robustness": 50.0 }, "hard": { - "accuracy": 27.3, + "accuracy": 7.7, "cost": 0.0, - "robustness": 0 + "robustness": 0.0 }, "all": { - "accuracy": 84.5, + "accuracy": 90.0, "cost": 0.0, + "robustness": 46.7 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 72.7 + }, + "medium": { + "accuracy": 79.2, + "cost": 0.0001, + "robustness": 63.6 + }, + "hard": { + "accuracy": 12.4, + "cost": 0.0002, "robustness": 80.0 + }, + "all": { + "accuracy": 77.5, + "cost": 0.0001, + "robustness": 70.4 } } }, - "Animals (Zoology)": { + "Physics": { "metrics": { "easy": { "accuracy": 100.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 }, "medium": { - "accuracy": 80.0, - "cost": 0.0, - "robustness": 0 + "accuracy": 83.3, + "cost": 0.0001, + "robustness": 50.0 }, "hard": { - "accuracy": 0, - "cost": 0, + "accuracy": 27.3, + "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 96.0, + "accuracy": 84.5, "cost": 0.0, - "robustness": 0.0 + "robustness": 80.0 } } }, @@ -12605,345 +12777,221 @@ } } }, - "Technology": { + "Social Science": { "metrics": { "easy": { - "accuracy": 94.7, - "cost": 0.0001, - "robustness": 55.8 + "accuracy": 95.3, + "cost": 0.0, + "robustness": 55.6 }, "medium": { - "accuracy": 64.3, + "accuracy": 60.5, "cost": 0.0001, - "robustness": 57.1 + "robustness": 83.3 }, "hard": { - "accuracy": 13.4, + "accuracy": 8.7, "cost": 0.0001, - "robustness": 71.4 + "robustness": 28.6 }, "all": { - "accuracy": 76.4, + "accuracy": 64.3, "cost": 0.0001, - "robustness": 57.7 + "robustness": 59.5 } }, "subcategories": { - "Engineering": { + "Economics": { "metrics": { "easy": { - "accuracy": 95.2, - "cost": 0.0001, - "robustness": 20.0 + "accuracy": 97.8, + "cost": 0.0, + "robustness": 66.7 }, "medium": { - "accuracy": 79.0, + "accuracy": 66.3, "cost": 0.0001, - "robustness": 53.8 + "robustness": 83.3 }, "hard": { - "accuracy": 31.0, + "accuracy": 12.5, "cost": 0.0001, - "robustness": 100.0 + "robustness": 66.7 }, "all": { - "accuracy": 79.1, + "accuracy": 69.7, "cost": 0.0001, - "robustness": 47.4 + "robustness": 71.4 } } }, - "Medicine and health": { + "Law": { "metrics": { "easy": { - "accuracy": 94.2, + "accuracy": 87.7, "cost": 0.0001, - "robustness": 58.8 + "robustness": 66.7 }, "medium": { - "accuracy": 53.3, + "accuracy": 54.0, "cost": 0.0001, - "robustness": 71.4 + "robustness": 75.0 }, "hard": { - "accuracy": 7.1, + "accuracy": 6.7, "cost": 0.0001, - "robustness": 66.7 + "robustness": 0.0 }, "all": { - "accuracy": 74.3, + "accuracy": 58.9, "cost": 0.0001, - "robustness": 61.7 + "robustness": 55.6 } } }, - "Management and public relations": { + "Social problems": { "metrics": { "easy": { - "accuracy": 98.3, + "accuracy": 95.7, "cost": 0.0, - "robustness": 75.0 + "robustness": 0.0 }, "medium": { - "accuracy": 81.8, + "accuracy": 0.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0 - }, - "all": { - "accuracy": 93.2, + "accuracy": 20.0, "cost": 0.0, - "robustness": 60.0 - } - } - } - } - }, - "Arts & recreation": { - "metrics": { - "easy": { - "accuracy": 94.6, - "cost": 0.0001, - "robustness": 50.0 - }, - "medium": { - "accuracy": 53.7, - "cost": 0.0002, - "robustness": 72.7 - }, - "hard": { - "accuracy": 6.1, - "cost": 0.0001, - "robustness": 0.0 - }, - "all": { - "accuracy": 61.1, - "cost": 0.0001, - "robustness": 50.0 - } - }, - "subcategories": { - "Sports, games and entertainment": { - "metrics": { - "easy": { - "accuracy": 92.1, - "cost": 0.0001, - "robustness": 100.0 - }, - "medium": { - "accuracy": 54.0, - "cost": 0.0001, - "robustness": 75.0 - }, - "hard": { - "accuracy": 6.1, - "cost": 0.0001, "robustness": 0.0 }, "all": { - "accuracy": 61.1, - "cost": 0.0001, - "robustness": 62.5 - } - } - }, - "Music": { - "metrics": { - "easy": { - "accuracy": 98.0, - "cost": 0.0001, - "robustness": 40.0 - }, - "medium": { - "accuracy": 43.1, - "cost": 0.0002, - "robustness": 60.0 - }, - "hard": { - "accuracy": 8.0, - "cost": 0.0003, + "accuracy": 75.4, + "cost": 0.0, "robustness": 0.0 - }, - "all": { - "accuracy": 62.1, - "cost": 0.0002, - "robustness": 45.5 } } }, - "Arts": { + "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 94.9, + "accuracy": 100.0, "cost": 0.0, - "robustness": 0.0 + "robustness": 0 }, "medium": { - "accuracy": 84.6, + "accuracy": 61.1, "cost": 0.0, "robustness": 100.0 }, "hard": { - "accuracy": 5.4, + "accuracy": 0.0, "cost": 0.0, "robustness": 0.0 }, "all": { - "accuracy": 59.7, + "accuracy": 25.5, "cost": 0.0, - "robustness": 40.0 + "robustness": 66.7 } } } } }, - "Literature": { + "Technology": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 94.7, + "cost": 0.0001, + "robustness": 55.8 }, "medium": { - "accuracy": 56.2, + "accuracy": 64.3, "cost": 0.0001, - "robustness": 50.0 + "robustness": 57.1 }, "hard": { - "accuracy": 30.1, + "accuracy": 13.4, "cost": 0.0001, - "robustness": 50.0 + "robustness": 71.4 }, "all": { - "accuracy": 44.0, + "accuracy": 76.4, "cost": 0.0001, - "robustness": 51.7 + "robustness": 57.7 } }, "subcategories": { - "Literature, rhetoric and criticism": { + "Engineering": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 95.2, + "cost": 0.0001, + "robustness": 20.0 }, "medium": { - "accuracy": 56.2, + "accuracy": 79.0, "cost": 0.0001, - "robustness": 50.0 + "robustness": 53.8 }, "hard": { - "accuracy": 30.1, + "accuracy": 31.0, "cost": 0.0001, - "robustness": 50.0 + "robustness": 100.0 }, "all": { - "accuracy": 44.0, + "accuracy": 79.1, "cost": 0.0001, - "robustness": 51.7 + "robustness": 47.4 } } - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 96.8, - "cost": 0.0, - "robustness": 52.2 - }, - "medium": { - "accuracy": 62.1, - "cost": 0.0001, - "robustness": 16.7 - }, - "hard": { - "accuracy": 10.1, - "cost": 0.0001, - "robustness": 30.0 }, - "all": { - "accuracy": 69.6, - "cost": 0.0001, - "robustness": 41.0 - } - }, - "subcategories": { - "Geography": { + "Management and public relations": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 98.3, "cost": 0.0, - "robustness": 37.5 + "robustness": 75.0 }, "medium": { - "accuracy": 51.7, + "accuracy": 81.8, "cost": 0.0, "robustness": 0.0 }, "hard": { - "accuracy": 8.3, - "cost": 0.0001, + "accuracy": 0.0, + "cost": 0.0, "robustness": 0 }, "all": { - "accuracy": 83.3, + "accuracy": 93.2, "cost": 0.0, - "robustness": 33.3 + "robustness": 60.0 } } }, - "History": { + "Medicine and health": { "metrics": { "easy": { - "accuracy": 96.0, + "accuracy": 94.2, "cost": 0.0001, - "robustness": 54.5 + "robustness": 58.8 }, "medium": { - "accuracy": 63.3, + "accuracy": 53.3, "cost": 0.0001, - "robustness": 20.0 + "robustness": 71.4 }, "hard": { - "accuracy": 10.3, + "accuracy": 7.1, "cost": 0.0001, - "robustness": 30.0 + "robustness": 66.7 }, "all": { - "accuracy": 63.5, + "accuracy": 74.3, "cost": 0.0001, - "robustness": 38.5 - } - } - }, - "Biography and genealogy": { - "metrics": { - "easy": { - "accuracy": 92.9, - "cost": 0.0, - "robustness": 75.0 - }, - "medium": { - "accuracy": 77.8, - "cost": 0.0, - "robustness": 0 - }, - "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0 - }, - "all": { - "accuracy": 88.5, - "cost": 0.0, - "robustness": 75.0 + "robustness": 61.7 } } } @@ -12975,54 +13023,128 @@ } }, "categories": { - "Computer science, information, and general works": { + "Arts & recreation": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0001, - "robustness": 42.3 + "accuracy": 93.0, + "cost": 0.0, + "robustness": 62.5 }, "medium": { - "accuracy": 77.5, - "cost": 0.0002, - "robustness": 57.1 + "accuracy": 48.6, + "cost": 0.0001, + "robustness": 63.6 }, "hard": { - "accuracy": 31.4, - "cost": 0.0005, - "robustness": 60.0 + "accuracy": 11.6, + "cost": 0.0001, + "robustness": 80.0 }, "all": { - "accuracy": 79.4, - "cost": 0.0002, - "robustness": 51.6 + "accuracy": 59.7, + "cost": 0.0001, + "robustness": 66.7 } }, "subcategories": { - "Library and information sciences": { + "Arts": { "metrics": { "easy": { - "accuracy": 97.5, + "accuracy": 96.6, "cost": 0.0001, - "robustness": 38.5 + "robustness": 0.0 }, "medium": { - "accuracy": 55.3, - "cost": 0.0001, - "robustness": 25.0 + "accuracy": 74.4, + "cost": 0.0, + "robustness": 50.0 }, "hard": { - "accuracy": 5.6, + "accuracy": 5.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 57.8, + "cost": 0.0001, + "robustness": 60.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 93.9, + "cost": 0.0, + "robustness": 80.0 + }, + "medium": { + "accuracy": 31.9, + "cost": 0.0001, + "robustness": 40.0 + }, + "hard": { + "accuracy": 24.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 56.7, "cost": 0.0001, + "robustness": 54.5 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 90.7, + "cost": 0.0, "robustness": 50.0 }, + "medium": { + "accuracy": 58.0, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 12.1, + "cost": 0.0001, + "robustness": 100.0 + }, "all": { - "accuracy": 80.1, + "accuracy": 63.1, "cost": 0.0001, - "robustness": 36.8 + "robustness": 87.5 } } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 42.3 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0002, + "robustness": 57.1 + }, + "hard": { + "accuracy": 31.4, + "cost": 0.0005, + "robustness": 60.0 }, + "all": { + "accuracy": 79.4, + "cost": 0.0002, + "robustness": 51.6 + } + }, + "subcategories": { "Computer science, knowledge, and systems": { "metrics": { "easy": { @@ -13046,298 +13168,348 @@ "robustness": 58.1 } } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0001, + "robustness": 38.5 + }, + "medium": { + "accuracy": 55.3, + "cost": 0.0001, + "robustness": 25.0 + }, + "hard": { + "accuracy": 5.6, + "cost": 0.0001, + "robustness": 50.0 + }, + "all": { + "accuracy": 80.1, + "cost": 0.0001, + "robustness": 36.8 + } + } } } }, - "Philosophy and psychology": { + "History": { "metrics": { "easy": { - "accuracy": 92.9, + "accuracy": 94.1, "cost": 0.0001, - "robustness": 51.9 + "robustness": 30.4 }, "medium": { "accuracy": 62.1, "cost": 0.0001, - "robustness": 30.0 + "robustness": 33.3 }, "hard": { - "accuracy": 12.3, + "accuracy": 9.4, "cost": 0.0001, - "robustness": 28.6 + "robustness": 70.0 }, "all": { - "accuracy": 74.3, + "accuracy": 68.0, "cost": 0.0001, - "robustness": 43.2 + "robustness": 41.0 } }, "subcategories": { - "Ethics": { + "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 89.2, + "accuracy": 97.6, "cost": 0.0, - "robustness": 50.0 + "robustness": 25.0 }, "medium": { - "accuracy": 32.9, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0 }, "hard": { "accuracy": 0.0, - "cost": 0.0001, - "robustness": 0.0 + "cost": 0.0, + "robustness": 0 }, "all": { - "accuracy": 68.2, - "cost": 0.0001, - "robustness": 33.3 + "accuracy": 96.2, + "cost": 0.0, + "robustness": 25.0 } } }, - "Philosophy": { + "Geography": { "metrics": { "easy": { - "accuracy": 86.5, + "accuracy": 97.2, "cost": 0.0, - "robustness": 33.3 + "robustness": 37.5 }, "medium": { - "accuracy": 70.0, - "cost": 0.0001, + "accuracy": 48.3, + "cost": 0.0, "robustness": 0.0 }, "hard": { - "accuracy": 17.8, - "cost": 0.0001, - "robustness": 66.7 + "accuracy": 16.7, + "cost": 0.0, + "robustness": 0 }, "all": { - "accuracy": 55.7, - "cost": 0.0001, - "robustness": 37.5 + "accuracy": 81.3, + "cost": 0.0, + "robustness": 33.3 } } }, - "Psychology": { + "History": { "metrics": { "easy": { - "accuracy": 96.7, + "accuracy": 91.9, "cost": 0.0001, - "robustness": 83.3 + "robustness": 27.3 }, "medium": { - "accuracy": 85.4, + "accuracy": 62.6, "cost": 0.0001, - "robustness": 0 + "robustness": 40.0 }, "hard": { - "accuracy": 5.6, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 8.8, + "cost": 0.0001, + "robustness": 70.0 }, "all": { - "accuracy": 85.0, + "accuracy": 61.0, "cost": 0.0001, - "robustness": 62.5 + "robustness": 46.2 } } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 88.5, + "cost": 0.0001, + "robustness": 63.6 }, - "Philosophical logic": { + "medium": { + "accuracy": 44.9, + "cost": 0.0001, + "robustness": 44.4 + }, + "hard": { + "accuracy": 48.4, + "cost": 0.0002, + "robustness": 80.0 + }, + "all": { + "accuracy": 59.6, + "cost": 0.0001, + "robustness": 68.9 + } + }, + "subcategories": { + "Language": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 88.5, "cost": 0.0001, - "robustness": 25.0 + "robustness": 63.6 }, "medium": { - "accuracy": 89.5, + "accuracy": 44.9, "cost": 0.0001, - "robustness": 100.0 + "robustness": 44.4 }, "hard": { - "accuracy": 20.0, - "cost": 0.0001, - "robustness": 0 + "accuracy": 48.4, + "cost": 0.0002, + "robustness": 80.0 }, "all": { - "accuracy": 92.6, + "accuracy": 59.6, "cost": 0.0001, - "robustness": 57.1 + "robustness": 68.9 } } } } }, - "Social Science": { + "Literature": { "metrics": { "easy": { - "accuracy": 94.3, + "accuracy": 99.1, "cost": 0.0001, - "robustness": 50.0 + "robustness": 33.3 }, "medium": { - "accuracy": 63.6, + "accuracy": 67.2, "cost": 0.0001, - "robustness": 25.0 + "robustness": 50.0 }, "hard": { - "accuracy": 7.4, + "accuracy": 27.4, "cost": 0.0002, - "robustness": 42.9 + "robustness": 45.8 }, "all": { - "accuracy": 64.7, - "cost": 0.0001, - "robustness": 40.5 + "accuracy": 42.9, + "cost": 0.0002, + "robustness": 44.8 } }, "subcategories": { - "Economics": { + "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 97.3, + "accuracy": 99.1, "cost": 0.0001, - "robustness": 41.7 + "robustness": 33.3 }, "medium": { - "accuracy": 73.6, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 67.2, + "cost": 0.0001, + "robustness": 50.0 }, "hard": { - "accuracy": 8.8, - "cost": 0.0003, - "robustness": 33.3 + "accuracy": 27.4, + "cost": 0.0002, + "robustness": 45.8 }, "all": { - "accuracy": 71.6, - "cost": 0.0001, - "robustness": 38.1 + "accuracy": 42.9, + "cost": 0.0002, + "robustness": 44.8 } } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 92.9, + "cost": 0.0001, + "robustness": 51.9 }, - "Law": { + "medium": { + "accuracy": 62.1, + "cost": 0.0001, + "robustness": 30.0 + }, + "hard": { + "accuracy": 12.3, + "cost": 0.0001, + "robustness": 28.6 + }, + "all": { + "accuracy": 74.3, + "cost": 0.0001, + "robustness": 43.2 + } + }, + "subcategories": { + "Ethics": { "metrics": { "easy": { - "accuracy": 87.7, - "cost": 0.0002, - "robustness": 33.3 + "accuracy": 89.2, + "cost": 0.0, + "robustness": 50.0 }, "medium": { - "accuracy": 47.6, - "cost": 0.0002, + "accuracy": 32.9, + "cost": 0.0001, "robustness": 0.0 }, "hard": { - "accuracy": 6.7, - "cost": 0.0002, - "robustness": 50.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0.0 }, "all": { - "accuracy": 56.3, - "cost": 0.0002, - "robustness": 22.2 + "accuracy": 68.2, + "cost": 0.0001, + "robustness": 33.3 } } }, - "Social sciences, sociology, and anthropology": { + "Philosophical logic": { "metrics": { "easy": { - "accuracy": 66.7, - "cost": 0.0, - "robustness": 0 + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 25.0 }, "medium": { - "accuracy": 44.4, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 89.5, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 5.9, - "cost": 0.0, - "robustness": 0.0 + "accuracy": 20.0, + "cost": 0.0001, + "robustness": 0 }, "all": { - "accuracy": 21.8, - "cost": 0.0, - "robustness": 33.3 + "accuracy": 92.6, + "cost": 0.0001, + "robustness": 57.1 } } }, - "Social problems": { + "Philosophy": { "metrics": { "easy": { - "accuracy": 93.6, + "accuracy": 86.5, "cost": 0.0, - "robustness": 100.0 + "robustness": 33.3 }, "medium": { - "accuracy": 33.3, - "cost": 0.0, - "robustness": 0 + "accuracy": 70.0, + "cost": 0.0001, + "robustness": 0.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 17.8, + "cost": 0.0001, + "robustness": 66.7 }, "all": { - "accuracy": 77.0, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 55.7, + "cost": 0.0001, + "robustness": 37.5 } } - } - } - }, - "Language": { - "metrics": { - "easy": { - "accuracy": 88.5, - "cost": 0.0001, - "robustness": 63.6 - }, - "medium": { - "accuracy": 44.9, - "cost": 0.0001, - "robustness": 44.4 - }, - "hard": { - "accuracy": 48.4, - "cost": 0.0002, - "robustness": 80.0 }, - "all": { - "accuracy": 59.6, - "cost": 0.0001, - "robustness": 68.9 - } - }, - "subcategories": { - "Language": { + "Psychology": { "metrics": { "easy": { - "accuracy": 88.5, + "accuracy": 96.7, "cost": 0.0001, - "robustness": 63.6 + "robustness": 83.3 }, "medium": { - "accuracy": 44.9, + "accuracy": 85.4, "cost": 0.0001, - "robustness": 44.4 + "robustness": 0 }, "hard": { - "accuracy": 48.4, - "cost": 0.0002, - "robustness": 80.0 + "accuracy": 5.6, + "cost": 0.0, + "robustness": 0.0 }, "all": { - "accuracy": 59.6, + "accuracy": 85.0, "cost": 0.0001, - "robustness": 68.9 + "robustness": 62.5 } } } @@ -13367,51 +13539,27 @@ } }, "subcategories": { - "Mathematics": { - "metrics": { - "easy": { - "accuracy": 96.2, - "cost": 0.0001, - "robustness": 18.2 - }, - "medium": { - "accuracy": 79.6, - "cost": 0.0001, - "robustness": 63.6 - }, - "hard": { - "accuracy": 28.1, - "cost": 0.0006, - "robustness": 20.0 - }, - "all": { - "accuracy": 79.2, - "cost": 0.0002, - "robustness": 37.0 - } - } - }, - "Earth sciences and geology": { + "Animals (Zoology)": { "metrics": { "easy": { - "accuracy": 97.3, - "cost": 0.0001, - "robustness": 37.5 + "accuracy": 95.0, + "cost": 0.0, + "robustness": 50.0 }, "medium": { - "accuracy": 67.8, - "cost": 0.0001, - "robustness": 16.7 + "accuracy": 80.0, + "cost": 0.0, + "robustness": 0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 0, + "cost": 0, + "robustness": 0 }, "all": { - "accuracy": 88.2, - "cost": 0.0001, - "robustness": 33.3 + "accuracy": 92.0, + "cost": 0.0, + "robustness": 50.0 } } }, @@ -13463,51 +13611,75 @@ } } }, - "Physics": { + "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0, - "robustness": 66.7 + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 37.5 }, "medium": { - "accuracy": 73.8, + "accuracy": 67.8, "cost": 0.0001, - "robustness": 100.0 + "robustness": 16.7 }, "hard": { - "accuracy": 36.4, - "cost": 0.0, - "robustness": 0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 80.4, + "accuracy": 88.2, "cost": 0.0001, - "robustness": 80.0 + "robustness": 33.3 } } }, - "Animals (Zoology)": { + "Mathematics": { "metrics": { "easy": { - "accuracy": 95.0, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 96.2, + "cost": 0.0001, + "robustness": 18.2 }, "medium": { - "accuracy": 80.0, - "cost": 0.0, - "robustness": 0 + "accuracy": 79.6, + "cost": 0.0001, + "robustness": 63.6 }, "hard": { - "accuracy": 0, - "cost": 0, - "robustness": 0 + "accuracy": 28.1, + "cost": 0.0006, + "robustness": 20.0 }, "all": { - "accuracy": 92.0, + "accuracy": 79.2, + "cost": 0.0002, + "robustness": 37.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, "cost": 0.0, - "robustness": 50.0 + "robustness": 66.7 + }, + "medium": { + "accuracy": 73.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 36.4, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 80.4, + "cost": 0.0001, + "robustness": 80.0 } } }, @@ -13537,376 +13709,350 @@ } } }, - "Technology": { + "Social Science": { "metrics": { "easy": { - "accuracy": 93.8, + "accuracy": 94.3, "cost": 0.0001, - "robustness": 55.8 + "robustness": 50.0 }, "medium": { - "accuracy": 62.9, + "accuracy": 63.6, "cost": 0.0001, - "robustness": 42.9 + "robustness": 25.0 }, "hard": { - "accuracy": 18.5, + "accuracy": 7.4, "cost": 0.0002, - "robustness": 14.3 + "robustness": 42.9 }, "all": { - "accuracy": 76.1, + "accuracy": 64.7, "cost": 0.0001, - "robustness": 47.9 + "robustness": 40.5 } }, "subcategories": { - "Engineering": { + "Economics": { "metrics": { "easy": { - "accuracy": 96.0, + "accuracy": 97.3, "cost": 0.0001, - "robustness": 60.0 + "robustness": 41.7 }, "medium": { - "accuracy": 80.2, + "accuracy": 73.6, "cost": 0.0002, - "robustness": 46.2 + "robustness": 33.3 }, "hard": { - "accuracy": 42.9, + "accuracy": 8.8, "cost": 0.0003, - "robustness": 100.0 + "robustness": 33.3 }, "all": { - "accuracy": 81.5, - "cost": 0.0002, - "robustness": 52.6 + "accuracy": 71.6, + "cost": 0.0001, + "robustness": 38.1 } } }, - "Medicine and health": { + "Law": { "metrics": { "easy": { - "accuracy": 92.9, - "cost": 0.0001, - "robustness": 52.9 + "accuracy": 87.7, + "cost": 0.0002, + "robustness": 33.3 }, "medium": { - "accuracy": 50.8, - "cost": 0.0001, - "robustness": 42.9 + "accuracy": 47.6, + "cost": 0.0002, + "robustness": 0.0 }, "hard": { - "accuracy": 9.7, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 6.7, + "cost": 0.0002, + "robustness": 50.0 }, "all": { - "accuracy": 73.2, - "cost": 0.0001, - "robustness": 44.7 + "accuracy": 56.3, + "cost": 0.0002, + "robustness": 22.2 } } }, - "Management and public relations": { + "Social problems": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0001, - "robustness": 75.0 + "accuracy": 93.6, + "cost": 0.0, + "robustness": 100.0 }, "medium": { - "accuracy": 63.6, + "accuracy": 33.3, "cost": 0.0, - "robustness": 0.0 + "robustness": 0 }, "hard": { "accuracy": 0.0, "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0, + "robustness": 100.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 66.7, + "cost": 0.0, "robustness": 0 }, + "medium": { + "accuracy": 44.4, + "cost": 0.0, + "robustness": 50.0 + }, + "hard": { + "accuracy": 5.9, + "cost": 0.0, + "robustness": 0.0 + }, "all": { - "accuracy": 90.4, - "cost": 0.0001, - "robustness": 60.0 + "accuracy": 21.8, + "cost": 0.0, + "robustness": 33.3 } } } } }, - "Arts & recreation": { + "Technology": { "metrics": { "easy": { - "accuracy": 93.0, - "cost": 0.0, - "robustness": 62.5 + "accuracy": 93.8, + "cost": 0.0001, + "robustness": 55.8 }, "medium": { - "accuracy": 48.6, + "accuracy": 62.9, "cost": 0.0001, - "robustness": 63.6 + "robustness": 42.9 }, "hard": { - "accuracy": 11.6, - "cost": 0.0001, - "robustness": 80.0 + "accuracy": 18.5, + "cost": 0.0002, + "robustness": 14.3 }, "all": { - "accuracy": 59.7, + "accuracy": 76.1, "cost": 0.0001, - "robustness": 66.7 + "robustness": 47.9 } }, "subcategories": { - "Sports, games and entertainment": { + "Engineering": { "metrics": { "easy": { - "accuracy": 90.7, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 96.0, + "cost": 0.0001, + "robustness": 60.0 }, "medium": { - "accuracy": 58.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 80.2, + "cost": 0.0002, + "robustness": 46.2 }, "hard": { - "accuracy": 12.1, - "cost": 0.0001, + "accuracy": 42.9, + "cost": 0.0003, "robustness": 100.0 }, "all": { - "accuracy": 63.1, - "cost": 0.0001, - "robustness": 87.5 + "accuracy": 81.5, + "cost": 0.0002, + "robustness": 52.6 } } }, - "Music": { + "Management and public relations": { "metrics": { "easy": { - "accuracy": 93.9, - "cost": 0.0, - "robustness": 80.0 + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 75.0 }, "medium": { - "accuracy": 31.9, - "cost": 0.0001, - "robustness": 40.0 + "accuracy": 63.6, + "cost": 0.0, + "robustness": 0.0 }, "hard": { - "accuracy": 24.0, - "cost": 0.0001, - "robustness": 0.0 + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0 }, "all": { - "accuracy": 56.7, + "accuracy": 90.4, "cost": 0.0001, - "robustness": 54.5 + "robustness": 60.0 } } }, - "Arts": { + "Medicine and health": { "metrics": { "easy": { - "accuracy": 96.6, + "accuracy": 92.9, "cost": 0.0001, - "robustness": 0.0 + "robustness": 52.9 }, "medium": { - "accuracy": 74.4, - "cost": 0.0, - "robustness": 50.0 + "accuracy": 50.8, + "cost": 0.0001, + "robustness": 42.9 }, "hard": { - "accuracy": 5.4, + "accuracy": 9.7, "cost": 0.0001, - "robustness": 100.0 + "robustness": 0.0 }, "all": { - "accuracy": 57.8, + "accuracy": 73.2, "cost": 0.0001, - "robustness": 60.0 + "robustness": 44.7 } } } } + } + } + }, + "sqwish-router": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0001, + "robustness": 100.0 }, - "Literature": { + "medium": { + "accuracy": 77.4, + "cost": 0.0002, + "robustness": 100.0 + }, + "hard": { + "accuracy": 34.1, + "cost": 0.0002, + "robustness": 100.0 + }, + "all": { + "accuracy": 76.4, + "cost": 0.0002, + "robustness": 100.0 + } + }, + "categories": { + "Arts & recreation": { "metrics": { "easy": { - "accuracy": 99.1, + "accuracy": 94.0, "cost": 0.0001, - "robustness": 33.3 + "robustness": 100.0 }, "medium": { - "accuracy": 67.2, - "cost": 0.0001, - "robustness": 50.0 + "accuracy": 65.1, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 27.4, - "cost": 0.0002, - "robustness": 45.8 + "accuracy": 14.3, + "cost": 0.0003, + "robustness": 100.0 }, "all": { - "accuracy": 42.9, - "cost": 0.0002, - "robustness": 44.8 + "accuracy": 66.7, + "cost": 0.0003, + "robustness": 100.0 } }, "subcategories": { - "Literature, rhetoric and criticism": { + "Arts": { "metrics": { "easy": { - "accuracy": 99.1, + "accuracy": 98.3, "cost": 0.0001, - "robustness": 33.3 + "robustness": 100.0 }, "medium": { - "accuracy": 67.2, + "accuracy": 87.2, "cost": 0.0001, - "robustness": 50.0 - }, - "hard": { - "accuracy": 27.4, - "cost": 0.0002, - "robustness": 45.8 - }, - "all": { - "accuracy": 42.9, - "cost": 0.0002, - "robustness": 44.8 - } - } - } - } - }, - "History": { - "metrics": { - "easy": { - "accuracy": 94.1, - "cost": 0.0001, - "robustness": 30.4 - }, - "medium": { - "accuracy": 62.1, - "cost": 0.0001, - "robustness": 33.3 - }, - "hard": { - "accuracy": 9.4, - "cost": 0.0001, - "robustness": 70.0 - }, - "all": { - "accuracy": 68.0, - "cost": 0.0001, - "robustness": 41.0 - } - }, - "subcategories": { - "Geography": { - "metrics": { - "easy": { - "accuracy": 97.2, - "cost": 0.0, - "robustness": 37.5 - }, - "medium": { - "accuracy": 48.3, - "cost": 0.0, - "robustness": 0.0 + "robustness": 100.0 }, "hard": { - "accuracy": 16.7, + "accuracy": 12.5, "cost": 0.0, - "robustness": 0 + "robustness": 100.0 }, "all": { - "accuracy": 81.3, - "cost": 0.0, - "robustness": 33.3 + "accuracy": 64.3, + "cost": 0.0001, + "robustness": 100.0 } } }, - "History": { + "Music": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0001, - "robustness": 27.3 + "accuracy": 93.9, + "cost": 0.0002, + "robustness": 100.0 }, "medium": { - "accuracy": 62.6, - "cost": 0.0001, - "robustness": 40.0 + "accuracy": 53.4, + "cost": 0.0007, + "robustness": 100.0 }, "hard": { - "accuracy": 8.8, - "cost": 0.0001, - "robustness": 70.0 + "accuracy": 12.0, + "cost": 0.0013, + "robustness": 100.0 }, "all": { - "accuracy": 61.0, - "cost": 0.0001, - "robustness": 46.2 + "accuracy": 65.8, + "cost": 0.0005, + "robustness": 100.0 } } }, - "Biography and genealogy": { + "Sports, games and entertainment": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0, - "robustness": 25.0 + "accuracy": 92.1, + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 100.0, - "cost": 0.0, - "robustness": 0 + "accuracy": 70.0, + "cost": 0.0002, + "robustness": 100.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0, - "robustness": 0 + "accuracy": 16.7, + "cost": 0.0002, + "robustness": 100.0 }, "all": { - "accuracy": 96.2, - "cost": 0.0, - "robustness": 25.0 + "accuracy": 68.6, + "cost": 0.0002, + "robustness": 100.0 } } } } - } - } - }, - "sqwish-router": { - "metrics": { - "easy": { - "accuracy": 96.7, - "cost": 0.0001, - "robustness": 100.0 - }, - "medium": { - "accuracy": 77.4, - "cost": 0.0002, - "robustness": 100.0 - }, - "hard": { - "accuracy": 34.1, - "cost": 0.0002, - "robustness": 100.0 }, - "all": { - "accuracy": 76.4, - "cost": 0.0002, - "robustness": 100.0 - } - }, - "categories": { "Computer science, information, and general works": { "metrics": { "easy": { @@ -13931,343 +14077,369 @@ } }, "subcategories": { - "Library and information sciences": { + "Computer science, knowledge, and systems": { "metrics": { "easy": { - "accuracy": 95.6, - "cost": 0.0, + "accuracy": 98.8, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 70.6, - "cost": 0.0001, + "accuracy": 87.0, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 11.1, - "cost": 0.0, + "accuracy": 38.8, + "cost": 0.0004, "robustness": 100.0 }, "all": { - "accuracy": 82.6, - "cost": 0.0, + "accuracy": 81.9, + "cost": 0.0002, "robustness": 100.0 } } }, - "Computer science, knowledge, and systems": { + "Library and information sciences": { "metrics": { "easy": { - "accuracy": 98.8, - "cost": 0.0001, + "accuracy": 95.6, + "cost": 0.0, "robustness": 100.0 }, "medium": { - "accuracy": 87.0, - "cost": 0.0002, + "accuracy": 70.6, + "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 38.8, - "cost": 0.0004, + "accuracy": 11.1, + "cost": 0.0, "robustness": 100.0 }, "all": { - "accuracy": 81.9, - "cost": 0.0002, + "accuracy": 82.6, + "cost": 0.0, "robustness": 100.0 } } } } }, - "Philosophy and psychology": { + "History": { "metrics": { "easy": { - "accuracy": 96.6, + "accuracy": 97.1, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 84.4, + "accuracy": 74.0, "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 38.3, + "accuracy": 14.1, "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 86.1, + "accuracy": 73.6, "cost": 0.0001, "robustness": 100.0 } }, "subcategories": { - "Ethics": { + "Biography and genealogy": { "metrics": { "easy": { - "accuracy": 95.1, + "accuracy": 97.6, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 78.8, + "accuracy": 77.8, "cost": 0.0001, - "robustness": 100.0 + "robustness": 0 }, "hard": { - "accuracy": 84.6, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 }, "all": { - "accuracy": 89.8, + "accuracy": 92.3, "cost": 0.0001, "robustness": 100.0 } } }, - "Philosophy": { + "Geography": { "metrics": { "easy": { - "accuracy": 97.3, + "accuracy": 98.2, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 77.5, + "accuracy": 62.1, "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 26.7, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 25.0, + "cost": 0.0, + "robustness": 0 }, "all": { - "accuracy": 64.8, + "accuracy": 85.3, "cost": 0.0001, "robustness": 100.0 } } }, - "Psychology": { + "History": { "metrics": { "easy": { - "accuracy": 96.7, + "accuracy": 96.4, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 89.6, + "accuracy": 76.3, "cost": 0.0001, - "robustness": 0 + "robustness": 100.0 }, "hard": { - "accuracy": 22.2, + "accuracy": 13.2, "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 87.7, + "accuracy": 68.1, "cost": 0.0001, "robustness": 100.0 } } - }, - "Philosophical logic": { + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0, + "robustness": 100.0 + }, + "medium": { + "accuracy": 74.9, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 59.2, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 72.7, + "cost": 0.0001, + "robustness": 100.0 + } + }, + "subcategories": { + "Language": { "metrics": { "easy": { - "accuracy": 100.0, - "cost": 0.0001, + "accuracy": 91.9, + "cost": 0.0, "robustness": 100.0 }, "medium": { - "accuracy": 97.4, - "cost": 0.0002, + "accuracy": 74.9, + "cost": 0.0, "robustness": 100.0 }, "hard": { - "accuracy": 80.0, - "cost": 0.0005, - "robustness": 0 + "accuracy": 59.2, + "cost": 0.0001, + "robustness": 100.0 }, "all": { - "accuracy": 98.1, - "cost": 0.0002, + "accuracy": 72.7, + "cost": 0.0001, "robustness": 100.0 } } } } }, - "Social Science": { + "Literature": { "metrics": { "easy": { - "accuracy": 96.6, + "accuracy": 100.0, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 74.3, - "cost": 0.0003, + "accuracy": 77.5, + "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 17.4, - "cost": 0.0003, + "accuracy": 35.9, + "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 71.7, - "cost": 0.0002, + "accuracy": 50.3, + "cost": 0.0001, "robustness": 100.0 } }, "subcategories": { - "Economics": { + "Literature, rhetoric and criticism": { "metrics": { "easy": { - "accuracy": 97.8, + "accuracy": 100.0, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 79.1, - "cost": 0.0003, + "accuracy": 77.5, + "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 18.8, - "cost": 0.0004, + "accuracy": 35.9, + "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 75.8, - "cost": 0.0002, + "accuracy": 50.3, + "cost": 0.0001, "robustness": 100.0 } } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 96.6, + "cost": 0.0001, + "robustness": 100.0 }, - "Law": { + "medium": { + "accuracy": 84.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 38.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 86.1, + "cost": 0.0001, + "robustness": 100.0 + } + }, + "subcategories": { + "Ethics": { "metrics": { "easy": { - "accuracy": 95.4, + "accuracy": 95.1, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 71.4, - "cost": 0.0003, + "accuracy": 78.8, + "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 26.7, - "cost": 0.0003, + "accuracy": 84.6, + "cost": 0.0, "robustness": 100.0 }, "all": { - "accuracy": 72.8, - "cost": 0.0002, + "accuracy": 89.8, + "cost": 0.0001, "robustness": 100.0 } } }, - "Social sciences, sociology, and anthropology": { + "Philosophical logic": { "metrics": { "easy": { "accuracy": 100.0, - "cost": 0.0, - "robustness": 0 + "cost": 0.0001, + "robustness": 100.0 }, "medium": { - "accuracy": 61.1, - "cost": 0.0, + "accuracy": 97.4, + "cost": 0.0002, "robustness": 100.0 }, "hard": { - "accuracy": 8.8, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 80.0, + "cost": 0.0005, + "robustness": 0 }, "all": { - "accuracy": 30.9, - "cost": 0.0, + "accuracy": 98.1, + "cost": 0.0002, "robustness": 100.0 } } }, - "Social problems": { + "Philosophy": { "metrics": { "easy": { - "accuracy": 93.6, + "accuracy": 97.3, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 33.3, - "cost": 0.0, - "robustness": 0 + "accuracy": 77.5, + "cost": 0.0001, + "robustness": 100.0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0, + "accuracy": 26.7, + "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 77.0, + "accuracy": 64.8, "cost": 0.0001, "robustness": 100.0 } } - } - } - }, - "Language": { - "metrics": { - "easy": { - "accuracy": 91.9, - "cost": 0.0, - "robustness": 100.0 - }, - "medium": { - "accuracy": 74.9, - "cost": 0.0, - "robustness": 100.0 - }, - "hard": { - "accuracy": 59.2, - "cost": 0.0001, - "robustness": 100.0 }, - "all": { - "accuracy": 72.7, - "cost": 0.0001, - "robustness": 100.0 - } - }, - "subcategories": { - "Language": { + "Psychology": { "metrics": { "easy": { - "accuracy": 91.9, - "cost": 0.0, + "accuracy": 96.7, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 74.9, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 89.6, + "cost": 0.0001, + "robustness": 0 }, "hard": { - "accuracy": 59.2, + "accuracy": 22.2, "cost": 0.0001, "robustness": 100.0 }, "all": { - "accuracy": 72.7, + "accuracy": 87.7, "cost": 0.0001, "robustness": 100.0 } @@ -14299,49 +14471,25 @@ } }, "subcategories": { - "Mathematics": { - "metrics": { - "easy": { - "accuracy": 98.9, - "cost": 0.0001, - "robustness": 100.0 - }, - "medium": { - "accuracy": 84.8, - "cost": 0.0004, - "robustness": 100.0 - }, - "hard": { - "accuracy": 47.2, - "cost": 0.001, - "robustness": 100.0 - }, - "all": { - "accuracy": 85.4, - "cost": 0.0004, - "robustness": 100.0 - } - } - }, - "Earth sciences and geology": { + "Animals (Zoology)": { "metrics": { "easy": { - "accuracy": 98.8, + "accuracy": 100.0, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 72.9, + "accuracy": 60.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 0 }, "hard": { - "accuracy": 15.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 0, + "cost": 0, + "robustness": 0 }, "all": { - "accuracy": 90.9, + "accuracy": 92.0, "cost": 0.0001, "robustness": 100.0 } @@ -14395,50 +14543,74 @@ } } }, - "Physics": { + "Earth sciences and geology": { "metrics": { "easy": { - "accuracy": 97.7, - "cost": 0.0002, + "accuracy": 98.8, + "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 88.1, - "cost": 0.0004, + "accuracy": 72.9, + "cost": 0.0001, "robustness": 100.0 }, "hard": { - "accuracy": 45.5, - "cost": 0.0003, - "robustness": 0 + "accuracy": 15.4, + "cost": 0.0001, + "robustness": 100.0 }, "all": { - "accuracy": 87.6, - "cost": 0.0003, + "accuracy": 90.9, + "cost": 0.0001, "robustness": 100.0 } } }, - "Animals (Zoology)": { + "Mathematics": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 98.9, "cost": 0.0001, "robustness": 100.0 }, "medium": { - "accuracy": 60.0, - "cost": 0.0001, - "robustness": 0 + "accuracy": 84.8, + "cost": 0.0004, + "robustness": 100.0 }, "hard": { - "accuracy": 0, - "cost": 0, + "accuracy": 47.2, + "cost": 0.001, + "robustness": 100.0 + }, + "all": { + "accuracy": 85.4, + "cost": 0.0004, + "robustness": 100.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 88.1, + "cost": 0.0004, + "robustness": 100.0 + }, + "hard": { + "accuracy": 45.5, + "cost": 0.0003, "robustness": 0 }, "all": { - "accuracy": 92.0, - "cost": 0.0001, + "accuracy": 87.6, + "cost": 0.0003, "robustness": 100.0 } } @@ -14469,6 +14641,128 @@ } } }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 96.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 74.3, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 17.4, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 71.7, + "cost": 0.0002, + "robustness": 100.0 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.8, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 79.1, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 18.8, + "cost": 0.0004, + "robustness": 100.0 + }, + "all": { + "accuracy": 75.8, + "cost": 0.0002, + "robustness": 100.0 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 95.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 71.4, + "cost": 0.0003, + "robustness": 100.0 + }, + "hard": { + "accuracy": 26.7, + "cost": 0.0003, + "robustness": 100.0 + }, + "all": { + "accuracy": 72.8, + "cost": 0.0002, + "robustness": 100.0 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 33.3, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0 + }, + "medium": { + "accuracy": 61.1, + "cost": 0.0, + "robustness": 100.0 + }, + "hard": { + "accuracy": 8.8, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 30.9, + "cost": 0.0, + "robustness": 100.0 + } + } + } + } + }, "Technology": { "metrics": { "easy": { @@ -14517,6 +14811,30 @@ } } }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0001, + "robustness": 100.0 + }, + "medium": { + "accuracy": 63.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 90.4, + "cost": 0.0001, + "robustness": 100.0 + } + } + }, "Medicine and health": { "metrics": { "easy": { @@ -14540,274 +14858,2800 @@ "robustness": 100.0 } } + } + } + } + } + }, + "agentforge": { + "metrics": { + "easy": { + "accuracy": 96.4, + "cost": 0.0001, + "robustness": 40.9 + }, + "medium": { + "accuracy": 73.0, + "cost": 0.0002, + "robustness": 42.5 + }, + "hard": { + "accuracy": 33.0, + "cost": 0.0002, + "robustness": 37.6 + }, + "all": { + "accuracy": 74.8, + "cost": 0.0001, + "robustness": 40.5 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0001, + "robustness": 62.5 }, - "Management and public relations": { + "medium": { + "accuracy": 55.7, + "cost": 0.0002, + "robustness": 27.3 + }, + "hard": { + "accuracy": 14.3, + "cost": 0.0002, + "robustness": 20.0 + }, + "all": { + "accuracy": 64.0, + "cost": 0.0001, + "robustness": 37.5 + } + }, + "subcategories": { + "Arts": { "metrics": { "easy": { - "accuracy": 98.3, + "accuracy": 96.6, "cost": 0.0001, - "robustness": 100.0 + "robustness": 0.0 + }, + "medium": { + "accuracy": 82.1, + "cost": 0.0, + "robustness": 50.0 + }, + "hard": { + "accuracy": 8.9, + "cost": 0.0, + "robustness": 50.0 + }, + "all": { + "accuracy": 61.0, + "cost": 0.0001, + "robustness": 40.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 98.0, + "cost": 0.0001, + "robustness": 80.0 + }, + "medium": { + "accuracy": 45.7, + "cost": 0.0003, + "robustness": 40.0 + }, + "hard": { + "accuracy": 24.0, + "cost": 0.0003, + "robustness": 0.0 + }, + "all": { + "accuracy": 65.0, + "cost": 0.0002, + "robustness": 54.5 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 57.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 15.2, + "cost": 0.0003, + "robustness": 0.0 + }, + "all": { + "accuracy": 64.7, + "cost": 0.0002, + "robustness": 12.5 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0001, + "robustness": 34.6 + }, + "medium": { + "accuracy": 82.0, + "cost": 0.0002, + "robustness": 33.3 + }, + "hard": { + "accuracy": 39.6, + "cost": 0.0004, + "robustness": 46.7 + }, + "all": { + "accuracy": 82.6, + "cost": 0.0001, + "robustness": 37.1 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 98.6, + "cost": 0.0001, + "robustness": 23.1 + }, + "medium": { + "accuracy": 85.4, + "cost": 0.0002, + "robustness": 29.4 + }, + "hard": { + "accuracy": 44.5, + "cost": 0.0004, + "robustness": 46.2 + }, + "all": { + "accuracy": 82.4, + "cost": 0.0002, + "robustness": 32.6 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0, + "robustness": 46.2 + }, + "medium": { + "accuracy": 67.1, + "cost": 0.0, + "robustness": 50.0 + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0, + "robustness": 50.0 + }, + "all": { + "accuracy": 83.1, + "cost": 0.0, + "robustness": 47.4 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.8, + "cost": 0.0001, + "robustness": 60.9 + }, + "medium": { + "accuracy": 71.8, + "cost": 0.0001, + "robustness": 66.7 + }, + "hard": { + "accuracy": 13.4, + "cost": 0.0001, + "robustness": 60.0 + }, + "all": { + "accuracy": 72.7, + "cost": 0.0001, + "robustness": 61.5 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 94.2, + "cost": 0.0001, + "robustness": 50.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0001, + "robustness": 87.5 + }, + "medium": { + "accuracy": 58.6, + "cost": 0.0001, + "robustness": 100.0 + }, + "hard": { + "accuracy": 33.3, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 85.3, + "cost": 0.0001, + "robustness": 88.9 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 95.5, + "cost": 0.0001, + "robustness": 45.5 + }, + "medium": { + "accuracy": 74.1, + "cost": 0.0001, + "robustness": 60.0 + }, + "hard": { + "accuracy": 11.8, + "cost": 0.0001, + "robustness": 60.0 + }, + "all": { + "accuracy": 66.7, + "cost": 0.0001, + "robustness": 53.8 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 90.9, + "cost": 0.0, + "robustness": 27.3 + }, + "medium": { + "accuracy": 65.3, + "cost": 0.0001, + "robustness": 77.8 + }, + "hard": { + "accuracy": 58.7, + "cost": 0.0001, + "robustness": 32.0 + }, + "all": { + "accuracy": 69.9, + "cost": 0.0, + "robustness": 40.0 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 90.9, + "cost": 0.0, + "robustness": 27.3 + }, + "medium": { + "accuracy": 65.3, + "cost": 0.0001, + "robustness": 77.8 + }, + "hard": { + "accuracy": 58.7, + "cost": 0.0001, + "robustness": 32.0 + }, + "all": { + "accuracy": 69.9, + "cost": 0.0, + "robustness": 40.0 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 76.8, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 33.9, + "cost": 0.0001, + "robustness": 25.0 + }, + "all": { + "accuracy": 48.7, + "cost": 0.0001, + "robustness": 24.1 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 76.8, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 33.9, + "cost": 0.0001, + "robustness": 25.0 + }, + "all": { + "accuracy": 48.7, + "cost": 0.0001, + "robustness": 24.1 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 95.1, + "cost": 0.0001, + "robustness": 33.3 + }, + "medium": { + "accuracy": 74.4, + "cost": 0.0001, + "robustness": 30.0 + }, + "hard": { + "accuracy": 34.6, + "cost": 0.0001, + "robustness": 28.6 + }, + "all": { + "accuracy": 81.9, + "cost": 0.0001, + "robustness": 31.8 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 91.4, + "cost": 0.0, + "robustness": 35.7 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0001, + "robustness": 40.0 + }, + "hard": { + "accuracy": 76.9, + "cost": 0.0, + "robustness": 50.0 + }, + "all": { + "accuracy": 81.3, + "cost": 0.0001, + "robustness": 38.1 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 98.5, + "cost": 0.0001, + "robustness": 25.0 + }, + "medium": { + "accuracy": 94.7, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 80.0, + "cost": 0.0005, + "robustness": 0 + }, + "all": { + "accuracy": 96.3, + "cost": 0.0002, + "robustness": 14.3 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 33.3 + }, + "medium": { + "accuracy": 75.0, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0001, + "robustness": 33.3 + }, + "all": { + "accuracy": 63.1, + "cost": 0.0001, + "robustness": 37.5 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0001, + "robustness": 33.3 + }, + "medium": { + "accuracy": 83.3, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 86.6, + "cost": 0.0001, + "robustness": 25.0 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.4, + "cost": 0.0001, + "robustness": 43.6 + }, + "medium": { + "accuracy": 82.6, + "cost": 0.0002, + "robustness": 38.1 + }, + "hard": { + "accuracy": 27.8, + "cost": 0.0005, + "robustness": 66.7 + }, + "all": { + "accuracy": 83.8, + "cost": 0.0002, + "robustness": 44.9 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 92.0, + "cost": 0.0001, + "robustness": 50.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 57.1 + }, + "medium": { + "accuracy": 76.5, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0003, + "robustness": 0 + }, + "all": { + "accuracy": 92.3, + "cost": 0.0001, + "robustness": 57.1 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 33.3 + }, + "medium": { + "accuracy": 94.1, + "cost": 0.0002, + "robustness": 50.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 84.4, + "cost": 0.0001, + "robustness": 40.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 98.4, + "cost": 0.0001, + "robustness": 62.5 + }, + "medium": { + "accuracy": 72.9, + "cost": 0.0001, + "robustness": 50.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0001, + "robustness": 100.0 + }, + "all": { + "accuracy": 90.6, + "cost": 0.0001, + "robustness": 60.0 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 36.4 + }, + "medium": { + "accuracy": 84.0, + "cost": 0.0002, + "robustness": 36.4 + }, + "hard": { + "accuracy": 44.9, + "cost": 0.001, + "robustness": 60.0 + }, + "all": { + "accuracy": 84.2, + "cost": 0.0003, + "robustness": 40.7 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 90.5, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 36.4, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 88.7, + "cost": 0.0002, + "robustness": 0.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0, + "robustness": 40.0 + }, + "medium": { + "accuracy": 72.0, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 6.8, + "cost": 0.0, + "robustness": 66.7 + }, + "all": { + "accuracy": 44.7, + "cost": 0.0, + "robustness": 50.0 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0001, + "robustness": 27.8 + }, + "medium": { + "accuracy": 68.0, + "cost": 0.0001, + "robustness": 41.7 + }, + "hard": { + "accuracy": 12.8, + "cost": 0.0002, + "robustness": 42.9 + }, + "all": { + "accuracy": 68.0, + "cost": 0.0001, + "robustness": 35.1 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0001, + "robustness": 16.7 + }, + "medium": { + "accuracy": 74.2, + "cost": 0.0001, + "robustness": 33.3 + }, + "hard": { + "accuracy": 13.8, + "cost": 0.0002, + "robustness": 33.3 + }, + "all": { + "accuracy": 72.8, + "cost": 0.0001, + "robustness": 23.8 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 90.8, + "cost": 0.0001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 58.7, + "cost": 0.0002, + "robustness": 50.0 + }, + "hard": { + "accuracy": 16.7, + "cost": 0.0002, + "robustness": 50.0 + }, + "all": { + "accuracy": 63.9, + "cost": 0.0001, + "robustness": 55.6 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 95.7, + "cost": 0.0, + "robustness": 33.3 + }, + "medium": { + "accuracy": 11.1, + "cost": 0.0, + "robustness": 0 + }, + "hard": { + "accuracy": 20.0, + "cost": 0.0, + "robustness": 0.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0, + "robustness": 25.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0, + "robustness": 0 + }, + "medium": { + "accuracy": 72.2, + "cost": 0.0, + "robustness": 50.0 + }, + "hard": { + "accuracy": 5.9, + "cost": 0.0, + "robustness": 100.0 + }, + "all": { + "accuracy": 32.7, + "cost": 0.0, + "robustness": 66.7 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0001, + "robustness": 44.2 + }, + "medium": { + "accuracy": 69.0, + "cost": 0.0002, + "robustness": 47.6 + }, + "hard": { + "accuracy": 28.0, + "cost": 0.0003, + "robustness": 28.6 + }, + "all": { + "accuracy": 79.7, + "cost": 0.0001, + "robustness": 43.7 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 98.4, + "cost": 0.0002, + "robustness": 60.0 + }, + "medium": { + "accuracy": 85.0, + "cost": 0.0004, + "robustness": 46.2 + }, + "hard": { + "accuracy": 54.8, + "cost": 0.0007, + "robustness": 100.0 + }, + "all": { + "accuracy": 86.3, + "cost": 0.0003, + "robustness": 52.6 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0, + "robustness": 50.0 + }, + "medium": { + "accuracy": 36.4, + "cost": 0.0001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0, + "robustness": 0 + }, + "all": { + "accuracy": 86.3, + "cost": 0.0, + "robustness": 40.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 94.0, + "cost": 0.0001, + "robustness": 41.2 + }, + "medium": { + "accuracy": 59.5, + "cost": 0.0001, + "robustness": 57.1 + }, + "hard": { + "accuracy": 18.6, + "cost": 0.0001, + "robustness": 16.7 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 40.4 + } + } + } + } + } + } + }, + "nadir": { + "metrics": { + "easy": { + "accuracy": 96.3, + "cost": 0.0003, + "robustness": 28.8 + }, + "medium": { + "accuracy": 73.6, + "cost": 0.0003, + "robustness": 23.0 + }, + "hard": { + "accuracy": 34.1, + "cost": 0.0005, + "robustness": 22.0 + }, + "all": { + "accuracy": 75.2, + "cost": 0.0004, + "robustness": 25.5 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 93.6, + "cost": 0.0003, + "robustness": 25.0 + }, + "medium": { + "accuracy": 60.4, + "cost": 0.0003, + "robustness": 18.2 + }, + "hard": { + "accuracy": 12.2, + "cost": 0.0005, + "robustness": 40.0 + }, + "all": { + "accuracy": 64.4, + "cost": 0.0003, + "robustness": 25.0 + } + }, + "subcategories": { + "Arts": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0004, + "robustness": 0.0 + }, + "medium": { + "accuracy": 82.1, + "cost": 0.0004, + "robustness": 0.0 + }, + "hard": { + "accuracy": 8.9, + "cost": 0.0006, + "robustness": 50.0 + }, + "all": { + "accuracy": 62.3, + "cost": 0.0005, + "robustness": 20.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 94.9, + "cost": 0.0003, + "robustness": 20.0 + }, + "medium": { + "accuracy": 50.9, + "cost": 0.0003, + "robustness": 40.0 + }, + "hard": { + "accuracy": 16.0, + "cost": 0.0004, + "robustness": 0.0 + }, + "all": { + "accuracy": 65.4, + "cost": 0.0003, + "robustness": 27.3 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 90.0, + "cost": 0.0003, + "robustness": 50.0 + }, + "medium": { + "accuracy": 63.0, + "cost": 0.0003, + "robustness": 0.0 + }, + "hard": { + "accuracy": 13.6, + "cost": 0.0004, + "robustness": 50.0 + }, + "all": { + "accuracy": 64.7, + "cost": 0.0003, + "robustness": 25.0 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0004, + "robustness": 23.1 + }, + "medium": { + "accuracy": 81.6, + "cost": 0.0003, + "robustness": 14.3 + }, + "hard": { + "accuracy": 51.8, + "cost": 0.0004, + "robustness": 6.7 + }, + "all": { + "accuracy": 84.4, + "cost": 0.0004, + "robustness": 16.1 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 97.8, + "cost": 0.0004, + "robustness": 46.2 + }, + "medium": { + "accuracy": 86.2, + "cost": 0.0002, + "robustness": 17.6 + }, + "hard": { + "accuracy": 59.8, + "cost": 0.0004, + "robustness": 0.0 + }, + "all": { + "accuracy": 85.6, + "cost": 0.0003, + "robustness": 20.9 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 97.5, + "cost": 0.0006, + "robustness": 0.0 + }, + "medium": { + "accuracy": 61.2, + "cost": 0.0004, + "robustness": 0.0 + }, + "hard": { + "accuracy": 5.6, + "cost": 0.0005, + "robustness": 50.0 + }, + "all": { + "accuracy": 81.3, + "cost": 0.0005, + "robustness": 5.3 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.9, + "cost": 0.0003, + "robustness": 34.8 + }, + "medium": { + "accuracy": 71.2, + "cost": 0.0004, + "robustness": 33.3 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0006, + "robustness": 30.0 + }, + "all": { + "accuracy": 73.6, + "cost": 0.0004, + "robustness": 33.3 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 25.0 + }, + "medium": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0017, + "robustness": 0 + }, + "all": { + "accuracy": 98.1, + "cost": 0.0003, + "robustness": 25.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0005, + "robustness": 25.0 + }, + "medium": { + "accuracy": 69.0, + "cost": 0.0004, + "robustness": 0.0 + }, + "hard": { + "accuracy": 41.7, + "cost": 0.0004, + "robustness": 0 + }, + "all": { + "accuracy": 88.7, + "cost": 0.0005, + "robustness": 22.2 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 96.9, + "cost": 0.0002, + "robustness": 45.5 + }, + "medium": { + "accuracy": 69.8, + "cost": 0.0004, + "robustness": 40.0 + }, + "hard": { + "accuracy": 13.2, + "cost": 0.0006, + "robustness": 30.0 + }, + "all": { + "accuracy": 66.5, + "cost": 0.0004, + "robustness": 38.5 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0005, + "robustness": 27.3 + }, + "medium": { + "accuracy": 64.7, + "cost": 0.0008, + "robustness": 44.4 + }, + "hard": { + "accuracy": 54.9, + "cost": 0.0005, + "robustness": 12.0 + }, + "all": { + "accuracy": 68.3, + "cost": 0.0006, + "robustness": 22.2 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 91.9, + "cost": 0.0005, + "robustness": 27.3 + }, + "medium": { + "accuracy": 64.7, + "cost": 0.0008, + "robustness": 44.4 + }, + "hard": { + "accuracy": 54.9, + "cost": 0.0005, + "robustness": 12.0 + }, + "all": { + "accuracy": 68.3, + "cost": 0.0006, + "robustness": 22.2 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 0.0 + }, + "medium": { + "accuracy": 66.8, + "cost": 0.0003, + "robustness": 0.0 + }, + "hard": { + "accuracy": 33.4, + "cost": 0.0004, + "robustness": 29.2 + }, + "all": { + "accuracy": 47.4, + "cost": 0.0004, + "robustness": 24.1 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0003, + "robustness": 0.0 + }, + "medium": { + "accuracy": 66.8, + "cost": 0.0003, + "robustness": 0.0 + }, + "hard": { + "accuracy": 33.4, + "cost": 0.0004, + "robustness": 29.2 + }, + "all": { + "accuracy": 47.4, + "cost": 0.0004, + "robustness": 24.1 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 95.1, + "cost": 0.0003, + "robustness": 40.7 + }, + "medium": { + "accuracy": 78.7, + "cost": 0.0003, + "robustness": 30.0 + }, + "hard": { + "accuracy": 32.1, + "cost": 0.0004, + "robustness": 42.9 + }, + "all": { + "accuracy": 82.9, + "cost": 0.0003, + "robustness": 38.6 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 92.4, + "cost": 0.0005, + "robustness": 35.7 + }, + "medium": { + "accuracy": 70.6, + "cost": 0.0004, + "robustness": 20.0 + }, + "hard": { + "accuracy": 61.5, + "cost": 0.0003, + "robustness": 50.0 + }, + "all": { + "accuracy": 84.5, + "cost": 0.0005, + "robustness": 33.3 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0001, + "robustness": 50.0 + }, + "medium": { + "accuracy": 84.2, + "cost": 0.0002, + "robustness": 66.7 + }, + "hard": { + "accuracy": 60.0, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 92.6, + "cost": 0.0001, + "robustness": 57.1 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 94.6, + "cost": 0.0003, + "robustness": 0.0 + }, + "medium": { + "accuracy": 80.0, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 24.4, + "cost": 0.0003, + "robustness": 33.3 + }, + "all": { + "accuracy": 63.9, + "cost": 0.0003, + "robustness": 12.5 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0002, + "robustness": 66.7 + }, + "medium": { + "accuracy": 87.5, + "cost": 0.0003, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0005, + "robustness": 50.0 + }, + "all": { + "accuracy": 87.2, + "cost": 0.0003, + "robustness": 62.5 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.3, + "cost": 0.0002, + "robustness": 35.9 + }, + "medium": { + "accuracy": 81.0, + "cost": 0.0003, + "robustness": 23.8 + }, + "hard": { + "accuracy": 30.9, + "cost": 0.0005, + "robustness": 22.2 + }, + "all": { + "accuracy": 83.6, + "cost": 0.0003, + "robustness": 30.4 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0006, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 92.0, + "cost": 0.0005, + "robustness": 0.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0002, + "robustness": 71.4 + }, + "medium": { + "accuracy": 70.6, + "cost": 0.0002, + "robustness": 0 + }, + "hard": { + "accuracy": 33.3, + "cost": 0.0005, + "robustness": 0 + }, + "all": { + "accuracy": 91.6, + "cost": 0.0003, + "robustness": 71.4 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0002, + "robustness": 100.0 + }, + "medium": { + "accuracy": 94.1, + "cost": 0.0002, + "robustness": 0.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 84.4, + "cost": 0.0002, + "robustness": 60.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0003, + "robustness": 12.5 + }, + "medium": { + "accuracy": 69.5, + "cost": 0.0003, + "robustness": 0.0 + }, + "hard": { + "accuracy": 7.7, + "cost": 0.0003, + "robustness": 0.0 + }, + "all": { + "accuracy": 89.1, + "cost": 0.0003, + "robustness": 6.7 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 98.9, + "cost": 0.0001, + "robustness": 36.4 + }, + "medium": { + "accuracy": 84.4, + "cost": 0.0003, + "robustness": 36.4 + }, + "hard": { + "accuracy": 50.6, + "cost": 0.0004, + "robustness": 40.0 + }, + "all": { + "accuracy": 85.7, + "cost": 0.0002, + "robustness": 37.0 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0001, + "robustness": 0.0 + }, + "medium": { + "accuracy": 83.3, + "cost": 0.0002, + "robustness": 50.0 + }, + "hard": { + "accuracy": 36.4, + "cost": 0.0002, + "robustness": 0 + }, + "all": { + "accuracy": 84.5, + "cost": 0.0001, + "robustness": 20.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0003, + "robustness": 20.0 + }, + "medium": { + "accuracy": 64.0, + "cost": 0.0014, + "robustness": 0 + }, + "hard": { + "accuracy": 8.5, + "cost": 0.0009, + "robustness": 0.0 + }, + "all": { + "accuracy": 43.9, + "cost": 0.0008, + "robustness": 12.5 + } + } + } + } + }, + "Social Science": { + "metrics": { + "easy": { + "accuracy": 95.6, + "cost": 0.0002, + "robustness": 38.9 + }, + "medium": { + "accuracy": 68.8, + "cost": 0.0003, + "robustness": 25.0 + }, + "hard": { + "accuracy": 11.4, + "cost": 0.0008, + "robustness": 0.0 + }, + "all": { + "accuracy": 68.0, + "cost": 0.0004, + "robustness": 27.0 + } + }, + "subcategories": { + "Economics": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0002, + "robustness": 33.3 + }, + "medium": { + "accuracy": 78.5, + "cost": 0.0004, + "robustness": 0.0 + }, + "hard": { + "accuracy": 11.2, + "cost": 0.0011, + "robustness": 0.0 + }, + "all": { + "accuracy": 73.7, + "cost": 0.0004, + "robustness": 19.0 + } + } + }, + "Law": { + "metrics": { + "easy": { + "accuracy": 92.3, + "cost": 0.0002, + "robustness": 33.3 + }, + "medium": { + "accuracy": 55.6, + "cost": 0.0002, + "robustness": 50.0 + }, + "hard": { + "accuracy": 13.3, + "cost": 0.0002, + "robustness": 0.0 + }, + "all": { + "accuracy": 62.7, + "cost": 0.0002, + "robustness": 33.3 + } + } + }, + "Social problems": { + "metrics": { + "easy": { + "accuracy": 97.9, + "cost": 0.0001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 11.1, + "cost": 0.0001, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0.0 + }, + "all": { + "accuracy": 77.0, + "cost": 0.0001, + "robustness": 50.0 + } + } + }, + "Social sciences, sociology, and anthropology": { + "metrics": { + "easy": { + "accuracy": 66.7, + "cost": 0.0, + "robustness": 0 + }, + "medium": { + "accuracy": 55.6, + "cost": 0.0006, + "robustness": 50.0 + }, + "hard": { + "accuracy": 11.8, + "cost": 0.0006, + "robustness": 0.0 + }, + "all": { + "accuracy": 29.1, + "cost": 0.0006, + "robustness": 33.3 + } + } + } + } + }, + "Technology": { + "metrics": { + "easy": { + "accuracy": 95.1, + "cost": 0.0003, + "robustness": 14.0 + }, + "medium": { + "accuracy": 71.4, + "cost": 0.0003, + "robustness": 19.0 + }, + "hard": { + "accuracy": 30.6, + "cost": 0.0005, + "robustness": 42.9 + }, + "all": { + "accuracy": 80.8, + "cost": 0.0003, + "robustness": 18.3 + } + }, + "subcategories": { + "Engineering": { + "metrics": { + "easy": { + "accuracy": 97.6, + "cost": 0.0002, + "robustness": 20.0 + }, + "medium": { + "accuracy": 89.8, + "cost": 0.0002, + "robustness": 15.4 + }, + "hard": { + "accuracy": 59.5, + "cost": 0.0004, + "robustness": 100.0 + }, + "all": { + "accuracy": 89.0, + "cost": 0.0002, + "robustness": 21.1 + } + } + }, + "Management and public relations": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0006, + "robustness": 25.0 + }, + "medium": { + "accuracy": 54.5, + "cost": 0.0006, + "robustness": 0.0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0008, + "robustness": 0 + }, + "all": { + "accuracy": 90.4, + "cost": 0.0006, + "robustness": 20.0 + } + } + }, + "Medicine and health": { + "metrics": { + "easy": { + "accuracy": 94.2, + "cost": 0.0002, + "robustness": 11.8 + }, + "medium": { + "accuracy": 59.5, + "cost": 0.0003, + "robustness": 28.6 + }, + "hard": { + "accuracy": 20.4, + "cost": 0.0005, + "robustness": 33.3 + }, + "all": { + "accuracy": 77.3, + "cost": 0.0003, + "robustness": 17.0 + } + } + } + } + } + } + }, + "orcarouter": { + "metrics": { + "easy": { + "accuracy": 97.1, + "cost": 0.0007, + "robustness": 22.7 + }, + "medium": { + "accuracy": 74.9, + "cost": 0.0013, + "robustness": 15.9 + }, + "hard": { + "accuracy": 33.6, + "cost": 0.0019, + "robustness": 29.4 + }, + "all": { + "accuracy": 75.8, + "cost": 0.0011, + "robustness": 22.6 + } + }, + "categories": { + "Arts & recreation": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0008, + "robustness": 0.0 + }, + "medium": { + "accuracy": 62.0, + "cost": 0.0021, + "robustness": 45.5 + }, + "hard": { + "accuracy": 13.6, + "cost": 0.0027, + "robustness": 40.0 + }, + "all": { + "accuracy": 66.9, + "cost": 0.0017, + "robustness": 29.2 + } + }, + "subcategories": { + "Arts": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 71.8, + "cost": 0.0014, + "robustness": 0.0 + }, + "hard": { + "accuracy": 10.7, + "cost": 0.0024, + "robustness": 50.0 + }, + "all": { + "accuracy": 60.4, + "cost": 0.0014, + "robustness": 20.0 + } + } + }, + "Music": { + "metrics": { + "easy": { + "accuracy": 97.0, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 53.4, + "cost": 0.0021, + "robustness": 80.0 + }, + "hard": { + "accuracy": 20.0, + "cost": 0.0023, + "robustness": 0.0 + }, + "all": { + "accuracy": 67.9, + "cost": 0.0015, + "robustness": 36.4 + } + } + }, + "Sports, games and entertainment": { + "metrics": { + "easy": { + "accuracy": 96.4, + "cost": 0.0011, + "robustness": 0.0 + }, + "medium": { + "accuracy": 68.0, + "cost": 0.0024, + "robustness": 25.0 + }, + "hard": { + "accuracy": 13.6, + "cost": 0.0031, + "robustness": 50.0 + }, + "all": { + "accuracy": 69.3, + "cost": 0.002, + "robustness": 25.0 + } + } + } + } + }, + "Computer science, information, and general works": { + "metrics": { + "easy": { + "accuracy": 98.7, + "cost": 0.0006, + "robustness": 23.1 + }, + "medium": { + "accuracy": 84.2, + "cost": 0.0015, + "robustness": 9.5 + }, + "hard": { + "accuracy": 44.1, + "cost": 0.004, + "robustness": 60.0 + }, + "all": { + "accuracy": 84.4, + "cost": 0.0015, + "robustness": 27.4 + } + }, + "subcategories": { + "Computer science, knowledge, and systems": { + "metrics": { + "easy": { + "accuracy": 99.0, + "cost": 0.0007, + "robustness": 23.1 + }, + "medium": { + "accuracy": 89.4, + "cost": 0.0016, + "robustness": 11.8 + }, + "hard": { + "accuracy": 49.8, + "cost": 0.0046, + "robustness": 61.5 + }, + "all": { + "accuracy": 85.2, + "cost": 0.0018, + "robustness": 30.2 + } + } + }, + "Library and information sciences": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0005, + "robustness": 23.1 + }, + "medium": { + "accuracy": 61.2, + "cost": 0.001, + "robustness": 0.0 + }, + "hard": { + "accuracy": 11.1, + "cost": 0.0004, + "robustness": 50.0 + }, + "all": { + "accuracy": 82.3, + "cost": 0.0006, + "robustness": 21.1 + } + } + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.1, + "cost": 0.0007, + "robustness": 30.4 + }, + "medium": { + "accuracy": 74.0, + "cost": 0.0012, + "robustness": 0.0 + }, + "hard": { + "accuracy": 16.8, + "cost": 0.0015, + "robustness": 10.0 + }, + "all": { + "accuracy": 74.1, + "cost": 0.001, + "robustness": 20.5 + } + }, + "subcategories": { + "Biography and genealogy": { + "metrics": { + "easy": { + "accuracy": 90.5, + "cost": 0.0006, + "robustness": 25.0 + }, + "medium": { + "accuracy": 77.8, + "cost": 0.0003, + "robustness": 0 + }, + "hard": { + "accuracy": 0.0, + "cost": 0.0001, + "robustness": 0 + }, + "all": { + "accuracy": 86.5, + "cost": 0.0005, + "robustness": 25.0 + } + } + }, + "Geography": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0005, + "robustness": 25.0 + }, + "medium": { + "accuracy": 72.4, + "cost": 0.0008, + "robustness": 0.0 + }, + "hard": { + "accuracy": 25.0, + "cost": 0.0013, + "robustness": 0 + }, + "all": { + "accuracy": 88.0, + "cost": 0.0006, + "robustness": 22.2 + } + } + }, + "History": { + "metrics": { + "easy": { + "accuracy": 97.3, + "cost": 0.0008, + "robustness": 36.4 + }, + "medium": { + "accuracy": 74.1, + "cost": 0.0014, + "robustness": 0.0 + }, + "hard": { + "accuracy": 16.2, + "cost": 0.0015, + "robustness": 10.0 + }, + "all": { + "accuracy": 68.7, + "cost": 0.0012, + "robustness": 19.2 + } + } + } + } + }, + "Language": { + "metrics": { + "easy": { + "accuracy": 93.8, + "cost": 0.0006, + "robustness": 18.2 + }, + "medium": { + "accuracy": 58.7, + "cost": 0.0007, + "robustness": 11.1 + }, + "hard": { + "accuracy": 61.6, + "cost": 0.0012, + "robustness": 16.0 + }, + "all": { + "accuracy": 70.5, + "cost": 0.0009, + "robustness": 15.6 + } + }, + "subcategories": { + "Language": { + "metrics": { + "easy": { + "accuracy": 93.8, + "cost": 0.0006, + "robustness": 18.2 + }, + "medium": { + "accuracy": 58.7, + "cost": 0.0007, + "robustness": 11.1 + }, + "hard": { + "accuracy": 61.6, + "cost": 0.0012, + "robustness": 16.0 + }, + "all": { + "accuracy": 70.5, + "cost": 0.0009, + "robustness": 15.6 + } + } + } + } + }, + "Literature": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0006, + "robustness": 33.3 + }, + "medium": { + "accuracy": 76.7, + "cost": 0.0013, + "robustness": 0.0 + }, + "hard": { + "accuracy": 32.0, + "cost": 0.0015, + "robustness": 16.7 + }, + "all": { + "accuracy": 47.1, + "cost": 0.0013, + "robustness": 17.2 + } + }, + "subcategories": { + "Literature, rhetoric and criticism": { + "metrics": { + "easy": { + "accuracy": 99.1, + "cost": 0.0006, + "robustness": 33.3 + }, + "medium": { + "accuracy": 76.7, + "cost": 0.0013, + "robustness": 0.0 + }, + "hard": { + "accuracy": 32.0, + "cost": 0.0015, + "robustness": 16.7 + }, + "all": { + "accuracy": 47.1, + "cost": 0.0013, + "robustness": 17.2 + } + } + } + } + }, + "Philosophy and psychology": { + "metrics": { + "easy": { + "accuracy": 96.1, + "cost": 0.0007, + "robustness": 33.3 + }, + "medium": { + "accuracy": 66.4, + "cost": 0.001, + "robustness": 20.0 + }, + "hard": { + "accuracy": 27.2, + "cost": 0.0013, + "robustness": 42.9 + }, + "all": { + "accuracy": 79.1, + "cost": 0.0009, + "robustness": 31.8 + } + }, + "subcategories": { + "Ethics": { + "metrics": { + "easy": { + "accuracy": 94.1, + "cost": 0.0006, + "robustness": 28.6 + }, + "medium": { + "accuracy": 40.0, + "cost": 0.0008, + "robustness": 20.0 + }, + "hard": { + "accuracy": 15.4, + "cost": 0.0006, + "robustness": 0.0 + }, + "all": { + "accuracy": 74.2, + "cost": 0.0006, + "robustness": 23.8 + } + } + }, + "Philosophical logic": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0009, + "robustness": 0.0 + }, + "medium": { + "accuracy": 89.5, + "cost": 0.0012, + "robustness": 33.3 + }, + "hard": { + "accuracy": 60.0, + "cost": 0.0027, + "robustness": 0 + }, + "all": { + "accuracy": 94.4, + "cost": 0.0011, + "robustness": 14.3 + } + } + }, + "Philosophy": { + "metrics": { + "easy": { + "accuracy": 89.2, + "cost": 0.001, + "robustness": 66.7 + }, + "medium": { + "accuracy": 77.5, + "cost": 0.0011, + "robustness": 0.0 + }, + "hard": { + "accuracy": 28.9, + "cost": 0.0016, + "robustness": 66.7 + }, + "all": { + "accuracy": 63.1, + "cost": 0.0013, + "robustness": 50.0 + } + } + }, + "Psychology": { + "metrics": { + "easy": { + "accuracy": 99.2, + "cost": 0.0007, + "robustness": 50.0 + }, + "medium": { + "accuracy": 85.4, + "cost": 0.001, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0007, + "robustness": 50.0 + }, + "all": { + "accuracy": 88.2, + "cost": 0.0008, + "robustness": 50.0 + } + } + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 98.2, + "cost": 0.0006, + "robustness": 10.3 + }, + "medium": { + "accuracy": 81.7, + "cost": 0.0012, + "robustness": 14.3 + }, + "hard": { + "accuracy": 28.4, + "cost": 0.0021, + "robustness": 44.4 + }, + "all": { + "accuracy": 83.4, + "cost": 0.001, + "robustness": 15.9 + } + }, + "subcategories": { + "Animals (Zoology)": { + "metrics": { + "easy": { + "accuracy": 95.0, + "cost": 0.0003, + "robustness": 0.0 + }, + "medium": { + "accuracy": 60.0, + "cost": 0.0011, + "robustness": 0 + }, + "hard": { + "accuracy": 0, + "cost": 0, + "robustness": 0 + }, + "all": { + "accuracy": 88.0, + "cost": 0.0004, + "robustness": 0.0 + } + } + }, + "Biology": { + "metrics": { + "easy": { + "accuracy": 97.4, + "cost": 0.0006, + "robustness": 14.3 + }, + "medium": { + "accuracy": 82.4, + "cost": 0.0007, + "robustness": 0 + }, + "hard": { + "accuracy": 22.2, + "cost": 0.0011, + "robustness": 0 + }, + "all": { + "accuracy": 90.9, + "cost": 0.0006, + "robustness": 14.3 + } + } + }, + "Chemistry": { + "metrics": { + "easy": { + "accuracy": 100.0, + "cost": 0.0008, + "robustness": 33.3 + }, + "medium": { + "accuracy": 94.1, + "cost": 0.0015, + "robustness": 0.0 + }, + "hard": { + "accuracy": 30.8, + "cost": 0.001, + "robustness": 0 + }, + "all": { + "accuracy": 87.8, + "cost": 0.0011, + "robustness": 20.0 + } + } + }, + "Earth sciences and geology": { + "metrics": { + "easy": { + "accuracy": 98.1, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 74.6, + "cost": 0.0009, + "robustness": 0.0 + }, + "hard": { + "accuracy": 23.1, + "cost": 0.0005, + "robustness": 100.0 + }, + "all": { + "accuracy": 90.9, + "cost": 0.0006, + "robustness": 6.7 + } + } + }, + "Mathematics": { + "metrics": { + "easy": { + "accuracy": 98.9, + "cost": 0.0007, + "robustness": 18.2 + }, + "medium": { + "accuracy": 84.0, + "cost": 0.0012, + "robustness": 27.3 + }, + "hard": { + "accuracy": 38.2, + "cost": 0.003, + "robustness": 60.0 + }, + "all": { + "accuracy": 83.7, + "cost": 0.0013, + "robustness": 29.6 + } + } + }, + "Physics": { + "metrics": { + "easy": { + "accuracy": 97.7, + "cost": 0.0005, + "robustness": 0.0 + }, + "medium": { + "accuracy": 76.2, + "cost": 0.0015, + "robustness": 0.0 + }, + "hard": { + "accuracy": 63.6, + "cost": 0.0021, + "robustness": 0 + }, + "all": { + "accuracy": 84.5, + "cost": 0.0011, + "robustness": 0.0 + } + } + }, + "Science": { + "metrics": { + "easy": { + "accuracy": 96.7, + "cost": 0.0005, + "robustness": 0.0 }, "medium": { - "accuracy": 63.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 72.0, + "cost": 0.0015, + "robustness": 0 }, "hard": { - "accuracy": 0.0, - "cost": 0.0001, - "robustness": 0 + "accuracy": 8.5, + "cost": 0.0015, + "robustness": 0.0 }, "all": { - "accuracy": 90.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 45.6, + "cost": 0.0012, + "robustness": 0.0 } } } } }, - "Arts & recreation": { + "Social Science": { "metrics": { "easy": { - "accuracy": 94.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 95.0, + "cost": 0.0008, + "robustness": 27.8 }, "medium": { - "accuracy": 65.1, - "cost": 0.0004, - "robustness": 100.0 + "accuracy": 77.1, + "cost": 0.0015, + "robustness": 8.3 }, "hard": { - "accuracy": 14.3, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 18.1, + "cost": 0.0016, + "robustness": 42.9 }, "all": { - "accuracy": 66.7, - "cost": 0.0003, - "robustness": 100.0 + "accuracy": 72.1, + "cost": 0.0012, + "robustness": 24.3 } }, "subcategories": { - "Sports, games and entertainment": { + "Economics": { "metrics": { "easy": { - "accuracy": 92.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.2, + "cost": 0.0008, + "robustness": 25.0 }, "medium": { - "accuracy": 70.0, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 79.8, + "cost": 0.0013, + "robustness": 16.7 }, "hard": { - "accuracy": 16.7, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 18.8, + "cost": 0.0019, + "robustness": 0.0 }, "all": { - "accuracy": 68.6, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 75.4, + "cost": 0.0012, + "robustness": 19.0 } } }, - "Music": { + "Law": { "metrics": { "easy": { - "accuracy": 93.9, - "cost": 0.0002, - "robustness": 100.0 + "accuracy": 92.3, + "cost": 0.0013, + "robustness": 33.3 }, "medium": { - "accuracy": 53.4, - "cost": 0.0007, - "robustness": 100.0 + "accuracy": 77.8, + "cost": 0.002, + "robustness": 0.0 }, "hard": { - "accuracy": 12.0, - "cost": 0.0013, + "accuracy": 20.0, + "cost": 0.001, "robustness": 100.0 }, "all": { - "accuracy": 65.8, - "cost": 0.0005, - "robustness": 100.0 + "accuracy": 72.8, + "cost": 0.0015, + "robustness": 33.3 } } }, - "Arts": { + "Social problems": { "metrics": { "easy": { - "accuracy": 98.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 95.7, + "cost": 0.0003, + "robustness": 33.3 }, "medium": { - "accuracy": 87.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 44.4, + "cost": 0.0007, + "robustness": 0 }, "hard": { - "accuracy": 12.5, - "cost": 0.0, - "robustness": 100.0 + "accuracy": 0.0, + "cost": 0.0002, + "robustness": 0.0 }, "all": { - "accuracy": 64.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 80.3, + "cost": 0.0004, + "robustness": 25.0 } } - } - } - }, - "Literature": { - "metrics": { - "easy": { - "accuracy": 100.0, - "cost": 0.0001, - "robustness": 100.0 - }, - "medium": { - "accuracy": 77.5, - "cost": 0.0001, - "robustness": 100.0 - }, - "hard": { - "accuracy": 35.9, - "cost": 0.0001, - "robustness": 100.0 }, - "all": { - "accuracy": 50.3, - "cost": 0.0001, - "robustness": 100.0 - } - }, - "subcategories": { - "Literature, rhetoric and criticism": { + "Social sciences, sociology, and anthropology": { "metrics": { "easy": { - "accuracy": 100.0, + "accuracy": 66.7, "cost": 0.0001, - "robustness": 100.0 + "robustness": 0 }, "medium": { - "accuracy": 77.5, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 66.7, + "cost": 0.0016, + "robustness": 0.0 }, "hard": { - "accuracy": 35.9, - "cost": 0.0001, + "accuracy": 17.6, + "cost": 0.0017, "robustness": 100.0 }, "all": { - "accuracy": 50.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 36.4, + "cost": 0.0016, + "robustness": 33.3 } } } } }, - "History": { + "Technology": { "metrics": { "easy": { - "accuracy": 97.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.6, + "cost": 0.0006, + "robustness": 25.6 }, "medium": { - "accuracy": 74.0, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 75.0, + "cost": 0.0011, + "robustness": 19.0 }, "hard": { - "accuracy": 14.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 24.2, + "cost": 0.0013, + "robustness": 28.6 }, "all": { - "accuracy": 73.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 82.0, + "cost": 0.0009, + "robustness": 23.9 } }, "subcategories": { - "Geography": { + "Engineering": { "metrics": { "easy": { - "accuracy": 98.2, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 97.6, + "cost": 0.0009, + "robustness": 0.0 }, "medium": { - "accuracy": 62.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 88.0, + "cost": 0.0013, + "robustness": 15.4 }, "hard": { - "accuracy": 25.0, - "cost": 0.0, - "robustness": 0 + "accuracy": 35.7, + "cost": 0.0021, + "robustness": 0.0 }, "all": { - "accuracy": 85.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 85.1, + "cost": 0.0013, + "robustness": 10.5 } } }, - "History": { + "Management and public relations": { "metrics": { "easy": { - "accuracy": 96.4, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.7, + "cost": 0.0006, + "robustness": 25.0 }, "medium": { - "accuracy": 76.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 63.6, + "cost": 0.0005, + "robustness": 0.0 }, "hard": { - "accuracy": 13.2, + "accuracy": 0.0, "cost": 0.0001, - "robustness": 100.0 + "robustness": 0 }, "all": { - "accuracy": 68.1, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 89.0, + "cost": 0.0006, + "robustness": 20.0 } } }, - "Biography and genealogy": { + "Medicine and health": { "metrics": { "easy": { - "accuracy": 97.6, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 96.4, + "cost": 0.0006, + "robustness": 29.4 }, "medium": { - "accuracy": 77.8, - "cost": 0.0001, - "robustness": 0 + "accuracy": 66.5, + "cost": 0.0011, + "robustness": 28.6 }, "hard": { - "accuracy": 0.0, - "cost": 0.0001, - "robustness": 0 + "accuracy": 20.4, + "cost": 0.001, + "robustness": 33.3 }, "all": { - "accuracy": 92.3, - "cost": 0.0001, - "robustness": 100.0 + "accuracy": 80.4, + "cost": 0.0007, + "robustness": 29.8 } } } diff --git a/src/data/routerMetrics/leaderboard.json b/src/data/routerMetrics/leaderboard.json index 6d8b373..02d85b4 100644 --- a/src/data/routerMetrics/leaderboard.json +++ b/src/data/routerMetrics/leaderboard.json @@ -10,6 +10,28 @@ "Accuracy": 76.4, "Cost per 1k": 0.18 }, + { + "Router Name": "AgentForge Router", + "Arena Score": 74.13, + "Optimal Selection Score": 17.84, + "Optimal Cost Score": 52.47, + "Optimal Acc. Score": 98.68, + "Robustness Score": 40.48, + "Latency Score": null, + "Accuracy": 74.72, + "Cost per 1k": 0.13 + }, + { + "Router Name": "Nadir Router", + "Arena Score": 73.33, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 25.48, + "Latency Score": null, + "Accuracy": 74.87, + "Cost per 1k": 0.29 + }, { "Router Name": "Weave Router", "Arena Score": 72.82, @@ -21,6 +43,28 @@ "Accuracy": 76.32, "Cost per 1k": 0.94 }, + { + "Router Name": "OrcaRouter-Adaptive", + "Arena Score": 72.08, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 22.62, + "Latency Score": null, + "Accuracy": 75.54, + "Cost per 1k": 1.0 + }, + { + "Router Name": "azure", + "Arena Score": 71.87, + "Optimal Selection Score": null, + "Optimal Cost Score": null, + "Optimal Acc. Score": null, + "Robustness Score": 71.43, + "Latency Score": null, + "Accuracy": 72.82, + "Cost per 1k": 0.22 + }, { "Router Name": "R2-Router", "Arena Score": 71.6, @@ -58,32 +102,21 @@ "Router Name": "mirt_bert", "Arena Score": 66.89, "Optimal Selection Score": 3.44, - "Optimal Cost Score": 19.6178, + "Optimal Cost Score": 19.62, "Optimal Acc. Score": 78.18, "Robustness Score": 61.19, - "Latency Score": 27.027, + "Latency Score": 27.03, "Accuracy": 66.88, "Cost per 1k": 0.15 }, - { - "Router Name": "azure", - "Arena Score": 71.87, - "Optimal Selection Score": null, - "Optimal Cost Score": null, - "Optimal Acc. Score": null, - "Robustness Score": 71.43, - "Latency Score": null, - "Accuracy": 72.82, - "Cost per 1k": 0.22 - }, { "Router Name": "nirt_bert", "Arena Score": 66.12, "Optimal Selection Score": 3.83, - "Optimal Cost Score": 14.039, + "Optimal Cost Score": 14.04, "Optimal Acc. Score": 77.88, "Robustness Score": 49.29, - "Latency Score": 10.4167, + "Latency Score": 10.42, "Accuracy": 66.34, "Cost per 1k": 0.21 }, @@ -102,10 +135,10 @@ "Router Name": "carrot", "Arena Score": 63.87, "Optimal Selection Score": 2.68, - "Optimal Cost Score": 6.7697, + "Optimal Cost Score": 6.77, "Optimal Acc. Score": 78.63, "Robustness Score": 89.05, - "Latency Score": 1.4993, + "Latency Score": 1.5, "Accuracy": 67.21, "Cost per 1k": 2.06 }, @@ -124,10 +157,10 @@ "Router Name": "routerbench_mlp", "Arena Score": 57.56, "Optimal Selection Score": 13.39, - "Optimal Cost Score": 24.4499, + "Optimal Cost Score": 24.45, "Optimal Acc. Score": 83.32, "Robustness Score": 80.0, - "Latency Score": 90.9091, + "Latency Score": 90.91, "Accuracy": 61.62, "Cost per 1k": 4.83 }, @@ -135,7 +168,7 @@ "Router Name": "notdiamond", "Arena Score": 57.29, "Optimal Selection Score": 1.55, - "Optimal Cost Score": 2.1367, + "Optimal Cost Score": 2.14, "Optimal Acc. Score": 76.81, "Robustness Score": 55.91, "Latency Score": null, @@ -146,10 +179,10 @@ "Router Name": "graphrouter", "Arena Score": 57.22, "Optimal Selection Score": 4.73, - "Optimal Cost Score": 38.3347, + "Optimal Cost Score": 38.33, "Optimal Acc. Score": 74.25, "Robustness Score": 94.29, - "Latency Score": 2.6954, + "Latency Score": 2.7, "Accuracy": 57.0, "Cost per 1k": 0.34 }, @@ -157,10 +190,10 @@ "Router Name": "routerbench_knn", "Arena Score": 55.48, "Optimal Selection Score": 13.09, - "Optimal Cost Score": 25.4887, + "Optimal Cost Score": 25.49, "Optimal Acc. Score": 78.77, "Robustness Score": 83.33, - "Latency Score": 1.328, + "Latency Score": 1.33, "Accuracy": 58.69, "Cost per 1k": 4.27 }, @@ -168,10 +201,10 @@ "Router Name": "routellm", "Arena Score": 48.07, "Optimal Selection Score": 99.72, - "Optimal Cost Score": 99.6314, + "Optimal Cost Score": 99.63, "Optimal Acc. Score": 68.76, "Robustness Score": 100.0, - "Latency Score": 0.4016, + "Latency Score": 0.4, "Accuracy": 47.04, "Cost per 1k": 0.27 }, @@ -179,10 +212,10 @@ "Router Name": "RouterDC", "Arena Score": 33.75, "Optimal Selection Score": 39.84, - "Optimal Cost Score": 72.998, + "Optimal Cost Score": 73.0, "Optimal Acc. Score": 49.05, "Robustness Score": 85.24, - "Latency Score": 10.7527, + "Latency Score": 10.75, "Accuracy": 32.01, "Cost per 1k": 0.07 }