diff --git a/Makefile b/Makefile index db49845..e765669 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ floor: --python-version 3.11 --quiet --output-file "$$requirements" && \ uv run --isolated --no-project --python 3.11 --with-requirements "$$requirements" \ --with-editable . --with pytest --with pytest-socket --with tiktoken \ + --with json-repair --with scipy \ pytest --disable-socket -q -p no:cacheprovider integration: diff --git a/bench/README.md b/bench/README.md index 9c08e6f..9a53cd7 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,7 +1,8 @@ # The benchmark runner This folder runs hotmemory on LongMemEval-S, a benchmark of 500 questions over long chat -histories. It is not part of the hotmemory package. [docs/benchmarks.md](../docs/benchmarks.md) +histories, and on BEAM, a benchmark of long conversations in many domains. It is not part +of the hotmemory package. [docs/benchmarks.md](../docs/benchmarks.md) reports the numbers, and each number names the result file that it comes from. ## Before a run @@ -188,6 +189,73 @@ questions that the rows file holds. It refuses a rows file of another configurat run without `--resume` refuses to start over a rows file. When every question has finished, the runner writes the result file and deletes the rows file. +## BEAM + +BEAM (arXiv 2510.27246) holds conversations of about 100,000 to 1 million tokens, each +with 20 probing questions, 2 for each of 10 abilities. `bench/beam.py` reads the splits +`100K` and `1M` of the Hugging Face dataset `Mohammadta/BEAM` at a pinned revision, and +checks the SHA-256 of each file. The data license is CC BY-SA 4.0, so the files stay in +`bench/data/`, which git ignores. This command downloads a split and prints its counts: + +```sh +uv run --group bench python -m bench.beam --split 1M +``` + +The loader cuts each batch of a conversation into windows of whole turn pairs, at most +10,000 characters each, with the date of the batch. A turn pair longer than that is cut +at blank lines. It strips the index markers, such as `->-> 1,1`, from the user turns. +Samples A and B are 5 conversations each of the `100K` split, from 5 different domains, +in `bench/sample-beam-a.json` and `bench/sample-beam-b.json`. The smoke sample is the +conversation of sample A with the fewest windows. + +This command prints the cost estimate and the expected time of a run, and calls no model: + +```sh +uv run --group bench --env-file .env python -m bench.beam_e2e --split 100K --sample a --workers 5 +``` + +Add `--run` to run it. The options are these: + +| Option | Values | Default | +|---|---|---| +| `--split` | `100K` or `1M` | `100K` | +| `--sample` | `smoke`, `a`, or `b` of the `100K` split, or `all` | `smoke` | +| `--arms` | `memory`, `chunks`, or both, comma-separated | both | +| `--readers` | `mini` (`gpt-4.1-mini`), the one reader | `mini` | +| `--workers` | conversations at a time, each in its own process | 1 | +| `--reuse` | read from the kept databases, with no `capture` | off | +| `--resume` | go on from the rows file of an earlier run | off | +| `--replay` | call no model, and fail a conversation at a reply that the cache does not hold | off | +| `--extract-rate`, `--window-seconds` | the rates of the estimate | $0.0005 and 8.3 s | +| `--limit`, `--tag`, `--url` | as for `bench.e2e` | | + +Each conversation runs over one local database, named +`hotmemory-kept-beam---chunks1000-vector--`. +The database keeps the memory after the run, and the runner never deletes it. A worker +takes the conversation with the most windows first. It captures each window in order with +the `detail` extractor, as chunks of 1,000 characters. Then, for each probe, it builds one +block of at most 16,000 characters for each arm: + +- `memory` calls `recall` with `k` 30 and `episodes="search"`. +- `chunks` searches the chunks alone by vector, `k` 30, as the RAG baseline. + +Each reader answers from each block with the official RAG answer prompt of BEAM. The +scoring follows the official code at commit `b2da22e`: one `gpt-4.1-mini` judge call for +each nugget of the rubric, and for event ordering, the line match and Kendall tau-b. The +official judge prompt does not see the question (upstream issue +mohammadtavakoli78/BEAM#15). The prompts are copied in `bench/beam_prompts/` under their +MIT license. The score of a split is the mean over the 10 abilities. + +The rows file holds one row for each captured conversation and one row for each finished +probe. If a conversation fails, the others go on, and the runner writes no result file. +Run the same command again with `--resume`. It skips the finished probes and the captured +conversations, and a conversation that stopped during capture goes on over its kept +database. `capture` skips each window whose chunks are stored, with no extractor call. A +conversation stops when it spends twice its estimate. The result file is +`bench/results/beam--[-].json`. Each probe row records the windows +that hold its source turns, and the share of them that each block shows, in +`source_coverage`. + ## The checks of the results This command compares the vector rankings of two retrieval result files, as the driver diff --git a/bench/beam.py b/bench/beam.py new file mode 100644 index 0000000..de276f9 --- /dev/null +++ b/bench/beam.py @@ -0,0 +1,467 @@ +"""The BEAM data, splits `100K` and `1M`: the download, its checks, and the windows of capture. + +One row of a split is one conversation: dated batches of user and assistant turns, and 20 +probing questions, 2 for each of the 10 abilities. Print the counts of a split with: + + uv run --group bench python -m bench.beam --split 1M + +With `--split 100K --samples`, it also writes the files of samples A and B. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import random +import re +import statistics +import urllib.request +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from bench.data import DATA_DIR, sha256 + +HERE = Path(__file__).resolve().parent + +DATA_REVISION = "3205395e897e7318c7b094ef4e6047b9b82dbb03" +DATA_URL = f"https://huggingface.co/datasets/Mohammadta/BEAM/resolve/{DATA_REVISION}/data/" +SPLITS = { + "100K": ( + "100K-00000-of-00001.parquet", + "c0519be25907005ba873c927c50877471d550873039d96c041554d0075a78ace", + ), + "1M": ( + "1M-00000-of-00001.parquet", + "41b5acbbb55a586b1305514ef9d9fb03365d9b3331b598a1c2dd7603d93ef533", + ), +} +ABILITIES = ( + "abstention", + "contradiction_resolution", + "event_ordering", + "information_extraction", + "instruction_following", + "knowledge_update", + "multi_session_reasoning", + "preference_following", + "summarization", + "temporal_reasoning", +) +"""The 10 abilities, in the column order of the official report.""" +GOLD_FIELDS = { + "abstention": "ideal_response", + "contradiction_resolution": "ideal_answer", + "event_ordering": "answer", + "information_extraction": "answer", + "instruction_following": "expected_compliance", + "knowledge_update": "answer", + "multi_session_reasoning": "answer", + "preference_following": "expected_compliance", + "summarization": "ideal_summary", + "temporal_reasoning": "answer", +} +"""The field of each ability that holds the gold answer.""" +WINDOW_CHARS = 10_000 +SAMPLE_SPLIT = "100K" +SAMPLE_SIZE = 5 +DATE_FORMAT = "%B-%d-%Y" +_MARKER = re.compile(r"\s*->->\s*\d+\s*,\s*\d+\s*$") +_PARAGRAPH = "\n\n" + + +@dataclass(frozen=True) +class Message: + """One turn of a batch: its id in the conversation, its role, and its text.""" + + id: int + role: str + content: str + + +@dataclass(frozen=True) +class Batch: + """One batch of a conversation: its number from 1, its date in UTC, and its turns.""" + + number: int + date: datetime + messages: tuple[Message, ...] + + +@dataclass(frozen=True) +class Probe: + """One probing question, with its gold answer, its rubric, and its source turn ids.""" + + id: str + conversation: str + ability: str + question: str + gold: str + rubric: tuple[str, ...] + difficulty: str + sources: tuple[int, ...] + + +@dataclass(frozen=True) +class Window: + """One `capture` call: whole turn pairs of one batch, as `role: content` paragraphs. + + `messages` holds the ids of the turns in the window. A turn pair longer than the + limit is cut into pieces, so its ids can be in more than one window. + """ + + batch: int + number: int + date: datetime + text: str + messages: tuple[int, ...] + + @property + def key(self) -> str: + """The episode key: `window--`.""" + return f"window-{self.batch:02d}-{self.number:04d}" + + +@dataclass(frozen=True) +class Conversation: + """One row of a split: the domain, the batches in order, and the probes.""" + + id: str + domain: str + batches: tuple[Batch, ...] + probes: tuple[Probe, ...] + + @property + def chars(self) -> int: + """The characters of every turn.""" + return sum(len(m.content) for batch in self.batches for m in batch.messages) + + def windows(self, limit: int = WINDOW_CHARS) -> list[Window]: + """Return the windows of every batch, in order, numbered from 1 in each batch. + + A window holds whole turn pairs, a user turn and the turns after it, of at most + `limit` characters in all. A pair longer than `limit` is cut at blank lines into + pieces of at most `limit` characters, and each piece is packed as a pair. + """ + found = [] + for batch in self.batches: + parts = [ + (piece, ids) + for text, ids in _pairs(batch.messages) + for piece in (cut(text, limit) if len(text) > limit else [text]) + ] + for number, (text, ids) in enumerate(_pack(parts, limit), start=1): + found.append(Window(batch.number, number, batch.date, text, ids)) + return found + + +def strip_marker(text: str) -> str: + """Return `text` without the index marker at its end, such as `->-> 1,1`.""" + return _MARKER.sub("", text) + + +def cut(text: str, limit: int) -> list[str]: + """Cut `text` at blank lines into pieces of at most `limit` characters. + + A paragraph longer than `limit` is cut every `limit` characters. + """ + pieces: list[str] = [] + current = "" + for paragraph in text.split(_PARAGRAPH): + while len(paragraph) > limit: + if current: + pieces.append(current) + current = "" + pieces.append(paragraph[:limit]) + paragraph = paragraph[limit:] + joined = current + _PARAGRAPH + paragraph if current else paragraph + if len(joined) > limit: + pieces.append(current) + current = paragraph + else: + current = joined + if current: + pieces.append(current) + return pieces + + +def _pairs(messages: Sequence[Message]) -> Iterator[tuple[str, tuple[int, ...]]]: + """Yield the text and the turn ids of each turn pair, a user turn and the turns after it.""" + pair: list[Message] = [] + for message in messages: + if message.role == "user" and pair: + yield _paragraphs(pair), tuple(m.id for m in pair) + pair = [] + pair.append(message) + if pair: + yield _paragraphs(pair), tuple(m.id for m in pair) + + +def _paragraphs(messages: Sequence[Message]) -> str: + """Return `messages` as `role: content` paragraphs, as `bench.prompts.session_text` does.""" + return _PARAGRAPH.join(f"{m.role}: {m.content.strip()}" for m in messages) + + +def _pack( + parts: Sequence[tuple[str, tuple[int, ...]]], limit: int +) -> Iterator[tuple[str, tuple[int, ...]]]: + """Join `parts` in order into windows of at most `limit` characters, one blank line apart.""" + text = "" + ids: tuple[int, ...] = () + for part, part_ids in parts: + if text and len(text) + len(_PARAGRAPH) + len(part) > limit: + yield text, ids + text, ids = "", () + text = text + _PARAGRAPH + part if text else part + ids = ids + tuple(i for i in part_ids if i not in ids) + if text: + yield text, ids + + +def parse_date(value: str) -> datetime: + """Return a date such as `March-15-2024` as midnight UTC.""" + return datetime.strptime(value, DATE_FORMAT).replace(tzinfo=UTC) + + +def parse_conversation(row: Mapping[str, Any]) -> Conversation: + """Return the conversation of one row, with the index markers stripped from user turns. + + The date of a batch is the `time_anchor` of its first turn that has one. Raises + ValueError if a batch has no date or two dates, or if a probe has no gold field. + """ + conversation = str(row["conversation_id"]) + batches = [] + for number, turns in enumerate(row["chat"], start=1): + anchors = {t["time_anchor"] for t in turns if t["time_anchor"]} + if len(anchors) != 1: + raise ValueError(f"conversation {conversation} batch {number}: dates {anchors}") + messages = tuple( + Message( + int(t["id"]), + t["role"], + strip_marker(t["content"]) if t["role"] == "user" else t["content"], + ) + for t in turns + ) + batches.append(Batch(number, parse_date(anchors.pop()), messages)) + probes = _probes(conversation, row["probing_questions"]) + return Conversation(conversation, row["conversation_seed"]["category"], tuple(batches), probes) + + +def _probes(conversation: str, value: str | Mapping[str, Any]) -> tuple[Probe, ...]: + """Return the probes of `value`, a dict or its Python literal, in the order of ABILITIES.""" + by_ability = ast.literal_eval(value) if isinstance(value, str) else value + if set(by_ability) != set(ABILITIES): + raise ValueError(f"conversation {conversation}: abilities {sorted(by_ability)}") + probes = [] + for ability in ABILITIES: + for number, entry in enumerate(by_ability[ability], start=1): + field = GOLD_FIELDS[ability] + if field not in entry: + raise ValueError(f"conversation {conversation} {ability} {number}: no {field}") + probes.append( + Probe( + id=f"{conversation}-{ability}-{number}", + conversation=conversation, + ability=ability, + question=entry["question"], + gold=str(entry[field]), + rubric=tuple(str(item) for item in entry["rubric"]), + difficulty=str(entry.get("difficulty", "")), + sources=_ids(entry.get("source_chat_ids")), + ) + ) + return tuple(probes) + + +def _ids(value: Any) -> tuple[int, ...]: + """Return the turn ids in `value`, at any depth of lists and dicts, sorted, without repeats. + + An item that is not a whole number, such as a note, is left out. + """ + found: set[int] = set() + + def walk(item: Any) -> None: + if isinstance(item, Mapping): + for inner in item.values(): + walk(inner) + elif isinstance(item, list | tuple): + for inner in item: + walk(inner) + elif isinstance(item, int) or (isinstance(item, str) and item.strip().isdigit()): + found.add(int(item)) + + walk(value) + return tuple(sorted(found)) + + +def download(split: str, directory: Path = DATA_DIR) -> Path: + """Download the file of `split` into `directory` unless it is there, and return its path. + + Raises ValueError, and keeps no file, if the download has another SHA-256. + """ + name, expected = SPLITS[split] + path = directory / f"beam-{name}" + if path.exists(): + return path + directory.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(".part") + with urllib.request.urlopen(DATA_URL + name) as response, partial.open("wb") as file: + while block := response.read(1 << 20): + file.write(block) + found = sha256(partial) + if found != expected: + partial.unlink() + raise ValueError(f"{DATA_URL + name} has SHA-256 {found}, expected {expected}") + partial.rename(path) + return path + + +def load(path: Path, expected_sha256: str | None) -> list[Conversation]: + """Return the conversations of the parquet file at `path`, in file order. + + Raises ValueError if `expected_sha256` is not None and the file has another SHA-256, + or if two conversations share an id. + """ + import pyarrow.parquet + + if expected_sha256 is not None: + found = sha256(path) + if found != expected_sha256: + raise ValueError(f"{path} has SHA-256 {found}, expected {expected_sha256}") + rows = pyarrow.parquet.read_table(path).to_pylist() + conversations = [parse_conversation(row) for row in rows] + ids = [conversation.id for conversation in conversations] + if len(set(ids)) != len(ids): + raise ValueError(f"{path} repeats a conversation id") + return conversations + + +def load_split(split: str, directory: Path = DATA_DIR) -> list[Conversation]: + """Download the file of `split` if it is not there, and return its conversations.""" + return load(download(split, directory), SPLITS[split][1]) + + +@dataclass(frozen=True) +class Sample: + """A fixed sample of SAMPLE_SPLIT: its file, its seed, and the samples it leaves out.""" + + file: str + seed: int + excludes: tuple[str, ...] = () + + +SAMPLES = { + "a": Sample("bench/sample-beam-a.json", 20261026), + "b": Sample("bench/sample-beam-b.json", 20261027, excludes=("a",)), +} + + +def choose_sample( + conversations: Sequence[Conversation], + seed: int, + exclude: Sequence[str] = (), + size: int = SAMPLE_SIZE, +) -> list[Conversation]: + """Return `size` conversations from `size` different domains, drawn with `seed`. + + The draw first picks the domains, then one conversation of each, and leaves out the + ids in `exclude`. The result is sorted by id. Raises ValueError if fewer than `size` + domains are left. + """ + rng = random.Random(seed) + left = [c for c in conversations if c.id not in set(exclude)] + domains = sorted({c.domain for c in left}) + if len(domains) < size: + raise ValueError(f"only {len(domains)} domains are left, {size} needed") + chosen = [] + for domain in rng.sample(domains, size): + chosen.append(rng.choice(sorted((c for c in left if c.domain == domain), key=_order))) + return sorted(chosen, key=_order) + + +def _order(conversation: Conversation) -> tuple[int, str]: + """Sort by the conversation id as a number, then as text.""" + return (int(conversation.id), "") if conversation.id.isdigit() else (0, conversation.id) + + +def sample_ids( + name: str, conversations: Sequence[Conversation], root: Path = HERE.parent +) -> list[str]: + """Return the conversation ids of sample `name`, and write its file first if it is missing. + + The file lists each conversation with its domain and its window count. The draw of a + sample leaves out every id of the samples that it excludes. + """ + sample = SAMPLES[name] + path = root / sample.file + if not path.exists(): + exclude = [i for other in sample.excludes for i in sample_ids(other, conversations, root)] + chosen = choose_sample(conversations, sample.seed, exclude) + entries = [ + {"conversation": c.id, "domain": c.domain, "windows": len(c.windows())} for c in chosen + ] + path.write_text(json.dumps(entries, indent=1) + "\n") + return [entry["conversation"] for entry in json.loads(path.read_text())] + + +def counts(conversations: Sequence[Conversation], limit: int = WINDOW_CHARS) -> dict[str, Any]: + """Return the counts of `conversations`: questions, nuggets, characters, and windows.""" + windows = [conversation.windows(limit) for conversation in conversations] + per_conversation = [len(found) for found in windows] + sizes = [len(window.text) for found in windows for window in found] + probes = [probe for conversation in conversations for probe in conversation.probes] + long_pairs = sum( + 1 + for conversation in conversations + for batch in conversation.batches + for text, _ in _pairs(batch.messages) + if len(text) > limit + ) + return { + "conversations": len(conversations), + "questions": len(probes), + "nuggets": sum(len(probe.rubric) for probe in probes), + "characters": sum(conversation.chars for conversation in conversations), + "windows": sum(per_conversation), + "windows_per_conversation": { + "min": min(per_conversation), + "median": statistics.median(per_conversation), + "mean": round(statistics.mean(per_conversation), 1), + "max": max(per_conversation), + }, + "window_chars": { + "median": statistics.median(sizes), + "mean": round(statistics.mean(sizes)), + "max": max(sizes), + }, + "long_pairs": long_pairs, + "domains": sorted({conversation.domain for conversation in conversations}), + } + + +def main(argv: Sequence[str] | None = None) -> None: + """Print the counts of one split, and write the sample files with `--samples`.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else "") + parser.add_argument("--split", choices=sorted(SPLITS), required=True) + parser.add_argument( + "--samples", + action="store_true", + help=f"write the sample files from the {SAMPLE_SPLIT} split if they do not exist", + ) + args = parser.parse_args(argv) + conversations = load_split(args.split) + for name, value in counts(conversations).items(): + print(f"{name}: {value}") + if args.samples: + if args.split != SAMPLE_SPLIT: + parser.error(f"--samples needs --split {SAMPLE_SPLIT}") + for sample in SAMPLES: + print(f"sample {sample}: {', '.join(sample_ids(sample, conversations))}") + + +if __name__ == "__main__": + main() diff --git a/bench/beam_e2e.py b/bench/beam_e2e.py new file mode 100644 index 0000000..99e61ba --- /dev/null +++ b/bench/beam_e2e.py @@ -0,0 +1,839 @@ +"""The BEAM end-to-end score: capture each conversation once, then recall, read, and judge. + +Run it from the root of the repository, with the local stack up. Without `--run`, it prints +the cost estimate and the expected time, and stops. With `--replay`, it calls no model: + + uv run --group bench --env-file .env python -m bench.beam_e2e --split 100K --sample smoke + uv run --group bench --env-file .env python -m bench.beam_e2e --split 100K --sample a --run + uv run --group bench --env-file .env python -m bench.beam_e2e --split 1M --sample all \\ + --workers 8 --run + +Each conversation gets one local database that keeps its memory after the run, named +`-`. A worker takes one conversation at a time, the one with the +most windows first. It captures each window in order with the `detail` extractor, then, for +each probe, recalls once for each arm, reads the block with each reader, and judges each +answer with the official BEAM scoring. The reads and the judge calls of a conversation run +in a thread pool. + +It appends one row for each captured conversation and one row for each finished probe to a +rows file. `--resume` skips the finished probes and the captured conversations. A +conversation that stopped during capture keeps its database, and its capture runs again: +`capture` skips each window whose chunks are stored, with no extractor call. `--reuse` +reads from the kept databases with no capture. The runner deletes no database. +""" + +from __future__ import annotations + +import argparse +import hashlib +import heapq +import json +import multiprocessing +import statistics +import threading +import time +import uuid +from collections.abc import Callable, Mapping, Sequence +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from bench import embed +from bench.beam import ( + DATA_REVISION, + SAMPLE_SPLIT, + SAMPLES, + SPLITS, + WINDOW_CHARS, + Conversation, + Probe, + Window, + load_split, + sample_ids, +) +from bench.beam_score import BEAM_COMMIT, EVENT_ORDERING, JUDGE, JUDGE_MAX_TOKENS, answer_prompt +from bench.beam_score import score as score_probe +from bench.beam_score import summarize as summarize_scores +from bench.e2e import ( + COST_LIMIT, + Config, + WriteTimer, + append_row, + existing_databases, + extractor, + kept_name, + library_commit, + local_store, +) +from bench.embed import CACHE_DIR +from bench.models import ( + BEAM_MINI, + EMBEDDING_PRICE, + EXTRACTOR_MAX_TOKENS, + PRICES, + CachedModels, + Complete, + Reply, +) +from bench.retrieval import RESULTS_DIR, _commit +from hotmemory import Filter, Memory, Store +from hotmemory.contract import render_episode + +ARMS = ("memory", "chunks") +READERS = {"mini": BEAM_MINI} +READER_MAX_TOKENS = 2000 +CONFIG = Config( + k=30, + budget=16_000, + extractor="detail", + session_episodes=True, + episodes="search", + chunk_chars=1000, +) +"""The phase 10 configuration of the memory arm.""" +NAMESPACE = ("bench", "beam") +ACTOR = "beam-extractor" +JUDGE_THREADS = 8 +SMOKE = "smoke" +ALL = "all" +SPEND_FACTOR = 2.0 +WINDOW_KEY_CHARS = len("window-01-0001") + + +@dataclass(frozen=True) +class Rates: + """The rates of the estimate. `docs/internal/plan.md` gives the derivation of each.""" + + extract_window: float = 0.0005 + window_seconds: float = 8.3 + embed_tokens_per_char: float = 2.1 / 4.4 + reader_in: int = 3850 + reader_out: int = 80 + judge_in: int = 1200 + judge_out: int = 80 + match_calls: int = 85 + match_in: int = 150 + match_out: int = 2 + call_seconds: float = 0.7 + + +@dataclass(frozen=True) +class Estimate: + """The expected cost of a run in US dollars, by part, and its expected time in seconds.""" + + parts: dict[str, float] + by_conversation: dict[str, float] + seconds: float + + @property + def total(self) -> float: + return sum(self.parts.values()) + + +def estimate( + conversations: Sequence[Conversation], + arms: Sequence[str], + readers: Sequence[str], + workers: int, + rates: Rates | None = None, + capture: bool = True, +) -> Estimate: + """Return the expected cost and time of a run over `conversations`. + + The time is the longest worker when each worker takes the conversation with the most + windows first. A conversation takes its windows times `window_seconds` for capture, + and its model calls of the read phase times `call_seconds`, over JUDGE_THREADS threads. + """ + rates = rates or Rates() + judge_in, judge_out = PRICES[JUDGE] + nugget_call = (rates.judge_in * judge_in + rates.judge_out * judge_out) / 1e6 + match_call = (rates.match_in * judge_in + rates.match_out * judge_out) / 1e6 + parts = dict.fromkeys(("extraction", "embeddings", "readers", "judge"), 0.0) + by_conversation: dict[str, float] = {} + seconds: list[float] = [] + combos = len(arms) * len(readers) + for conversation in conversations: + windows = len(conversation.windows()) if capture else 0 + extraction = windows * rates.extract_window + embeddings = ( + conversation.chars * rates.embed_tokens_per_char * EMBEDDING_PRICE / 1e6 + if capture + else 0.0 + ) + reading = 0.0 + for reader in readers: + price_in, price_out = PRICES[READERS[reader]] + call = (rates.reader_in * price_in + rates.reader_out * price_out) / 1e6 + reading += call * len(arms) * len(conversation.probes) + nuggets = sum(len(probe.rubric) for probe in conversation.probes) + orderings = sum(1 for probe in conversation.probes if probe.ability == EVENT_ORDERING) + judging = combos * (nuggets * nugget_call + orderings * rates.match_calls * match_call) + parts["extraction"] += extraction + parts["embeddings"] += embeddings + parts["readers"] += reading + parts["judge"] += judging + by_conversation[conversation.id] = extraction + embeddings + reading + judging + calls = combos * (len(conversation.probes) + nuggets + orderings * rates.match_calls) + seconds.append(windows * rates.window_seconds + calls * rates.call_seconds / JUDGE_THREADS) + return Estimate(parts, by_conversation, longest_worker(seconds, workers)) + + +def longest_worker(seconds: Sequence[float], workers: int) -> float: + """Return the time of the busiest worker when each takes the longest job left first.""" + loads = [0.0] * max(1, workers) + for job in sorted(seconds, reverse=True): + heapq.heapreplace(loads, loads[0] + job) + return max(loads) + + +class SpendLimit(RuntimeError): + """A conversation spent more than its limit.""" + + +class ThreadModels: + """A `CachedModels` for each thread, over one cache file, with a spend limit. + + `spent` and `calls` sum every thread. A call raises SpendLimit once `spent` passes + `limit`, and makes no model call after that. + """ + + def __init__(self, complete: Complete, path: Path, limit: float = float("inf")) -> None: + self._complete = complete + self._path = path + self._limit = limit + self._local = threading.local() + self._lock = threading.Lock() + self._all: list[CachedModels] = [] + + def __call__( + self, + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None = None, + system: str | None = None, + ) -> Reply: + models: CachedModels | None = getattr(self._local, "models", None) + if models is None: + models = CachedModels(self._complete, self._path) + self._local.models = models + with self._lock: + self._all.append(models) + self._check() + reply = models(model, prompt, max_tokens, schema, system) + self._check() + return reply + + def _check(self) -> None: + if self.spent > self._limit: + raise SpendLimit(f"spent ${self.spent:.2f}, over the limit of ${self._limit:.2f}") + + @property + def spent(self) -> float: + with self._lock: + return sum(models.spent for models in self._all) + + @property + def calls(self) -> int: + with self._lock: + return sum(models.calls for models in self._all) + + def close(self) -> None: + with self._lock: + for models in self._all: + models.close() + + +@dataclass(frozen=True) +class Block: + """The block of one arm for one probe, and what it shows.""" + + text: str + episodes: tuple[str, ...] + facts: int + + +def memory_block(store: Store, namespace: Sequence[str], question: str) -> Block: + """Return the block of the memory arm: `recall` with the configuration of CONFIG.""" + records, text = Memory(store).recall( + question, [namespace], budget=CONFIG.budget, k=CONFIG.k, episodes="search" + ) + episodes = tuple(record.key for record in records if record.kind == "episode") + return Block(text, episodes, len(records) - len(episodes)) + + +def chunks_block(store: Store, namespace: Sequence[str], question: str) -> Block: + """Return the block of the chunks arm: the top `k` chunks for `question`, in the budget. + + The chunks are rendered as `recall` renders episodes, and the block stops before the + first chunk that would pass the budget. + """ + hits = store.search(question, [namespace], Filter(kind="episode"), k=CONFIG.k) + parts: list[str] = [] + shown: list[str] = [] + for hit in hits: + more = [*parts, render_episode(hit.record)] + if len("\n".join(more)) > CONFIG.budget: + break + parts, shown = more, [*shown, hit.record.key] + return Block("\n".join(parts), tuple(shown), 0) + + +BLOCKS: dict[str, Callable[[Store, Sequence[str], str], Block]] = { + "memory": memory_block, + "chunks": chunks_block, +} + + +def source_windows(windows: Sequence[Window], probe: Probe) -> set[str]: + """Return the keys of the windows that hold a source turn of `probe`.""" + sources = set(probe.sources) + return {window.key for window in windows if sources & set(window.messages)} + + +def coverage(found: set[str], block: Block) -> float | None: + """Return the share of the windows in `found` that the block shows a chunk of.""" + if not found: + return None + shown = {key[:WINDOW_KEY_CHARS] for key in block.episodes} + return len(found & shown) / len(found) + + +@dataclass(frozen=True) +class Job: + """What every worker of one run shares.""" + + url: str + arms: tuple[str, ...] + readers: tuple[str, ...] + replay: bool + kept: str + rows_file: Path + + +@dataclass +class Done: + """What one conversation did: its finished probes, what it cost, and its failure.""" + + conversation: str + probes: int = 0 + spent: float = 0.0 + calls: int = 0 + embedder_misses: int = 0 + failure: str | None = None + seconds: float = 0.0 + capture: dict[str, Any] = field(default_factory=dict) + + +def capture_conversation( + store: Store, models: ThreadModels, windows: Sequence[Window], namespace: Sequence[str] +) -> dict[str, Any]: + """Capture each window in order, and return what the capture did and how long it took.""" + memory = Memory(store) + timer = WriteTimer(store) + at_limit: list[str] = [] + seconds: list[float] = [] + written = 0 + for window in windows: + started = time.perf_counter() + ids = memory.capture( + window.text, + namespace, + extractor(models, CONFIG.variant, "", at_limit), + actor=ACTOR, + observed_at=window.date, + episode=window.key, + chunk_chars=CONFIG.chunk_chars, + ) + seconds.append(time.perf_counter() - started) + written += len(ids) + ordered = sorted(seconds) + return { + "windows": len(windows), + "windows_at_limit": len(at_limit), + "facts_written": written, + "window_s": { + "p50": round(statistics.median(ordered), 3) if ordered else None, + "p95": round(ordered[int(len(ordered) * 0.95)], 3) if ordered else None, + "highest": round(ordered[-1], 3) if ordered else None, + "sum": round(sum(ordered), 1), + }, + "writes": timer.take(), + } + + +def table_rows(store: Store) -> dict[str, int]: + """Return the row count of each record table of a HotdataStore.""" + from hotmemory.hotdata import CUE_TABLE, RECORD_TABLES + + client, database = store.client, store.database # type: ignore[attr-defined] + counts = {} + for table in (*RECORD_TABLES, CUE_TABLE): + result = client.execute_sql( + f"SELECT COUNT(*) FROM default.public.{table}", database=database + ) + counts[table] = int(result.rows[0][0]) + return counts + + +def answer_and_score( + models: ThreadModels, probe: Probe, block: Block, reader: str +) -> dict[str, Any]: + """Read `block` with `reader`, judge the answer, and return the cell of the row.""" + reply = models(READERS[reader], answer_prompt(block.text, probe.question), READER_MAX_TOKENS) + found = score_probe(models, probe, reply.text) + return { + "answer": reply.text, + "reader_input_tokens": reply.input_tokens, + "reader_output_tokens": reply.output_tokens, + "score": found.score, + "nugget": found.nugget, + "nuggets": list(found.nuggets), + "tau_norm": found.tau_norm, + "f1": found.f1, + "tau_undefined": found.tau_undefined, + } + + +def run_on_store( + job: Job, + conversation: Conversation, + store: Store, + models: ThreadModels, + capture: bool, + finished: frozenset[str], + count_rows: Callable[[Store], dict[str, int]] | None = None, +) -> Done: + """Capture `conversation` into `store` if `capture`, then run each probe not in `finished`. + + Appends a capture row and each probe row to `job.rows_file`. With `count_rows`, the + capture row holds the row count of each table after the capture. + """ + started = time.perf_counter() + done = Done(conversation.id) + namespace = (*NAMESPACE, conversation.id) + windows = conversation.windows() + if capture: + done.capture = capture_conversation(store, models, windows, namespace) + if count_rows is not None: + done.capture["rows"] = count_rows(store) + done.capture["seconds"] = round(time.perf_counter() - started, 1) + append_row( + job.rows_file, {"kind": "capture", "conversation": conversation.id, **done.capture} + ) + print( + f"{conversation.id} captured {len(windows)} windows, " + f"{done.capture['windows_at_limit']} at the limit, " + f"rows {done.capture.get('rows')}, spent ${models.spent:.2f}", + flush=True, + ) + probes = [probe for probe in conversation.probes if probe.id not in finished] + blocks = { + (probe.id, arm): BLOCKS[arm](store, namespace, probe.question) + for probe in probes + for arm in job.arms + } + tasks = [(probe, arm, reader) for probe in probes for arm in job.arms for reader in job.readers] + cells: dict[tuple[str, str, str], dict[str, Any]] = {} + with ThreadPoolExecutor(JUDGE_THREADS) as pool: + futures = { + pool.submit(answer_and_score, models, probe, blocks[(probe.id, arm)], reader): ( + probe.id, + arm, + reader, + ) + for probe, arm, reader in tasks + } + for future in as_completed(futures): + cells[futures[future]] = future.result() + for probe in probes: + append_row(job.rows_file, probe_row(conversation, probe, windows, blocks, cells, job)) + done.probes += 1 + print(f"{conversation.id} finished {len(probes)} probes, spent ${models.spent:.2f}", flush=True) + return done + + +def probe_row( + conversation: Conversation, + probe: Probe, + windows: Sequence[Window], + blocks: Mapping[tuple[str, str], Block], + cells: Mapping[tuple[str, str, str], dict[str, Any]], + job: Job, +) -> dict[str, Any]: + """Return the row of the result file for one probe.""" + found = source_windows(windows, probe) + arms = {} + for arm in job.arms: + block = blocks[(probe.id, arm)] + arms[arm] = { + "context_chars": len(block.text), + "block_sha256": hashlib.sha256(block.text.encode()).hexdigest(), + "facts_in_block": block.facts, + "episodes_in_block": list(block.episodes), + "source_windows": sorted(found), + "source_coverage": coverage(found, block), + } + return { + "kind": "probe", + "id": probe.id, + "conversation": conversation.id, + "domain": conversation.domain, + "ability": probe.ability, + "arms": arms, + "combos": { + f"{arm}/{reader}": cells[(probe.id, arm, reader)] + for arm in job.arms + for reader in job.readers + }, + } + + +def run_conversation( + job: Job, + conversation: Conversation, + limit: float, + capture: bool, + finished: frozenset[str], +) -> Done: + """Run `conversation` over its kept local database, as `run_on_store` does. + + Returns every error as the text of `failure`, so that no exception crosses the process + pool. The database is kept in every case. + """ + from bench.models import Offline, Remote + + started = time.perf_counter() + done = Done(conversation.id) + models = ThreadModels( + Offline() if job.replay else Remote(), CACHE_DIR / "replies.sqlite", limit + ) + embedder = embed.openai_embedder() + try: + name = f"{job.kept}-{conversation.id}" + with local_store(job.url, name, embedder, "never") as (stores, _): + store = stores[CONFIG.recall_with] + done = run_on_store(job, conversation, store, models, capture, finished, table_rows) + except Exception as error: + done.failure = f"{conversation.id}: {type(error).__name__}: {error}" + print(f"{conversation.id} failed: {done.failure}", flush=True) + finally: + done.spent, done.calls = models.spent, models.calls + done.embedder_misses = embedder.misses + done.seconds = round(time.perf_counter() - started, 1) + models.close() + embedder.close() + return done + + +def read_rows(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: + """Return the header, the capture rows, and the probe rows of a rows file. + + A last line that is not complete JSON is left out. + """ + lines = path.read_text().splitlines() + header: dict[str, Any] = json.loads(lines[0])["header"] + captures: list[dict[str, Any]] = [] + probes: list[dict[str, Any]] = [] + for line in lines[1:]: + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + (captures if row["kind"] == "capture" else probes).append(row) + return header, captures, probes + + +def summarize( + rows: Sequence[Mapping[str, Any]], arms: Sequence[str], readers: Sequence[str] +) -> dict[str, Any]: + """Return the scores of each arm and reader over `rows`, the probe rows of a run.""" + return { + f"{arm}/{reader}": summarize_scores( + [ + { + "ability": row["ability"], + "domain": row["domain"], + "score": row["combos"][f"{arm}/{reader}"]["score"], + } + for row in rows + ] + ) + for arm in arms + for reader in readers + } + + +def choose(split: str, sample: str, conversations: Sequence[Conversation]) -> list[Conversation]: + """Return the conversations of `sample` in `split`, in the order of the split. + + The smoke sample is the conversation of sample A with the fewest windows. + """ + if sample == ALL: + return list(conversations) + if split != SAMPLE_SPLIT: + raise ValueError(f"sample {sample} needs --split {SAMPLE_SPLIT}") + ids = set(sample_ids("a" if sample == SMOKE else sample, conversations)) + chosen = [conversation for conversation in conversations if conversation.id in ids] + if sample == SMOKE: + chosen = [min(chosen, key=lambda c: len(c.windows()))] + return chosen + + +def main(argv: Sequence[str] | None = None) -> Path | None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else "") + parser.add_argument("--split", choices=sorted(SPLITS), default=SAMPLE_SPLIT) + parser.add_argument("--sample", choices=(*SAMPLES, SMOKE, ALL), default=SMOKE) + parser.add_argument("--arms", default=",".join(ARMS), help="a comma-separated list of arms") + parser.add_argument( + "--readers", default=",".join(READERS), help="a comma-separated list of readers" + ) + parser.add_argument( + "--run", action="store_true", help="call the models; without it, only estimate" + ) + parser.add_argument( + "--replay", action="store_true", help="call no model: use cached replies, fail at a miss" + ) + parser.add_argument( + "--reuse", action="store_true", help="read from the kept databases, with no capture" + ) + parser.add_argument( + "--resume", + action="store_true", + help="skip what the rows file of an earlier run of this command holds", + ) + parser.add_argument("--workers", type=int, default=1, help="conversations at a time") + parser.add_argument("--limit", type=int, default=None, help="the first N conversations") + parser.add_argument( + "--extract-rate", + type=float, + default=Rates.extract_window, + help="US dollars of extraction for each window, for the estimate", + ) + parser.add_argument( + "--window-seconds", + type=float, + default=Rates.window_seconds, + help="worker seconds for each window, for the estimate", + ) + parser.add_argument("--tag", default="", help="a suffix for the name of the result file") + parser.add_argument("--url", default="http://localhost:3000") + args = parser.parse_args(argv) + arms = tuple(arm for arm in args.arms.split(",") if arm) + readers = tuple(reader for reader in args.readers.split(",") if reader) + if not arms or not set(arms) <= set(ARMS): + parser.error(f"arms must be among {', '.join(ARMS)}") + if not readers or not set(readers) <= set(READERS): + parser.error(f"readers must be among {', '.join(READERS)}") + if args.workers < 1: + parser.error("workers must be at least 1") + try: + conversations = choose(args.split, args.sample, load_split(args.split)) + except ValueError as error: + parser.error(str(error)) + conversations = conversations[: args.limit] + + rates = Rates(extract_window=args.extract_rate, window_seconds=args.window_seconds) + expected = estimate(conversations, arms, readers, args.workers, rates, not args.reuse) + for part, cost in expected.parts.items(): + print(f"estimate {part}: ${cost:.2f}") + print( + f"estimate in total, {len(conversations)} conversations, " + f"{sum(len(c.probes) for c in conversations)} probes: ${expected.total:.2f}, " + f"and the limit is ${COST_LIMIT:.0f}" + ) + print( + f"expected time with {args.workers} workers: {expected.seconds / 60:.0f} minutes; " + f"a conversation stops at {SPEND_FACTOR:g} times its estimate" + ) + if expected.total > COST_LIMIT: + print("the estimate is over the limit: stop") + return None + if not args.run and not args.replay: + print("no model was called; pass --run to run, or --replay to run from the cache") + return None + + try: + kept = kept_name(f"beam-{args.split.lower()}", CONFIG, library_commit()) + except RuntimeError as error: + parser.error(str(error)) + name = f"beam-{args.split}-{args.sample}" + name += f"-{args.tag}" if args.tag else "" + name += f"-first{args.limit}" if args.limit is not None else "" + path = RESULTS_DIR / f"{name}.json" + rows_file = RESULTS_DIR / f"{name}.rows.jsonl" + header = { + "split": args.split, + "sample": args.sample, + "limit": args.limit, + "arms": list(arms), + "readers": list(readers), + "config": asdict(CONFIG), + "kept": kept, + "reuse": args.reuse, + } + captured: set[str] = set() + finished: set[str] = set() + if args.resume: + if not rows_file.exists(): + parser.error(f"--resume found no rows file {rows_file}") + found_header, earlier_captures, earlier_probes = read_rows(rows_file) + if found_header != header: + parser.error(f"the rows file {rows_file} is of another run: {found_header}") + captured = {row["conversation"] for row in earlier_captures} + finished = {row["id"] for row in earlier_probes} + elif rows_file.exists(): + parser.error(f"the rows file {rows_file} exists: pass --resume, or delete it") + left = [c for c in conversations if any(p.id not in finished for p in c.probes)] + if args.resume: + print(f"resuming: {len(finished)} probes finished, {len(left)} conversations left") + + names = [f"{kept}-{conversation.id}" for conversation in conversations] + found = existing_databases(args.url, names) + if args.reuse: + missing = sorted(set(names) - set(found)) + if missing: + parser.error(f"{len(missing)} kept databases do not exist, for example {missing[0]}") + elif found and not args.resume: + parser.error( + f"{len(found)} kept databases exist already, for example {found[0]}: " + "pass --reuse to read from them, --resume to go on, or delete them" + ) + print(f"{'reusing' if args.reuse else 'keeping'} the memory as {kept}-") + + commit = _commit() + started = datetime.now(UTC) + run_id = f"{started:%Y%m%d%H%M%S}-{uuid.uuid4().hex[:6]}" + clock_start = time.perf_counter() + RESULTS_DIR.mkdir(exist_ok=True) + if not args.resume: + rows_file.write_text(json.dumps({"header": header}) + "\n") + job = Job(args.url, arms, readers, args.replay, kept, rows_file) + order = sorted(left, key=lambda c: len(c.windows()), reverse=True) + frozen = frozenset(finished) + calls = [ + ( + job, + conversation, + SPEND_FACTOR * expected.by_conversation[conversation.id], + not args.reuse and conversation.id not in captured, + frozen, + ) + for conversation in order + ] + workers = min(args.workers, len(calls)) if calls else 0 + if workers <= 1: + done = [run_conversation(*call) for call in calls] + else: + context = multiprocessing.get_context("spawn") + with ProcessPoolExecutor(workers, mp_context=context) as pool: + done = list(pool.map(run_conversation, *zip(*calls, strict=True))) + spent = sum(found_done.spent for found_done in done) + failed = [found_done.failure for found_done in done if found_done.failure] + if failed: + print(f"{len(failed)} conversations failed; spent ${spent:.2f}") + for failure in failed: + print(f" {failure}") + print(f"the finished rows are in {rows_file}; run the command again with --resume") + return None + + _, captures, probe_rows = read_rows(rows_file) + by_id = {row["id"]: row for row in probe_rows} + rows = [by_id[probe.id] for conversation in conversations for probe in conversation.probes] + capture_of = {row["conversation"]: row for row in captures} + summaries = summarize(rows, arms, readers) + data_name, data_sha256 = SPLITS[args.split] + result_file = { + "config": { + "benchmark": "BEAM", + "split": args.split, + "data_file": data_name, + "data_revision": DATA_REVISION, + "data_sha256": data_sha256, + "sample": ( + {"name": ALL, "conversations": len(conversations)} + if args.sample == ALL + else { + "name": args.sample, + "file": SAMPLES["a" if args.sample == SMOKE else args.sample].file, + "seed": SAMPLES["a" if args.sample == SMOKE else args.sample].seed, + "conversations": [c.id for c in conversations], + } + ), + "readers": { + reader: { + "model": READERS[reader], + "temperature": 0, + "max_tokens": READER_MAX_TOKENS, + "prompt": f"answer_generation_for_rag, BEAM {BEAM_COMMIT}", + } + for reader in readers + }, + "judge": { + "model": JUDGE, + "temperature": 0, + "max_tokens": JUDGE_MAX_TOKENS, + "prompt": f"unified_llm_judge_base_prompt, BEAM {BEAM_COMMIT}", + "event_ordering": "tau_norm of the official line match", + "question_shown": False, + }, + "extractor": { + "variant": CONFIG.extractor, + "model": CONFIG.variant.model, + "max_tokens": EXTRACTOR_MAX_TOKENS, + }, + "embedder": {"model": embed.MODEL, "dimensions": embed.DIMENSIONS}, + "driver": "hotdata", + "location": "local", + "capture": { + "window_chars": WINDOW_CHARS, + "chunk_chars": CONFIG.chunk_chars, + "kept": kept, + "reused": args.reuse, + }, + "recall": { + "ranking": CONFIG.recall_with, + "k": CONFIG.k, + "budget": CONFIG.budget, + "episodes": CONFIG.episodes, + "as_of": None, + }, + "arms": { + "memory": "recall of facts, then chunks found by search, in the budget", + "chunks": f"the top {CONFIG.k} chunks by vector search, in the budget", + }, + "conversations": len(conversations), + "probes": len(rows), + "estimate_usd": round(expected.total, 2), + "estimate_s": round(expected.seconds), + "replay": args.replay, + "workers": workers, + "spent_usd": round(spent, 2), + "resumed_probes": len(finished), + "model_calls": sum(found_done.calls for found_done in done), + "embedder_misses": sum(found_done.embedder_misses for found_done in done), + "windows_at_limit": sum(row["windows_at_limit"] for row in captures), + "hotmemory_commit": commit, + "run_id": run_id, + "started_at": started.isoformat(), + "duration_s": round(time.perf_counter() - clock_start, 1), + }, + "summary": summaries, + "captures": [capture_of[c.id] for c in conversations if c.id in capture_of], + "probes": rows, + } + path.write_text(json.dumps(result_file, indent=1) + "\n") + rows_file.unlink() + for combo, found_summary in summaries.items(): + print( + f"{combo}: mean over abilities {found_summary['mean']:.3f}, " + f"over questions {found_summary['questions']:.3f}" + ) + print(f"spent ${spent:.2f}; wrote {path}") + return path + + +if __name__ == "__main__": + main() diff --git a/bench/beam_prompts/LICENSE b/bench/beam_prompts/LICENSE new file mode 100644 index 0000000..3f9b9d0 --- /dev/null +++ b/bench/beam_prompts/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Mohammad Tavakoli + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bench/beam_prompts/answer.txt b/bench/beam_prompts/answer.txt new file mode 100644 index 0000000..0797b7f --- /dev/null +++ b/bench/beam_prompts/answer.txt @@ -0,0 +1,18 @@ + +You are an assistant that MUST answer questions using ONLY the information provided in the context below. + +STRICT INSTRUCTIONS: +1. Answer ONLY based on the provided context +2. Do NOT use your internal knowledge + +CONTEXT: + + +QUESTION: + + +ANSWER REQUIREMENTS: +- Be direct and concise +- Only output the answer to the question without any explanation + +RESPONSE: diff --git a/bench/beam_prompts/judge.txt b/bench/beam_prompts/judge.txt new file mode 100644 index 0000000..0938076 --- /dev/null +++ b/bench/beam_prompts/judge.txt @@ -0,0 +1,69 @@ + +You are an expert evaluator tasked with judging whether the LLM's response demonstrates compliance with the specified RUBRIC CRITERION. + +## EVALUATION INPUTS +- RUBRIC CRITERION (what to check): +- RESPONSE TO EVALUATE: + +## EVALUATION RUBRIC: +The rubric defines a specific requirement, constraint, or expected behavior that the LLM response should demonstrate. + +**IMPORTANT**: Pay careful attention to whether the rubric specifies: +- **Positive requirements** (things the response SHOULD include/do) +- **Negative constraints** (things the response SHOULD NOT include/do, often indicated by "no", "not", "avoid", "absent") + +## RESPONSIVENESS REQUIREMENT +A compliant response must be **on-topic** and attempt to answer it. +- If the response does not address the QUESTION, score **0.0** and stop. +- For negative constraints, both must hold: (a) the response is responsive to the QUESTION, and (b) the prohibited element is absent. + +## SEMANTIC TOLERANCE RULES: +Judge by meaning, not exact wording. +- Accept **paraphrases** and **synonyms** that preserve intent. +- **Case/punctuation/whitespace** differences must be ignored. +- **Numbers/currencies/dates** may appear in equivalent forms (e.g., “$68,000”, “68k”, “68,000 USD”, or “sixty-eight thousand dollars”). Treat them as equal when numerically equivalent. +- If the rubric expects a number or duration, prefer **normalized comparison** (extract and compare values) over string matching. + +## STYLE NEUTRALITY (prevents style contamination): +Ignore tone, politeness, length, and flourish unless the rubric explicitly requires a format/structure (e.g., “itemized list”, “no citations”, “one sentence”). +- Do **not** penalize hedging, voice, or verbosity if content satisfies the rubric. +- Only evaluate format when the rubric **explicitly** mandates it. + +## SCORING SCALE: +- **1.0 (Complete Compliance)**: Fully complies with the rubric criterion. + - Positive: required element present, accurate, properly executed (allowing semantic equivalents). + - Negative: prohibited element **absent** AND response is **responsive**. + +- **0.5 (Partial Compliance)**: Partially complies. + - Positive: element present but minor inaccuracies/incomplete execution. + - Negative: generally responsive and mostly avoids the prohibited element but with minor/edge violations. + +- **0.0 (No Compliance)**: Fails to comply. + - Positive: required element missing or incorrect. + - Negative: prohibited element present **or** response is non-responsive/evasive even if the element is absent. + +## EVALUATION INSTRUCTIONS: +1. **Understand the Requirement**: Determine if the rubric is asking for something to be present (positive) or absent (negative/constraint). + +2. **Parse Compound Statements**: If the rubric contains multiple elements connected by "and" or commas, evaluate whether: + - **All elements** must be present for full compliance (1.0) + - **Some elements** present indicates partial compliance (0.5) + - **No elements** present indicates no compliance (0.0) + +3. **Check Compliance**: + - For positive requirements: Look for the presence and quality of the required element + - For negative constraints: Look for the absence of the prohibited element + +4. **Assign Score**: Based on compliance with the specific rubric criterion according to the scoring scale above. + +5. **Provide Reasoning**: Explain whether the rubric criterion was satisfied and justify the score. + +## OUTPUT FORMAT: +Return your evaluation in JSON format with two fields: + +{ + "score": [your score: 1.0, 0.5, or 0.0], + "reason": "[detailed explanation of whether the rubric criterion was satisfied and why this justified the assigned score]" +} + +NOTE: ONLY output the json object, without any explanation before or after that diff --git a/bench/beam_prompts/match_system.txt b/bench/beam_prompts/match_system.txt new file mode 100644 index 0000000..a5c9784 --- /dev/null +++ b/bench/beam_prompts/match_system.txt @@ -0,0 +1,6 @@ + + You are a binary classifier. + If the TWO snippets describe the SAME event/fact, reply **YES** + Otherwise reply **NO**. No extra words. + DO NOT provide any exaplanation. + \ No newline at end of file diff --git a/bench/beam_prompts/match_user.txt b/bench/beam_prompts/match_user.txt new file mode 100644 index 0000000..fb95413 --- /dev/null +++ b/bench/beam_prompts/match_user.txt @@ -0,0 +1,4 @@ +First snippet: {first} + + Second snippet: {second} + \ No newline at end of file diff --git a/bench/beam_score.py b/bench/beam_score.py new file mode 100644 index 0000000..1adb7a2 --- /dev/null +++ b/bench/beam_score.py @@ -0,0 +1,206 @@ +"""The BEAM answer prompt and score, as the official code at commit b2da22e gives them. + +The prompts in `bench/beam_prompts/` are copied from github.com/mohammadtavakoli78/BEAM at +commit b2da22e, under the MIT license in `bench/beam_prompts/LICENSE`. The functions here +follow `src/evaluation/compute_metrics.py` and `src/evaluation/report_results.py` there: + +- Each nugget of the rubric gets one judge call with the unified judge prompt, which sees + the nugget and the response but not the question. The reply is parsed as JSON, and its + `score` is 0, 0.5, or 1. The nugget score of a question is the mean. +- Event ordering also splits the response on newlines, matches each line to a nugget with + one binary call for each unmatched nugget, and computes Kendall tau-b over the matched + lists. The report scores event ordering by `tau_norm`, (tau-b + 1) / 2, alone. +- The aggregate is the mean over the 10 abilities. +""" + +from __future__ import annotations + +import json +import math +import re +import statistics +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from json_repair import repair_json +from scipy.stats import kendalltau + +from bench.beam import ABILITIES, Probe +from bench.models import BEAM_MINI, Reply + +BEAM_COMMIT = "b2da22e" +PROMPTS = Path(__file__).resolve().parent / "beam_prompts" +ANSWER_PROMPT = (PROMPTS / "answer.txt").read_text() +JUDGE_PROMPT = (PROMPTS / "judge.txt").read_text() +MATCH_SYSTEM = (PROMPTS / "match_system.txt").read_text() +MATCH_USER = (PROMPTS / "match_user.txt").read_text() +JUDGE = BEAM_MINI +JUDGE_MAX_TOKENS = 1000 +MATCH_MAX_TOKENS = 50 +EVENT_ORDERING = "event_ordering" +Models = Callable[..., Reply] + + +@dataclass(frozen=True) +class Score: + """The score of one response to one probe. + + `score` is the official score: `tau_norm` for event ordering, else `nugget`. `nugget` + is the mean of `nuggets`, the judge score of each nugget. For event ordering, `f1` is + the F1 of the line match, and `tau_undefined` is True if Kendall tau-b was not defined, + which the official code leaves as NaN and this score counts as 0. + """ + + score: float + nugget: float + nuggets: tuple[float, ...] + tau_norm: float | None = None + f1: float | None = None + tau_undefined: bool = False + + +def answer_prompt(context: str, question: str) -> str: + """Return the official RAG answer prompt for `question` over `context`.""" + return ANSWER_PROMPT.replace("", context).replace("", question) + + +def judge_prompt(item: str, response: str) -> str: + """Return the official judge prompt for one rubric item and one response.""" + return JUDGE_PROMPT.replace("", item).replace("", response) + + +def parse_json_response(response: str) -> Any: + """Return the JSON in `response`, as the official `parse_json_response` finds it. + + Raises ValueError if no JSON is found. + """ + response = response.strip() + if response.startswith("```"): + match = re.search(r"```(?:json)?\s*(\[.*\]|\{.*\})\s*```", response, re.DOTALL) + if match: + response = match.group(1).strip() + try: + return json.loads(response) + except json.JSONDecodeError: + pass + match = re.search(r"(\{.*?\}|\[.*?\])", response, re.DOTALL) + if match: + try: + return json.loads(match.group(1)) + except Exception as error: + raise ValueError(f"Found possible JSON but failed to parse it: {error}") from error + raise ValueError("No valid JSON found in response.") + + +def judge_reply(text: str) -> float: + """Return the score of one judge reply, with the official parse and its repair fallback.""" + try: + parsed = parse_json_response(text.strip()) + except ValueError: + parsed = json.loads(repair_json(text.strip())) + return float(parsed["score"]) + + +def nugget_scores(models: Models, rubric: Sequence[str], response: str) -> tuple[float, ...]: + """Return the judge score of each rubric item for `response`, one call each.""" + return tuple( + judge_reply(models(JUDGE, judge_prompt(item, response), JUDGE_MAX_TOKENS).text) + for item in rubric + ) + + +def same_event(models: Models, first: str, second: str) -> bool: + """Return True if the binary classifier says the two snippets name the same event.""" + prompt = MATCH_USER.replace("{first}", first, 1).replace("{second}", second, 1) + reply = models(JUDGE, prompt, MATCH_MAX_TOKENS, None, system=MATCH_SYSTEM) + return "yes" in reply.text.lower() + + +def align( + models: Models, reference: Sequence[str], system: Sequence[str] +) -> tuple[list[str], list[str]]: + """Return `reference` and `system` with each line replaced by its first unused match. + + Each line of `system` is compared with each reference item that no earlier line + matched, in order, until one matches. A line with no match stays as it is. + """ + used: set[int] = set() + out = [] + for line in system: + matched = None + for index, item in enumerate(reference): + if index not in used and same_event(models, item, line): + matched = index + break + if matched is None: + out.append(line) + else: + out.append(reference[matched]) + used.add(matched) + return list(reference), out + + +def order_score(reference: Sequence[str], system: Sequence[str]) -> dict[str, float]: + """Return the precision, recall, F1, `tau_norm`, and final score of two aligned lists. + + `tau_norm` is NaN if Kendall tau-b is not defined, as in the official code. + """ + tp = len(set(reference) & set(system)) + fp = len([x for x in system if x not in reference]) + fn = len([x for x in reference if x not in system]) + precision = tp / (tp + fp) if tp + fp else 0 + recall = tp / (tp + fn) if tp + fn else 0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0 + union = list(dict.fromkeys([*reference, *system])) + tie_rank = len(union) + 1 + + def to_rank(seq: Sequence[str]) -> list[int]: + ranks = {item: i + 1 for i, item in enumerate(seq)} + return [ranks.get(item, tie_rank) for item in union] + + tau_b = kendalltau(to_rank(reference), to_rank(system), variant="b", method="auto")[0] + tau_norm = (float(tau_b) + 1) / 2 + return { + "precision": precision, + "recall": recall, + "f1": f1, + "tau_norm": tau_norm, + "final_score": tau_norm * f1, + } + + +def score(models: Models, probe: Probe, response: str) -> Score: + """Return the official score of `response` to `probe`.""" + nuggets = nugget_scores(models, probe.rubric, response) + nugget = sum(nuggets) / len(nuggets) + if probe.ability != EVENT_ORDERING: + return Score(nugget, nugget, nuggets) + reference, system = align(models, probe.rubric, response.split("\n")) + order = order_score(reference, system) + undefined = math.isnan(order["tau_norm"]) + tau_norm = 0.0 if undefined else order["tau_norm"] + return Score(tau_norm, nugget, nuggets, tau_norm, order["f1"], undefined) + + +def summarize(rows: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Return the scores of `rows`, each with `ability`, `domain`, and `score`. + + `mean` is the mean over the abilities that `rows` hold, the aggregate of the official + report. `questions` is the mean over every row. + """ + by_ability = { + ability: statistics.mean(r["score"] for r in rows if r["ability"] == ability) + for ability in ABILITIES + if any(r["ability"] == ability for r in rows) + } + domains = sorted({r["domain"] for r in rows}) + by_domain = {d: statistics.mean(r["score"] for r in rows if r["domain"] == d) for d in domains} + return { + "mean": statistics.mean(by_ability.values()) if by_ability else 0.0, + "questions": statistics.mean(r["score"] for r in rows) if rows else 0.0, + "count": len(rows), + "by_ability": by_ability, + "by_domain": by_domain, + } diff --git a/bench/models.py b/bench/models.py index 7848589..0f6fdbc 100644 --- a/bench/models.py +++ b/bench/models.py @@ -16,11 +16,18 @@ READER_MAX_TOKENS = 800 JUDGE_MAX_TOKENS = 10 EXTRACTOR_MAX_TOKENS = 4000 +BEAM_MINI = "gpt-4.1-mini-2025-04-14" PRICES = { READER: (2.50, 10.00), EXTRACTOR: (0.15, 0.60), + BEAM_MINI: (0.40, 1.60), } -"""US dollars for one million input tokens and one million output tokens, read on 2026-10-10.""" +"""US dollars for one million input tokens and one million output tokens. + +The first two were read on 2026-10-10, and `gpt-4.1-mini` on 2026-10-26. +""" +EMBEDDING_PRICE = 0.02 +"""US dollars for one million tokens of `text-embedding-3-small`, read on 2026-10-26.""" @dataclass(frozen=True) @@ -43,17 +50,24 @@ def cost(self, model: str) -> float: class Complete(Protocol): - """Sends one prompt to one model and returns the reply.""" + """Sends one prompt to one model, after `system` as a system message if given, and returns + the reply.""" def __call__( - self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None + self, + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None, + system: str | None = None, ) -> Reply: ... class CachedModels: """Calls `complete` for each prompt that the cache does not hold, and keeps the reply. - The cache key is the SHA-256 of the model, the token limit, the schema, and the prompt. + The cache key is the SHA-256 of the model, the token limit, the schema, and the prompt, + and of the system message if there is one. `spent` sums the price of the replies that came from a model call in this process, and `calls` counts those calls. """ @@ -63,19 +77,25 @@ def __init__(self, complete: Complete, path: Path) -> None: self.calls = 0 self._complete = complete path.parent.mkdir(parents=True, exist_ok=True) - self._db = sqlite3.connect(path, timeout=60) + self._db = sqlite3.connect(path, timeout=60, check_same_thread=False) self._db.execute("CREATE TABLE IF NOT EXISTS replies (key TEXT PRIMARY KEY, reply TEXT)") def __call__( - self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None = None + self, + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None = None, + system: str | None = None, ) -> Reply: - key = hashlib.sha256( - json.dumps([model, max_tokens, schema, prompt], sort_keys=True).encode() - ).hexdigest() + parts: list[Any] = [model, max_tokens, schema, prompt] + if system is not None: + parts.append(system) + key = hashlib.sha256(json.dumps(parts, sort_keys=True).encode()).hexdigest() row = self._db.execute("SELECT reply FROM replies WHERE key = ?", (key,)).fetchone() if row is not None: return Reply(**json.loads(row[0])) - reply = self._complete(model, prompt, max_tokens, schema) + reply = self._complete(model, prompt, max_tokens, schema, system) self.calls += 1 self.spent += reply.cost(model) with self._db: @@ -98,7 +118,12 @@ class Offline: """ def __call__( - self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None + self, + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None, + system: str | None = None, ) -> Reply: raise CacheMiss(f"the cache holds no reply of {model} to this prompt: {prompt[:80]!r}") @@ -123,8 +148,16 @@ def __init__(self) -> None: ) def __call__( - self, model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None + self, + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None, + system: str | None = None, ) -> Reply: + messages: list[Any] = [{"role": "user", "content": prompt}] + if system is not None: + messages.insert(0, {"role": "system", "content": system}) extra: dict[str, Any] = {} if schema is not None: extra["response_format"] = { @@ -133,7 +166,7 @@ def __call__( } completion = self._openai.chat.completions.create( model=model, - messages=[{"role": "user", "content": prompt}], + messages=messages, n=1, temperature=0, max_tokens=max_tokens, diff --git a/bench/sample-beam-a.json b/bench/sample-beam-a.json new file mode 100644 index 0000000..1b7e741 --- /dev/null +++ b/bench/sample-beam-a.json @@ -0,0 +1,27 @@ +[ + { + "conversation": "2", + "domain": "Coding", + "windows": 69 + }, + { + "conversation": "6", + "domain": "Writing Assistant & Learning", + "windows": 80 + }, + { + "conversation": "12", + "domain": "Philosophical or Ethical Discussion", + "windows": 128 + }, + { + "conversation": "17", + "domain": "Lifestyle", + "windows": 68 + }, + { + "conversation": "18", + "domain": "Therapy and Emotional Support", + "windows": 99 + } +] diff --git a/bench/sample-beam-b.json b/bench/sample-beam-b.json new file mode 100644 index 0000000..347bd28 --- /dev/null +++ b/bench/sample-beam-b.json @@ -0,0 +1,27 @@ +[ + { + "conversation": "1", + "domain": "Coding", + "windows": 74 + }, + { + "conversation": "4", + "domain": "Math", + "windows": 76 + }, + { + "conversation": "7", + "domain": "Writing Assistant & Learning", + "windows": 74 + }, + { + "conversation": "14", + "domain": "Asking Recommendation", + "windows": 58 + }, + { + "conversation": "19", + "domain": "Legal and Administrative", + "windows": 104 + } +] diff --git a/bench/tests/test_beam.py b/bench/tests/test_beam.py new file mode 100644 index 0000000..da454fd --- /dev/null +++ b/bench/tests/test_beam.py @@ -0,0 +1,199 @@ +import hashlib +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from bench.beam import ( + ABILITIES, + GOLD_FIELDS, + Conversation, + choose_sample, + counts, + cut, + load, + parse_conversation, + sample_ids, + strip_marker, +) + + +def turn(id: int, role: str, content: str, anchor: str | None = None) -> dict[str, Any]: + return { + "id": id, + "role": role, + "content": content, + "time_anchor": anchor, + "index": None, + "question_type": None, + } + + +def probes() -> dict[str, list[dict[str, Any]]]: + found = {} + for ability in ABILITIES: + found[ability] = [ + { + "question": f"{ability} question {n}?", + GOLD_FIELDS[ability]: f"{ability} gold {n}", + "rubric": [f"{ability} nugget {n}a", f"{ability} nugget {n}b"], + "difficulty": "easy", + "source_chat_ids": {"first": [3, 1], "second": [[1, 5]]} if n == 1 else [2], + } + for n in (1, 2) + ] + return found + + +def row(conversation: str = "7", chat: list[list[dict[str, Any]]] | None = None) -> dict[str, Any]: + if chat is None: + chat = [ + [ + turn(0, "user", "First question? ->-> 1,1", "March-15-2024"), + turn(1, "assistant", "First answer ->-> 1,1"), + turn(2, "user", "Second question ->->2,3"), + turn(3, "assistant", "Second answer"), + ], + [ + turn(4, "user", "Later question", "April-02-2024"), + turn(5, "assistant", "Later answer"), + ], + ] + return { + "conversation_id": conversation, + "conversation_seed": {"category": "Coding", "id": 1, "theme": "t", "title": "t"}, + "chat": chat, + "probing_questions": repr(probes()), + } + + +@pytest.fixture +def conversation() -> Conversation: + return parse_conversation(row()) + + +def test_strip_marker_removes_only_the_end_marker() -> None: + assert strip_marker("Hello ->-> 1,1") == "Hello" + assert strip_marker("Hello ->->12, 3 ") == "Hello" + assert strip_marker("a ->-> b is kept") == "a ->-> b is kept" + + +def test_batches_have_their_dates_and_stripped_user_turns(conversation: Conversation) -> None: + assert conversation.id == "7" + assert conversation.domain == "Coding" + first, later = conversation.batches + assert (first.number, first.date) == (1, datetime(2024, 3, 15, tzinfo=UTC)) + assert later.date == datetime(2024, 4, 2, tzinfo=UTC) + assert [m.content for m in first.messages] == [ + "First question?", + "First answer ->-> 1,1", + "Second question", + "Second answer", + ] + + +def test_probes_follow_the_abilities_with_one_gold_field(conversation: Conversation) -> None: + assert len(conversation.probes) == 20 + assert [p.ability for p in conversation.probes[::2]] == list(ABILITIES) + first = conversation.probes[0] + assert first.id == "7-abstention-1" + assert first.gold == "abstention gold 1" + assert first.rubric == ("abstention nugget 1a", "abstention nugget 1b") + assert first.sources == (1, 3, 5) + assert conversation.probes[1].sources == (2,) + for probe in conversation.probes: + assert probe.gold == f"{probe.ability} gold {probe.id[-1]}" + + +def test_a_batch_needs_exactly_one_date() -> None: + with pytest.raises(ValueError, match="dates"): + parse_conversation(row(chat=[[turn(0, "user", "q"), turn(1, "assistant", "a")]])) + two = [[turn(0, "user", "q", "March-01-2024"), turn(1, "assistant", "a", "March-02-2024")]] + with pytest.raises(ValueError, match="dates"): + parse_conversation(row(chat=two)) + + +def test_a_probe_needs_its_gold_field() -> None: + found = probes() + del found["summarization"][0]["ideal_summary"] + with pytest.raises(ValueError, match="no ideal_summary"): + parse_conversation({**row(), "probing_questions": repr(found)}) + + +def test_windows_hold_whole_pairs_under_the_limit(conversation: Conversation) -> None: + pair = len("user: First question?\n\nassistant: First answer ->-> 1,1") + windows = conversation.windows(limit=pair + 5) + assert [(w.batch, w.number, w.key) for w in windows] == [ + (1, 1, "window-01-0001"), + (1, 2, "window-01-0002"), + (2, 1, "window-02-0001"), + ] + assert windows[0].text == "user: First question?\n\nassistant: First answer ->-> 1,1" + assert windows[0].messages == (0, 1) + assert windows[1].messages == (2, 3) + assert windows[2].date == datetime(2024, 4, 2, tzinfo=UTC) + whole = conversation.windows() + assert len(whole) == 2 + assert whole[0].text.count("user: ") == 2 + + +def test_a_long_pair_is_cut_into_pieces() -> None: + long = "\n\n".join(["x" * 30] * 4) + chat = [[turn(0, "user", "q", "March-01-2024"), turn(1, "assistant", long)]] + windows = parse_conversation(row(chat=chat)).windows(limit=70) + assert len(windows) > 1 + assert all(len(w.text) <= 70 for w in windows) + assert all(w.messages == (0, 1) for w in windows) + assert "".join(w.text for w in windows).count("x") == 120 + + +def test_cut_splits_at_blank_lines_and_then_hard() -> None: + assert cut("aa\n\nbb\n\ncc", 6) == ["aa\n\nbb", "cc"] + assert cut("abcdefgh", 3) == ["abc", "def", "gh"] + assert all(len(piece) <= 5 for piece in cut("a\n\n" + "b" * 12 + "\n\nc", 5)) + + +def test_load_checks_the_hash_and_the_ids(tmp_path: Path) -> None: + import pyarrow + import pyarrow.parquet + + path = tmp_path / "beam.parquet" + pyarrow.parquet.write_table(pyarrow.Table.from_pylist([row("1"), row("2")]), path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert [c.id for c in load(path, digest)] == ["1", "2"] + with pytest.raises(ValueError, match="SHA-256"): + load(path, "0" * 64) + pyarrow.parquet.write_table(pyarrow.Table.from_pylist([row("1"), row("1")]), path) + with pytest.raises(ValueError, match="repeats a conversation id"): + load(path, None) + + +def test_counts(conversation: Conversation) -> None: + found = counts([conversation]) + assert found["conversations"] == 1 + assert found["questions"] == 20 + assert found["nuggets"] == 40 + assert found["windows"] == 2 + assert found["long_pairs"] == 0 + assert found["domains"] == ["Coding"] + + +def test_a_sample_takes_one_conversation_from_each_of_five_domains(tmp_path: Path) -> None: + domains = ["A", "A", "B", "B", "C", "C", "D", "D", "E", "E", "F", "F"] + conversations = [ + parse_conversation({**row(str(i)), "conversation_seed": {"category": domain, "id": i}}) + for i, domain in enumerate(domains, start=1) + ] + (tmp_path / "bench").mkdir() + a = sample_ids("a", conversations, tmp_path) + b = sample_ids("b", conversations, tmp_path) + by_id = {c.id: c for c in conversations} + assert len(a) == len(b) == 5 + assert len({by_id[i].domain for i in a}) == 5 + assert len({by_id[i].domain for i in b}) == 5 + assert not set(a) & set(b) + assert a == sample_ids("a", conversations[::-1], tmp_path) + assert choose_sample(conversations, 1) == choose_sample(conversations[::-1], 1) + with pytest.raises(ValueError, match="domains are left"): + choose_sample(conversations[:8], 1) diff --git a/bench/tests/test_beam_e2e.py b/bench/tests/test_beam_e2e.py new file mode 100644 index 0000000..669c7a5 --- /dev/null +++ b/bench/tests/test_beam_e2e.py @@ -0,0 +1,222 @@ +import json +import threading +from pathlib import Path +from typing import Any + +import pytest + +from bench.beam import Conversation, parse_conversation +from bench.beam_e2e import ( + CONFIG, + NAMESPACE, + Block, + Job, + SpendLimit, + ThreadModels, + capture_conversation, + choose, + chunks_block, + coverage, + estimate, + longest_worker, + read_rows, + run_on_store, + source_windows, + summarize, +) +from bench.beam_score import JUDGE, MATCH_SYSTEM +from bench.models import BEAM_MINI, EXTRACTOR, Reply +from bench.tests.test_beam import row +from hotmemory import Memory, MemoryStore +from tests.conftest import FakeClock, fake_embedder + + +class FakeModels: + """Facts from each user line, a fixed reader answer, and full marks from the judge.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + self.answers = 0 + self.lock = threading.Lock() + + def __call__( + self, + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None = None, + system: str | None = None, + ) -> Reply: + with self.lock: + self.calls.append(model) + if model == EXTRACTOR: + session = prompt.split("Session:\n", 1)[1] + facts = [ + {"subject": "user", "content": line.removeprefix("user: ")} + for line in session.split("\n\n") + if line.startswith("user: ") + ] + return Reply(json.dumps({"facts": facts}), 100, 10) + if model == JUDGE and system == MATCH_SYSTEM: + return Reply("NO", 10, 1) + if model == JUDGE and "RUBRIC CRITERION" in prompt: + return Reply('{"score": 1.0, "reason": "ok"}', 100, 10) + with self.lock: + self.answers += 1 + return Reply("The answer.", 1000, 5) + + +@pytest.fixture +def conversation() -> Conversation: + return parse_conversation(row()) + + +def job(tmp_path: Path) -> Job: + rows_file = tmp_path / "rows.jsonl" + rows_file.write_text(json.dumps({"header": {"run": 1}}) + "\n") + return Job("", ("memory", "chunks"), ("mini",), False, "kept", rows_file) + + +def store() -> MemoryStore: + return MemoryStore(embedder=fake_embedder, clock=FakeClock()) + + +def test_a_conversation_is_captured_once_and_every_probe_is_scored( + tmp_path: Path, conversation: Conversation +) -> None: + fake = FakeModels() + models = ThreadModels(fake, tmp_path / "replies.sqlite") + found = job(tmp_path) + memory = store() + done = run_on_store(found, conversation, memory, models, True, frozenset()) + assert done.probes == 20 + assert done.capture["windows"] == 2 + assert done.capture["windows_at_limit"] == 0 + assert done.capture["facts_written"] == 3 + header, captures, probes = read_rows(found.rows_file) + assert header == {"run": 1} + assert [c["conversation"] for c in captures] == ["7"] + assert len(probes) == 20 + first = probes[0] + assert set(first["combos"]) == {"memory/mini", "chunks/mini"} + assert first["arms"]["memory"]["facts_in_block"] == 3 + assert first["arms"]["chunks"]["facts_in_block"] == 0 + assert first["arms"]["chunks"]["episodes_in_block"] + assert first["arms"]["memory"]["source_windows"] == ["window-01-0001", "window-02-0001"] + nugget_rows = [p for p in probes if p["ability"] != "event_ordering"] + assert all(c["score"] == 1.0 for p in nugget_rows for c in p["combos"].values()) + ordering = [p for p in probes if p["ability"] == "event_ordering"] + assert all(c["tau_norm"] is not None for p in ordering for c in p["combos"].values()) + assert fake.calls.count(EXTRACTOR) == 2 + assert fake.answers == 40 + summaries = summarize(probes, ("memory", "chunks"), ("mini",)) + assert summaries["memory/mini"]["count"] == 20 + assert summaries["memory/mini"]["by_domain"].keys() == {"Coding"} + models.close() + + +def test_a_resumed_conversation_runs_only_the_probes_left( + tmp_path: Path, conversation: Conversation +) -> None: + fake = FakeModels() + models = ThreadModels(fake, tmp_path / "replies.sqlite") + memory = store() + capture_conversation(memory, models, conversation.windows(), (*NAMESPACE, "7")) + calls = len(fake.calls) + finished = frozenset(probe.id for probe in conversation.probes[:18]) + found = job(tmp_path) + done = run_on_store(found, conversation, memory, models, False, finished) + assert done.probes == 2 + assert done.capture == {} + assert EXTRACTOR not in fake.calls[calls:] + _, captures, probes = read_rows(found.rows_file) + assert captures == [] + assert [p["id"] for p in probes] == [p.id for p in conversation.probes[18:]] + models.close() + + +def test_a_second_capture_skips_the_stored_windows( + tmp_path: Path, conversation: Conversation +) -> None: + fake = FakeModels() + models = ThreadModels(fake, tmp_path / "replies.sqlite") + memory = store() + namespace = (*NAMESPACE, "7") + capture_conversation(memory, models, conversation.windows(), namespace) + again = capture_conversation(memory, models, conversation.windows(), namespace) + assert again["facts_written"] == 0 + assert fake.calls.count(EXTRACTOR) == 2 + models.close() + + +def test_thread_models_sum_every_thread_and_stop_at_the_limit(tmp_path: Path) -> None: + models = ThreadModels(FakeModels(), tmp_path / "replies.sqlite", limit=1.0) + threads = [threading.Thread(target=models, args=(BEAM_MINI, f"p{n}", 10)) for n in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert models.calls == 4 + assert models.spent == pytest.approx(4 * (1000 * 0.40 + 5 * 1.60) / 1e6) + paid: list[str] = [] + + def costly(model: str, prompt: str, *_: Any, **__: Any) -> Reply: + paid.append(prompt) + return Reply("x", 10_000_000, 0) + + big = ThreadModels(costly, tmp_path / "r2.sqlite", 0.5) + with pytest.raises(SpendLimit): + big(BEAM_MINI, "p", 10) + with pytest.raises(SpendLimit): + big(BEAM_MINI, "q", 10) + assert paid == ["p"] + models.close() + big.close() + + +def test_the_chunks_block_keeps_to_the_budget(tmp_path: Path, conversation: Conversation) -> None: + memory = store() + namespace = (*NAMESPACE, "7") + Memory(memory).capture( + "\n\n".join(["word " * 190] * 25), + namespace, + lambda text, at, known: [], + episode="window-01-0001", + chunk_chars=CONFIG.chunk_chars, + ) + block = chunks_block(memory, namespace, "word") + assert 0 < len(block.text) <= CONFIG.budget + assert len(block.episodes) < 25 + assert all(key.startswith("window-01-0001-") for key in block.episodes) + + +def test_coverage_counts_the_source_windows_in_the_block(conversation: Conversation) -> None: + windows = conversation.windows(limit=60) + probe = conversation.probes[0] + found = source_windows(windows, probe) + assert found == {"window-01-0001", "window-01-0002", "window-02-0001"} + block = Block("", ("window-01-0001-0001", "window-01-0001-0002", "window-02-0001-0001"), 0) + assert coverage(found, block) == pytest.approx(2 / 3) + assert coverage(set(), block) is None + + +def test_the_estimate_counts_each_part(conversation: Conversation) -> None: + both = estimate([conversation], ("memory", "chunks"), ("mini",), 1) + assert both.parts["extraction"] == pytest.approx(2 * 0.0005) + assert both.parts["readers"] > 0 and both.parts["judge"] > 0 + assert both.total == pytest.approx(sum(both.by_conversation.values())) + reused = estimate([conversation], ("memory",), ("mini",), 1, capture=False) + assert reused.parts["extraction"] == reused.parts["embeddings"] == 0 + assert reused.parts["judge"] == pytest.approx(both.parts["judge"] / 2) + + +def test_the_longest_worker_takes_the_longest_jobs_first() -> None: + assert longest_worker([3, 5, 3, 4], 2) == 8 + assert longest_worker([3, 5, 3, 4], 1) == 15 + assert longest_worker([], 4) == 0 + + +def test_a_sample_needs_the_sample_split(conversation: Conversation) -> None: + assert choose("1M", "all", [conversation]) == [conversation] + with pytest.raises(ValueError, match="needs --split 100K"): + choose("1M", "a", [conversation]) diff --git a/bench/tests/test_beam_score.py b/bench/tests/test_beam_score.py new file mode 100644 index 0000000..d9983f5 --- /dev/null +++ b/bench/tests/test_beam_score.py @@ -0,0 +1,291 @@ +import hashlib +import json +import math +import random +import re +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +import pytest +from json_repair import repair_json +from scipy.stats import kendalltau + +from bench import beam_score +from bench.beam import Probe +from bench.beam_score import ( + JUDGE, + MATCH_SYSTEM, + align, + answer_prompt, + judge_prompt, + judge_reply, + order_score, + parse_json_response, + score, + summarize, +) +from bench.models import Reply + +PROMPT_SHA256 = { + "answer.txt": "ed89b98d1024432a46bd297345961cf89d2429746daec37befb50db46e0c914a", + "judge.txt": "593373c642a288a7b590577d8a8fc92c3f9a2b70e2f64ad6e59a040a6c56b7f5", + "match_system.txt": "0863c78c1b27d36c97bb7bfd2d4d6b88ef69b093e92bb4719e665527722413e2", + "match_user.txt": "8686f5ce5009350f1d04fccc06fc23a234cfcd956603c5d54bb2161c2d2c6045", +} + + +# The reference below is copied from github.com/mohammadtavakoli78/BEAM at commit b2da22e, +# src/evaluation/compute_metrics.py, under the MIT license in bench/beam_prompts/LICENSE. +# Only the type hints and the line breaks differ. +def official_parse_json_response(response: str) -> Any: + response = response.strip() + if response.startswith("```"): + match = re.search(r"```(?:json)?\s*(\[.*\]|\{.*\})\s*```", response, re.DOTALL) + if match: + response = match.group(1).strip() + try: + return json.loads(response) + except json.JSONDecodeError: + pass + match = re.search(r"(\{.*?\}|\[.*?\])", response, re.DOTALL) + if match: + json_part = match.group(1) + try: + return json.loads(json_part) + except Exception as e: + raise ValueError(f"Found possible JSON but failed to parse it: {e}") from e + raise ValueError("No valid JSON found in response.") + + +def official_llm_equivalence(first_paragraph: str, second_paragraph: str, llm: Any) -> bool: + system_msg = { + "role": "system", + "content": """ + You are a binary classifier. + If the TWO snippets describe the SAME event/fact, reply **YES** + Otherwise reply **NO**. No extra words. + DO NOT provide any exaplanation. + """, + } + user_msg = { + "role": "user", + "content": f"""First snippet: {first_paragraph} \n + Second snippet: {second_paragraph} + """, + } + messages = [system_msg, user_msg] + response: str = llm.invoke(messages).content.lower() + return "yes" in response + + +def official_align_with_llm( + reference: list[str], system: list[str], llm: Any +) -> tuple[list[str], list[str]]: + used: set[int] = set() + system_out = [] + for s in system: + matched_index = None + for index, r in enumerate(reference): + if index in used: + continue + if official_llm_equivalence(first_paragraph=r, second_paragraph=s, llm=llm): + matched_index = index + break + if matched_index is not None: + system_out.append(reference[matched_index]) + used.add(matched_index) + else: + system_out.append(s) + return reference, system_out + + +def official_event_ordering_score( + reference_list: list[str], system_list: list[str], llm: Any +) -> dict[str, float]: + reference_canon, system_canon = official_align_with_llm(reference_list, system_list, llm) + tp = len(set(reference_canon) & set(system_canon)) + fp = len([x for x in system_canon if x not in reference_canon]) + fn = len([x for x in reference_canon if x not in system_canon]) + precision = tp / (tp + fp) if tp + fp else 0 + recall = tp / (tp + fn) if tp + fn else 0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0 + union = list(dict.fromkeys(reference_canon + system_canon)) + tie_rank = len(union) + 1 + + def to_rank(seq: list[str]) -> list[int]: + r = {item: i + 1 for i, item in enumerate(seq)} + return [r.get(u, tie_rank) for u in union] + + tau_b, p_value = kendalltau( + to_rank(reference_canon), to_rank(system_canon), variant="b", method="auto" + ) + tau_b_norm = (tau_b + 1) / 2 if tau_b is not None else 0 + final_score = tau_b_norm * f1 + return dict( + precision=precision, recall=recall, f1=f1, tau_norm=tau_b_norm, final_score=final_score + ) + + +def official_nugget_score(rubric: list[str], llm_response: str, llm: Any) -> float: + score = 0.0 + for item in rubric: + prompt = beam_score.JUDGE_PROMPT.replace("", item).replace( + "", llm_response + ) + response = llm.invoke(prompt).content.strip() + try: + response = official_parse_json_response(response=response) + except: # noqa: E722 + response = json.loads(repair_json(response)) + score += float(response["score"]) + return score / len(rubric) + + +# End of the copied reference. + + +class Content: + def __init__(self, content: str) -> None: + self.content = content + + +class Oracle: + """A fake model that answers a match call by whether both snippets name the same event. + + An event is a word `e`. A judge call gets the next reply of `judge`. + """ + + def __init__(self, judge: Sequence[str] = ()) -> None: + self.judge = list(judge) + self.matches: list[tuple[str | None, str]] = [] + + def answer(self, system: str | None, prompt: str) -> str: + if system is None: + return self.judge.pop(0) + self.matches.append((system, prompt)) + first, second = prompt.split("Second snippet:", 1) + left = set(re.findall(r"e\d+", first)) + right = set(re.findall(r"e\d+", second)) + return "YES" if left and left == right else "NO" + + def invoke(self, prompt: str | list[dict[str, str]]) -> Content: + if isinstance(prompt, str): + return Content(self.answer(None, prompt)) + return Content(self.answer(prompt[0]["content"], prompt[1]["content"])) + + def __call__( + self, + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None = None, + system: str | None = None, + ) -> Reply: + assert model == JUDGE + return Reply(self.answer(system, prompt), 1, 1) + + +def probe(ability: str, rubric: Sequence[str]) -> Probe: + return Probe("1-x-1", "1", ability, "q?", "gold", tuple(rubric), "easy", ()) + + +def test_the_prompts_are_the_official_files() -> None: + for name, digest in PROMPT_SHA256.items(): + path = Path(beam_score.PROMPTS) / name + assert hashlib.sha256(path.read_bytes()).hexdigest() == digest, name + assert "MIT License" in (Path(beam_score.PROMPTS) / "LICENSE").read_text() + + +def test_the_prompts_fill_their_slots() -> None: + prompt = answer_prompt("the block", "the question?") + assert "CONTEXT:\nthe block\n" in prompt + assert "QUESTION:\nthe question?\n" in prompt + judged = judge_prompt("LLM response should state: 7 days", "It is 7 days.") + assert "- RUBRIC CRITERION (what to check): LLM response should state: 7 days" in judged + assert "- RESPONSE TO EVALUATE: It is 7 days." in judged + assert "<" + "rubric_item>" not in judged + + +REPLIES = [ + '{"score": 1.0, "reason": "ok"}', + '```json\n{"score": 0.5, "reason": "partial"}\n```', + 'Here is my verdict: {"score": 0.0, "reason": "missing"} done', + '{"score": 0.5, "reason": "a reason with an unclosed quote}', + ' {"score": 1, "reason": "spaces"} ', +] + + +@pytest.mark.parametrize("text", REPLIES) +def test_the_judge_parse_matches_the_official_one(text: str) -> None: + try: + expected = float(official_parse_json_response(text)["score"]) + except ValueError: + expected = float(json.loads(repair_json(text))["score"]) + assert judge_reply(text) == expected + + +def test_parse_raises_without_json() -> None: + with pytest.raises(ValueError, match="No valid JSON"): + parse_json_response("no json here") + + +def test_a_nugget_question_scores_the_mean_as_the_official_code() -> None: + rubric = ["n1", "n2", "n3", "n4", "n5"] + found = score(Oracle(REPLIES), probe("temporal_reasoning", rubric), "resp") + assert found.nuggets == (1.0, 0.5, 0.0, 0.5, 1.0) + assert found.score == found.nugget == official_nugget_score(rubric, "resp", Oracle(REPLIES)) + assert found.tau_norm is None + + +def random_case(rng: random.Random) -> tuple[list[str], str]: + events = [f"e{n}" for n in range(rng.randint(2, 8))] + reference = [f"Event {e} happened" for e in events] + shown = rng.sample(events, rng.randint(0, len(events))) + lines = [f"- then {e}" for e in shown] + for _ in range(rng.randint(0, 3)): + lines.insert(rng.randint(0, len(lines)), rng.choice(["", "Here is the order:", "e99"])) + if rng.random() < 0.2 and lines: + lines.append(lines[0]) + return reference, "\n".join(lines) + + +def test_event_ordering_matches_the_official_code_on_many_replies() -> None: + rng = random.Random(20261026) + for _ in range(300): + reference, response = random_case(rng) + ours, theirs = Oracle(), Oracle() + reference_out, system_out = align(ours, reference, response.split("\n")) + expected = official_event_ordering_score(reference, response.split("\n"), theirs) + found = order_score(reference_out, system_out) + assert ours.matches == theirs.matches + for key, value in expected.items(): + if math.isnan(value): + assert math.isnan(found[key]) + else: + assert found[key] == pytest.approx(value), (key, reference, response) + + +def test_event_ordering_scores_tau_norm_and_keeps_the_nugget_mean() -> None: + rubric = ["Event e1 happened", "Event e2 happened", "Event e3 happened"] + response = "Order:\n- e2\n- e1\n- e3" + model = Oracle(['{"score": 1}', '{"score": 1}', '{"score": 0}']) + found = score(model, probe("event_ordering", rubric), response) + expected = official_event_ordering_score(rubric, response.split("\n"), Oracle()) + assert found.score == found.tau_norm == pytest.approx(expected["tau_norm"]) + assert found.f1 == pytest.approx(expected["f1"]) + assert found.nugget == pytest.approx(2 / 3) + assert all(system == MATCH_SYSTEM for system, _ in model.matches) + + +def test_summarize_takes_the_mean_over_the_abilities() -> None: + rows = [ + {"ability": "abstention", "domain": "Coding", "score": 1.0}, + {"ability": "abstention", "domain": "Math", "score": 0.0}, + {"ability": "summarization", "domain": "Coding", "score": 1.0}, + ] + found = summarize(rows) + assert found["by_ability"] == {"abstention": 0.5, "summarization": 1.0} + assert found["mean"] == 0.75 + assert found["questions"] == pytest.approx(2 / 3) + assert found["by_domain"] == {"Coding": 1.0, "Math": 0.0} diff --git a/bench/tests/test_e2e.py b/bench/tests/test_e2e.py index 5a32c9c..6b265a7 100644 --- a/bench/tests/test_e2e.py +++ b/bench/tests/test_e2e.py @@ -1,5 +1,7 @@ +import hashlib import json import os +import sqlite3 import subprocess import uuid from collections.abc import Sequence @@ -267,7 +269,13 @@ def truncate(self, text: str, limit: int) -> str: def test_cached_models_call_once(tmp_path: Path) -> None: calls: list[str] = [] - def complete(model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None) -> Reply: + def complete( + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None, + system: str | None = None, + ) -> Reply: calls.append(prompt) return Reply("yes", 1_000_000, 0) @@ -279,6 +287,30 @@ def complete(model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | assert models.spent == pytest.approx(5.0) +def test_a_system_message_is_passed_and_keys_the_cache(tmp_path: Path) -> None: + systems: list[str | None] = [] + + def complete( + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None, + system: str | None = None, + ) -> Reply: + systems.append(system) + return Reply(system or "none", 1, 1) + + path = tmp_path / "replies.sqlite" + models = CachedModels(complete, path) + assert models(READER, "p", 10).text == "none" + assert models(READER, "p", 10, system="s").text == "s" + assert models(READER, "p", 10, system="s").text == "s" + assert systems == [None, "s"] + legacy = hashlib.sha256(json.dumps([READER, 10, None, "p"]).encode()).hexdigest() + keys = [row[0] for row in sqlite3.connect(path).execute("SELECT key FROM replies")] + assert legacy in keys + + def test_shards_deal_every_question_once(questions: dict[str, Question]) -> None: base = questions["q_one"] pool = [replace(base, id=f"q{n}") for n in range(10)] @@ -368,7 +400,13 @@ def stopped(model: str, prompt: str, max_tokens: int, schema: Any = None) -> Rep def test_cached_models_keep_a_stopped_reply(tmp_path: Path) -> None: - def complete(model: str, prompt: str, max_tokens: int, schema: dict[str, Any] | None) -> Reply: + def complete( + model: str, + prompt: str, + max_tokens: int, + schema: dict[str, Any] | None, + system: str | None = None, + ) -> Reply: return Reply("{", 1, max_tokens, stopped=True) path = tmp_path / "replies.sqlite" diff --git a/pyproject.toml b/pyproject.toml index 2d1f5fa..9470e06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,13 @@ dev = [ "ruff", { include-group = "bench" }, ] -bench = ["hotmemory[hotdata,openai]", "numpy", "tiktoken"] +bench = [ + "hotmemory[hotdata,openai]", + "json-repair>=0.64.2", + "numpy", + "scipy>=1.17.1", + "tiktoken", +] [tool.ruff] line-length = 100 @@ -68,7 +74,7 @@ strict = true files = ["src", "tests", "skills/hotmemory/scripts", "bench"] [[tool.mypy.overrides]] -module = ["pyarrow", "pyarrow.*"] +module = ["pyarrow", "pyarrow.*", "scipy", "scipy.*"] ignore_missing_imports = true [tool.pytest.ini_options] diff --git a/uv.lock b/uv.lock index 52e06b9..3cbcc26 100644 --- a/uv.lock +++ b/uv.lock @@ -322,18 +322,24 @@ openai = [ [package.dev-dependencies] bench = [ { name = "hotmemory", extra = ["hotdata", "openai"] }, + { name = "json-repair" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "tiktoken" }, ] dev = [ { name = "hotmemory", extra = ["hotdata", "openai"] }, + { name = "json-repair" }, { name = "mypy" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pytest" }, { name = "pytest-socket" }, { name = "ruff" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "tiktoken" }, ] @@ -348,16 +354,20 @@ provides-extras = ["hotdata", "openai"] [package.metadata.requires-dev] bench = [ { name = "hotmemory", extras = ["hotdata", "openai"] }, + { name = "json-repair", specifier = ">=0.64.2" }, { name = "numpy" }, + { name = "scipy", specifier = ">=1.17.1" }, { name = "tiktoken" }, ] dev = [ { name = "hotmemory", extras = ["hotdata", "openai"] }, + { name = "json-repair", specifier = ">=0.64.2" }, { name = "mypy" }, { name = "numpy" }, { name = "pytest" }, { name = "pytest-socket" }, { name = "ruff" }, + { name = "scipy", specifier = ">=1.17.1" }, { name = "tiktoken" }, ] @@ -531,6 +541,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/8b/d76219ebdbcf3d4209d9d21a0810db4c8d0a6f88e3ee87d30bdea4e90d30/jiter-0.17.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c0bf24c72fd0491405dce5d40194f2070e9021ce648c1a1d46234b93d848ff", size = 307147, upload-time = "2026-09-12T15:14:12.897Z" }, ] +[[package]] +name = "json-repair" +version = "0.64.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/b2/52bdd0fca981b289a4d9966366e1dfe50fac315f2c87440fa70726bdcb69/json_repair-0.64.2.tar.gz", hash = "sha256:3f8e76631edab40e7e0c9b859e64c06992a39488b4bbb8285f0e9c3e61d684e9", size = 54227, upload-time = "2026-10-23T05:35:23.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/79/91b83df91d03b9647ee759f286900957f0d113f49a606e28c8a43d39365b/json_repair-0.64.2-py3-none-any.whl", hash = "sha256:1dec7868c05abff0c52e91dc896e01f28fda6ba1c9f2a9ede18ab520dde637e1", size = 52688, upload-time = "2026-10-23T05:35:21.978Z" }, +] + [[package]] name = "librt" version = "0.16.0" @@ -1368,6 +1387,164 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/05/c16957eb287c3fc062e032619a25868d93d408a844b725bcf514f7a378ff/ruff-0.16.10-py3-none-win_arm64.whl", hash = "sha256:25a65fe998c4e6861ec079ada5826a2fc605e6cbccbe9dcd7fac1f54e791621b", size = 10366440, upload-time = "2026-10-01T18:03:19.04Z" }, ] +[[package]] +name = "scipy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/75/b4ce781849931fef6fd529afa6b63711d5a733065722d0c3e2724af9e40a/scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec", size = 31613675, upload-time = "2026-02-23T00:16:00.13Z" }, + { url = "https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696", size = 28162057, upload-time = "2026-02-23T00:16:09.456Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ee/18146b7757ed4976276b9c9819108adbc73c5aad636e5353e20746b73069/scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee", size = 20334032, upload-time = "2026-02-23T00:16:17.358Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e6/cef1cf3557f0c54954198554a10016b6a03b2ec9e22a4e1df734936bd99c/scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd", size = 22709533, upload-time = "2026-02-23T00:16:25.791Z" }, + { url = "https://files.pythonhosted.org/packages/4d/60/8804678875fc59362b0fb759ab3ecce1f09c10a735680318ac30da8cd76b/scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c", size = 33062057, upload-time = "2026-02-23T00:16:36.931Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4", size = 35349300, upload-time = "2026-02-23T00:16:49.108Z" }, + { url = "https://files.pythonhosted.org/packages/b4/3d/7ccbbdcbb54c8fdc20d3b6930137c782a163fa626f0aef920349873421ba/scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444", size = 35127333, upload-time = "2026-02-23T00:17:01.293Z" }, + { url = "https://files.pythonhosted.org/packages/e8/19/f926cb11c42b15ba08e3a71e376d816ac08614f769b4f47e06c3580c836a/scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082", size = 37741314, upload-time = "2026-02-23T00:17:12.576Z" }, + { url = "https://files.pythonhosted.org/packages/95/da/0d1df507cf574b3f224ccc3d45244c9a1d732c81dcb26b1e8a766ae271a8/scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff", size = 36607512, upload-time = "2026-02-23T00:17:23.424Z" }, + { url = "https://files.pythonhosted.org/packages/68/7f/bdd79ceaad24b671543ffe0ef61ed8e659440eb683b66f033454dcee90eb/scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d", size = 24599248, upload-time = "2026-02-23T00:17:34.561Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" }, + { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" }, + { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, + { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, + { url = "https://files.pythonhosted.org/packages/76/27/07ee1b57b65e92645f219b37148a7e7928b82e2b5dbeccecb4dff7c64f0b/scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c", size = 31590199, upload-time = "2026-02-23T00:19:17.192Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ae/db19f8ab842e9b724bf5dbb7db29302a91f1e55bc4d04b1025d6d605a2c5/scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f", size = 28154001, upload-time = "2026-02-23T00:19:22.241Z" }, + { url = "https://files.pythonhosted.org/packages/5b/58/3ce96251560107b381cbd6e8413c483bbb1228a6b919fa8652b0d4090e7f/scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d", size = 20325719, upload-time = "2026-02-23T00:19:26.329Z" }, + { url = "https://files.pythonhosted.org/packages/b2/83/15087d945e0e4d48ce2377498abf5ad171ae013232ae31d06f336e64c999/scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b", size = 22683595, upload-time = "2026-02-23T00:19:30.304Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" }, + { url = "https://files.pythonhosted.org/packages/35/e5/d6d0e51fc888f692a35134336866341c08655d92614f492c6860dc45bb2c/scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448", size = 36510943, upload-time = "2026-02-23T00:20:50.89Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fd/3be73c564e2a01e690e19cc618811540ba5354c67c8680dce3281123fb79/scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87", size = 24545621, upload-time = "2026-02-23T00:20:55.871Z" }, + { url = "https://files.pythonhosted.org/packages/6f/6b/17787db8b8114933a66f9dcc479a8272e4b4da75fe03b0c282f7b0ade8cd/scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a", size = 31936708, upload-time = "2026-02-23T00:19:58.694Z" }, + { url = "https://files.pythonhosted.org/packages/38/2e/524405c2b6392765ab1e2b722a41d5da33dc5c7b7278184a8ad29b6cb206/scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0", size = 28570135, upload-time = "2026-02-23T00:20:03.934Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c3/5bd7199f4ea8556c0c8e39f04ccb014ac37d1468e6cfa6a95c6b3562b76e/scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce", size = 20741977, upload-time = "2026-02-23T00:20:07.935Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b8/8ccd9b766ad14c78386599708eb745f6b44f08400a5fd0ade7cf89b6fc93/scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6", size = 23029601, upload-time = "2026-02-23T00:20:12.161Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" }, + { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" }, + { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" }, + { url = "https://files.pythonhosted.org/packages/06/1c/1172a88d507a4baaf72c5a09bb6c018fe2ae0ab622e5830b703a46cc9e44/scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c", size = 36562980, upload-time = "2026-02-23T00:20:40.575Z" }, + { url = "https://files.pythonhosted.org/packages/70/b0/eb757336e5a76dfa7911f63252e3b7d1de00935d7705cf772db5b45ec238/scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49", size = 24856543, upload-time = "2026-02-23T00:20:45.313Z" }, + { url = "https://files.pythonhosted.org/packages/cf/83/333afb452af6f0fd70414dc04f898647ee1423979ce02efa75c3b0f2c28e/scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717", size = 31584510, upload-time = "2026-02-23T00:21:01.015Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a6/d05a85fd51daeb2e4ea71d102f15b34fedca8e931af02594193ae4fd25f7/scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9", size = 28170131, upload-time = "2026-02-23T00:21:05.888Z" }, + { url = "https://files.pythonhosted.org/packages/db/7b/8624a203326675d7746a254083a187398090a179335b2e4a20e2ddc46e83/scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b", size = 20342032, upload-time = "2026-02-23T00:21:09.904Z" }, + { url = "https://files.pythonhosted.org/packages/c9/35/2c342897c00775d688d8ff3987aced3426858fd89d5a0e26e020b660b301/scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866", size = 22678766, upload-time = "2026-02-23T00:21:14.313Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" }, + { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" }, + { url = "https://files.pythonhosted.org/packages/4b/39/f0e8ea762a764a9dc52aa7dabcfad51a354819de1f0d4652b6a1122424d6/scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19", size = 37290984, upload-time = "2026-02-23T00:22:35.023Z" }, + { url = "https://files.pythonhosted.org/packages/7c/56/fe201e3b0f93d1a8bcf75d3379affd228a63d7e2d80ab45467a74b494947/scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293", size = 25192877, upload-time = "2026-02-23T00:22:39.798Z" }, + { url = "https://files.pythonhosted.org/packages/96/ad/f8c414e121f82e02d76f310f16db9899c4fcde36710329502a6b2a3c0392/scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6", size = 31949750, upload-time = "2026-02-23T00:21:42.289Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b0/c741e8865d61b67c81e255f4f0a832846c064e426636cd7de84e74d209be/scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1", size = 28585858, upload-time = "2026-02-23T00:21:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1b/3985219c6177866628fa7c2595bfd23f193ceebbe472c98a08824b9466ff/scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39", size = 20757723, upload-time = "2026-02-23T00:21:52.039Z" }, + { url = "https://files.pythonhosted.org/packages/c0/19/2a04aa25050d656d6f7b9e7b685cc83d6957fb101665bfd9369ca6534563/scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca", size = 23043098, upload-time = "2026-02-23T00:21:56.185Z" }, + { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" }, + { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" }, + { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" }, + { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" }, + { url = "https://files.pythonhosted.org/packages/11/2f/b29eafe4a3fbc3d6de9662b36e028d5f039e72d345e05c250e121a230dd4/scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484", size = 37345327, upload-time = "2026-02-23T00:22:24.442Z" }, + { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, +] + +[[package]] +name = "scipy" +version = "1.18.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/f7/240c110c08693826b4513a52f5717d62ec7c7af72f2920821247c03b17b3/scipy-1.18.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:457fd7a2a8edeb044ab6ffbc0aa03ff6cd18491356e5e0c834d76ce621b916d1", size = 31111061, upload-time = "2026-08-21T23:23:44.522Z" }, + { url = "https://files.pythonhosted.org/packages/05/4a/78c6285577c375e7cf27277ea8ee6961224327f1e1a0c44af5f17f23635c/scipy-1.18.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:e708533e8b2ae2497d65346538a7dcc92814410b25b81432eac66de0f2af8265", size = 28733332, upload-time = "2026-08-21T23:23:50.015Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f6/a5b82f8abbe14d134691b8b903696f701d25a081353a29dc655c364d9e62/scipy-1.18.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:7bbf207c4453ce1ad2e00b17313852b33310b83090c2311bdaf97f93c0380d12", size = 20475078, upload-time = "2026-08-21T23:23:54.138Z" }, + { url = "https://files.pythonhosted.org/packages/23/22/0858a0bbd6b3e825ceb8cd9baf9eaf3b2f2b1d77727eb6be40500bcdc92f/scipy-1.18.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:78c0665edead396b1abb4897c41a5c1d9bf090c8a637a4c20a61678e0a264e66", size = 23108904, upload-time = "2026-08-21T23:23:57.824Z" }, + { url = "https://files.pythonhosted.org/packages/75/9a/2e71719f31eaefe0e3a1706c4a1ded94e664bfd95ffca2b219a671faee01/scipy-1.18.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c085faa2cfa879c5141df483f836f4d691045a078224a670fa570fa01612d89", size = 34025113, upload-time = "2026-08-21T23:24:02.209Z" }, + { url = "https://files.pythonhosted.org/packages/df/64/ff35eb9e54894cf471ff4716abd3c81eb0a0626869217ce3e6ba4ccf17d7/scipy-1.18.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f55fa87b6c612ecd6b058f167c53231b1d14e412efe361d3d6e38b3631c73218", size = 35344199, upload-time = "2026-08-21T23:24:07.844Z" }, + { url = "https://files.pythonhosted.org/packages/d3/af/c5538be1792f7034c12c7db6ee67cace58253c7b87b122d68253eaf5de89/scipy-1.18.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c35d74ce0e193ff740c2f2be2ac913ddc232fe6c1ff40b26cfecb9c670c63314", size = 35639587, upload-time = "2026-08-21T23:24:13.05Z" }, + { url = "https://files.pythonhosted.org/packages/91/4c/075e4f66471bac101141ac739e9e135549be1bae584571bd03a530c056e1/scipy-1.18.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2924a03db38dc2e848bca2fe9f077dafb891480b91a00a0963a8cf86dfc31c1", size = 37480330, upload-time = "2026-08-21T23:24:19.608Z" }, + { url = "https://files.pythonhosted.org/packages/39/e7/979fd14e75008623df31ba70d6bb144700f68feadcea042021c06a05bf82/scipy-1.18.1-cp312-cp312-win_amd64.whl", hash = "sha256:5e4d44984abc0020154ea81b247adeddcc3ac5527b975ff798bd1ba0adc513c2", size = 36658278, upload-time = "2026-08-21T23:24:25.463Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0b/e1525354ff9d7d5feb6d1b31af6d14072e5c91e9607b421fa1ec889660b3/scipy-1.18.1-cp312-cp312-win_arm64.whl", hash = "sha256:d65d448389b8436493abcf629cc94ad0cf32aecaf06e1acca1de53cc795f2f12", size = 24400588, upload-time = "2026-08-21T23:24:30.579Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/4540ee0f9c42a9ad7109d0d1a8cc70de54c3572b01c6693a2b1c70e90ceb/scipy-1.18.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:3ab3523da44749156e1f68b464dc56af11ae4cbc5c739a49d05f32b982eca9f3", size = 31089958, upload-time = "2026-08-21T23:24:35.8Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f5/769f36d14922b8071a43e95d24d18b6bdafad10d7f5cf647867e1ac052bc/scipy-1.18.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6fb6a55cc0ba97b59a1f288fb86dc6fce8bdfc0fffcbfd015e3a954bf2a2d93", size = 28715106, upload-time = "2026-08-21T23:24:40.775Z" }, + { url = "https://files.pythonhosted.org/packages/9a/d7/21d890274f75ea37a8209d5519e72da3da90302e3b9fb8397a0918386a62/scipy-1.18.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ea324d9dd34c38bfb9bec8ca4d1b407db97dbb74029f566b8e322b1b6fe56fe6", size = 20456846, upload-time = "2026-08-21T23:24:45.066Z" }, + { url = "https://files.pythonhosted.org/packages/ec/01/798430ecea2e78ec7c02663d5f71c007bb6abeca931080debd40d7fa55ea/scipy-1.18.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:75b00eb8fb802090aa903f4ea1c7f5a584779f967361e68b7e98e531cc2d7174", size = 23087986, upload-time = "2026-08-21T23:24:49.539Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5f/4634e9d35c68496e4e34cb6946eafab044458e6cedab42b40b6588e475b6/scipy-1.18.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d416b16cccfd70fbf62400e84d0bb2f4e6af519a45557f1692c749b37f14b315", size = 33998146, upload-time = "2026-08-21T23:24:54.714Z" }, + { url = "https://files.pythonhosted.org/packages/41/48/6450ed9243315322bbc19ac57b9b70d66a20bf1d38d124c96bc4bf6af9ea/scipy-1.18.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdaf5ea890a6183d0565f51a61799d67081bd5b1cf03c5f4b3fd3732108625c9", size = 35312578, upload-time = "2026-08-21T23:25:00.44Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/bf5a4be6a3525676499f6dff307991739ff6fdcad1481b1aeb6745339f58/scipy-1.18.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c825cef2f49e46753726a7181a8e199804a912b29519ada542c6ebc654951899", size = 35612621, upload-time = "2026-08-21T23:25:06.144Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4e/3c45c33e00a77996c4b1cb707929f833ba7b1d522ee29f882512c330676d/scipy-1.18.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e3b417bf8c2c7c16e8f58ad91db17783ec911ac16e7b50eb6eab6e809b4f5b07", size = 37457323, upload-time = "2026-08-21T23:25:12.483Z" }, + { url = "https://files.pythonhosted.org/packages/93/0e/e0348fbc0dbab65c114cf78957e7dfeb49f8e8b556b4d930cc12ff195e18/scipy-1.18.1-cp313-cp313-win_amd64.whl", hash = "sha256:559ed65f60c1af5a03f3912605a1b5114f522c7c32fb23c3376ae8f03219fe28", size = 36622841, upload-time = "2026-08-21T23:25:18.722Z" }, + { url = "https://files.pythonhosted.org/packages/50/a8/6a77f5f267c555108f0a864b6db714363dab567a8266422a79a385f9232b/scipy-1.18.1-cp313-cp313-win_arm64.whl", hash = "sha256:cd479fc04dd9401e3b4f49e76518768ef99c4f517a98c284eb091fd725719adf", size = 24399315, upload-time = "2026-08-21T23:25:23.458Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/d8eb4e280ddb56a4ab2c6f02ee49b56b23f6e977cf0802fd6d68dbef14f5/scipy-1.18.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:83de5453a7799afc9048b4616bd085cef126e36412f0ea2f6370c36a2a3a51e7", size = 31090936, upload-time = "2026-08-21T23:25:28.686Z" }, + { url = "https://files.pythonhosted.org/packages/2a/49/59ea385dc3a62ff498ddf3cfff7c2b41b0f9f9d3c4122b3f1dcb6d6327fe/scipy-1.18.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:9554bcc6d715ee87a633a3cc8e7703c6628b100dd29cb8a2efc4c0533c7ff729", size = 28725221, upload-time = "2026-08-21T23:25:33.244Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/6b0c288c50942d78193696c9f15f9a0874f5178aa0ddf40f83d9924b3e8d/scipy-1.18.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc", size = 20466839, upload-time = "2026-08-21T23:25:37.516Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e0/54fd3793c729e3b936782f181b59cbb1205bf250ab605a16cb1ba61cdd5e/scipy-1.18.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:88f0e784020649f88ea48c9f5ddfa403bf9205820667c0914740b392035afb82", size = 23089121, upload-time = "2026-08-21T23:25:42.019Z" }, + { url = "https://files.pythonhosted.org/packages/0b/56/030af62bea3cf878e0028515dff78c123b01633606a879b63f42d2db99cc/scipy-1.18.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d3ab0e8c69a17dd3559eab8cbb88f258e285c94d572c2719033f90f83290c89", size = 34053851, upload-time = "2026-08-21T23:25:47.998Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/2a844506d49651e9aa1af6ef95b6bd8031cb1d5a4375edec6155037e04cf/scipy-1.18.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac0333bdf38309aa3dcbe7e3fa7ea29e7a2c37c6ea306a757b700ded8e4596ad", size = 35329183, upload-time = "2026-08-21T23:25:53.522Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/c7370c3640e92ac9613cbf26cb3f729f9b12ddf1727b55b94b53b24d6f48/scipy-1.18.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:911de823097db8b63f034299d12662db93344e6ffa0b881cbb57748974b70168", size = 35672551, upload-time = "2026-08-21T23:25:59.387Z" }, + { url = "https://files.pythonhosted.org/packages/24/16/ec8536f351421f8bf60a1120930638f83790f4710b8230446aca3d6159d4/scipy-1.18.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:95298364e251be3e60249facbeeca03631d3bb7584f85879516ec55ac717b81f", size = 37469416, upload-time = "2026-08-21T23:26:05.432Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/d73da0d28f16c45bb9b0a5691b91610b0275c5ef0eb5e43c87cf2dc1bf31/scipy-1.18.1-cp314-cp314-win_amd64.whl", hash = "sha256:78a0d7c918e74a232394117160e7e3db503377572a45bcef8826e4ab8a35feba", size = 37362755, upload-time = "2026-08-21T23:26:11.366Z" }, + { url = "https://files.pythonhosted.org/packages/89/25/e996e4dc74e10e227b1e14db5eaf6608bb6dd33884a64851c38f18dd4249/scipy-1.18.1-cp314-cp314-win_arm64.whl", hash = "sha256:cbf38d043c1aa4ab306e1ada6ab6eddacc3322a20b7af1b30bc93254b366fe09", size = 25036090, upload-time = "2026-08-21T23:26:15.887Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c9/c00213f92309d753b48903e6a451b87eb52ff5b7a16e789d1568bbf221c4/scipy-1.18.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0fcb3c93519f27bb4f0c4b0f7802cdcaca7fcf93267b75edda2e9f4e8a55cbd7", size = 31485550, upload-time = "2026-08-21T23:26:20.776Z" }, + { url = "https://files.pythonhosted.org/packages/74/b2/e3067c487982d4eeab2938928529410370c06fea84a4d3f4925e7d96647d/scipy-1.18.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ddef79fb382df40104a19bb7151b3b23e57c1778fcf857c71ceecd9bd264513f", size = 29174642, upload-time = "2026-08-21T23:26:25.395Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ab/374c9fe2d1ec014e576c781a4b5d8e1ba340e8f6b4638c16f711d2b194f0/scipy-1.18.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:0e82073ecc7acc6436fac4b31674109c7e1d3e596789767eda01258a8c9e8123", size = 20916357, upload-time = "2026-08-21T23:26:30.112Z" }, + { url = "https://files.pythonhosted.org/packages/90/38/223915c88a17317cafbf8ca2a42b11c265a9fb1e804aa665544132b5fe8a/scipy-1.18.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:8bcf3c1ba5d6456e2effd30fcbd3459b044d683fcdac79a2e6830f0bdf7de487", size = 23482611, upload-time = "2026-08-21T23:26:34.846Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d1/db0948da8ca57a80b36520ef0a768b967d99f3af65f4b6f1bf6362ad4dd4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cfbf154f2ba187f2ed6cce2639efff7d105f1140573642c0161615b6d91d6a87", size = 34143202, upload-time = "2026-08-21T23:26:40.4Z" }, + { url = "https://files.pythonhosted.org/packages/87/53/39d046cc7574ed6acacb6bd5723e220107ece80bff12faaf3efc4ddeede4/scipy-1.18.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1d33a7836f7ddc1993427966a0823468ec41bcbdb1a9f9942d1d7e57f803ba3", size = 35380876, upload-time = "2026-08-21T23:26:46.1Z" }, + { url = "https://files.pythonhosted.org/packages/f9/da/32e0e799d875a85ca57d9bde6c78148afcc0e38276df683d95854eadc8c3/scipy-1.18.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f4b8bc363b6d65ee2152bec57568e3c52639bb34c46057b09857a307ed5e21d", size = 35770885, upload-time = "2026-08-21T23:26:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/88/2e/f97a666d362fee68b18f41c9c30ed502ca5c98b549749bfcb52a8b74d1eb/scipy-1.18.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11c423f1049c5755ad4409af52a9ada1cff96fe9b50795d4af3619f292901239", size = 37525424, upload-time = "2026-08-21T23:26:56.751Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d5/a9e765a84654ebba8479a1fd1b059ced1af72b168a3b2a3a46540ea38d20/scipy-1.18.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c24acac1e18912761c4700239bbc1fd32f615af690f1584d49b35859be51324d", size = 37416961, upload-time = "2026-08-21T23:27:01.546Z" }, + { url = "https://files.pythonhosted.org/packages/ee/16/e79e0d1c63ef698879d85439d37e9fb434e3b804e506a6991038d086ebd9/scipy-1.18.1-cp314-cp314t-win_arm64.whl", hash = "sha256:9f2897bf7737392ad0d5213ea7b6add72a4edf5679b3153106aeb88b6507b3b9", size = 25331848, upload-time = "2026-08-21T23:27:05.884Z" }, + { url = "https://files.pythonhosted.org/packages/be/4f/1bd37c883b67163e2ca1f60977a399500e6879c15defecac62831c8d078d/scipy-1.18.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:eb0dfcf4e28a99c12c999744a2ff67c9b06200e20401c7c88186e33552a46331", size = 31091484, upload-time = "2026-08-21T23:27:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c5/ba929d7feb9b2332f96827c12e0e924b61973b59b4dea383b603372c65ce/scipy-1.18.1-cp315-cp315-macosx_12_0_arm64.whl", hash = "sha256:30f464bee641fa8e282577c7dce027308403213c6ca8270bba73285c91024bc5", size = 28725057, upload-time = "2026-08-21T23:27:15.9Z" }, + { url = "https://files.pythonhosted.org/packages/a4/19/68f1c50f609d955d230e66d25d02bd3e1e167ec540232135354fb9a4b9e3/scipy-1.18.1-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:1bca3b943fc2567ea49cd02c99abde49da4d5178ec46f624bd8255cda8755beb", size = 20466734, upload-time = "2026-08-21T23:27:20.044Z" }, + { url = "https://files.pythonhosted.org/packages/ef/6d/319fa29b73d1802fa80b32a6eaf3f5be456ef81526da2716a9493bcb5501/scipy-1.18.1-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:c9d18a33309122074ea483dd92dd444189166b8b2ec429fe9ed5ac73c7a0aa23", size = 23089664, upload-time = "2026-08-21T23:27:24.345Z" }, + { url = "https://files.pythonhosted.org/packages/b7/db/30992f9b51a63de671daf3888ffd18378b6cb9ec9f2c972264238ffa7fd6/scipy-1.18.1-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82f201b4c878551d48558337aab270d3c6cca5507b8737c8d8a608d234cccde0", size = 34054035, upload-time = "2026-08-21T23:27:29.409Z" }, + { url = "https://files.pythonhosted.org/packages/91/d4/bf3e735dc0b9d5a8ff45079d2540e17d3aff7a2f0048dd8f552ffd031d2b/scipy-1.18.1-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5", size = 35333883, upload-time = "2026-08-21T23:27:34.293Z" }, + { url = "https://files.pythonhosted.org/packages/19/93/12d78ce9f871fe945fca588d32644e6e63f553c2a35c564d73f3b22a3313/scipy-1.18.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ceb30a00ce7c92d459819443d29ca486d882b83fb6738bdcbb2a1cce94ac5daa", size = 35673124, upload-time = "2026-08-21T23:27:39.059Z" }, + { url = "https://files.pythonhosted.org/packages/70/cd/886219313a1012a48e6ae0ec4f302c837151beb92e1ff0d709ef8fdfc488/scipy-1.18.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f29633129f9fa7e88a3f0fca835de2d030bfc9643f7799e1a0c46cee24d38fc7", size = 37470753, upload-time = "2026-08-21T23:27:44.435Z" }, + { url = "https://files.pythonhosted.org/packages/17/6c/a776888ce618bee54fbde26172f0f46ac1da70d27b63861797fe78e1904b/scipy-1.18.1-cp315-cp315-win_amd64.whl", hash = "sha256:92c14f5bdbfb6216315ce33e78080474082de8b3830122ba97809bfbe65f75c0", size = 37361483, upload-time = "2026-08-21T23:27:49.334Z" }, + { url = "https://files.pythonhosted.org/packages/ab/09/97b651691322ebee97999b017ffc18a15a0b815103844c97e8da9d469731/scipy-1.18.1-cp315-cp315-win_arm64.whl", hash = "sha256:e402cf31eb68f453dbb2d36fc6d722b33f24a55d68b2ae1d92fa6305ca71c298", size = 25035883, upload-time = "2026-08-21T23:27:53.596Z" }, + { url = "https://files.pythonhosted.org/packages/ed/0f/9ec20467bbabd0d44e2a77d0fd3d124f884b4d67df92af82c91d2d6a486f/scipy-1.18.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:2a0b02f9fc46f8520330c23d45e6560db7e3a0d927232139427637f98943e11d", size = 31474926, upload-time = "2026-08-21T23:27:57.993Z" }, + { url = "https://files.pythonhosted.org/packages/8a/58/dcb79161e56efbedc50079fcd2f5fe427a0ebb53022eb476aa73c015ad8f/scipy-1.18.1-cp315-cp315t-macosx_12_0_arm64.whl", hash = "sha256:1d73131e358976663dd969e1fb4ed1404b815cd977eaaedc3b3a133ba2d81c35", size = 29164940, upload-time = "2026-08-21T23:28:03.062Z" }, + { url = "https://files.pythonhosted.org/packages/71/d3/1eeea80c817fcb8ef7bd4a05a58824977a0e57a375cfc3d7ea7c911c01ad/scipy-1.18.1-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:bff0b729edd992766136b34e39cc76bc2fad905aa58897ee72a9cd000a6d8443", size = 20906742, upload-time = "2026-08-21T23:28:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/54/46/e59350428b6099301a20128108c995e2eb175a43f383af9a346e38824f9b/scipy-1.18.1-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:10ac20c69d880f77f375db44c22e3e6a644f9fefa291d4cd2fb9790a89fc99fd", size = 23472183, upload-time = "2026-08-21T23:28:12.109Z" }, + { url = "https://files.pythonhosted.org/packages/89/31/cc91623fa98f0621766a0f0aaaadb2c66de74a7ea7e3837164f6e4354260/scipy-1.18.1-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33a834464fdabc0f26a45508df31b3cc5d028e04dbf6c5ed398541418e0a12fe", size = 34130796, upload-time = "2026-08-21T23:28:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/fc/3e/8572ef536957ddb8aa81bb4090d9e25f257e3b4e05d97deb54319deb8a3a/scipy-1.18.1-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49023963c193dacee096301452f223ee24d86ec5807f8df93c0f7221d119e305", size = 35374253, upload-time = "2026-08-21T23:28:23.732Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c6/59fdeffb4f1435299f93d9dc8140b43ad2916e6cfc944be6c3041fcec86d/scipy-1.18.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:d84a09d0dad90ba6525d8ac1c2334b33e64bf3ccfe9e841f02feb867a22681e4", size = 35758543, upload-time = "2026-08-21T23:28:29.431Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d9/135be205d9de8783193aff9cc3bf483a03a38e4b29432c954e8cb66ac14e/scipy-1.18.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:179ce34a8d0fe273d8883ba59e17e052247d08973dfcb743ca52bb1cce2d60b0", size = 37521946, upload-time = "2026-08-21T23:28:35.245Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a2/5b7d5270621ab7cfa3f7766067bf95dc360b5efb6394694e8143b4156e2b/scipy-1.18.1-cp315-cp315t-win_amd64.whl", hash = "sha256:5632e3ae3d09197c446310cd5187de63e28448ce22f0f67b2b93d97503c0c230", size = 37408295, upload-time = "2026-08-21T23:28:40.724Z" }, + { url = "https://files.pythonhosted.org/packages/63/ad/741c19fcb66755ff953daf9243af8480e4bf3d7fbe57583c178c7d2b6b51/scipy-1.18.1-cp315-cp315t-win_arm64.whl", hash = "sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a", size = 25319710, upload-time = "2026-08-21T23:28:45.713Z" }, +] + [[package]] name = "six" version = "1.17.0"