diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index da3f6b0..0189f08 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,6 +21,8 @@ jobs: run: hatch fmt --linter --check - name: Type Check run: hatch run mypy:check + - name: Check learner release quality + run: hatch run python extensions/english-learner/check_release.py - name: Check Buildable run: hatch build diff --git a/CHANGELOG.md b/CHANGELOG.md index b05a4f3..58901c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ ## [Unreleased][unreleased] +## [v1.9.0] + +**Release date: 2026-10-01** + +### Added + +* Additive English learner annotations on existing words and meanings, + preserving original definitions and exposing scoped notes and provenance + in the Python and web APIs. ([sign/wn#8](https://github.com/sign/wn/pull/8)) +* Reviewed extension for the pinned top 5,000 English frequency words: + 14,048 example sentences across 6,424 concepts and 12,643 plain-language + explanations, plus optional domain, register, usage and grammar notes. + All 17,047 published sense additions have independent Astra approval; + 100 uncertain senses remain withheld. Review is AI-based, not human verification. +* Offline generation and a calibrated, content-bound quality gate that + recompiles the reviewed source in CI and Docker builds. + +Docker includes the extension files but installation remains opt-in with +`--build-arg INSTALL_ENGLISH_LEARNER=true` until the dictionary exposes AI +attribution. The extension is also available as a downloadable release asset. + ## [v1.8.0] diff --git a/Dockerfile b/Dockerfile index 6bb17c7..7bd9f4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,16 @@ COPY extensions/wikidata-lexemes/output ./extensions/wikidata-lexemes/output COPY extensions/wikidata-lexemes/merge_extension.py ./extensions/wikidata-lexemes/merge_extension.py RUN python extensions/wikidata-lexemes/merge_extension.py extensions/wikidata-lexemes/output/*.xml +# Reviewed learner content is opt-in until the dictionary exposes AI attribution. +ARG INSTALL_ENGLISH_LEARNER=false +COPY extensions/english-learner/ ./extensions/english-learner/ +RUN python extensions/english-learner/check_release.py && \ + case "$INSTALL_ENGLISH_LEARNER" in \ + true) python -m wn add extensions/english-learner/rylo-en-learner.xml.gz ;; \ + false) echo "Learner content not installed" ;; \ + *) echo "INSTALL_ENGLISH_LEARNER must be true or false" >&2; exit 1 ;; \ + esac + # Run ANALYZE so SQLite has query planner statistics baked into the image RUN python -c "from wn._db import connect; c = connect(); c.execute('ANALYZE')" diff --git a/extensions/english-learner/.gitattributes b/extensions/english-learner/.gitattributes new file mode 100644 index 0000000..1de6afd --- /dev/null +++ b/extensions/english-learner/.gitattributes @@ -0,0 +1,3 @@ +# Release hashes must survive Windows checkouts without newline conversion. +*.json text eol=lf +*.jsonl text eol=lf diff --git a/extensions/english-learner/README.md b/extensions/english-learner/README.md new file mode 100644 index 0000000..a5770f0 --- /dev/null +++ b/extensions/english-learner/README.md @@ -0,0 +1,229 @@ +# English learner extension + +This is an additive extension of `omw-en:1.4` for senses reached by the pinned +5,000-word English frequency list. It preserves the original words, senses, +synsets, and definitions. It supplies examples where a concept has none, plus +optional learner information where useful. It does not introduce new concepts. + +## Using the extension + +```python +import wn + +wn.add('extensions/english-learner/rylo-en-learner.xml.gz') +english = wn.Wordnet('omw-en:1.4') +synset = english.synset('omw-en-09849598-n') +print(synset.definition()) # original WordNet definition +print(synset.examples()) # original plus extension examples +print(synset.learner_data()) # scoped annotations and provenance +``` + +`Word`, `Sense`, and `Synset` expose `learner_data()`. The web API includes the +same records under `attributes.learner_data`. Generated examples also expose +their provenance under `attributes.example_details`, while `examples` remains +a list of strings. Removing `rylo-en-learner:1.0` +removes its additions without rewriting the base lexicon. Unlike the separate +Wikidata lexeme import, this extension does not need a destructive database +merge. + +Docker builds validate the packaged release but do not install it by default. +Use `--build-arg INSTALL_ENGLISH_LEARNER=true` to explicitly opt in. + +## Data contract + +The package uses standard WN-LMF `LexiconExtension`, `ExternalLexicalEntry`, +`ExternalSense`, and `ExternalSynset` elements. Examples are ordinary `Example` +elements on the existing sense and concept. Example metadata identifies the +source sense, so a synonym's example is not presented as another word's usage. +Original concept examples take precedence: the builder only adds examples to +concepts whose original examples are empty. + +Learner annotations are ordinary additional `Definition` elements, marked with +`dc:type="learner-enrichment"`. Their text is a plain-language explanation, or +the unchanged reference definition when only structured notes are supplied. +`dc:description` contains a validated JSON object: + +```json +{ + "schema_version": 1, + "scope": "sense", + "domains": ["computing"], + "grammar_notes": ["Usually used as an uncountable noun."] +} +``` + +Optional fields are `domains`, `register`, `usage`, `grammar_notes`, +`irregular_forms`, and `usage_tips`. List fields contain strings; +`irregular_forms` maps form types to lists of spellings. Omit fields that add +no useful information. No automatic etymologies or redundant comparisons with +other meanings are included. + +- `synset` scope supplies a shared explanation for the concept. +- `sense` scope requires native `sourceSense` and applies only to that word's + meaning; grammatical or register notes must not spread to every synonym. +- Irregular forms default to `sense` scope, because conjugation can depend on + meaning (for example, lie/lay/lain versus lie/lied/lied). Explicit `word` + scope requires `word_id` and is available for forms shared by every sense. + +`wn.learner.validate_learner_fields` validates this structure. Invalid marked +annotations fail import. Original `.definition()` and `.definitions()` never expose the +annotation container. Native import/export retains source-sense references. + +## Generation and review + +Generation is offline. There is no request-time model call or UI button. +The input contains the exact lexical form, part of speech, original definition, +existing examples, and native domain/usage evidence. Prior generated training +text is an unreviewed candidate, not a factual authority. The prompt requires +sense-specific examples and permits empty enrichment for straightforward +meanings. Missing, truncated, malformed, or incorrectly targeted outputs are +rejected and recorded for retry. + +Native `dc:creator`, `dc:source`, and `status` retain origin and review state. +Model generation and AI review are separate from human review; callers must not +present generated material as human-verified. The release manifest records +coverage and input hashes. The builder refuses to release a corpus with any +remaining example-less target sense or concept. A synonym's examples cannot +satisfy another target word's coverage requirement. + +If a meaning cannot be illustrated reliably, keep its generated content out of +the package and list its exact `sense_id` and editorial `reason` in a separate +JSONL file. The builder's explicit `--withheld` input allows these exceptions +and records every one in the manifest. They are not counted as covered or as +published enrichment. Unlisted gaps and records marked unresolved still fail +the build. A package with withheld senses needs further editorial review. + +## Reproducing inputs and building + +Use the pinned frequency list in this directory. Matching mirrors the dictionary: +case/diacritic-insensitive matching of stored forms, with no inferred morphology. +An uppercase abbreviation is therefore included when it matches a frequent +lowercase form, but generation uses its actual spelling and meaning. Unmatched +frequency words are reported rather than invented as new entries. + +The pinned list matches 3,887 frequency words, reaching 21,043 existing senses +in 17,643 concepts. The other 1,113 frequency words have no matching stored form +in this WordNet release. Before enrichment, 6,499 of these concepts have no +examples. Published additions and withheld senses are reported in `manifest.json`. +`quality-audit.json` summarizes the independent Astra audit; its rejection rate +is a review outcome, not a measured accuracy estimate. +`quality-repairs.json` records the correction totals and recovery of previously +withheld senses. Writing attempts, including unsuccessful ones, are preserved in +`quality-repair-attempts.jsonl.xz`. + +```sh +python extensions/english-learner/prepare_inventory.py --help +python extensions/english-learner/prepare_candidates.py --help +python extensions/english-learner/generation.py --help +python extensions/english-learner/build_extension.py --help +``` + +To rebuild the release from its reviewed source records: + +```sh +python extensions/english-learner/prepare_inventory.py --output /tmp/learner-inventory +python extensions/english-learner/build_extension.py \ + --inventory /tmp/learner-inventory/inventory.jsonl \ + --records extensions/english-learner/reviewed-records.jsonl.xz \ + --withheld extensions/english-learner/withheld.jsonl \ + --semantic-reviews extensions/english-learner/quality-reviews.jsonl.xz \ + --calibration extensions/english-learner/quality-calibration-decisions.jsonl \ + --output extensions/english-learner/rylo-en-learner.xml.gz +``` + +`generation.py` supports Ollama and OpenAI-compatible vLLM endpoints, concurrent +batches, deterministic shards, and resume checkpoints keyed by input, prompt, +and model. Each exact sense is assessed once even if several frequency forms +reach it. Generation output is flat JSONL with existing identity fields, +provenance, and optional enrichment. Empty records mean no extra annotation was +needed; they are still useful coverage checkpoints. + +`build_extension.py` accepts an inventory and one or more JSONL (or JSONL.xz) +record files. Generated content requires a separate, content-bound Astra approval; +a model name or earlier Luna checkpoint alone cannot release it. Native WordNet +topic labels are checked against original relations and retain their source. +It writes deterministic compressed WN-LMF and a coverage manifest. +The database remains untouched until a caller explicitly installs the package. + +## Enforced semantic quality gate + +The independent critic receives every proposed field alongside the exact lemma, +part of speech, definition, synonyms, original examples and syntactic frames. +It checks sense fit, grammatical role, factual claims and usage restrictions. +Each review includes a verdict, a sense-specific rationale and field-level issues. +The SHA-256 binds the decision to the content, context, scope and review rubric. +Editing any of these requires another review. + +Reviewers return one JSONL record per input, using the same `review_run` for +their calibration and corpus decisions: + +```json +{ + "sense_id": "input ID", + "input_sha256": "input hash", + "reviewer": "gpt-6-astra", + "review_run": "run ID", + "rubric_version": "exact-sense-2", + "decision": "accept", + "rationale": "Why every proposed field fits this sense.", + "issues": [] +} +``` + +`reject` and `uncertain` decisions require one or more issues, each with a +learner `field` name and a specific `reason`. Accepted decisions have no issues. + +Each review run must first pass twelve calibration cases: eight known errors +and four valid controls. Passing these checks does not establish a general +accuracy rate. Astra review is still AI review, not human verification. + +`apply_reviews.py` requires complete review coverage. It quarantines an entire +sense if any field is rejected or uncertain; it never silently rewrites and +approves a correction. Unreleased rows remain in +`quarantined-records.jsonl.xz`, with the final verdicts in `quality-audit.jsonl.xz`. +The first audit and its rejected source are preserved separately in +`quality-initial-audit.jsonl.xz` and `quality-original-rejected-records.jsonl.xz`; +`quality-repair-history.jsonl.xz` records the subsequent review decisions. +Corrections need fresh review before they can enter a later release. +Native WordNet definitions remain available for withheld senses. + +`prepare_corrections.py` verifies the original review hashes and prepares exact +sense context with the flagged fields and reasons. Unsupported optional fields +can be removed with `remove_flagged_fields`; missing examples need new authored +sentences. Both remain unreviewed until another review approves the revised +content. Even deleting all optional fields requires a fresh decision. Generation +receives original WordNet examples to disambiguate short definitions and supports +targeted correction context without creating new meanings. +New Astra-authored corrections record an `author_run` in the source. This identity +is bound into the review hash; the gate rejects decisions from that same run. + +To review a new candidate set, prepare exact inputs with `quality_gate.py`, have +independent Astra reviewers return decisions using its rubric and calibration +inputs, then run `apply_reviews.py` before building. Use `--calibration-output` +to prepare blind calibration inputs; do not give reviewers the expected answers +from `quality-calibration.json`. These are offline steps; +CI never calls a model. The committed calibration decisions and compact +`quality-context.jsonl.xz` make the release gate reproducible without downloading +WordNet or accessing the generation machines. + +```sh +python extensions/english-learner/check_release.py +``` + +CI and Docker run this command. It rejects missing, stale, negative or +uncalibrated reviews, validates native topic evidence and coverage, and rebuilds +the XML from approved records to verify the actual packaged text. Changing only +a manifest hash cannot bypass the content review. + +## Sources + +- Base concepts and definitions: OMW English WordNet 1.4 / Princeton WordNet 3.0. + Original WordNet attribution and license continue to apply to source material; + the complete source notices are retained in `SOURCE-LICENSES.txt`. +- Frequency: `first20hours/google-10000-english`, via the pinned dictionary + snapshot identified in `frequency-source.json`. +- Prior candidate text: `sign/word-sense-disambiguation`, + `training/data/generated.tar.xz`, generated with `gpt-oss:120b`. + That repository is MIT-licensed, copyright 2026 Nagish Inc. +- Newly generated material retains the actual model and prompt version in each + record; native WordNet labels retain separate source attribution. diff --git a/extensions/english-learner/SOURCE-LICENSES.txt b/extensions/english-learner/SOURCE-LICENSES.txt new file mode 100644 index 0000000..14a6485 --- /dev/null +++ b/extensions/english-learner/SOURCE-LICENSES.txt @@ -0,0 +1,70 @@ +Source notices for the English learner extension + +Base definitions and WordNet-derived data: + +WordNet Release 3.0 + +This software and database is being provided to you, the LICENSEE, by +Princeton University under the following license. By obtaining, using +and/or copying this software and database, you agree that you have +read, understood, and will comply with these terms and conditions.: + +Permission to use, copy, modify and distribute this software and +database and its documentation for any purpose and without fee or +royalty is hereby granted, provided that you agree to comply with +the following copyright notice and statements, including the disclaimer, +and that the same appear on ALL copies of the software, database and +documentation, including modifications that you make for internal +use or for distribution. + +WordNet 3.0 Copyright 2006 by Princeton University. All rights reserved. + +THIS SOFTWARE AND DATABASE IS PROVIDED "AS IS" AND PRINCETON +UNIVERSITY MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PRINCETON +UNIVERSITY MAKES NO REPRESENTATIONS OR WARRANTIES OF MERCHANT- +ABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE +OF THE LICENSED SOFTWARE, DATABASE OR DOCUMENTATION WILL NOT +INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR +OTHER RIGHTS. + +The name of Princeton University or Princeton may not be used in +advertising or publicity pertaining to distribution of the software +and/or database. Title to copyright in this software, database and +any associated documentation shall at all times remain with +Princeton University and LICENSEE agrees to preserve same. + +Prior generated training candidates (sign/word-sense-disambiguation): + +MIT License + +Copyright (c) 2026 Nagish Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Frequency-list provenance: + +The pinned top-5,000 list is taken from the existing Rylo dictionary snapshot +identified in frequency-source.json, originally from first20hours/google-10000-english. +Upstream describes it as derived from Google's Web Trillion Word Corpus, +with contributions from Peter Novig and editing by Josh Kaufman. +The upstream licensing statement distinguishes research/educational use from +commercial use and points commercial users to the Linguistic Data Consortium. +See https://github.com/first20hours/google-10000-english/blob/master/LICENSE.md +for the complete upstream statement; no additional rights are granted here. diff --git a/extensions/english-learner/apply_reviews.py b/extensions/english-learner/apply_reviews.py new file mode 100644 index 0000000..c7dcad6 --- /dev/null +++ b/extensions/english-learner/apply_reviews.py @@ -0,0 +1,116 @@ +"""Quarantine failed semantic reviews without editing or silently approving text.""" + +import argparse +import hashlib +import importlib.util +import json +import lzma +from collections import Counter +from pathlib import Path + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location( + "learner_quality", HERE / "quality_gate.py" +) +quality = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(quality) + + +def write_rows(path, rows): + data = "".join( + json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + "\n" + for row in rows + ).encode() + path.write_bytes(lzma.compress(data) if path.suffix == ".xz" else data) + + +def select_reviewed(records, inventory, decisions, fixtures, calibration, withheld): + items = quality.review_items(records, inventory) + reviewed = quality.validate_decisions(items, decisions, require_accept=False) + for run in sorted({d["review_run"] for d in decisions}): + quality.validate_calibration( + fixtures, [d for d in calibration if d["review_run"] == run] + ) + blocked = {sid for sid, d in reviewed.items() if d["decision"] != "accept"} + existing = {r["sense_id"] for r in withheld} + if existing & {r["sense_id"] for r in records}: + raise ValueError("previously withheld content must not re-enter without review") + new_withheld = [ + { + "sense_id": sid, + "reason": "Semantic review: " + + "; ".join( + f"{issue['field']}: {issue['reason']}" + for issue in reviewed[sid]["issues"] + ), + } + for sid in sorted(blocked) + ] + accepted = [r for r in records if r["sense_id"] not in blocked] + # Revised content cannot inherit its old review stamp. Only the validated + # fresh approval above confers this checkpoint, including intentional removals. + accepted = [ + {**r, "reviewer": quality.REVIEWER, "review_status": "ai-reviewed"} + if r.get("requires_semantic_review") + else r + for r in accepted + ] + return { + "accepted": accepted, + "quarantined": [r for r in records if r["sense_id"] in blocked], + "withheld": sorted([*withheld, *new_withheld], key=lambda r: r["sense_id"]), + "reviews": [d for d in decisions if d["decision"] == "accept"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inventory", type=Path, required=True) + parser.add_argument("--records", type=Path, required=True) + parser.add_argument("--decisions", type=Path, nargs="+", required=True) + parser.add_argument("--calibration", type=Path, nargs="+", required=True) + parser.add_argument("--withheld", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + records = quality.read_jsonl(args.records) + decisions = [d for p in args.decisions for d in quality.read_jsonl(p)] + calibration = [d for p in args.calibration for d in quality.read_jsonl(p)] + selected = select_reviewed( + records, + quality.read_jsonl(args.inventory), + decisions, + json.loads((HERE / "quality-calibration.json").read_text(encoding="utf-8")), + calibration, + quality.read_jsonl(args.withheld), + ) + args.output_dir.mkdir(parents=True, exist_ok=True) + for name, rows in { + "reviewed-records.jsonl.xz": selected["accepted"], + "quarantined-records.jsonl.xz": selected["quarantined"], + "quality-reviews.jsonl.xz": selected["reviews"], + "quality-audit.jsonl.xz": decisions, + "quality-calibration-decisions.jsonl": calibration, + "withheld.jsonl": selected["withheld"], + }.items(): + write_rows(args.output_dir / name, rows) + report = { + "rubric_version": quality.RUBRIC_VERSION, + "reviewer": quality.REVIEWER, + "source_records_sha256": hashlib.sha256(args.records.read_bytes()).hexdigest(), + "reviewed_senses": len(decisions), + "decisions": dict(sorted(Counter(d["decision"] for d in decisions).items())), + "issues_by_field": dict( + sorted(Counter(i["field"] for d in decisions for i in d["issues"]).items()) + ), + "withheld_senses": len(selected["withheld"]), + "note": "AI review, not human verification or a measured accuracy guarantee.", + } + (args.output_dir / "quality-audit.json").write_text( + json.dumps(report, indent=2) + "\n" + ) + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/extensions/english-learner/build_extension.py b/extensions/english-learner/build_extension.py new file mode 100644 index 0000000..7f0ecc3 --- /dev/null +++ b/extensions/english-learner/build_extension.py @@ -0,0 +1,409 @@ +"""Build an additive WN-LMF extension from scoped, provenance-bearing records. + +No lexical entries, senses, or synsets are created. All referenced identities +must already exist in the base lexicon. Examples are attached to both their +source sense and its existing concept; existing concept examples are preserved. +""" + +import argparse +import gzip +import hashlib +import importlib.util +import json +import lzma +import sqlite3 +from collections import Counter +from pathlib import Path + +from wn import lmf +from wn.learner import ANNOTATION_TYPE, validate_learner_fields + +BASE = {"id": "omw-en", "version": "1.4"} +FIELDS = ("domains", "register", "usage", "grammar_notes", "usage_tips") + +QUALITY_PATH = Path(__file__).with_name("quality_gate.py") +QUALITY_SPEC = importlib.util.spec_from_file_location("learner_quality", QUALITY_PATH) +quality = importlib.util.module_from_spec(QUALITY_SPEC) +QUALITY_SPEC.loader.exec_module(quality) + + +def read_records(path): + opener = lzma.open if path.suffix == ".xz" else open + with opener(path, "rt", encoding="utf-8") as stream: + for number, line in enumerate(stream, 1): + if line.strip(): + row = json.loads(line) + if not isinstance(row, dict): + raise ValueError(f"{path}:{number}: expected an object") + yield row + + +def provenance(row): + for field in ("creator", "source", "status"): + if not isinstance(row.get(field), str) or not row[field].strip(): + raise ValueError(f"missing provenance: {field}") + meta = {key: row[key] for key in ("creator", "source", "status")} + if row.get("reviewer"): + meta["contributor"] = row["reviewer"] + meta["note"] = "AI review: " + row.get("review_status", "unspecified") + if row.get("semantic_reviewer"): + meta["contributor"] = "; ".join(dict.fromkeys(filter(None, [ + meta.get("contributor"), row["semantic_reviewer"], + ]))) + meta["note"] = "AI review: " + quality.RUBRIC_VERSION + "; not human-reviewed" + return meta + + +def definition(row, scope="sense"): + fields = {"schema_version": 1, "scope": scope} + if scope == "word": + fields.update(word_id=row["word_id"], irregular_forms=row["irregular_forms"]) + else: + fields.update({key: row[key] for key in FIELDS if row.get(key)}) + if row.get("irregular_forms"): + fields["irregular_forms"] = row["irregular_forms"] + validate_learner_fields(fields) + meta = provenance(row) + meta.update( + type=ANNOTATION_TYPE, description=json.dumps(fields, ensure_ascii=False) + ) + data = { + "text": row.get("plain_language") or row["base_definition"], + "language": "en", + "meta": meta, + } + if scope == "sense": + data["sourceSense"] = row["sense_id"] + return data + + +def entities(row, entries, senses, synsets): + for key in ("word_id", "sense_id", "synset_id", "base_definition"): + if not isinstance(row.get(key), str) or not row[key].strip(): + raise ValueError(f"missing identity/reference: {key}") + entry = entries.setdefault( + row["word_id"], + { + "id": row["word_id"], + "external": True, + "senses": [], + }, + ) + if row["sense_id"] not in senses: + sense = {"id": row["sense_id"], "external": True, "examples": []} + senses[row["sense_id"]] = (row["word_id"], row["synset_id"], sense) + entry["senses"].append(sense) + word_id, synset_id, sense = senses[row["sense_id"]] + if (word_id, synset_id) != (row["word_id"], row["synset_id"]): + raise ValueError(f"conflicting sense identity: {row['sense_id']}") + synset = synsets.setdefault( + row["synset_id"], + { + "id": row["synset_id"], + "external": True, + "definitions": [], + "examples": [], + }, + ) + return sense, synset + + +def annotations(row, seen_irregular): + result = [] + if row.get("plain_language"): + plain = {**row, "source": row.get("plain_language_source", row["source"])} + for field in FIELDS: + plain.pop(field, None) + plain.pop("irregular_forms", None) + result.append(definition(plain, row.get("plain_language_scope", "sense"))) + if any(row.get(key) for key in FIELDS): + scoped = {key: value for key, value in row.items() if key != "plain_language"} + scoped.pop("irregular_forms", None) + result.append(definition(scoped, row.get("scope", "sense"))) + if row.get("irregular_forms"): + scope = row.get("irregular_forms_scope", "sense") + identity = row["word_id"] if scope == "word" else row["sense_id"] + previous = seen_irregular.get(identity) + if previous is not None and previous != row["irregular_forms"]: + raise ValueError(f"conflicting irregular forms: {identity}") + if previous is None: + irregular = { + key: value + for key, value in row.items() + if key not in (*FIELDS, "plain_language") + } + result.append(definition(irregular, scope)) + seen_irregular[identity] = row["irregular_forms"] + return result + + +def add_examples(row, sense, synset, seen, counts): + for text in row.get("examples", []): + if not isinstance(text, str) or not text.strip(): + raise ValueError("example must be a nonempty string") + key = (row["sense_id"], text.strip()) + if key in seen: + continue + seen.add(key) + example = { + "text": text.strip(), + "language": "en", + "meta": { + **provenance(row), + "source": row.get("example_source", row["source"]), + "identifier": row["sense_id"], + "type": "learner-example", + }, + } + sense["examples"].append(example) + synset["examples"].append(example) + counts["examples"] += 1 + + +def build_resource( + records, original_examples, required_example_senses=None, + withheld_example_concepts=(), +): + entries, senses, synsets = {}, {}, {} + counts = Counter() + seen_examples, seen_definitions, seen_irregular = set(), set(), {} + for row in records: + if row.get("unresolved_reason"): + raise ValueError( + f"unresolved content for {row.get('sense_id')}: " + f"{row['unresolved_reason']}" + ) + if row.get("synset_id") not in original_examples: + raise ValueError( + f"concept outside frequency inventory: {row.get('synset_id')}" + ) + provenance(row) + sense, synset = entities(row, entries, senses, synsets) + for item in annotations(row, seen_irregular): + key = (row["synset_id"], json.dumps(item, sort_keys=True)) + if key in seen_definitions: + continue + synset["definitions"].append(item) + seen_definitions.add(key) + counts["annotations"] += 1 + fields = json.loads(item["meta"]["description"]) + counts.update( + key for key in (*FIELDS, "irregular_forms") if fields.get(key) + ) + if item["text"] != row["base_definition"] and fields["scope"] != "word": + counts["plain_language"] += 1 + if not original_examples[row["synset_id"]]: + add_examples(row, sense, synset, seen_examples, counts) + missing = [ + key + for key, examples in original_examples.items() + if not examples and not synsets.get(key, {}).get("examples") + and key not in withheld_example_concepts + ] + if missing: + raise ValueError(f"{len(missing)} concepts still lack examples: {missing[:5]}") + missing_senses = [ + sid + for sid in sorted(required_example_senses or ()) + if sid not in senses or not senses[sid][2]["examples"] + ] + if missing_senses: + raise ValueError( + f"{len(missing_senses)} target senses still lack examples: " + f"{missing_senses[:5]}" + ) + counts["concepts_with_added_examples"] = sum( + bool(s["examples"]) for s in synsets.values() + ) + counts["referenced_senses"] = len(senses) + counts["referenced_words"] = len(entries) + counts["referenced_synsets"] = len(synsets) + counts["new_words"] = counts["new_senses"] = counts["new_synsets"] = 0 + resource = { + "lmf_version": "1.4", + "lexicons": [ + { + "id": "rylo-en-learner", + "version": "1.0", + "label": "English learner enrichment for frequent words", + "language": "en", + "email": "amit@nagish.com", + "license": ( + "https://github.com/sign/wn/blob/main/extensions/" + "english-learner/SOURCE-LICENSES.txt" + ), + "url": "https://github.com/sign/wn/tree/main/extensions/english-learner", + "meta": { + "description": ( + "AI-generated learner text plus sourced WordNet labels. " + "Independent AI review is recorded per annotation; " + "AI review is not human review." + ) + }, + "extends": BASE, + "entries": [entries[key] for key in sorted(entries)], + "synsets": [synsets[key] for key in sorted(synsets)], + } + ], + } + return resource, dict(sorted(counts.items())) + + +def validate_identities(records, database): + """Check source-sense membership against the read-only base database.""" + with sqlite3.connect(database.resolve().as_uri() + "?mode=ro", uri=True) as conn: + definitions = {} + for synset_id, text in conn.execute( + "SELECT ss.id,d.definition FROM definitions d " + "JOIN synsets ss ON ss.rowid=d.synset_rowid " + "JOIN lexicons l ON l.rowid=d.lexicon_rowid " + "WHERE l.specifier='omw-en:1.4'" + ): + definitions.setdefault(synset_id, set()).add(text) + known = dict( + conn.execute( + "SELECT s.id, e.id || char(9) || ss.id FROM senses s " + "JOIN entries e ON e.rowid=s.entry_rowid " + "JOIN synsets ss ON ss.rowid=s.synset_rowid " + "JOIN lexicons l ON l.rowid=s.lexicon_rowid " + "WHERE l.specifier='omw-en:1.4'" + ) + ) + for row in records: + expected = row["word_id"] + "\t" + row["synset_id"] + if known.get(row["sense_id"]) != expected: + raise ValueError(f"unknown or mis-targeted base sense: {row['sense_id']}") + if row["base_definition"] not in definitions.get(row["synset_id"], set()): + raise ValueError( + f"stale or incorrect reference definition: {row['sense_id']}" + ) + + +def validate_review(records): + """Keep generation checkpoints out of the reviewed release package.""" + assessed = set() + for row in records: + if row.get("creator") == "WordNet" and row.get("source") == "omw-en:1.4": + continue + if not row.get("reviewer") or row.get("review_status") != "ai-reviewed": + raise ValueError(f"missing independent AI review: {row['sense_id']}") + assessed.add(row["sense_id"]) + return assessed + + + +def mark_semantic_review(records): + return [ + {**row, "semantic_reviewer": quality.REVIEWER} + if not quality.native(row) and any(row.get(k) for k in quality.FIELDS) + else row for row in records + ] + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inventory", type=Path, required=True) + parser.add_argument("--records", type=Path, nargs="+", required=True) + parser.add_argument( + "--withheld", type=Path, + help="Explicit sense IDs and reasons requiring further editorial review", + ) + parser.add_argument( + "--semantic-reviews", type=Path, required=True, + help="Exact-content Astra decisions for all generated additions", + ) + parser.add_argument("--calibration", type=Path, required=True, + help="Passing regression decisions for every reviewer run") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--database", type=Path, default=Path.home() / ".wn_data/wn.db") + args = parser.parse_args() + inventory = list(read_records(args.inventory)) + original_examples = {row["synset_id"]: row["examples"] for row in inventory} + records = [row for path in args.records for row in read_records(path)] + validate_identities(records, args.database) + fixtures_path = Path(__file__).with_name("quality-calibration.json") + gate = quality.validate_release( + records, inventory, quality.read_jsonl(args.semantic_reviews), + json.loads(fixtures_path.read_text(encoding="utf-8")), + quality.read_jsonl(args.calibration), + ) + # Only the validated model decisions confer the stronger review provenance. + records = mark_semantic_review(records) + assessed_senses = validate_review(records) + expected_senses = {s["sense_id"] for r in inventory for s in r["senses"]} + if {row["sense_id"] for row in records} - expected_senses: + raise ValueError("release records contain senses outside the inventory") + withheld = list(read_records(args.withheld)) if args.withheld else [] + withheld_ids = {row["sense_id"] for row in withheld} + if len(withheld_ids) != len(withheld) or withheld_ids - expected_senses: + raise ValueError("withheld senses must be unique members of the inventory") + if withheld_ids & {row["sense_id"] for row in records}: + raise ValueError("withheld senses must not be published in release records") + if any(not isinstance(r.get("reason"), str) or not r["reason"].strip() + for r in withheld): + raise ValueError("every withheld sense requires an editorial reason") + missing_senses = expected_senses - assessed_senses + missing_senses -= withheld_ids + if missing_senses: + raise ValueError(f"{len(missing_senses)} target senses have not been assessed") + required_examples = { + s["sense_id"] for row in inventory if not row["examples"] for s in row["senses"] + } + concept_senses = {} + for row in inventory: + concept_senses.setdefault(row["synset_id"], set()).update( + s["sense_id"] for s in row["senses"] + ) + withheld_concepts = { + ssid for ssid, senses in concept_senses.items() if senses <= withheld_ids + } + resource, counts = build_resource( + records, original_examples, required_examples - withheld_ids, + withheld_concepts, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + xml_path = ( + args.output.with_suffix("") if args.output.suffix == ".gz" else args.output + ) + lmf.dump(resource, xml_path) + if args.output.suffix == ".gz": + args.output.write_bytes(gzip.compress(xml_path.read_bytes(), mtime=0)) + xml_path.unlink() + context_path = args.output.with_name("quality-context.jsonl.xz") + context_text = "".join(json.dumps(row, ensure_ascii=False) + "\n" + for row in quality.compact_inventory(inventory)) + context_path.write_bytes(lzma.compress(context_text.encode())) + report = { + "base_lexicon": "omw-en:1.4", + "extension": "rylo-en-learner:1.0", + "frequency_cutoff": 5000, + "matched_frequency_words": len({row["word"] for row in inventory}), + "word_meaning_pairs": len(inventory), + "unique_senses": len({s["sense_id"] for r in inventory for s in r["senses"]}), + "unique_synsets": len(original_examples), + "status": ( + "AI-generated, not human-reviewed; " + "native labels retain WordNet provenance" + ), + "assessed_senses": len(assessed_senses), + "quality_gate": gate, + "review_status_counts": dict(Counter( + row.get("review_status", "not-ai-reviewed") for row in records + )), + "withheld_senses": withheld, + "counts": counts, + "inputs": { + str(p.name): hashlib.sha256(p.read_bytes()).hexdigest() + for p in [args.inventory, *args.records, args.semantic_reviews, + args.calibration, fixtures_path, context_path, + *([args.withheld] if args.withheld else [])] + }, + "extension_sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), + } + args.output.with_name("manifest.json").write_text( + json.dumps(report, indent=2) + "\n" + ) + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/extensions/english-learner/check_release.py b/extensions/english-learner/check_release.py new file mode 100644 index 0000000..22a97c7 --- /dev/null +++ b/extensions/english-learner/check_release.py @@ -0,0 +1,114 @@ +"""Verify the committed source, reviews and compiled package agree; no model call.""" + +import argparse +import gzip +import hashlib +import importlib.util +import json +import tempfile +from pathlib import Path + +from wn import lmf + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location( + "learner_build", HERE / "build_extension.py" +) +builder = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(builder) +quality = builder.quality + + +def check_release(directory): + records_path = directory / "reviewed-records.jsonl.xz" + reviews_path = directory / "quality-reviews.jsonl.xz" + calibration_path = directory / "quality-calibration-decisions.jsonl" + context_path = directory / "quality-context.jsonl.xz" + withheld_path = directory / "withheld.jsonl" + fixtures_path = directory / "quality-calibration.json" + package_path = directory / "rylo-en-learner.xml.gz" + manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8")) + for path in ( + records_path, + reviews_path, + calibration_path, + context_path, + withheld_path, + fixtures_path, + ): + if ( + manifest["inputs"].get(path.name) + != hashlib.sha256(path.read_bytes()).hexdigest() + ): + raise ValueError(f"changed release input: {path.name}") + if ( + manifest["extension_sha256"] + != hashlib.sha256(package_path.read_bytes()).hexdigest() + ): + raise ValueError("changed compiled extension") + records = quality.read_jsonl(records_path) + inventory = quality.read_jsonl(context_path) + gate = quality.validate_release( + records, + inventory, + quality.read_jsonl(reviews_path), + json.loads(fixtures_path.read_text(encoding="utf-8")), + quality.read_jsonl(calibration_path), + ) + if manifest.get("quality_gate") != gate: + raise ValueError("manifest quality gate disagrees with exact-content decisions") + withheld = quality.read_jsonl(withheld_path) + withheld_ids = {r["sense_id"] for r in withheld} + expected = {s["sense_id"] for r in inventory for s in r["senses"]} + if ( + len(withheld_ids) != len(withheld) + or withheld_ids - expected + or any(not r.get("reason", "").strip() for r in withheld) + or withheld_ids & {r["sense_id"] for r in records} + ): + raise ValueError("invalid withheld scope") + assessed = builder.validate_review(records) + if assessed | withheld_ids != expected: + raise ValueError("release target coverage changed") + if manifest["withheld_senses"] != withheld: + raise ValueError("manifest withheld senses changed") + examples = {row["synset_id"]: row["examples"] for row in inventory} + required = { + s["sense_id"] for r in inventory if not r["examples"] for s in r["senses"] + } + excluded = { + r["synset_id"] + for r in inventory + if {s["sense_id"] for s in r["senses"]} <= withheld_ids + } + resource, counts = builder.build_resource( + builder.mark_semantic_review(records), + examples, + required - withheld_ids, + excluded, + ) + if counts != manifest["counts"]: + raise ValueError("manifest content counts changed") + # Recompile instead of trusting an editable status/hash in the manifest. + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "extension.xml" + lmf.dump(resource, path) + original = ( + gzip.decompress(package_path.read_bytes()) + .decode("utf-8") + .replace("\r\n", "\n") + ) + if path.read_text(encoding="utf-8") != original: + raise ValueError("compiled extension differs from approved source") + return gate + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--directory", type=Path, default=HERE) + args = parser.parse_args() + print(json.dumps(check_release(args.directory), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/extensions/english-learner/frequency-en-5000.json b/extensions/english-learner/frequency-en-5000.json new file mode 100644 index 0000000..d7f68e6 --- /dev/null +++ b/extensions/english-learner/frequency-en-5000.json @@ -0,0 +1,5002 @@ +[ + "the", + "of", + "and", + "to", + "a", + "in", + "for", + "is", + "on", + "that", + "by", + "this", + "with", + "i", + "you", + "it", + "not", + "or", + "be", + "are", + "from", + "at", + "as", + "your", + "all", + "have", + "new", + "more", + "an", + "was", + "we", + "will", + "home", + "can", + "us", + "about", + "if", + "page", + "my", + "has", + "search", + "free", + "but", + "our", + "one", + "other", + "do", + "no", + "information", + "time", + "they", + "site", + "he", + "up", + "may", + "what", + "which", + "their", + "news", + "out", + "use", + "any", + "there", + "see", + "only", + "so", + "his", + "when", + "contact", + "here", + "business", + "who", + "web", + "also", + "now", + "help", + "get", + "pm", + "view", + "online", + "c", + "e", + "first", + "am", + "been", + "would", + "how", + "were", + "me", + "s", + "services", + "some", + "these", + "click", + "its", + "like", + "service", + "x", + "than", + "find", + "price", + "date", + "back", + "top", + "people", + "had", + "list", + "name", + "just", + "over", + "state", + "year", + "day", + "into", + "email", + "two", + "health", + "n", + "world", + "re", + "next", + "used", + "go", + "b", + "work", + "last", + "most", + "products", + "music", + "buy", + "data", + "make", + "them", + "should", + "product", + "system", + "post", + "her", + "city", + "t", + "add", + "policy", + "number", + "such", + "please", + "available", + "copyright", + "support", + "message", + "after", + "best", + "software", + "then", + "jan", + "good", + "video", + "well", + "d", + "where", + "info", + "rights", + "public", + "books", + "high", + "school", + "through", + "m", + "each", + "links", + "she", + "review", + "years", + "order", + "very", + "privacy", + "book", + "items", + "company", + "r", + "read", + "group", + "sex", + "need", + "many", + "user", + "said", + "de", + "does", + "set", + "under", + "general", + "research", + "university", + "january", + "mail", + "full", + "map", + "reviews", + "program", + "life", + "know", + "games", + "way", + "days", + "management", + "p", + "part", + "could", + "great", + "united", + "hotel", + "real", + "f", + "item", + "international", + "center", + "ebay", + "must", + "store", + "travel", + "comments", + "made", + "development", + "report", + "off", + "member", + "details", + "line", + "terms", + "before", + "hotels", + "did", + "send", + "right", + "type", + "because", + "local", + "those", + "using", + "results", + "office", + "education", + "national", + "car", + "design", + "take", + "posted", + "internet", + "address", + "community", + "within", + "states", + "area", + "want", + "phone", + "dvd", + "shipping", + "reserved", + "subject", + "between", + "forum", + "family", + "l", + "long", + "based", + "w", + "code", + "show", + "o", + "even", + "black", + "check", + "special", + "prices", + "website", + "index", + "being", + "women", + "much", + "sign", + "file", + "link", + "open", + "today", + "technology", + "south", + "case", + "project", + "same", + "pages", + "uk", + "version", + "section", + "own", + "found", + "sports", + "house", + "related", + "security", + "both", + "g", + "county", + "american", + "photo", + "game", + "members", + "power", + "while", + "care", + "network", + "down", + "computer", + "systems", + "three", + "total", + "place", + "end", + "following", + "download", + "h", + "him", + "without", + "per", + "access", + "think", + "north", + "resources", + "current", + "posts", + "big", + "media", + "law", + "control", + "water", + "history", + "pictures", + "size", + "art", + "personal", + "since", + "including", + "guide", + "shop", + "directory", + "board", + "location", + "change", + "white", + "text", + "small", + "rating", + "rate", + "government", + "children", + "during", + "usa", + "return", + "students", + "v", + "shopping", + "account", + "times", + "sites", + "level", + "digital", + "profile", + "previous", + "form", + "events", + "love", + "old", + "john", + "main", + "call", + "hours", + "image", + "department", + "title", + "description", + "non", + "k", + "y", + "insurance", + "another", + "why", + "shall", + "property", + "class", + "cd", + "still", + "money", + "quality", + "every", + "listing", + "content", + "country", + "private", + "little", + "visit", + "save", + "tools", + "low", + "reply", + "customer", + "december", + "compare", + "movies", + "include", + "college", + "value", + "article", + "york", + "man", + "card", + "jobs", + "provide", + "j", + "food", + "source", + "author", + "different", + "press", + "u", + "learn", + "sale", + "around", + "print", + "course", + "job", + "canada", + "process", + "teen", + "room", + "stock", + "training", + "too", + "credit", + "point", + "join", + "science", + "men", + "categories", + "advanced", + "west", + "sales", + "look", + "english", + "left", + "team", + "estate", + "box", + "conditions", + "select", + "windows", + "photos", + "gay", + "thread", + "week", + "category", + "note", + "live", + "large", + "gallery", + "table", + "register", + "however", + "june", + "october", + "november", + "market", + "library", + "really", + "action", + "start", + "series", + "model", + "features", + "air", + "industry", + "plan", + "human", + "provided", + "tv", + "yes", + "required", + "second", + "hot", + "accessories", + "cost", + "movie", + "forums", + "march", + "la", + "september", + "better", + "say", + "questions", + "july", + "yahoo", + "going", + "medical", + "test", + "friend", + "come", + "dec", + "server", + "pc", + "study", + "application", + "cart", + "staff", + "articles", + "san", + "feedback", + "again", + "play", + "looking", + "issues", + "april", + "never", + "users", + "complete", + "street", + "topic", + "comment", + "financial", + "things", + "working", + "against", + "standard", + "tax", + "person", + "below", + "mobile", + "less", + "got", + "blog", + "party", + "payment", + "equipment", + "login", + "student", + "let", + "programs", + "offers", + "legal", + "above", + "recent", + "park", + "stores", + "side", + "act", + "problem", + "red", + "give", + "memory", + "performance", + "social", + "q", + "august", + "quote", + "language", + "story", + "sell", + "options", + "experience", + "rates", + "create", + "key", + "body", + "young", + "america", + "important", + "field", + "few", + "east", + "paper", + "single", + "ii", + "age", + "activities", + "club", + "example", + "girls", + "additional", + "password", + "z", + "latest", + "something", + "road", + "gift", + "question", + "changes", + "night", + "ca", + "hard", + "texas", + "oct", + "pay", + "four", + "poker", + "status", + "browse", + "issue", + "range", + "building", + "seller", + "court", + "february", + "always", + "result", + "audio", + "light", + "write", + "war", + "nov", + "offer", + "blue", + "groups", + "al", + "easy", + "given", + "files", + "event", + "release", + "analysis", + "request", + "fax", + "china", + "making", + "picture", + "needs", + "possible", + "might", + "professional", + "yet", + "month", + "major", + "star", + "areas", + "future", + "space", + "committee", + "hand", + "sun", + "cards", + "problems", + "london", + "washington", + "meeting", + "rss", + "become", + "interest", + "id", + "child", + "keep", + "enter", + "california", + "porn", + "share", + "similar", + "garden", + "schools", + "million", + "added", + "reference", + "companies", + "listed", + "baby", + "learning", + "energy", + "run", + "delivery", + "net", + "popular", + "term", + "film", + "stories", + "put", + "computers", + "journal", + "reports", + "co", + "try", + "welcome", + "central", + "images", + "president", + "notice", + "god", + "original", + "head", + "radio", + "until", + "cell", + "color", + "self", + "council", + "away", + "includes", + "track", + "australia", + "discussion", + "archive", + "once", + "others", + "entertainment", + "agreement", + "format", + "least", + "society", + "months", + "log", + "safety", + "friends", + "sure", + "faq", + "trade", + "edition", + "cars", + "messages", + "marketing", + "tell", + "further", + "updated", + "association", + "able", + "having", + "provides", + "david", + "fun", + "already", + "green", + "studies", + "close", + "common", + "drive", + "specific", + "several", + "gold", + "feb", + "living", + "sep", + "collection", + "called", + "short", + "arts", + "lot", + "ask", + "display", + "limited", + "powered", + "solutions", + "means", + "director", + "daily", + "beach", + "past", + "natural", + "whether", + "due", + "et", + "electronics", + "five", + "upon", + "period", + "planning", + "database", + "says", + "official", + "weather", + "mar", + "land", + "average", + "done", + "technical", + "window", + "france", + "pro", + "region", + "island", + "record", + "direct", + "microsoft", + "conference", + "environment", + "records", + "st", + "district", + "calendar", + "costs", + "style", + "url", + "front", + "statement", + "update", + "parts", + "aug", + "ever", + "downloads", + "early", + "miles", + "sound", + "resource", + "present", + "applications", + "either", + "ago", + "document", + "word", + "works", + "material", + "bill", + "apr", + "written", + "talk", + "federal", + "hosting", + "rules", + "final", + "adult", + "tickets", + "thing", + "centre", + "requirements", + "via", + "cheap", + "nude", + "kids", + "finance", + "true", + "minutes", + "else", + "mark", + "third", + "rock", + "gifts", + "europe", + "reading", + "topics", + "bad", + "individual", + "tips", + "plus", + "auto", + "cover", + "usually", + "edit", + "together", + "videos", + "percent", + "fast", + "function", + "fact", + "unit", + "getting", + "global", + "tech", + "meet", + "far", + "economic", + "en", + "player", + "projects", + "lyrics", + "often", + "subscribe", + "submit", + "germany", + "amount", + "watch", + "included", + "feel", + "though", + "bank", + "risk", + "thanks", + "everything", + "deals", + "various", + "words", + "linux", + "jul", + "production", + "commercial", + "james", + "weight", + "town", + "heart", + "advertising", + "received", + "choose", + "treatment", + "newsletter", + "archives", + "points", + "knowledge", + "magazine", + "error", + "camera", + "jun", + "girl", + "currently", + "construction", + "toys", + "registered", + "clear", + "golf", + "receive", + "domain", + "methods", + "chapter", + "makes", + "protection", + "policies", + "loan", + "wide", + "beauty", + "manager", + "india", + "position", + "taken", + "sort", + "listings", + "models", + "michael", + "known", + "half", + "cases", + "step", + "engineering", + "florida", + "simple", + "quick", + "none", + "wireless", + "license", + "paul", + "friday", + "lake", + "whole", + "annual", + "published", + "later", + "basic", + "sony", + "shows", + "corporate", + "google", + "church", + "method", + "purchase", + "customers", + "active", + "response", + "practice", + "hardware", + "figure", + "materials", + "fire", + "holiday", + "chat", + "enough", + "designed", + "along", + "among", + "death", + "writing", + "speed", + "html", + "countries", + "loss", + "face", + "brand", + "discount", + "higher", + "effects", + "created", + "remember", + "standards", + "oil", + "bit", + "yellow", + "political", + "increase", + "advertise", + "kingdom", + "base", + "near", + "environmental", + "thought", + "stuff", + "french", + "storage", + "oh", + "japan", + "doing", + "loans", + "shoes", + "entry", + "stay", + "nature", + "orders", + "availability", + "africa", + "summary", + "turn", + "mean", + "growth", + "notes", + "agency", + "king", + "monday", + "european", + "activity", + "copy", + "although", + "drug", + "pics", + "western", + "income", + "force", + "cash", + "employment", + "overall", + "bay", + "river", + "commission", + "ad", + "package", + "contents", + "seen", + "players", + "engine", + "port", + "album", + "regional", + "stop", + "supplies", + "started", + "administration", + "bar", + "institute", + "views", + "plans", + "double", + "dog", + "build", + "screen", + "exchange", + "types", + "soon", + "sponsored", + "lines", + "electronic", + "continue", + "across", + "benefits", + "needed", + "season", + "apply", + "someone", + "held", + "ny", + "anything", + "printer", + "condition", + "effective", + "believe", + "organization", + "effect", + "asked", + "eur", + "mind", + "sunday", + "selection", + "casino", + "pdf", + "lost", + "tour", + "menu", + "volume", + "cross", + "anyone", + "mortgage", + "hope", + "silver", + "corporation", + "wish", + "inside", + "solution", + "mature", + "role", + "rather", + "weeks", + "addition", + "came", + "supply", + "nothing", + "certain", + "usr", + "executive", + "running", + "lower", + "necessary", + "union", + "jewelry", + "according", + "dc", + "clothing", + "mon", + "com", + "particular", + "fine", + "names", + "robert", + "homepage", + "hour", + "gas", + "skills", + "six", + "bush", + "islands", + "advice", + "career", + "military", + "rental", + "decision", + "leave", + "british", + "teens", + "pre", + "huge", + "sat", + "woman", + "facilities", + "zip", + "bid", + "kind", + "sellers", + "middle", + "move", + "cable", + "opportunities", + "taking", + "values", + "division", + "coming", + "tuesday", + "object", + "lesbian", + "appropriate", + "machine", + "logo", + "length", + "actually", + "nice", + "score", + "statistics", + "client", + "ok", + "returns", + "capital", + "follow", + "sample", + "investment", + "sent", + "shown", + "saturday", + "christmas", + "england", + "culture", + "band", + "flash", + "ms", + "lead", + "george", + "choice", + "went", + "starting", + "registration", + "fri", + "thursday", + "courses", + "consumer", + "hi", + "airport", + "foreign", + "artist", + "outside", + "furniture", + "levels", + "channel", + "letter", + "mode", + "phones", + "ideas", + "wednesday", + "structure", + "fund", + "summer", + "allow", + "degree", + "contract", + "button", + "releases", + "wed", + "homes", + "super", + "male", + "matter", + "custom", + "virginia", + "almost", + "took", + "located", + "multiple", + "asian", + "distribution", + "editor", + "inn", + "industrial", + "cause", + "potential", + "song", + "cnet", + "ltd", + "los", + "hp", + "focus", + "late", + "fall", + "featured", + "idea", + "rooms", + "female", + "responsible", + "inc", + "communications", + "win", + "associated", + "thomas", + "primary", + "cancer", + "numbers", + "reason", + "tool", + "browser", + "spring", + "foundation", + "answer", + "voice", + "eg", + "friendly", + "schedule", + "documents", + "communication", + "purpose", + "feature", + "bed", + "comes", + "police", + "everyone", + "independent", + "ip", + "approach", + "cameras", + "brown", + "physical", + "operating", + "hill", + "maps", + "medicine", + "deal", + "hold", + "ratings", + "chicago", + "forms", + "glass", + "happy", + "tue", + "smith", + "wanted", + "developed", + "thank", + "safe", + "unique", + "survey", + "prior", + "telephone", + "sport", + "ready", + "feed", + "animal", + "sources", + "mexico", + "population", + "pa", + "regular", + "secure", + "navigation", + "operations", + "therefore", + "ass", + "simply", + "evidence", + "station", + "christian", + "round", + "paypal", + "favorite", + "understand", + "option", + "master", + "valley", + "recently", + "probably", + "thu", + "rentals", + "sea", + "built", + "publications", + "blood", + "cut", + "worldwide", + "improve", + "connection", + "publisher", + "hall", + "larger", + "anti", + "networks", + "earth", + "parents", + "nokia", + "impact", + "transfer", + "introduction", + "kitchen", + "strong", + "tel", + "carolina", + "wedding", + "properties", + "hospital", + "ground", + "overview", + "ship", + "accommodation", + "owners", + "disease", + "tx", + "excellent", + "paid", + "italy", + "perfect", + "hair", + "opportunity", + "kit", + "classic", + "basis", + "command", + "cities", + "william", + "express", + "anal", + "award", + "distance", + "tree", + "peter", + "assessment", + "ensure", + "thus", + "wall", + "ie", + "involved", + "el", + "extra", + "especially", + "interface", + "pussy", + "partners", + "budget", + "rated", + "guides", + "success", + "maximum", + "ma", + "operation", + "existing", + "quite", + "selected", + "boy", + "amazon", + "patients", + "restaurants", + "beautiful", + "warning", + "wine", + "locations", + "horse", + "vote", + "forward", + "flowers", + "stars", + "significant", + "lists", + "technologies", + "owner", + "retail", + "animals", + "useful", + "directly", + "manufacturer", + "ways", + "est", + "son", + "providing", + "rule", + "mac", + "housing", + "takes", + "iii", + "gmt", + "bring", + "catalog", + "searches", + "max", + "trying", + "mother", + "authority", + "considered", + "told", + "xml", + "traffic", + "programme", + "joined", + "input", + "strategy", + "feet", + "agent", + "valid", + "bin", + "modern", + "senior", + "ireland", + "sexy", + "teaching", + "door", + "grand", + "testing", + "trial", + "charge", + "units", + "instead", + "canadian", + "cool", + "normal", + "wrote", + "enterprise", + "ships", + "entire", + "educational", + "md", + "leading", + "metal", + "positive", + "fl", + "fitness", + "chinese", + "opinion", + "mb", + "asia", + "football", + "abstract", + "uses", + "output", + "funds", + "mr", + "greater", + "likely", + "develop", + "employees", + "artists", + "alternative", + "processing", + "responsibility", + "resolution", + "java", + "guest", + "seems", + "publication", + "pass", + "relations", + "trust", + "van", + "contains", + "session", + "multi", + "photography", + "republic", + "fees", + "components", + "vacation", + "century", + "academic", + "assistance", + "completed", + "skin", + "graphics", + "indian", + "prev", + "ads", + "mary", + "il", + "expected", + "ring", + "grade", + "dating", + "pacific", + "mountain", + "organizations", + "pop", + "filter", + "mailing", + "vehicle", + "longer", + "consider", + "int", + "northern", + "behind", + "panel", + "floor", + "german", + "buying", + "match", + "proposed", + "default", + "require", + "iraq", + "boys", + "outdoor", + "deep", + "morning", + "otherwise", + "allows", + "rest", + "protein", + "plant", + "reported", + "hit", + "transportation", + "mm", + "pool", + "mini", + "politics", + "partner", + "disclaimer", + "authors", + "boards", + "faculty", + "parties", + "fish", + "membership", + "mission", + "eye", + "string", + "sense", + "modified", + "pack", + "released", + "stage", + "internal", + "goods", + "recommended", + "born", + "unless", + "richard", + "detailed", + "japanese", + "race", + "approved", + "background", + "target", + "except", + "character", + "usb", + "maintenance", + "ability", + "maybe", + "functions", + "ed", + "moving", + "brands", + "places", + "php", + "pretty", + "trademarks", + "phentermine", + "spain", + "southern", + "yourself", + "etc", + "winter", + "rape", + "battery", + "youth", + "pressure", + "submitted", + "boston", + "incest", + "debt", + "keywords", + "medium", + "television", + "interested", + "core", + "break", + "purposes", + "throughout", + "sets", + "dance", + "wood", + "msn", + "itself", + "defined", + "papers", + "playing", + "awards", + "fee", + "studio", + "reader", + "virtual", + "device", + "established", + "answers", + "rent", + "las", + "remote", + "dark", + "programming", + "external", + "apple", + "le", + "regarding", + "instructions", + "min", + "offered", + "theory", + "enjoy", + "remove", + "aid", + "surface", + "minimum", + "visual", + "host", + "variety", + "teachers", + "isbn", + "martin", + "manual", + "block", + "subjects", + "agents", + "increased", + "repair", + "fair", + "civil", + "steel", + "understanding", + "songs", + "fixed", + "wrong", + "beginning", + "hands", + "associates", + "finally", + "az", + "updates", + "desktop", + "classes", + "paris", + "ohio", + "gets", + "sector", + "capacity", + "requires", + "jersey", + "un", + "fat", + "fully", + "father", + "electric", + "saw", + "instruments", + "quotes", + "officer", + "driver", + "businesses", + "dead", + "respect", + "unknown", + "specified", + "restaurant", + "mike", + "trip", + "pst", + "worth", + "mi", + "procedures", + "poor", + "teacher", + "xxx", + "eyes", + "relationship", + "workers", + "farm", + "fucking", + "georgia", + "peace", + "traditional", + "campus", + "tom", + "showing", + "creative", + "coast", + "benefit", + "progress", + "funding", + "devices", + "lord", + "grant", + "sub", + "agree", + "fiction", + "hear", + "sometimes", + "watches", + "careers", + "beyond", + "goes", + "families", + "led", + "museum", + "themselves", + "fan", + "transport", + "interesting", + "blogs", + "wife", + "evaluation", + "accepted", + "former", + "implementation", + "ten", + "hits", + "zone", + "complex", + "th", + "cat", + "galleries", + "references", + "die", + "presented", + "jack", + "flat", + "flow", + "agencies", + "literature", + "respective", + "parent", + "spanish", + "michigan", + "columbia", + "setting", + "dr", + "scale", + "stand", + "economy", + "highest", + "helpful", + "monthly", + "critical", + "frame", + "musical", + "definition", + "secretary", + "angeles", + "networking", + "path", + "australian", + "employee", + "chief", + "gives", + "kb", + "bottom", + "magazines", + "packages", + "detail", + "francisco", + "laws", + "changed", + "pet", + "heard", + "begin", + "individuals", + "colorado", + "royal", + "clean", + "switch", + "russian", + "largest", + "african", + "guy", + "titles", + "relevant", + "guidelines", + "justice", + "connect", + "bible", + "dev", + "cup", + "basket", + "applied", + "weekly", + "vol", + "installation", + "described", + "demand", + "pp", + "suite", + "vegas", + "na", + "square", + "chris", + "attention", + "advance", + "skip", + "diet", + "army", + "auction", + "gear", + "lee", + "os", + "difference", + "allowed", + "correct", + "charles", + "nation", + "selling", + "lots", + "piece", + "sheet", + "firm", + "seven", + "older", + "illinois", + "regulations", + "elements", + "species", + "jump", + "cells", + "module", + "resort", + "facility", + "random", + "pricing", + "dvds", + "certificate", + "minister", + "motion", + "looks", + "fashion", + "directions", + "visitors", + "documentation", + "monitor", + "trading", + "forest", + "calls", + "whose", + "coverage", + "couple", + "giving", + "chance", + "vision", + "ball", + "ending", + "clients", + "actions", + "listen", + "discuss", + "accept", + "automotive", + "naked", + "goal", + "successful", + "sold", + "wind", + "communities", + "clinical", + "situation", + "sciences", + "markets", + "lowest", + "highly", + "publishing", + "appear", + "emergency", + "developing", + "lives", + "currency", + "leather", + "determine", + "milf", + "temperature", + "palm", + "announcements", + "patient", + "actual", + "historical", + "stone", + "bob", + "commerce", + "ringtones", + "perhaps", + "persons", + "difficult", + "scientific", + "satellite", + "fit", + "tests", + "village", + "accounts", + "amateur", + "ex", + "met", + "pain", + "xbox", + "particularly", + "factors", + "coffee", + "www", + "settings", + "cum", + "buyer", + "cultural", + "steve", + "easily", + "oral", + "ford", + "poster", + "edge", + "functional", + "root", + "au", + "fi", + "closed", + "holidays", + "ice", + "pink", + "zealand", + "balance", + "monitoring", + "graduate", + "replies", + "shot", + "nc", + "architecture", + "initial", + "label", + "thinking", + "scott", + "llc", + "sec", + "recommend", + "canon", + "hardcore", + "league", + "waste", + "minute", + "bus", + "provider", + "optional", + "dictionary", + "cold", + "accounting", + "manufacturing", + "sections", + "chair", + "fishing", + "effort", + "phase", + "fields", + "bag", + "fantasy", + "po", + "letters", + "motor", + "va", + "professor", + "context", + "install", + "shirt", + "apparel", + "generally", + "continued", + "foot", + "mass", + "crime", + "count", + "breast", + "techniques", + "ibm", + "rd", + "johnson", + "sc", + "quickly", + "dollars", + "websites", + "religion", + "claim", + "driving", + "permission", + "surgery", + "patch", + "heat", + "wild", + "measures", + "generation", + "kansas", + "miss", + "chemical", + "doctor", + "task", + "reduce", + "brought", + "himself", + "nor", + "component", + "enable", + "exercise", + "bug", + "santa", + "mid", + "guarantee", + "leader", + "diamond", + "israel", + "se", + "processes", + "soft", + "servers", + "alone", + "meetings", + "seconds", + "jones", + "arizona", + "keyword", + "interests", + "flight", + "congress", + "fuel", + "username", + "walk", + "fuck", + "produced", + "italian", + "paperback", + "classifieds", + "wait", + "supported", + "pocket", + "saint", + "rose", + "freedom", + "argument", + "competition", + "creating", + "jim", + "drugs", + "joint", + "premium", + "providers", + "fresh", + "characters", + "attorney", + "upgrade", + "di", + "factor", + "growing", + "thousands", + "km", + "stream", + "apartments", + "pick", + "hearing", + "eastern", + "auctions", + "therapy", + "entries", + "dates", + "generated", + "signed", + "upper", + "administrative", + "serious", + "prime", + "samsung", + "limit", + "began", + "louis", + "steps", + "errors", + "shops", + "bondage", + "del", + "efforts", + "informed", + "ga", + "ac", + "thoughts", + "creek", + "ft", + "worked", + "quantity", + "urban", + "practices", + "sorted", + "reporting", + "essential", + "myself", + "tours", + "platform", + "load", + "affiliate", + "labor", + "immediately", + "admin", + "nursing", + "defense", + "machines", + "designated", + "tags", + "heavy", + "covered", + "recovery", + "joe", + "guys", + "integrated", + "configuration", + "cock", + "merchant", + "comprehensive", + "expert", + "universal", + "protect", + "drop", + "solid", + "cds", + "presentation", + "languages", + "became", + "orange", + "compliance", + "vehicles", + "prevent", + "theme", + "rich", + "im", + "campaign", + "marine", + "improvement", + "vs", + "guitar", + "finding", + "pennsylvania", + "examples", + "ipod", + "saying", + "spirit", + "ar", + "claims", + "porno", + "challenge", + "motorola", + "acceptance", + "strategies", + "mo", + "seem", + "affairs", + "touch", + "intended", + "towards", + "sa", + "goals", + "hire", + "election", + "suggest", + "branch", + "charges", + "serve", + "affiliates", + "reasons", + "magic", + "mount", + "smart", + "talking", + "gave", + "ones", + "latin", + "multimedia", + "xp", + "tits", + "avoid", + "certified", + "manage", + "corner", + "rank", + "computing", + "oregon", + "element", + "birth", + "virus", + "abuse", + "interactive", + "requests", + "separate", + "quarter", + "procedure", + "leadership", + "tables", + "define", + "racing", + "religious", + "facts", + "breakfast", + "kong", + "column", + "plants", + "faith", + "chain", + "developer", + "identify", + "avenue", + "missing", + "died", + "approximately", + "domestic", + "sitemap", + "recommendations", + "moved", + "houston", + "reach", + "comparison", + "mental", + "viewed", + "moment", + "extended", + "sequence", + "inch", + "attack", + "sorry", + "centers", + "opening", + "damage", + "lab", + "reserve", + "recipes", + "cvs", + "gamma", + "plastic", + "produce", + "snow", + "placed", + "truth", + "counter", + "failure", + "follows", + "eu", + "weekend", + "dollar", + "camp", + "ontario", + "automatically", + "des", + "minnesota", + "films", + "bridge", + "native", + "fill", + "williams", + "movement", + "printing", + "baseball", + "owned", + "approval", + "draft", + "chart", + "played", + "contacts", + "cc", + "jesus", + "readers", + "clubs", + "lcd", + "wa", + "jackson", + "equal", + "adventure", + "matching", + "offering", + "shirts", + "profit", + "leaders", + "posters", + "institutions", + "assistant", + "variable", + "ave", + "dj", + "advertisement", + "expect", + "parking", + "headlines", + "yesterday", + "compared", + "determined", + "wholesale", + "workshop", + "russia", + "gone", + "codes", + "kinds", + "extension", + "seattle", + "statements", + "golden", + "completely", + "teams", + "fort", + "cm", + "wi", + "lighting", + "senate", + "forces", + "funny", + "brother", + "gene", + "turned", + "portable", + "tried", + "electrical", + "applicable", + "disc", + "returned", + "pattern", + "ct", + "hentai", + "boat", + "named", + "theatre", + "laser", + "earlier", + "manufacturers", + "sponsor", + "classical", + "icon", + "warranty", + "dedicated", + "indiana", + "direction", + "harry", + "basketball", + "objects", + "ends", + "delete", + "evening", + "assembly", + "nuclear", + "taxes", + "mouse", + "signal", + "criminal", + "issued", + "brain", + "sexual", + "wisconsin", + "powerful", + "dream", + "obtained", + "false", + "da", + "cast", + "flower", + "felt", + "personnel", + "passed", + "supplied", + "identified", + "falls", + "pic", + "soul", + "aids", + "opinions", + "promote", + "stated", + "stats", + "hawaii", + "professionals", + "appears", + "carry", + "flag", + "decided", + "nj", + "covers", + "hr", + "em", + "advantage", + "hello", + "designs", + "maintain", + "tourism", + "priority", + "newsletters", + "adults", + "clips", + "savings", + "iv", + "graphic", + "atom", + "payments", + "rw", + "estimated", + "binding", + "brief", + "ended", + "winning", + "eight", + "anonymous", + "iron", + "straight", + "script", + "served", + "wants", + "miscellaneous", + "prepared", + "void", + "dining", + "alert", + "integration", + "atlanta", + "dakota", + "tag", + "interview", + "mix", + "framework", + "disk", + "installed", + "queen", + "vhs", + "credits", + "clearly", + "fix", + "handle", + "sweet", + "desk", + "criteria", + "pubmed", + "dave", + "massachusetts", + "diego", + "hong", + "vice", + "associate", + "ne", + "truck", + "behavior", + "enlarge", + "ray", + "frequently", + "revenue", + "measure", + "changing", + "votes", + "du", + "duty", + "looked", + "discussions", + "bear", + "gain", + "festival", + "laboratory", + "ocean", + "flights", + "experts", + "signs", + "lack", + "depth", + "iowa", + "whatever", + "logged", + "laptop", + "vintage", + "train", + "exactly", + "dry", + "explore", + "maryland", + "spa", + "concept", + "nearly", + "eligible", + "checkout", + "reality", + "forgot", + "handling", + "origin", + "knew", + "gaming", + "feeds", + "billion", + "destination", + "scotland", + "faster", + "intelligence", + "dallas", + "bought", + "con", + "ups", + "nations", + "route", + "followed", + "specifications", + "broken", + "tripadvisor", + "frank", + "alaska", + "zoom", + "blow", + "battle", + "residential", + "anime", + "speak", + "decisions", + "industries", + "protocol", + "query", + "clip", + "partnership", + "editorial", + "nt", + "expression", + "es", + "equity", + "provisions", + "speech", + "wire", + "principles", + "suggestions", + "rural", + "shared", + "sounds", + "replacement", + "tape", + "strategic", + "judge", + "spam", + "economics", + "acid", + "bytes", + "cent", + "forced", + "compatible", + "fight", + "apartment", + "height", + "null", + "zero", + "speaker", + "filed", + "gb", + "netherlands", + "obtain", + "bc", + "consulting", + "recreation", + "offices", + "designer", + "remain", + "managed", + "pr", + "failed", + "marriage", + "roll", + "korea", + "banks", + "fr", + "participants", + "secret", + "bath", + "aa", + "kelly", + "leads", + "negative", + "austin", + "favorites", + "toronto", + "theater", + "springs", + "missouri", + "andrew", + "var", + "perform", + "healthy", + "translation", + "estimates", + "font", + "assets", + "injury", + "mt", + "joseph", + "ministry", + "drivers", + "lawyer", + "figures", + "married", + "protected", + "proposal", + "sharing", + "philadelphia", + "portal", + "waiting", + "birthday", + "beta", + "fail", + "gratis", + "banking", + "officials", + "brian", + "toward", + "won", + "slightly", + "assist", + "conduct", + "contained", + "lingerie", + "shemale", + "legislation", + "calling", + "parameters", + "jazz", + "serving", + "bags", + "profiles", + "miami", + "comics", + "matters", + "houses", + "doc", + "postal", + "relationships", + "tennessee", + "wear", + "controls", + "breaking", + "combined", + "ultimate", + "wales", + "representative", + "frequency", + "introduced", + "minor", + "finish", + "departments", + "residents", + "noted", + "displayed", + "mom", + "reduced", + "physics", + "rare", + "spent", + "performed", + "extreme", + "samples", + "davis", + "daniel", + "bars", + "reviewed", + "row", + "oz", + "forecast", + "removed", + "helps", + "singles", + "administrator", + "cycle", + "amounts", + "contain", + "accuracy", + "dual", + "rise", + "usd", + "sleep", + "mg", + "bird", + "pharmacy", + "brazil", + "creation", + "static", + "scene", + "hunter", + "addresses", + "lady", + "crystal", + "famous", + "writer", + "chairman", + "violence", + "fans", + "oklahoma", + "speakers", + "drink", + "academy", + "dynamic", + "gender", + "eat", + "permanent", + "agriculture", + "dell", + "cleaning", + "constitutes", + "portfolio", + "practical", + "delivered", + "collectibles", + "infrastructure", + "exclusive", + "seat", + "concerns", + "colour", + "vendor", + "originally", + "intel", + "utilities", + "philosophy", + "regulation", + "officers", + "reduction", + "aim", + "bids", + "referred", + "supports", + "nutrition", + "recording", + "regions", + "junior", + "toll", + "les", + "cape", + "ann", + "rings", + "meaning", + "tip", + "secondary", + "wonderful", + "mine", + "ladies", + "henry", + "ticket", + "announced", + "guess", + "agreed", + "prevention", + "whom", + "ski", + "soccer", + "math", + "import", + "posting", + "presence", + "instant", + "mentioned", + "automatic", + "healthcare", + "viewing", + "maintained", + "ch", + "increasing", + "majority", + "connected", + "christ", + "dan", + "dogs", + "sd", + "directors", + "aspects", + "austria", + "ahead", + "moon", + "participation", + "scheme", + "utility", + "preview", + "fly", + "manner", + "matrix", + "containing", + "combination", + "devel", + "amendment", + "despite", + "strength", + "guaranteed", + "turkey", + "libraries", + "proper", + "distributed", + "degrees", + "singapore", + "enterprises", + "delta", + "fear", + "seeking", + "inches", + "phoenix", + "rs", + "convention", + "shares", + "principal", + "daughter", + "standing", + "voyeur", + "comfort", + "colors", + "wars", + "cisco", + "ordering", + "kept", + "alpha", + "appeal", + "cruise", + "bonus", + "certification", + "previously", + "hey", + "bookmark", + "buildings", + "specials", + "beat", + "disney", + "household", + "batteries", + "adobe", + "smoking", + "bbc", + "becomes", + "drives", + "arms", + "alabama", + "tea", + "improved", + "trees", + "avg", + "achieve", + "positions", + "dress", + "subscription", + "dealer", + "contemporary", + "sky", + "utah", + "nearby", + "rom", + "carried", + "happen", + "exposure", + "panasonic", + "hide", + "permalink", + "signature", + "gambling", + "refer", + "miller", + "provision", + "outdoors", + "clothes", + "caused", + "luxury", + "babes", + "frames", + "viagra", + "certainly", + "indeed", + "newspaper", + "toy", + "circuit", + "layer", + "printed", + "slow", + "removal", + "easier", + "src", + "liability", + "trademark", + "hip", + "printers", + "faqs", + "nine", + "adding", + "kentucky", + "mostly", + "eric", + "spot", + "taylor", + "trackback", + "prints", + "spend", + "factory", + "interior", + "revised", + "grow", + "americans", + "optical", + "promotion", + "relative", + "amazing", + "clock", + "dot", + "hiv", + "identity", + "suites", + "conversion", + "feeling", + "hidden", + "reasonable", + "victoria", + "serial", + "relief", + "revision", + "broadband", + "influence", + "ratio", + "pda", + "importance", + "rain", + "onto", + "dsl", + "planet", + "webmaster", + "copies", + "recipe", + "zum", + "permit", + "seeing", + "proof", + "dna", + "diff", + "tennis", + "bass", + "prescription", + "bedroom", + "empty", + "instance", + "hole", + "pets", + "ride", + "licensed", + "orlando", + "specifically", + "tim", + "bureau", + "maine", + "sql", + "represent", + "conservation", + "pair", + "ideal", + "specs", + "recorded", + "don", + "pieces", + "finished", + "parks", + "dinner", + "lawyers", + "sydney", + "stress", + "cream", + "ss", + "runs", + "trends", + "yeah", + "discover", + "sexo", + "ap", + "patterns", + "boxes", + "louisiana", + "hills", + "javascript", + "fourth", + "nm", + "advisor", + "mn", + "marketplace", + "nd", + "evil", + "aware", + "wilson", + "shape", + "evolution", + "irish", + "certificates", + "objectives", + "stations", + "suggested", + "gps", + "op", + "remains", + "acc", + "greatest", + "firms", + "concerned", + "euro", + "operator", + "structures", + "generic", + "encyclopedia", + "usage", + "cap", + "ink", + "charts", + "continuing", + "mixed", + "census", + "interracial", + "peak", + "tn", + "competitive", + "exist", + "wheel", + "transit", + "dick", + "suppliers", + "salt", + "compact", + "poetry", + "lights", + "tracking", + "angel", + "bell", + "keeping", + "preparation", + "attempt", + "receiving", + "matches", + "accordance", + "width", + "noise", + "engines", + "forget", + "array", + "discussed", + "accurate", + "stephen", + "elizabeth", + "climate", + "reservations", + "pin", + "playstation", + "alcohol", + "greek", + "instruction", + "managing", + "annotation", + "sister", + "raw", + "differences", + "walking", + "explain", + "smaller", + "newest", + "establish", + "gnu", + "happened", + "expressed", + "jeff", + "extent", + "sharp", + "lesbians", + "ben", + "lane", + "paragraph", + "kill", + "mathematics", + "aol", + "compensation", + "ce", + "export", + "managers", + "aircraft", + "modules", + "sweden", + "conflict", + "conducted", + "versions", + "employer", + "occur", + "percentage", + "knows", + "mississippi", + "describe", + "concern", + "backup", + "requested", + "citizens", + "connecticut", + "heritage", + "personals", + "immediate", + "holding", + "trouble", + "spread", + "coach", + "kevin", + "agricultural", + "expand", + "supporting", + "audience", + "assigned", + "jordan", + "collections", + "ages", + "participate", + "plug", + "specialist", + "cook", + "affect", + "virgin", + "experienced", + "investigation", + "raised", + "hat", + "institution", + "directed", + "dealers", + "searching", + "sporting", + "helping", + "perl", + "affected", + "lib", + "bike", + "totally", + "plate", + "expenses", + "indicate", + "blonde", + "ab", + "proceedings", + "favourite", + "transmission", + "anderson", + "utc", + "characteristics", + "der", + "lose", + "organic", + "seek", + "experiences", + "albums", + "cheats", + "extremely", + "verzeichnis", + "contracts", + "guests", + "hosted", + "diseases", + "concerning", + "developers", + "equivalent", + "chemistry", + "tony", + "neighborhood", + "nevada", + "kits", + "thailand", + "variables", + "agenda", + "anyway", + "continues", + "tracks", + "advisory", + "cam", + "curriculum", + "logic", + "template", + "prince", + "circle", + "soil", + "grants", + "anywhere", + "psychology", + "responses", + "atlantic", + "wet", + "circumstances", + "edward", + "investor", + "identification", + "ram", + "leaving", + "wildlife", + "appliances", + "matt", + "elementary", + "cooking", + "speaking", + "sponsors", + "fox", + "unlimited", + "respond", + "sizes", + "plain", + "exit", + "entered", + "iran", + "arm", + "keys", + "launch", + "wave", + "checking", + "costa", + "belgium", + "printable", + "holy", + "acts", + "guidance", + "mesh", + "trail", + "enforcement", + "symbol", + "crafts", + "highway", + "buddy", + "hardcover", + "observed", + "dean", + "setup", + "poll", + "booking", + "glossary", + "fiscal", + "celebrity", + "styles", + "denver", + "unix", + "filled", + "bond", + "channels", + "ericsson", + "appendix", + "notify", + "blues", + "chocolate", + "pub", + "portion", + "scope", + "hampshire", + "supplier", + "cables", + "cotton", + "bluetooth", + "controlled", + "requirement", + "authorities", + "biology", + "dental", + "killed", + "border", + "ancient", + "debate", + "representatives", + "starts", + "pregnancy", + "causes", + "arkansas", + "biography", + "leisure", + "attractions", + "learned", + "transactions", + "notebook", + "explorer", + "historic", + "attached", + "opened", + "tm", + "husband", + "disabled", + "authorized", + "crazy", + "upcoming", + "britain", + "concert", + "retirement", + "scores", + "financing", + "efficiency", + "sp", + "comedy", + "adopted", + "efficient", + "weblog", + "linear", + "commitment", + "specialty", + "bears", + "jean", + "hop", + "carrier", + "edited", + "constant", + "visa", + "mouth", + "jewish", + "meter", + "linked", + "portland", + "interviews", + "concepts", + "nh", + "gun", + "reflect", + "pure", + "deliver", + "wonder", + "hell", + "lessons", + "fruit", + "begins", + "qualified", + "reform", + "lens", + "alerts", + "treated", + "discovery", + "draw", + "mysql", + "classified", + "relating", + "assume", + "confidence", + "alliance", + "fm", + "confirm", + "warm", + "neither", + "lewis", + "howard", + "offline", + "leaves", + "engineer", + "lifestyle", + "consistent", + "replace", + "clearance", + "connections", + "inventory", + "converter", + "suck", + "organisation", + "babe", + "checks", + "reached", + "becoming", + "blowjob", + "safari", + "objective", + "indicated", + "sugar", + "crew", + "legs", + "sam", + "stick", + "securities", + "allen", + "pdt", + "relation", + "enabled", + "genre", + "slide", + "montana", + "volunteer", + "tested", + "rear", + "democratic", + "enhance", + "switzerland", + "exact", + "bound", + "parameter", + "adapter", + "processor", + "node", + "formal", + "dimensions", + "contribute", + "lock", + "hockey", + "storm", + "micro", + "colleges", + "laptops", + "mile", + "showed", + "challenges", + "editors", + "mens", + "threads", + "bowl", + "supreme", + "brothers", + "recognition", + "presents", + "ref", + "tank", + "submission", + "dolls", + "estimate", + "encourage", + "navy", + "kid", + "regulatory", + "inspection", + "consumers", + "cancel", + "limits", + "territory", + "transaction", + "manchester", + "weapons", + "paint", + "delay", + "pilot", + "outlet", + "contributions", + "continuous", + "db", + "czech", + "resulting", + "cambridge", + "initiative", + "novel", + "pan", + "execution", + "disability", + "increases", + "ultra", + "winner", + "idaho", + "contractor", + "ph", + "episode", + "examination", + "potter", + "dish", + "plays", + "bulletin", + "ia", + "pt", + "indicates", + "modify", + "oxford", + "adam", + "truly", + "epinions", + "painting", + "committed", + "extensive", + "affordable", + "universe", + "candidate", + "databases", + "patent", + "slot", + "psp", + "outstanding", + "ha", + "eating", + "perspective", + "planned", + "watching", + "lodge", + "messenger", + "mirror", + "tournament", + "consideration", + "ds", + "discounts", + "sterling", + "sessions", + "kernel", + "boobs", + "stocks", + "buyers", + "journals", + "gray", + "catalogue", + "ea", + "jennifer", + "antonio", + "charged", + "broad", + "taiwan", + "und", + "chosen", + "demo", + "greece", + "lg", + "swiss", + "sarah", + "clark", + "labour", + "hate", + "terminal", + "publishers", + "nights", + "behalf", + "caribbean", + "liquid", + "rice", + "nebraska", + "loop", + "salary", + "reservation", + "foods", + "gourmet", + "guard", + "properly", + "orleans", + "saving", + "nfl", + "remaining", + "empire", + "resume", + "twenty", + "newly", + "raise", + "prepare", + "avatar", + "gary", + "depending", + "illegal", + "expansion", + "vary", + "hundreds", + "rome", + "arab", + "lincoln", + "helped", + "premier", + "tomorrow", + "purchased", + "milk", + "decide", + "consent", + "drama", + "visiting", + "performing", + "downtown", + "keyboard", + "contest", + "collected", + "nw", + "bands", + "boot", + "suitable", + "ff", + "absolutely", + "millions", + "lunch", + "dildo", + "audit", + "push", + "chamber", + "guinea", + "findings", + "muscle", + "featuring", + "iso", + "implement", + "clicking", + "scheduled", + "polls", + "typical", + "tower", + "yours", + "sum", + "misc", + "calculator", + "significantly", + "chicken", + "temporary", + "attend", + "shower", + "alan", + "sending", + "jason", + "tonight", + "dear", + "sufficient", + "holdem", + "shell", + "province", + "catholic", + "oak", + "vat", + "awareness", + "vancouver", + "governor", + "beer", + "seemed", + "contribution", + "measurement", + "swimming", + "spyware", + "formula", + "constitution", + "packaging", + "solar", + "jose", + "catch", + "jane", + "pakistan", + "ps", + "reliable", + "consultation", + "northwest", + "sir", + "doubt", + "earn", + "finder", + "unable", + "periods", + "classroom", + "tasks", + "democracy", + "attacks", + "kim", + "wallpaper", + "merchandise", + "const", + "resistance", + "doors", + "symptoms", + "resorts", + "biggest", + "memorial", + "visitor", + "twin", + "forth", + "insert", + "baltimore", + "gateway", + "ky", + "dont", + "alumni", + "drawing", + "candidates", + "charlotte", + "ordered", + "biological", + "fighting", + "transition", + "happens", + "preferences", + "spy", + "romance", + "instrument", + "bruce", + "split", + "themes", + "powers", + "heaven", + "br", + "bits", + "pregnant", + "twice", + "classification", + "focused", + "egypt", + "physician", + "hollywood", + "bargain", + "wikipedia", + "cellular", + "norway", + "vermont", + "asking", + "blocks", + "normally", + "lo", + "spiritual", + "hunting", + "diabetes", + "suit", + "ml", + "shift", + "chip", + "res", + "sit", + "bodies", + "photographs", + "cutting", + "wow", + "simon", + "writers", + "marks", + "flexible", + "loved", + "favourites", + "mapping", + "numerous", + "relatively", + "birds", + "satisfaction", + "represents", + "char", + "indexed", + "pittsburgh", + "superior", + "preferred", + "saved", + "paying", + "cartoon", + "shots", + "intellectual", + "moore", + "granted", + "choices", + "carbon", + "spending", + "comfortable", + "magnetic", + "interaction", + "listening", + "effectively", + "registry", + "crisis", + "outlook", + "massive", + "denmark", + "employed", + "bright", + "treat", + "header", + "cs", + "poverty", + "formed", + "piano", + "echo", + "que", + "grid", + "sheets", + "patrick", + "experimental", + "puerto", + "revolution", + "consolidation", + "displays", + "plasma", + "allowing", + "earnings", + "voip", + "mystery", + "landscape", + "dependent", + "mechanical", + "journey", + "delaware", + "bidding", + "consultants", + "risks", + "banner", + "applicant", + "charter", + "fig", + "barbara", + "cooperation", + "counties", + "acquisition", + "ports", + "implemented", + "sf", + "directories", + "recognized", + "dreams", + "blogger", + "notification", + "kg", + "licensing", + "stands", + "teach", + "occurred", + "textbooks", + "rapid", + "pull", + "hairy", + "diversity", + "cleveland", + "ut", + "reverse", + "deposit", + "seminar", + "investments", + "latina", + "nasa", + "wheels", + "sexcam", + "specify", + "accessibility", + "dutch", + "sensitive", + "templates", + "formats", + "tab", + "depends", + "boots", + "holds", + "router", + "concrete", + "si", + "editing", + "poland", + "folder", + "womens", + "css", + "completion", + "upload", + "pulse", + "universities", + "technique", + "contractors", + "milfhunter", + "voting", + "courts", + "notices", + "subscriptions", + "calculate", + "mc", + "detroit", + "alexander", + "broadcast", + "converted", + "metro", + "toshiba", + "anniversary", + "improvements", + "strip", + "specification", + "pearl", + "accident", + "nick", + "accessible", + "accessory", + "resident", + "plot", + "qty", + "possibly", + "airline", + "typically", + "representation", + "regard", + "pump", + "exists", + "arrangements", + "smooth", + "conferences", + "uniprotkb", + "beastiality", + "strike", + "consumption", + "birmingham", + "flashing", + "lp", + "narrow", + "afternoon", + "threat", + "surveys", + "sitting", + "putting", + "consultant", + "controller", + "ownership", + "committees", + "penis", + "legislative", + "researchers", + "vietnam", + "trailer", + "anne", + "castle", + "gardens", + "missed", + "malaysia", + "unsubscribe", + "antique", + "labels", + "willing", + "bio", + "molecular", + "upskirt", + "acting", + "heads", + "stored", + "exam", + "logos", + "residence", + "attorneys", + "milfs", + "antiques", + "density", + "hundred", + "ryan", + "operators", + "strange", + "sustainable", + "philippines", + "statistical", + "beds", + "breasts", + "mention", + "innovation", + "pcs", + "employers", + "grey", + "parallel", + "honda", + "amended", + "operate", + "bills", + "bold", + "bathroom", + "stable", + "opera", + "definitions", + "von", + "doctors", + "lesson", + "cinema", + "asset", + "ag", + "scan", + "elections", + "drinking", + "blowjobs", + "reaction", + "blank", + "enhanced", + "entitled", + "severe", + "generate", + "stainless", + "newspapers", + "hospitals", + "vi", + "deluxe", + "humor", + "aged", + "monitors", + "exception", + "lived", + "duration", + "bulk", + "successfully", + "indonesia", + "pursuant", + "sci", + "fabric", + "edt", + "visits", + "primarily", + "tight", + "domains", + "capabilities", + "pmid", + "contrast", + "recommendation", + "flying", + "recruitment", + "sin", + "berlin", + "cute", + "organized", + "ba", + "para", + "siemens", + "adoption", + "improving", + "cr", + "expensive", + "meant", + "capture", + "pounds", + "buffalo", + "organisations", + "plane", + "pg", + "explained", + "seed", + "programmes", + "desire", + "expertise", + "mechanism", + "camping", + "ee", + "jewellery", + "meets", + "welfare", + "peer", + "caught", + "eventually", + "marked", + "driven", + "measured", + "medline", + "bottle", + "agreements", + "considering", + "innovative", + "marshall", + "massage", + "rubber", + "conclusion", + "closing", + "tampa", + "thousand", + "meat", + "legend", + "grace", + "susan", + "ing", + "ks", + "adams", + "python", + "monster", + "alex", + "bang", + "villa", + "bone", + "columns", + "disorders", + "bugs", + "collaboration", + "hamilton", + "detection", + "ftp", + "cookies", + "inner", + "formation", + "tutorial", + "med", + "engineers", + "entity", + "cruises", + "gate", + "holder", + "proposals", + "moderator", + "sw", + "tutorials", + "settlement", + "portugal", + "lawrence", + "roman", + "duties", + "valuable", + "erotic", + "tone", + "collectables", + "ethics", + "forever", + "dragon", + "busy", + "captain", + "fantastic", + "imagine", + "brings", + "heating", + "leg", + "neck", + "hd", + "wing", + "governments", + "purchasing", + "scripts", + "abc", + "stereo", + "appointed", + "taste", + "dealing", + "commit", + "tiny", + "operational", + "rail", + "airlines", + "liberal", + "livecam", + "jay", + "trips", + "gap", + "sides", + "tube", + "turns", + "corresponding", + "descriptions", + "cache", + "belt", + "jacket", + "determination", + "animation", + "oracle", + "er", + "matthew", + "lease", + "productions", + "aviation", + "hobbies", + "proud", + "excess", + "disaster", + "console", + "commands", + "jr", + "telecommunications", + "instructor", + "giant", + "achieved", + "injuries", + "shipped", + "bestiality", + "seats", + "approaches", + "biz", + "alarm", + "voltage", + "anthony", + "nintendo", + "usual", + "loading", + "stamps", + "appeared", + "franklin", + "angle", + "rob", + "vinyl", + "highlights", + "mining", + "designers", + "melbourne", + "ongoing", + "worst", + "imaging", + "betting", + "scientists", + "liberty", + "wyoming", + "blackjack", + "argentina", + "era", + "convert", + "possibility", + "analyst", + "commissioner", + "dangerous", + "garage", + "exciting", + "reliability", + "thongs", + "gcc", + "unfortunately", + "respectively", + "volunteers", + "attachment", + "ringtone", + "finland", + "morgan", + "derived", + "pleasure", + "honor", + "asp", + "oriented", + "eagle", + "desktops", + "pants", + "columbus", + "nurse", + "prayer", + "appointment", + "workshops", + "hurricane", + "quiet", + "luck", + "postage", + "producer", + "represented", + "mortgages", + "dial", + "responsibilities", + "cheese", + "comic", + "carefully", + "jet", + "productivity", + "investors", + "crown", + "par", + "underground", + "diagnosis", + "maker", + "crack", + "principle", + "picks", + "vacations", + "gang", + "semester", + "calculated", + "cumshot", + "fetish", + "applies", + "casinos", + "appearance", + "smoke", + "apache", + "filters", + "incorporated", + "nv", + "craft", + "cake", + "notebooks", + "apart", + "fellow", + "blind", + "lounge", + "mad", + "algorithm", + "semi", + "coins", + "andy", + "gross", + "strongly", + "cafe", + "valentine", + "hilton", + "ken", + "proteins", + "horror", + "su", + "exp", + "familiar", + "capable", + "douglas", + "debian", + "till", + "involving", + "pen", + "investing", + "christopher", + "admission", + "epson", + "shoe", + "elected", + "carrying", + "victory", + "sand", + "madison", + "terrorism", + "joy", + "editions", + "cpu", + "mainly", + "ethnic", + "ran", + "parliament", + "actor", + "finds", + "seal", + "situations", + "fifth", + "allocated", + "citizen", + "vertical", + "corrections", + "structural", + "municipal", + "describes", + "prize", + "sr", + "occurs", + "jon", + "absolute", + "disabilities", + "consists", + "anytime", + "substance", + "prohibited", + "addressed", + "lies", + "pipe", + "soldiers", + "nr", + "guardian", + "lecture", + "simulation", + "layout", + "initiatives", + "ill", + "concentration", + "classics", + "lbs", + "lay", + "interpretation", + "horses", + "lol", + "dirty", + "deck", + "wayne", + "donate", + "taught", + "bankruptcy", + "mp", + "worker", + "optimization", + "alive", + "temple", + "substances", + "prove", + "discovered", + "wings", + "breaks", + "genetic", + "restrictions", + "participating", + "waters", + "promise", + "thin", + "exhibition", + "prefer", + "ridge", + "cabinet", + "modem", + "harris", + "mph", + "bringing", + "sick", + "dose", + "evaluate", + "tiffany", + "tropical", + "collect", + "bet", + "composition", + "toyota", + "streets", + "nationwide", + "vector", + "definitely", + "shaved", + "turning", + "buffer", + "purple", + "existence", + "commentary", + "larry", + "limousines", + "developments", + "def", + "immigration", + "destinations", + "lets", + "mutual", + "pipeline", + "necessarily", + "syntax", + "li", + "attribute", + "prison", + "skill", + "chairs", + "nl", + "everyday", + "apparently", + "surrounding", + "mountains", + "moves", + "popularity", + "inquiry", + "ethernet", + "checked", + "exhibit", + "throw", + "trend", + "sierra", + "visible", + "cats", + "desert", + "postposted", + "ya", + "oldest", + "rhode", + "nba", + "busty", + "coordinator", + "obviously", + "mercury", + "steven", + "handbook", + "greg", + "navigate", + "worse", + "summit", + "victims", + "epa", + "spaces", + "fundamental", + "burning", + "escape", + "coupons", + "somewhat", + "receiver", + "substantial", + "tr", + "progressive", + "cialis", + "bb", + "boats", + "glance", + "scottish", + "championship", + "arcade", + "richmond", + "sacramento", + "impossible", + "ron", + "russell", + "tells", + "obvious", + "fiber", + "depression", + "graph", + "covering", + "platinum", + "judgment", + "bedrooms", + "talks", + "filing", + "foster", + "modeling", + "passing", + "awarded", + "testimonials", + "trials", + "tissue", + "nz", + "memorabilia", + "clinton", + "masters", + "bonds", + "cartridge", + "alberta", + "explanation", + "folk", + "org", + "commons", + "cincinnati", + "subsection", + "fraud", + "electricity", + "permitted", + "spectrum", + "arrival", + "okay", + "pottery", + "emphasis", + "roger", + "aspect", + "workplace", + "awesome", + "mexican", + "confirmed", + "counts", + "priced", + "wallpapers", + "hist", + "crash", + "lift", + "desired", + "inter", + "closer", + "assumes", + "heights", + "shadow", + "riding", + "infection", + "firefox", + "lisa", + "expense", + "grove", + "eligibility", + "venture", + "clinic", + "korean", + "healing", + "princess", + "mall", + "entering", + "packet", + "spray", + "studios", + "involvement", + "dad", + "buttons", + "placement", + "observations", + "vbulletin", + "funded", + "thompson", + "winners", + "extend", + "roads", + "subsequent", + "pat", + "dublin", + "rolling", + "fell", + "motorcycle", + "yard", + "disclosure", + "establishment", + "memories", + "nelson", + "te", + "arrived", + "creates", + "faces", + "tourist", + "cocks", + "av", + "mayor", + "murder", + "sean", + "adequate", + "senator", + "yield", + "presentations", + "grades", + "cartoons", + "pour", + "digest", + "reg", + "lodging", + "tion", + "dust", + "hence", + "wiki", + "entirely", + "replaced", + "radar", + "rescue", + "undergraduate", + "losses", + "combat", + "reducing", + "stopped", + "occupation", + "lakes", + "butt", + "donations", + "associations", + "citysearch", + "closely", + "radiation", + "diary", + "seriously", + "kings", + "shooting", + "kent", + "adds", + "nsw", + "ear", + "flags", + "pci", + "baker", + "launched", + "elsewhere", + "pollution", + "conservative", + "guestbook", + "shock", + "effectiveness", + "walls", + "abroad", + "ebony", + "tie", + "ward", + "drawn", + "arthur", + "ian", + "visited", + "roof", + "walker", + "demonstrate", + "atmosphere", + "suggests", + "kiss", + "beast", + "ra", + "operated", + "experiment", + "targets", + "overseas", + "purchases", + "dodge", + "counsel", + "federation", + "pizza", + "invited", + "yards", + "assignment", + "chemicals", + "gordon", + "mod", + "farmers", + "rc", + "queries", + "bmw", + "rush", + "ukraine", + "absence", + "nearest", + "cluster", + "vendors", + "mpeg", + "whereas", + "yoga", + "serves", + "woods", + "surprise", + "lamp", + "rico", + "partial", + "shoppers", + "phil", + "everybody", + "couples", + "nashville", + "ranking", + "jokes", + "cst", + "http", + "ceo", + "simpson", + "twiki", + "sublime", + "counseling", + "palace", + "acceptable", + "satisfied", + "glad", + "wins", + "measurements", + "verify", + "globe", + "trusted", + "copper", + "milwaukee", + "rack", + "medication", + "warehouse", + "shareware", + "ec", + "rep", + "dicke", + "kerry", + "receipt", + "supposed", + "ordinary", + "nobody", + "ghost", + "violation", + "configure", + "stability", + "mit", + "applying", + "southwest", + "boss", + "pride", + "institutional", + "expectations", + "independence", + "knowing", + "reporter", + "metabolism", + "keith", + "champion", + "cloudy", + "linda", + "ross", + "personally", + "chile", + "anna", + "plenty", + "solo", + "sentence", + "throat", + "ignore", + "maria", + "uniform", + "excellence", + "wealth", + "tall", + "rm", + "somewhere", + "vacuum", + "dancing", + "attributes", + "recognize", + "brass", + "writes", + "plaza", + "pdas", + "outcomes", + "survival", + "quest", + "publish", + "sri", + "screening", + "toe", + "thumbnail", + "trans", + "jonathan", + "whenever", + "nova", + "lifetime", + "api", + "pioneer", + "booty", + "forgotten", + "acrobat", + "plates", + "acres", + "venue", + "athletic", + "thermal", + "essays", + "behaviour", + "vital", + "telling", + "fairly", + "coastal", + "config", + "cf", + "charity", + "intelligent", + "edinburgh", + "vt", + "excel", + "modes", + "obligation", + "campbell", + "wake", + "stupid", + "harbor", + "hungary", + "traveler", + "urw", + "segment", + "realize", + "regardless", + "lan", + "enemy", + "puzzle", + "rising", + "aluminum", + "wells", + "wishlist", + "opens", + "insight", + "sms", + "shit", + "restricted", + "republican", + "secrets", + "lucky", + "latter", + "merchants", + "thick", + "trailers", + "repeat", + "syndrome", + "philips", + "attendance", + "penalty", + "drum", + "glasses", + "enables", + "nec", + "iraqi", + "builder", + "vista", + "jessica", + "chips", + "terry", + "flood", + "foto", + "ease", + "arguments", + "amsterdam", + "orgy", + "arena", + "adventures", + "pupils", + "stewart", + "announcement", + "tabs", + "outcome", + "xx", + "appreciate", + "expanded", + "casual", + "grown", + "polish", + "lovely", + "extras", + "gm", + "centres", + "jerry", + "clause", + "smile", + "lands", + "ri", + "troops", + "indoor", + "bulgaria", + "armed", + "broker", + "charger", + "regularly", + "believed", + "pine", + "cooling", + "tend", + "gulf", + "rt", + "rick", + "trucks", + "cp", + "mechanisms", + "divorce", + "laura", + "shopper", + "tokyo", + "partly", + "nikon", + "customize", + "tradition", + "candy", + "pills", + "tiger", + "donald", + "folks", + "sensor", + "exposed", + "telecom", + "hunt", + "angels", + "deputy", + "indicators", + "sealed", + "thai", + "emissions", + "physicians", + "loaded", + "fred", + "complaint", + "scenes", + "experiments", + "balls", + "afghanistan", + "dd", + "boost", + "spanking", + "scholarship", + "governance", + "mill", + "founded", + "supplements", + "chronic", + "icons", + "tranny", + "moral", + "den", + "catering", + "aud", + "finger", + "keeps", + "pound", + "locate", + "camcorder", + "pl", + "trained", + "burn", + "implementing", + "roses", + "labs", + "ourselves", + "bread", + "tobacco", + "wooden", + "motors", + "tough", + "roberts", + "incident", + "gonna", + "dynamics", + "lie", + "crm", + "rf", + "conversation", + "decrease", + "cumshots", + "chest", + "pension", + "billy", + "revenues", + "emerging", + "worship", + "bukkake", + "capability", + "ak", + "fe", + "craig", + "herself", + "producing", + "churches", + "precision", + "damages", + "reserves", + "contributed", + "solve", + "shorts", + "reproduction", + "minority", + "td", + "diverse", + "amp", + "ingredients", + "sb", + "ah", + "johnny", + "sole", + "franchise", + "recorder", + "complaints", + "facing", + "sm", + "nancy", + "promotions", + "tones", + "passion", + "rehabilitation", + "maintaining", + "sight", + "laid", + "clay", + "defence", + "patches", + "weak", + "refund", + "usc", + "towns", + "environments", + "trembl", + "divided", + "blvd", + "reception", + "amd", + "wise", + "emails", + "cyprus", + "wv", + "odds", + "correctly", + "insider", + "seminars", + "consequences", + "makers" +] diff --git a/extensions/english-learner/frequency-source.json b/extensions/english-learner/frequency-source.json new file mode 100644 index 0000000..2cc3ca6 --- /dev/null +++ b/extensions/english-learner/frequency-source.json @@ -0,0 +1,10 @@ +{ + "upstream": "https://github.com/first20hours/google-10000-english", + "snapshot_repository": "https://github.com/talknagish/rylo-apps", + "snapshot_path": "apps/dictionary-web/src/data/frequency/en.json", + "snapshot_git_revision": "30e71817859c20740c7ceb84496b665aedee10ec", + "source_sha256": "70d214db48f254b4016a8182a9b8b740cbb0a426d2d3d2a3088a3554a741d611", + "cutoff": 5000, + "matching": "NFD, remove U+0300–U+036F and U+0591–U+05C7, lowercase; stored forms only; no inferred morphology", + "license": "See upstream repository for frequency-list provenance and licensing." +} diff --git a/extensions/english-learner/generation.py b/extensions/english-learner/generation.py new file mode 100644 index 0000000..e24c9fc --- /dev/null +++ b/extensions/english-learner/generation.py @@ -0,0 +1,593 @@ +"""Generate resumable, unreviewed enrichment with Ollama or an OpenAI-compatible server. + +Each output line records one exact lexical sense, its input hash, model, +source identifiers, and optional enrichment. Empty enrichment is intentional: +assessing a meaning does not require adding information to it. +""" + +import argparse +import hashlib +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +PROMPT_VERSION = "english-learner-6" +SYSTEM = """You are a careful English lexicographer enriching existing WordNet meanings. +Return one item for every input id, preserving order. Do not create or merge +meanings. The supplied definition identifies the intended sense; other_meanings are +distractors only and must not be illustrated or added as user-facing +comparisons. Lexical forms retain their exact capitalization (IT is not it). +Prior generated candidates are UNREVIEWED and may be wrong: reuse only text that +accurately expresses this precise sense. Reject invented facts, ambiguous +examples, and unnatural usage. Every example must use the target word in the +supplied part of speech and syntactic frame: a verb sense of check cannot be +illustrated by the noun in write a check, nor by the unrelated verb meaning +verify. Likewise preserve historical or specialist senses rather than replacing +them with common ones. If you cannot express the precise sense naturally and +accurately, return an empty enrichment and a concise unresolved_reason instead +of inventing usage. + +Add only information useful beyond the definition; empty enrichment is valid. +plain_language: one short explanation ONLY for an obscure/technical definition. +domains: conventional specialist labels (medical, legal, computing, finance, +biology, chemistry, physics, mathematics, linguistics, religion, music, +military, sport, etc.). Use source topic-domain evidence when given. Do not +label ordinary vocabulary with broad labels such as everyday life or +relationships. Domain is not register; leave unsupported labels absent. +register: ONLY marked register (informal, formal, slang, vulgar, offensive, +archaic, dated, literary). Never neutral. Familiar words such as bank are not +formal just because they have a financial meaning. usage: ONLY helpful region or +context restrictions, without guessing. grammar_notes: ONLY useful sense- +specific constraints such as countability or required complements; do not repeat +the part of speech or basic conjugation. usage_tips: at most two concise +idiomatic constructions or learner pitfalls; no generic advice, obvious +restatement, or comparison to other senses. word_enrichment.irregular_forms: +lexical irregular forms only, as an object mapping form type to a list of +strings (past, past_participle, plural, comparative, superlative). These +describe the exact target sense of the lemma, never the frequency surface word. +Distinguish sense-dependent forms such as lie/lay and lie/lied or hang/hung and +hang/hanged; they will be stored on that sense. Do not add regular forms. Omit +unless certain; do not guess proper-name forms. + +If needs_examples=true, add EXACTLY TWO short natural sentences that visibly use +the target word (or a listed inflection) in context, not merely quote or mention +its name. Use one of the supplied matching_forms with appropriate capitalization +and unambiguously express the supplied meaning. If false, OMIT examples. A +person meaning must describe a person, not a pet, emotion, object or different +sense. Medical/scientific examples must avoid making clinical recommendations or +unsupported factual assertions. Use non-graphic educational contexts for sexual +vocabulary; no erotic content. Describe offensive usage without endorsing +stereotypes or directing insults at people. Do not echo source_definition or +original IDs in prose. No invented etymologies, translations, or sign-language +advice. + +Quality examples (use these principles, not boilerplate): +- love, noun, beloved person: GOOD: "Good morning, my love," she said to her + husband. BAD: Her love would always be by his side. The bad sentence could + describe a feeling, so it does not identify the intended sense. +- bank, financial institution: domains=["finance"] is useful. Do NOT add formal + register, countable (ordinary and unhelpful), or advice to use "the". Return + only domains when nothing else is useful. +- AIDS: a medical domain does NOT imply formal register. Examples should show + contextual use such as a clinic supporting people living with AIDS. Do not + add claims about transmission, treatment, prognosis, or prevention. +- A clear definition needs no plain_language paraphrase. Most fields should be + absent for ordinary vocabulary. Never add generic advice about articles, + frequent collocations, or countability simply to fill a field. +- Existing candidates have deliberately been withheld in this pass: assess the + source definition independently. Report unresolved_reason rather than guess. + +Minimal output examples for needs_examples=false: +- outcome, noun, a result caused by something before it: + {"id": 0, "enrichment": {}} +- burn, verb, cause a stinging pain: + {"id": 1, "enrichment": {}} +- grown, adjective, fully developed: + {"id": 2, "enrichment": {}} +- dynamics, noun, branch of mechanics studying forces and motion: + {"id": 3, "enrichment": {"domains": ["physics"], + "grammar_notes": ["Takes a singular verb when naming this field of study."]}} +The first three are ordinary English, not formal, clinical, scientific, or +restricted to a profession. A word's use in a technical sentence does not give +the word a technical register or domain. Do not label adjectives scientific +because the definition mentions animals, or label results physics/chemistry. +Do not add "requires a direct object", "do not use as a noun", "use in context", +or rules about articles. Empty enrichment is usually the best answer. +Proper-name examples: use simple fictional reading, travel, or discussion +contexts, avoiding invented events, quotations, exact dates, or measurements. +For element symbols prefer a laboratory label or sample, not everyday phrases +such as "Fe beams". For state abbreviations prefer written addresses rather +than unnatural spoken sentences such as "we entered IN on the highway". + +Correction mode: when correction is supplied, repair EVERY reported issue and +return the COMPLETE replacement enrichment, not a patch. Keep useful unflagged +information only when defensible. Remove unsupported optional claims rather than +guessing a replacement. Avoid needless rewrites. Previous annotations are +untrusted, even when they carry earlier review stamps. A missing example still +requires two correct sentences; merely quoting or defining the word does not +illustrate its grammatical use. For noun medical, "She needed a medical before +joining the team" is valid; "She needed a medical examination" is not. +Do not narrow a definition with invented conditions such as "over time" or +"always". Each example must distinguish this meaning from neighboring meanings: +a general benefit is not necessarily operating profit. The base definition +identifies the intended sense but can contain dated or imprecise factual wording; +do not amplify a factual error into a new assertion. Omit that optional claim or +report unresolved_reason when the intended sense cannot be illustrated reliably. +Use existing_examples and syntactic frames to disambiguate short definitions. +For example, have defined as "have left" means still possess a remaining amount +("How many years do you have left?"), not the perfect tense of leaving a place. +Never reinterpret the target lemma as an auxiliary in a different phrase. +""" + +STRING = {"type": "string", "minLength": 1, "maxLength": 500} +LIST = {"type": "array", "items": STRING, "minItems": 1, "maxItems": 3} +FIELDS = { + "plain_language": STRING, + **dict.fromkeys( + ("domains", "register", "usage", "grammar_notes", "usage_tips"), LIST + ), +} +FIELDS["examples"] = {"type": "array", "items": STRING, "minItems": 2, "maxItems": 2} +SCHEMA = { + "type": "object", + "required": ["items"], + "additionalProperties": False, + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "enrichment"], + "additionalProperties": False, + "properties": { + "id": {"type": "integer"}, + "unresolved_reason": STRING, + "enrichment": { + "type": "object", + "properties": FIELDS, + "additionalProperties": False, + }, + "word_enrichment": { + "type": "object", + "additionalProperties": False, + "properties": { + "irregular_forms": { + "type": "object", + "additionalProperties": LIST, + } + }, + }, + }, + }, + } + }, +} + + +PROMPT_HASH = hashlib.sha256( + (PROMPT_VERSION + SYSTEM + json.dumps(SCHEMA, sort_keys=True)).encode() +).hexdigest() + + +def input_hash(row): + value = json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(value.encode()).hexdigest() + + +def compact(row, index): + """Bound source text while preserving exact lexical identity and evidence.""" + senses = row["senses"] + value = { + "id": index, + "word": senses[0]["lemma"], + "pos": row["pos"], + "definition": row["definitions"], + "other_meanings": row.get("other_meanings", []), + "needs_examples": not row["examples"], + "existing_examples": row["examples"], + "lexical_entries": [ + { + "lemma": s["lemma"], + "matching_forms": s["matching_forms"], + "forms": s["forms"], + "tags": s["tags"], + "syntactic_frames": s["syntactic_frames"], + } + for s in senses + ], + "topic_domains": [ + r + for r in row["relations"] + if r["type"] in ("domain_topic", "domain_region", "domain_usage") + ], + } + if row.get("correction"): + value["correction"] = row["correction"] + return value + + +def validate(item, row, index): + if not isinstance(item, dict): + raise ValueError("Response item must be an object") + if item.get("id") != index: + raise ValueError("Response ids do not match input order") + if item.get("unresolved_reason"): + raise ValueError("Semantic review needed: " + item["unresolved_reason"]) + fields = item.get("enrichment") + if not isinstance(fields, dict) or set(fields) - set(FIELDS): + raise ValueError("Invalid enrichment fields") + for key, value in fields.items(): + values = [value] if key == "plain_language" else value + if not isinstance(values, list) or not values or len(values) > 3: + raise ValueError(f"Invalid {key} list") + if any(not isinstance(v, str) or not v.strip() or len(v) > 500 for v in values): + raise ValueError(f"Invalid {key} text") + validate_examples(fields.get("examples", []), row) + validate_word(item.get("word_enrichment", {})) + + +def validate_examples(examples, row): + from wn.morphy import morphy + + if len({text.strip().casefold() for text in examples}) != len(examples): + raise ValueError("Duplicate examples") + if row["examples"] and examples: + raise ValueError("Examples supplied for meaning that already has examples") + if not row["examples"] and len(examples) != 2: + raise ValueError("Missing meaning requires two examples") + forms = [form for sense in row["senses"] for form in sense["matching_forms"]] + for example in examples: + # Morphy's suffix rules accept regular inflections (thank/thanked) while + # preserving the supplied part of speech and proper-name capitalization. + inflections = set() + for token in re.findall(r"\b[\w-]+\b", example): + for lemmas in morphy(token, pos=row.get("pos")).values(): + inflections.update(lemmas) + if token.istitle(): + for lemmas in morphy(token.lower(), pos=row.get("pos")).values(): + inflections.update(lemma for lemma in lemmas if lemma.islower()) + if not any( + re.search(r"(? 3: + raise ValueError("Invalid irregular form list") + if any(not isinstance(f, str) or not f.strip() or len(f) > 100 for f in forms): + raise ValueError("Invalid irregular form") + + +def generate(rows, args, error_hint=None): + source = json.dumps( + [compact(row, i) for i, row in enumerate(rows)], ensure_ascii=False + ) + if len(source) + len(SYSTEM) > 18000: + raise ValueError( + "Batch source too large for bounded context; reduce batch size" + ) + payload = { + "model": args.model, + "stream": False, + "think": "low", + "keep_alive": "30m", + "options": {"temperature": 0, "num_ctx": 8192, "num_predict": 2048}, + "format": SCHEMA, + "messages": [ + {"role": "system", "content": SYSTEM}, + {"role": "user", "content": source}, + ], + } + if not args.model.startswith("gpt-oss"): + payload.pop("think", None) + if error_hint: + payload["messages"].append( + { + "role": "user", + "content": "A previous response failed validation: " + + error_hint + + ". Correct that issue while preserving the supplied meaning.", + } + ) + endpoint_path = "/api/chat" + if args.api == "openai": + endpoint_path = "/v1/chat/completions" + payload = { + "model": args.model, + "messages": payload["messages"], + "temperature": 0, + "max_tokens": 2048, + "reasoning_effort": "low", + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "learner_enrichment", + "schema": SCHEMA, + }, + }, + } + request = Request( + args.endpoint.rstrip("/") + endpoint_path, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + ) + try: + with urlopen(request, timeout=args.timeout) as response: + result = json.load(response) + except HTTPError as error: + detail = error.read(1000).decode(errors="replace") + raise ValueError(f"Server HTTP {error.code}: {detail}") from error + result = normalize_response(result, args.api) + parsed = json.loads(result["message"]["content"]) + if not isinstance(parsed, dict) or not isinstance(parsed.get("items"), list): + raise ValueError("Response must contain an items list") + items = parsed["items"] + if len(items) != len(rows): + raise ValueError("Response does not cover every input") + for i, (item, row) in enumerate(zip(items, rows, strict=True)): + discard_surplus_examples(item, row) + validate(item, row, i) + return items, result + + +def discard_surplus_examples(item, row): + """Keep original examples when the model volunteers replacements.""" + if row["examples"] and isinstance(item, dict): + enrichment = item.get("enrichment") + if isinstance(enrichment, dict): + enrichment.pop("examples", None) + + +def normalize_response(result, api): + if not isinstance(result, dict): + raise ValueError("Server response must be an object") + if api == "openai": + if not isinstance(result.get("choices"), list) or not result["choices"]: + raise ValueError("Server choices must be a nonempty list") + choice = result["choices"][0] + if not isinstance(choice, dict): + raise ValueError("Server choice must be an object") + usage = result.get("usage") or {} + if not isinstance(usage, dict): + raise ValueError("Server usage must be an object") + result = { + "message": choice["message"], + "done_reason": choice["finish_reason"], + "eval_count": usage.get("completion_tokens", 0), + } + if result.get("done_reason") != "stop": + raise ValueError("Model response was truncated") + if not isinstance(result.get("message"), dict): + raise ValueError("Server message must be an object") + if not isinstance(result["message"].get("content"), str): + raise ValueError("Server message content must be text") + return result + + +def expand_inventory(rows, context_rows=None): + """Assess exact senses once; provide known forms and prompt-only distractors.""" + from wn.personalized_examples import _inflected_variant + + meanings = {} + for row in context_rows if context_rows is not None else rows: + for sense in row["senses"]: + meaning = { + "pos": row.get("pos"), + "definition": row.get("definitions", [""])[0], + } + if meaning not in meanings.setdefault(sense["lemma"], []): + meanings[sense["lemma"]].append(meaning) + selected = {} + for row in sorted(rows, key=lambda value: value["frequency_rank"]): + for sense in row["senses"]: + if sense["sense_id"] in selected: + continue + lemma = sense["lemma"] + forms = [ + lemma, + *sense["matching_forms"], + *[f["form"] for f in sense.get("forms", [])], + ] + if re.fullmatch(r"[a-z]+(?:-[a-z]+)*", lemma): + variant = _inflected_variant(lemma, row.get("pos")) + if variant: + forms.append(variant) + target = {**sense, "matching_forms": list(dict.fromkeys(forms))} + distractors = [ + m + for m in meanings[lemma] + if m["definition"] not in row.get("definitions", []) + ][:8] + selected[sense["sense_id"]] = { + **row, + "senses": [target], + "other_meanings": distractors, + } + return list(selected.values()) + + +def output_record(row, item, args): + sense = row["senses"][0] + return { + "word": sense["lemma"], + "frequency_word": row["word"], + "frequency_rank": row["frequency_rank"], + "word_id": sense["entry_id"], + "sense_id": sense["sense_id"], + "synset_id": row["synset_id"], + "base_definition": row["definitions"][0], + "input_hash": input_hash(row), + "prompt_version": PROMPT_VERSION, + "model": args.model, + "model_revision": args.model_revision, + "prompt_hash": PROMPT_HASH, + "creator": args.model, + "source": f"{args.api}:{args.model}/{PROMPT_VERSION}", + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "unreviewed", + **({"requires_semantic_review": True} if row.get("correction") else {}), + **item["enrichment"], + **item.get("word_enrichment", {}), + } + + +def process_batch(batch, args): + """Retry individual rejected inputs; one malformed response cannot stop a run.""" + try: + items, _ = generate(batch, args) + return [ + output_record(row, item, args) + for row, item in zip(batch, items, strict=True) + ], [] + except (ValueError, OSError, KeyError, TypeError, IndexError) as error: + errors = [] + records = [] + for row in batch: + last_error = error + for _ in range(2): + try: + items, _ = generate([row], args, error_hint=str(last_error)) + records.append(output_record(row, items[0], args)) + break + except ( + ValueError, + OSError, + KeyError, + TypeError, + IndexError, + ) as retry_error: + last_error = retry_error + else: + errors.append( + { + "sense_id": row["senses"][0]["sense_id"], + "input_hash": input_hash(row), + "error": str(last_error), + } + ) + return records, errors + + +def read_completed(path, model, model_revision=None): + if not path.exists(): + return set() + try: + records = [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line + ] + except json.JSONDecodeError as error: + raise ValueError( + f"Invalid checkpoint JSON in {path}; repair any interrupted trailing " + "line before resuming, so successful checkpoints are not overwritten" + ) from error + return { + r["input_hash"] + for r in records + if r["model"] == model + and r["prompt_version"] == PROMPT_VERSION + and r.get("prompt_hash") == PROMPT_HASH + and r.get("model_revision") == model_revision + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("inventory", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--context-inventory", type=Path) + parser.add_argument("--endpoint", default="http://127.0.0.1:11434") + parser.add_argument("--api", choices=("ollama", "openai"), default="ollama") + parser.add_argument("--model", default="gpt-oss:120b") + parser.add_argument("--model-revision") + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--workers", type=int, default=1) + parser.add_argument("--shard", type=int, default=0) + parser.add_argument("--shards", type=int, default=1) + parser.add_argument("--limit", type=int) + parser.add_argument("--timeout", type=int, default=600) + args = parser.parse_args() + if not 0 <= args.shard < args.shards or not 1 <= args.batch_size <= 8: + parser.error("Require 0 <= shard < shards and 1 <= batch-size <= 8") + if not 1 <= args.workers <= 16: + parser.error("Use 1-16 workers, within the server's configured concurrency") + rows = expand_inventory( + [ + json.loads(line) + for line in args.inventory.read_text(encoding="utf-8").splitlines() + if line + ], + [ + json.loads(line) + for line in args.context_inventory.read_text(encoding="utf-8").splitlines() + if line + ] + if args.context_inventory + else None, + ) + rows = [row for i, row in enumerate(rows) if i % args.shards == args.shard] + completed = read_completed(args.output, args.model, args.model_revision) + rows = [row for row in rows if input_hash(row) not in completed] + if args.limit: + rows = rows[: args.limit] + args.output.parent.mkdir(parents=True, exist_ok=True) + print(f"{len(rows)} pending senses; {len(completed)} checkpoints", flush=True) + batches = [ + rows[i : i + args.batch_size] for i in range(0, len(rows), args.batch_size) + ] + done, failures = 0, 0 + started = time.monotonic() + with ( + ThreadPoolExecutor(max_workers=args.workers) as pool, + args.output.open("a", encoding="utf-8") as output, + args.output.with_suffix(".errors.jsonl").open("a", encoding="utf-8") as errors, + ): + futures = [pool.submit(process_batch, batch, args) for batch in batches] + for future in as_completed(futures): + records, rejected = future.result() + for record in records: + output.write(json.dumps(record, ensure_ascii=False) + "\n") + for error in rejected: + errors.write(json.dumps(error, ensure_ascii=False) + "\n") + output.flush() + errors.flush() + done += len(records) + failures += len(rejected) + print( + f"{done}/{len(rows)} senses; {failures} rejected; " + f"{time.monotonic() - started:.1f}s", + flush=True, + ) + if failures: + raise SystemExit(f"{failures} senses require retry; successful outputs saved") + + +if __name__ == "__main__": + main() diff --git a/extensions/english-learner/manifest.json b/extensions/english-learner/manifest.json new file mode 100644 index 0000000..3ff925c --- /dev/null +++ b/extensions/english-learner/manifest.json @@ -0,0 +1,473 @@ +{ + "base_lexicon": "omw-en:1.4", + "extension": "rylo-en-learner:1.0", + "frequency_cutoff": 5000, + "matched_frequency_words": 3887, + "word_meaning_pairs": 22700, + "unique_senses": 21043, + "unique_synsets": 17643, + "status": "AI-generated, not human-reviewed; native labels retain WordNet provenance", + "assessed_senses": 20943, + "quality_gate": { + "status": "passed", + "rubric_version": "exact-sense-2", + "reviewer": "gpt-6-astra", + "reviewed_senses": 17047, + "content_sha256": "2687bed33c70fbc3605cb881127a055ffe07291cb08d8abb76791439fa1714d4", + "decisions_sha256": "e056404cc6148126f7ad486985934348431be9269c0048eca08e33cc5f150198", + "calibration": { + "astra-gate-1": { + "cases": 12, + "passed": 12, + "fixtures_sha256": "307394b616513b60907fb10ea66502597e68226d8cbf5ea6e39d9b32de892730", + "decisions_sha256": "53b4f29cb851719d2255e096e973bd15f5a23ccc65c711fb0fa007bd36e25d99" + }, + "astra-gate-2": { + "cases": 12, + "passed": 12, + "fixtures_sha256": "307394b616513b60907fb10ea66502597e68226d8cbf5ea6e39d9b32de892730", + "decisions_sha256": "b67b78601c9c2d0f847c71bd820f93841ce07f023dcf95152fcbe6239e9295b7" + }, + "astra-gate-3": { + "cases": 12, + "passed": 12, + "fixtures_sha256": "307394b616513b60907fb10ea66502597e68226d8cbf5ea6e39d9b32de892730", + "decisions_sha256": "70526d25670c1a57f4349dfb2ca6e1bbab3ae89c110067780c3949560b95b7fe" + } + } + }, + "review_status_counts": { + "ai-reviewed": 22439, + "not-ai-reviewed": 1235 + }, + "withheld_senses": [ + { + "reason": "The source lemma Massachusetts does not match the people name Massachusett; using the two as interchangeable would be inaccurate. Correction attempt: I cannot confidently use Massachusetts as a natural standalone noun for an individual member of this people; substituting Massachusett would change the required lemma.", + "sense_id": "omw-en-Massachusetts-09660373-n" + }, + { + "reason": "Semantic review: examples: Access code illustrates the compound synonym; network access is a separate permission sense, not noun access meaning code. Correction attempt: Standalone access normally denotes permission or entry rather than a character sequence; natural examples require access code, the compound synonym rejected in the prior issue.", + "sense_id": "omw-en-access-06354204-n" + }, + { + "reason": "Agriculture normally denotes the activity or sector, not a single enterprise; the source\u2019s count-noun sense is questionable. Correction attempt: I cannot produce a natural use of agriculture denoting one large-scale farming enterprise rather than farming activity or the industry; the required countable enterprise sense remains questionable.", + "sense_id": "omw-en-agriculture-01104406-n" + }, + { + "reason": "Semantic review: examples: Audio frequency and audio frequencies instantiate the compound term, not standalone noun audio meaning a frequency. Correction attempt: Standalone audio is not a natural count noun for one audible frequency; ordinary examples use audio frequency or shift to sound or recording.", + "sense_id": "omw-en-audio-05056386-n" + }, + { + "reason": "Semantic review: plain_language: The source defines a phosphorylated nucleoside; merely a pairing molecular component does not preserve that entity.; examples: DNA base and RNA base normally denote nucleobases, not specifically the supplied nucleotide sense. Correction attempt: The source equates base with nucleotide, although a nucleobase is not the whole phosphorylated nucleoside; natural base examples would reinforce this chemical conflation.", + "sense_id": "omw-en-base-14964590-n" + }, + { + "reason": "Semantic review: examples: Bidding settled on describes a sequence of actions; 3NT does not universally indicate a balanced high-point hand. Correction attempt: Natural bidding denotes the bridge auction or bidding activity; I cannot confidently make it denote the specific contracted number of tricks without substituting bid or conflating the senses.", + "sense_id": "omw-en-bidding-06737112-n" + }, + { + "reason": "Semantic review: examples: A rise in demand was a branch of population growth is not natural English for an outgrowth or consequence. Correction attempt: I cannot confidently use branch as a natural noun for a developmental consequence without slipping into the distinct organizational division or offshoot sense.", + "sense_id": "omw-en-branch-11415721-n" + }, + { + "reason": "Semantic review: examples: Deep burn and especially brown burn usually suggest injury or sound unnatural; reliable attestation is needed for these tanning-sense constructions. Correction attempt: A further correction attempt could not confidently distinguish natural burn meaning skin browning alone from burn meaning skin injury. The browning-only source sense needs editorial clarification; invented tanning collocations are withheld.", + "sense_id": "omw-en-burn-14290534-n" + }, + { + "reason": "Semantic review: examples: Button-like and button-shaped invoke resemblance to fasteners, not plant structures called buttons. Correction attempt: I cannot confidently name a specific plant part conventionally called a button without relying on an unsupported botanical claim or merely using button-shaped as a comparison.", + "sense_id": "omw-en-button-11531916-n" + }, + { + "reason": "Semantic review: examples: Calculating receipts or hours means computing totals, not the target sense keeping an account. Correction attempt: I cannot produce natural intransitive calculate meaning keep an account without shifting to computing totals or reckoning on something.", + "sense_id": "omw-en-calculate-02265231-v" + }, + { + "reason": "The supplied examples treat case as a font-style setting (\u201citalic case,\u201d \u201cbold case\u201d), which is not idiomatic. I could not find a natural example for this rare printing sense without confusing it with a physical type case or letter case. Correction attempt: Standalone case meaning a specific font is too uncertain to illustrate naturally; normal uses denote letter case or the physical container for type.", + "sense_id": "omw-en-case-06825399-n" + }, + { + "reason": "The supplied examples use charge as a spoken command, but that does not clearly illustrate the listed rare sense of a hunting dog lying down on command. I could not find a natural example that demonstrates the defined use. Correction attempt: I cannot confidently produce natural examples of charge meaning lie down on command without suggesting a dog's forward attack.", + "sense_id": "omw-en-charge-01985424-v" + }, + { + "reason": "The candidate's examples mistake this rare sense for inspecting or verifying a check; I could not confidently supply natural examples of writing a bank check as a verb. Correction attempt: I cannot confidently use intransitive check naturally to mean write a bank check; ordinary examples would imply inspection instead.", + "sense_id": "omw-en-check-01064999-v" + }, + { + "reason": "Semantic review: examples: Olympic weightlifting ranks successful lifted weight, not performance-quality marks for a flawless clean. Correction attempt: In standard weightlifting terminology a clean brings the bar to the shoulders; the source includes the subsequent jerk, so faithful whole-lift examples would teach a misleading sense.", + "sense_id": "omw-en-clean-00626838-n" + }, + { + "reason": "The base definition relies on obsolete racial categories; examples use a contemporary phrase but do not illustrate the source classification itself. Correction attempt: Standalone colour as a collective racial group is obsolete and cannot be illustrated confidently without conflating the group with skin pigmentation.", + "sense_id": "omw-en-colour-07968354-n" + }, + { + "reason": "The base uses \u201cconsideration\u201d for an advance retainer fee, but the supplied examples sound unnatural and this is not a familiar present-day use; I could not confirm an idiomatic example. Correction attempt: I cannot confidently distinguish natural consideration meaning an advance retainer from the broader payment or contractual-exchange senses.", + "sense_id": "omw-en-consideration-13323313-n" + }, + { + "reason": "The supplied examples use \u201ccontractor\u201d as an awkward metaphor for the heart and diaphragm; I could not find a natural use of this rare anatomical noun sense. Correction attempt: I cannot confidently illustrate contractor as a bodily organ without an unnatural anatomical label or a word-mention sentence; a specialist attestation is needed.", + "sense_id": "omw-en-contractor-05289601-n" + }, + { + "reason": "This act noun is rarely used; \u201cconvening\u201d is more natural in contemporary English. Correction attempt: I cannot confidently illustrate convention as the act of convening without using an unnatural construction or the meeting itself as the referent.", + "sense_id": "omw-en-convention-01231819-n" + }, + { + "reason": "Semantic review: examples: The document set out the copyright distinguishes the document from copyright, contrary to this document noun sense. Correction attempt: Copyright ordinarily denotes the legal right, not a document granting it; forcing the document sense would risk teaching a misleading legal concept.", + "sense_id": "omw-en-copyright-06473168-n" + }, + { + "reason": "Semantic review: examples: A hammer's crack and a crack heard from a pipe denote sharp noises, not cracking actions. Correction attempt: Natural noun examples of crack tend to denote the resulting split, a sound, or a blow; I cannot confidently isolate the act of cracking here.", + "sense_id": "omw-en-crack-00376994-n" + }, + { + "reason": "Semantic review: plain_language: A symbol is a noun description, not denoting a quantity of five hundred.; examples: Both examples mention D as a symbol instead of using it adjectivally for a quantity. Correction attempt: I cannot provide natural modern sentences using lowercase d adjectivally for five hundred without contriving a Roman-numeral inventory or merely mentioning the symbol.", + "sense_id": "omw-en-d-02198631-s" + }, + { + "reason": "Delta commonly names the Greek letter or triangular form, but the gloss \u201cequilateral triangle\u201d is overly specific and examples may be unnatural. Correction attempt: I cannot confidently use delta for a physical equilateral-triangle object without making it sound like a Greek letter or an invented label.", + "sense_id": "omw-en-delta-13880102-n" + }, + { + "reason": "Semantic review: examples: Demanded that the witness appear means required; was demanded in court is not a natural illustration of summoning someone. Correction attempt: Natural demand constructions require someone to appear or demand a person's presence; I cannot confidently use transitive demand itself to mean legally summon a person to court.", + "sense_id": "omw-en-demand-01068985-v" + }, + { + "reason": "Semantic review: examples: Dental consonant and the sound is dental illustrate adjective uses, not the target noun. Correction attempt: The source conflates dental and alveolar articulation; a noun example identifying a specific consonant risks reinforcing that phonetic ambiguity.", + "sense_id": "omw-en-dental-07115493-n" + }, + { + "reason": "Semantic review: examples: Deposit box and deposit area modify the named facility; deposit does not independently refer to a depository here. Correction attempt: I cannot confidently use deposit naturally as the storage facility itself rather than the stored material or an attributive modifier.", + "sense_id": "omw-en-deposit-03177349-n" + }, + { + "reason": "This rare intransitive use of \u201cdigest\u201d for softening or disintegration is difficult to illustrate naturally without risking a wrong sense. Correction attempt: I cannot confidently provide idiomatic intransitive digest examples for material softening under heat or moisture without shifting to biological or chemical treatment senses.", + "sense_id": "omw-en-digest-00397987-v" + }, + { + "reason": "Semantic review: examples: Discovering previously undocumented species uses the finding sense; the target is identifying or classifying a specimen. Correction attempt: Discover naturally means find or learn; I cannot make it mean identify a biological specimen without using discover that and shifting to the ordinary finding-out sense.", + "sense_id": "omw-en-discover-00652346-v" + }, + { + "reason": "Semantic review: examples: Diversity of weather patterns or software updates denotes assorted types, not variability as this synset specifies. Correction attempt: Natural diversity examples denote variety among things; I cannot isolate the rare condition-of-being-changeable sense without conflating those meanings.", + "sense_id": "omw-en-diversity-04735711-n" + }, + { + "reason": "Semantic review: plain_language: Asking a partner for a specific card or suit is not a general definition of doubling in bridge.; examples: The examples describe unspecified doubles without showing a demand for a particular card or suit. Correction attempt: The gloss does not identify a specific bridge convention; a generic double changes the contract stakes, while a demand for a suit requires context not supplied here.", + "sense_id": "omw-en-double-02300425-v" + }, + { + "reason": "The source gloss does not clearly distinguish a sexually arousing person from someone seeking sexual excitement, so natural, sense-specific examples cannot be written confidently. Correction attempt: The circular gloss an erotic person does not establish a precise enough meaning for natural noun examples without guessing whether it means arousing, sensual, or sexually preoccupied.", + "sense_id": "omw-en-erotic-10062716-n" + }, + { + "reason": "The source gloss uses a rare category for non-Abrahamic religion, while generated examples conflate ethnic identity and belief; retain only the gloss-level clarification. Correction attempt: I cannot confidently use archaic ethnic meaning pagan in natural critical educational examples without suggesting the modern cultural-identity sense.", + "sense_id": "omw-en-ethnic-01784401-s" + }, + { + "reason": "Semantic review: examples: Flowed over banks and flowed across ground describe motion; the target sense means cover something with water. Correction attempt: I cannot confidently write idiomatic transitive flow meaning flood a surface; natural flow examples instead describe water moving.", + "sense_id": "omw-en-flow-00217427-v" + }, + { + "reason": "Semantic review: plain_language: Calling any network-connected computer a guest may conflate guest systems, visiting clients, and ordinary nodes.; examples: The examples label a connected laptop a guest without establishing the specialized usage rather than merely asserting it. Correction attempt: A further attempt could not establish guest as an unrestricted network-node noun without importing guest-access or virtual-machine semantics. The source scope needs lexical clarification.", + "sense_id": "omw-en-guest-03827107-n" + }, + { + "reason": "Semantic review: examples: In hair cloth, hair identifies the fiber composing cloth; neither sentence uses hair alone to denote the cloth. Correction attempt: I cannot confidently use standalone hair to mean woven haircloth without suggesting the raw fiber or forcing an unfamiliar count construction.", + "sense_id": "omw-en-hair-03475674-n" + }, + { + "reason": "Semantic review: examples: Holy city and holy destination modify nouns instead of illustrating holy as the place noun. Correction attempt: I cannot confidently use standalone holy as a pilgrimage-place noun; natural examples require holy place or holy site, where holy is adjectival.", + "sense_id": "omw-en-holy-08585056-n" + }, + { + "reason": "Semantic review: examples: Ice the cake means apply icing and does not instantiate the target noun sense. Correction attempt: I cannot confidently use standalone ice as the sugar topping in natural contemporary prose without it being read as frozen water; the ordinary noun is icing.", + "sense_id": "omw-en-ice-07620822-n" + }, + { + "reason": "The supplied examples do not use \u201cil\u201d naturally as the number 49, and the abbreviation is not a standard way to write that number. I could not create an idiomatic example without inventing usage. Correction attempt: The lemma il is not a standard natural numeral for forty-nine; writing contrived quantities with it would teach unreliable notation.", + "sense_id": "omw-en-il-02191139-s" + }, + { + "reason": "The source labels \u201cincreasing\u201d as a musical adjective, but neither gloss nor examples establish a clear distinct music sense. Correction attempt: The source definition consists only of music and does not establish which distinct musical meaning increasing should illustrate.", + "sense_id": "omw-en-increasing-02537743-a" + }, + { + "reason": "This is a rare international-law use of \u201cinfection\u201d; I cannot form natural example sentences without risking an invented legal claim. Correction attempt: I cannot confidently illustrate the rare maritime-law sense without inventing a legal rule about when a ship or cargo becomes liable to seizure.", + "sense_id": "omw-en-infection-04811296-n" + }, + { + "reason": "Semantic review: examples: The source frame instruments somebody; instrument the notice to someone substitutes an unsupported argument pattern. Correction attempt: I cannot substantiate a natural construction instrument somebody meaning address a legal document to them; ordinary instrument examples would use unrelated senses.", + "sense_id": "omw-en-instrument-00991151-v" + }, + { + "reason": "Semantic review: examples: He wrote 12 k on the board is a metalinguistic mention, not an adjective modifying a quantity of items. Correction attempt: Standalone k is not naturally used as an adjective for one thousand; normal examples embed it in numerals such as 10k, leaving the requested lemma usage uncertain.", + "sense_id": "omw-en-k-02198752-s" + }, + { + "reason": "Semantic review: examples: Both examples discuss L as a symbol, without using it adjectivally to quantify anything.; plain_language: The number fifty is a nominal definition rather than the adjective meaning consisting of fifty units. Correction attempt: I cannot write natural English examples using lowercase l adjectivally for fifty without contriving a Roman-numeral quantity or merely naming the symbol.", + "sense_id": "omw-en-l-02191232-s" + }, + { + "reason": "Semantic review: examples: Large of the target and large of the bullseye may be mechanical substitutions for wide of rather than natural examples. Correction attempt: I cannot confidently use large as an adverb meaning wide of a target; the apparent examples sound like mechanical replacements for wide.", + "sense_id": "omw-en-large-00386393-r" + }, + { + "reason": "The lemma is a rare Roman-numeral form; the generated examples used it as if it were a common English word, so no examples are supplied. Correction attempt: I cannot produce natural ordinary sentences using lowercase li adjectivally for fifty-one without contriving Roman-numeral quantities.", + "sense_id": "omw-en-li-02191336-s" + }, + { + "reason": "The generated examples described a tsunami and an earthquake without context; the rare technical sense needs a sourced example. Correction attempt: I cannot confidently isolate lift as a wave itself from the more common upward-motion or lifting-force senses without a sourced technical usage.", + "sense_id": "omw-en-lift-07348399-n" + }, + { + "reason": "Semantic review: plain_language: A connecting reference in a list need not be a programming command joining modules or elements.; examples: The first example describes a list connection, while the second suggests control flow; neither clearly establishes the specified instruction sense. Correction attempt: A further attempt could not reliably isolate the source instruction sense from a connection, pointer, or named LINK command without unsupported technical usage.", + "sense_id": "omw-en-link-06586471-n" + }, + { + "reason": "Semantic review: examples: Completed the location of the sculpture is not a confidently idiomatic use for physically placing it; the first sentence can still mean selected position. Correction attempt: A further attempt could not reliably separate the act of physical placement from a selected position or the act of finding a location in natural examples.", + "sense_id": "omw-en-location-01051331-n" + }, + { + "reason": "Semantic review: examples: Lorded it over means dominated others; neither example makes someone a lord, and the captured lands construction is unnatural. Correction attempt: I cannot confidently use transitive lord naturally to mean confer a lordship; ordinary lorded constructions mean behave domineeringly.", + "sense_id": "omw-en-lord-02398854-v" + }, + { + "reason": "The provided examples use lose to mean suffer a loss or concede, not the rare sense \u201cretreat.\u201d I could not find two natural examples of lose meaning retreat. Correction attempt: I cannot confidently use intransitive lose to mean retreat without adding ground or another complement, which changes the supplied construction.", + "sense_id": "omw-en-lose-01113806-v" + }, + { + "reason": "Semantic review: examples: Inventory reached m makes m the nominal amount; it does not modify an item count as required. Correction attempt: I cannot naturally use lowercase m adjectivally for one thousand without contriving Roman-numeral quantities or inviting confusion with other abbreviations.", + "sense_id": "omw-en-m-02198752-s" + }, + { + "reason": "Semantic review: examples: Win the match denotes the contest; match point is a separate compound and does not demonstrate the score noun match. Correction attempt: Standalone match naturally names the contest; I cannot confidently use it as the required winning score without relying on the separate compound match point.", + "sense_id": "omw-en-match-13596673-n" + }, + { + "reason": "The gloss describes a noun meaning a low-gloss, low-contrast quality, but the candidate examples use the adjective matte or a person\u2019s name. I could not write natural examples using matt as a noun in this sense. Correction attempt: I cannot confidently use matt as a noun naming the low-gloss quality itself; natural examples use an adjective or a paint finish as a product.", + "sense_id": "omw-en-matt-04956110-n" + }, + { + "reason": "The definition describes metamorphosis, not metabolism; I cannot preserve the target sense without inventing or changing the lemma. Correction attempt: The supplied transformation sense belongs to metamorphosis in ordinary biological usage; I cannot confidently use metabolism for larval transformation without a specialist historical attestation.", + "sense_id": "omw-en-metabolism-13514880-n" + }, + { + "reason": "Semantic review: examples: Received a second mill and an initial coarse mill are not confidently idiomatic names for grinding passes rather than machines. Correction attempt: A further attempt could not substantiate a natural count-noun mill denoting a grinding operation rather than the machine or facility.", + "sense_id": "omw-en-mill-00358290-n" + }, + { + "reason": "Semantic review: plain_language: The source says usually smaller scale, not necessarily scaled down.; examples: A created cell model and an architect's model are representations produced, not the act of representing. Correction attempt: Standalone model naturally denotes the resulting representation rather than the act; the ordinary process noun is modeling, which would substitute the listed synonym.", + "sense_id": "omw-en-model-00898804-n" + }, + { + "reason": "Semantic review: examples: A clay or wax modeling standing on a bench is not confidently natural usage for a preliminary sculpture rather than its modeling process. Correction attempt: A further attempt could not substantiate natural modeling denoting a preliminary sculpture itself rather than its production or form.", + "sense_id": "omw-en-modeling-00937895-n" + }, + { + "reason": "The noun modeling usually names the process, while this gloss describes its sculptural product; generated examples used it unnaturally as a count noun, so examples are withheld. Correction attempt: I cannot confidently use modeling as a countable sculpture rather than the process of creating one without unnatural wording.", + "sense_id": "omw-en-modeling-03779621-n" + }, + { + "reason": "Semantic review: examples: A modified performance or edition is changed; added comments that it was mediocre do not give modified that meaning. Correction attempt: I cannot confidently use modified to mean mediocre; natural constructions mean altered, and adding mediocrity in context would not change that lexical sense.", + "sense_id": "omw-en-modified-01913642-s" + }, + { + "reason": "This rare legal sense is more commonly called a \u201cpresentment\u201d; the supplied examples use \u201cnotification\u201d unnaturally for an indictment, so I cannot confirm idiomatic exact-headword usage. Correction attempt: I cannot confidently use notification for a grand jury's self-initiated accusation without a legal attestation; ordinary notification means notice, not the accusation itself.", + "sense_id": "omw-en-notification-01187463-n" + }, + { + "reason": "Semantic review: examples: Parliament's order suggests an instruction; adhered to the order does not clearly illustrate a body of parliamentary rules. Correction attempt: I cannot confidently use standalone order to name the entire body of parliamentary rules without shifting to a command, procedural conformity, or sequence.", + "sense_id": "omw-en-order-06652878-n" + }, + { + "reason": "The examples read as awkward attempts to use \u201coutlook\u201d for a lookout or view; I could not find a natural sentence for the exact sense \u201cthe act of looking out.\u201d Correction attempt: I cannot confidently use outlook for the act of looking out without it being read as a view, prospect, or attitude; the action sense needs attestation.", + "sense_id": "omw-en-outlook-00881545-n" + }, + { + "reason": "The definition says this adjective means \u201cyielding a fair profit,\u201d but the supplied examples use paid to mean compensated or sponsored. I could not find a natural example that clearly uses paid in the profitability sense. Correction attempt: Paid normally means compensated or settled; a natural adjective meaning profitable would be paying, and substituting it would not illustrate this lemma.", + "sense_id": "omw-en-paid-01871473-s" + }, + { + "reason": "The gloss describes working as a paragrapher, but I could not write natural examples that use paragraph as a verb in this exact sense without shifting to the separate sense of dividing text into paragraphs. Correction attempt: I cannot confidently use intransitive paragraph to mean work as a writer of paragraphs; natural uses take text as an object and mean divide it into paragraphs.", + "sense_id": "omw-en-paragraph-01699539-v" + }, + { + "reason": "Semantic review: examples: Imposed a penalty and the penalty was intended to punish name a punishment imposed, without clearly illustrating the act of punishing. Correction attempt: Standalone penalty naturally denotes the punishment imposed, rather than the act of imposing it; I cannot write an unambiguous process example without substituting penalization.", + "sense_id": "omw-en-penalty-01160342-n" + }, + { + "reason": "The noun permit meaning the act of giving authorization is rare; the generated examples illustrate the separate document sense. Correction attempt: Standalone permit naturally names an authorization or its document, rather than the act of giving authorization; an exact process example would be forced.", + "sense_id": "omw-en-permit-01139194-n" + }, + { + "reason": "Semantic review: examples: The spear's point and mountain's point identify physical extremities rather than a shape's sharply tapered quality. Correction attempt: I cannot confidently use point to denote the abstract quality of pointedness rather than the physical tip or the act of shaping one.", + "sense_id": "omw-en-point-05071556-n" + }, + { + "reason": "Semantic review: examples: Position of the vase and adjusting the microphone's position concern spatial arrangement, not the placement action. Correction attempt: Position as a noun normally denotes a resulting location or arrangement; I cannot unambiguously demonstrate the act itself without using positioning or placement.", + "sense_id": "omw-en-position-01051331-n" + }, + { + "reason": "The supplied definition is a rare technical aeronautical sense; the candidate examples use race for a stream of air but do not establish natural usage of this noun sense. Correction attempt: I cannot confidently use race naturally for an aircraft propeller's backward airstream without relying on an unverified rare technical construction.", + "sense_id": "omw-en-race-11423197-n" + }, + { + "reason": "This rare nautical/weather sense is unclear, and the candidate examples invent unrelated sail and hay actions. Correction attempt: I cannot confidently use rack for running before a gale with the supplied human subject frame; the rare nautical construction needs an attested example.", + "sense_id": "omw-en-rack-01946138-v" + }, + { + "reason": "Semantic review: examples: A rise in the tide is a tidal-level increase, not the wave or swell identified by this noun sense. Correction attempt: I cannot confidently use rise for a wave itself rather than the upward motion or an increase in water level.", + "sense_id": "omw-en-rise-07348399-n" + }, + { + "reason": "This dated slang sense is difficult to illustrate naturally; examples are provisional and may conflate cheating with robbery. Correction attempt: I cannot distinguish the proposed underhanded selling or obtaining sense of roll from robbery in natural exact-lemma examples without an attested construction.", + "sense_id": "omw-en-roll-02322596-v" + }, + { + "reason": "The WordNet gloss describes a whorl, but \u201croll\u201d is not a natural learner-facing label for this botanical form. Correction attempt: I cannot confidently use roll naturally for the concentric whorl shape specified here without suggesting a cylinder or spirally rolled material.", + "sense_id": "omw-en-roll-13875970-n" + }, + { + "reason": "Semantic review: register: The source does not establish that the metal plate sense is specifically historical.; examples: The artillery uses of scale are specialized and cannot be confirmed as idiomatic from these examples alone. Correction attempt: Standalone scale for uniform metal sheathing is too uncertain to illustrate naturally without specialist evidence; familiar uses denote small overlapping plates or surface deposits.", + "sense_id": "omw-en-scale-03959701-n" + }, + { + "reason": "The verb is awkward in current standard English in this construction; \u201cdevise\u201d or \u201cdraw up\u201d is usual. Correction attempt: I cannot confidently use scheme transitively for devising a neutral system without an unnatural construction or the separate plotting sense.", + "sense_id": "omw-en-scheme-00708376-v" + }, + { + "reason": "This sense appears to use scheme for an evasive statement, a rare/dated meaning; verify source before retaining. Correction attempt: I cannot confidently use scheme to name the evasive statement itself rather than a plan for evasion; this rare sense needs attestation.", + "sense_id": "omw-en-scheme-06756680-n" + }, + { + "reason": "Semantic review: examples: Secretary of his private fears is not confidently idiomatic for a person entrusted with secrets, even in the historical framing. Correction attempt: A further attempt could not establish a natural secretary construction for a person entrusted with secrets independently of an administrative role; historical lexical evidence is still needed.", + "sense_id": "omw-en-secretary-10521928-n" + }, + { + "reason": "Semantic review: examples: Security alarm and security device name devices with security modifiers; security itself does not denote the device there. Correction attempt: Standalone security normally means protection, personnel, or arrangements; I cannot naturally use it for a particular electrical alarm device without a compound such as security system.", + "sense_id": "omw-en-security-04165945-n" + }, + { + "reason": "Semantic review: examples: Soft consonant is not generally synonymous with a hushing consonant; the unspecified language and soft sh classification lack sufficient context. Correction attempt: The source conflates palatalization with hissing or hushing sounds; I cannot choose an exact soft consonant example without a language-specific interpretation or specialist evidence.", + "sense_id": "omw-en-soft-01156925-a" + }, + { + "reason": "The gloss describes the state of being dirty, but the candidate examples use soil in unidiomatic phrases. I could not produce natural examples using soil as a noun for this state rather than for dirt or earth. Correction attempt: I cannot confidently use soil for the state of being dirty rather than the dirt itself; proposed abstract-state constructions sound unnatural.", + "sense_id": "omw-en-soil-14498096-n" + }, + { + "reason": "Semantic review: examples: Patience and attention to detail as specialties naturally suggest things someone excels at, leaving this distinctiveness sense unclear. Correction attempt: I cannot confidently use specialty for a distinguishing trait alone without it suggesting expertise or a characteristic product; this rare sense needs attestation.", + "sense_id": "omw-en-specialty-04763925-n" + }, + { + "reason": "Semantic review: examples: Crystal shape identifying a mineral and symptoms indicating a cause do not clearly illustrate the defining-character sense of specify. Correction attempt: The exact essential-quality sense is difficult to separate from stating details or identifying something; I cannot confidently supply natural examples with the source's inanimate-subject frame.", + "sense_id": "omw-en-specify-02611630-v" + }, + { + "reason": "Semantic review: examples: The speed of the cheetah and roller coaster denotes rate or swiftness, not the specified activity sense. Correction attempt: Standalone speed naturally denotes rate or rapidity rather than the activity of moving; an unambiguous act-noun example would require speeding or hurrying.", + "sense_id": "omw-en-speed-00330160-n" + }, + { + "reason": "Semantic review: plain_language: In wine terminology split commonly means a quarter-size bottle rather than half; the generated half-standard-volume claim needs justification.; examples: The wine example intensifies the unresolved conflict between the source's half-bottle definition and common split terminology. Correction attempt: The source defines a half-size bottle while common wine usage often means a quarter-size bottle; an accurate beverage-specific example requires evidence resolving the size convention.", + "sense_id": "omw-en-split-13770076-n" + }, + { + "reason": "This WordNet verb sense means patronize or shop at a business, but sponsor does not naturally express that meaning in contemporary English. Correction attempt: Sponsor normally means support or finance, not shop at; I cannot confidently write an exact-lemma patronage example without shifting to that support sense.", + "sense_id": "omw-en-sponsor-02466134-v" + }, + { + "reason": "Semantic review: examples: Proved to be a sport by trying out uses the willing participant or good-sport sense, not a sports practitioner. Correction attempt: I cannot confidently use standalone sport for an athlete without suggesting the different willing-participant or good-sport sense.", + "sense_id": "omw-en-sport-10638922-n" + }, + { + "reason": "Semantic review: examples: Sugar and other carbohydrates and sugar stored as starch distinguish simple sugars from other carbohydrates, leaving the broad saccharide-class sense unestablished. Correction attempt: The source conflates sugar with the broader carbohydrate class, including polysaccharides; ordinary natural examples cannot safely demonstrate that scope.", + "sense_id": "omw-en-sugar-14792703-n" + }, + { + "reason": "Semantic review: examples: Studying survival and adaptation does not necessarily study selection; it can concern individual persistence alone. Correction attempt: I cannot confidently make standalone survival mean the evolutionary process of natural selection rather than persistence of individual organisms.", + "sense_id": "omw-en-survival-13563746-n" + }, + { + "reason": "Semantic review: examples: The tab reported and he preferred the tab identify a publication; this sense is the journalistic approach, not the newspaper entity. Correction attempt: Bare tab naturally denotes a tabloid publication, not the sensationalist style of journalism specified by this source sense.", + "sense_id": "omw-en-tab-06267991-n" + }, + { + "reason": "Semantic review: examples: Her tone and his tone are objects of soften or raising; neither sentence uses the verb tone. Correction attempt: Using tone transitively to vary speech pitch is difficult to distinguish naturally from changing vocal quality; reliable examples need attestation.", + "sense_id": "omw-en-tone-00982293-v" + }, + { + "reason": "The base describes a tugboat, but the candidate\u2019s examples use \u201ctugboat,\u201d not the exact headword \u201ctower\u201d; I cannot confirm a natural current use of \u201ctower\u201d for this vessel. Correction attempt: I cannot confidently use tower for a tugboat without specialist attestation; ordinary readers would understand a tall structure or an unspecified towing agent.", + "sense_id": "omw-en-tower-04495843-n" + }, + { + "reason": "Semantic review: examples: Transportation on the bus cost money names the service being purchased; paid the transportation charged is not idiomatic evidence for the sum itself. Correction attempt: A further attempt could not clearly distinguish transportation as a fare amount from the transport service or its cost without forcing the wording.", + "sense_id": "omw-en-transportation-13308147-n" + }, + { + "reason": "Semantic review: plain_language: Universe of discourse commonly denotes the domain of objects under discussion, not a total set of propositions and assumptions.; examples: The philosopher's restriction to empirically verifiable statements reinforces the unresolved propositions-versus-domain distinction. Correction attempt: The source's everything stated or assumed conflicts with the usual domain-of-discourse interpretation; examples would need to resolve that distinction first.", + "sense_id": "omw-en-universe-05814162-n" + }, + { + "reason": "Semantic review: examples: V represents five and uses V denote numeral nouns, not a modifier meaning five items. Correction attempt: I cannot naturally use lowercase v adjectivally for five without contriving Roman-numeral quantities or merely discussing the numeral.", + "sense_id": "omw-en-v-02186750-s" + }, + { + "reason": "Semantic review: examples: These examples do not clearly establish variety alone as the show noun rather than an attributive entertainment category. Correction attempt: I cannot confidently use standalone variety for one particular show rather than the entertainment genre or a modifier in variety show.", + "sense_id": "omw-en-variety-07020239-n" + }, + { + "reason": "Semantic review: examples: Ordered vi cupcakes and requires vi pieces lack a Roman-numbering context and are not idiomatic ordinary sentences. Correction attempt: I cannot produce ordinary natural examples using lowercase vi adjectivally for six without contriving a Roman-numeral inventory.", + "sense_id": "omw-en-vi-02186833-s" + }, + { + "reason": "Semantic review: examples: Both sentences refer to intestinal gas, not the reflex or act of passing it specified by this sense.; register: Wind for intestinal gas remains ordinary British usage and is not inherently dated. Correction attempt: Wind naturally names intestinal gas; constructions such as breaking wind name the act through breaking, so two clear bare-noun examples of the specified reflex remain unresolved.", + "sense_id": "omw-en-wind-00839597-n" + }, + { + "reason": "The generated examples did not demonstrate this lemma naturally in the target sense; hold for editorial review. Correction attempt: I cannot naturally use lowercase xx adjectivally for twenty without contriving a Roman-numeral quantity or mentioning the symbol instead.", + "sense_id": "omw-en-xx-02188317-s" + }, + { + "reason": "The inventory defines \u201cyard\u201d as the number 1,000, but I cannot confidently verify or illustrate this rare usage. Correction attempt: I cannot verify yard as one thousand; financial slang commonly uses yard for one billion, so invented thousand-value examples would risk teaching the wrong quantity.", + "sense_id": "omw-en-yard-13750844-n" + }, + { + "reason": "Semantic review: examples: Seasonal yield being lower and factory yield increasing both readily denote output quantity, the neighboring noun sense, without establishing production itself. Correction attempt: Natural noun uses of yield readily denote the amount produced; I cannot cleanly isolate the specified production-process sense without forcing the wording.", + "sense_id": "omw-en-yield-00914632-n" + }, + { + "reason": "Semantic review: examples: A zero set ordinarily means points where a function vanishes and need not be empty; these examples assert emptiness without adequate mathematical context. Correction attempt: The adjective's claimed null-set sense lacks a natural unambiguous construction; zero set commonly means a function's roots and is not necessarily empty.", + "sense_id": "omw-en-zero-03145851-a" + } + ], + "counts": { + "annotations": 16036, + "concepts_with_added_examples": 6424, + "domains": 2698, + "examples": 14048, + "grammar_notes": 159, + "irregular_forms": 88, + "new_senses": 0, + "new_synsets": 0, + "new_words": 0, + "plain_language": 12643, + "referenced_senses": 20943, + "referenced_synsets": 17568, + "referenced_words": 6041, + "register": 340, + "usage": 241, + "usage_tips": 106 + }, + "inputs": { + "inventory.jsonl": "fdd96b6d0f2eb217b9fa7e8528a83a5a4f5666936d91e829558e2e11e3b0b243", + "reviewed-records.jsonl.xz": "7aff5189f8924d275bb039bcc0efd4d502dae23d42ac51a887f50cd15edd236b", + "quality-reviews.jsonl.xz": "f15f1a0ea4a1f1cee4a9707ff34f7b5d2fe6b5258d5a72ea7141ae7fd7c22adb", + "quality-calibration-decisions.jsonl": "2c0a4ed1801a66427d79899e462aeddc12d6828fc1b5aa0985b959582233da10", + "quality-calibration.json": "f127d032dff3de27b1675d2ec36d938af632f21e498566064482ba5d65fea30b", + "quality-context.jsonl.xz": "d7abfc3096b421d8c718a6caf2d8bd79d94eb48f2a210f86354adebbdf92a734", + "withheld.jsonl": "16147181ed1879ea217f40c57a343731daac2ca91cc06722a6352861375665dc" + }, + "extension_sha256": "f6bc021ed1e4f5229fb1119aee69ba9c058f671b7e9dd2facd7d00fb30079632" +} diff --git a/extensions/english-learner/prepare_candidates.py b/extensions/english-learner/prepare_candidates.py new file mode 100644 index 0000000..09ad36a --- /dev/null +++ b/extensions/english-learner/prepare_candidates.py @@ -0,0 +1,290 @@ +"""Select existing unreviewed learner candidates and expose authoring gaps. + +Selection checks identity and simple textual constraints, not semantic quality. +Every retained language-model candidate still needs linguistic review. +""" + +import argparse +import json +import re +import sqlite3 +import tarfile +from collections import defaultdict +from pathlib import Path + +CREATOR = "gpt-oss:120b" +SOURCE = "training/data/generated.tar.xz" +MANUAL_EXAMPLE_SYNSETS = {"omw-en-09849598-n"} + + +def normalized(text): + return " ".join(text.split()).casefold() + + +def short_text(value, limit=500): + return isinstance(value, str) and bool(value.strip()) and len(value) <= limit + + +def examples(candidate, word, synset_id): + if synset_id in MANUAL_EXAMPLE_SYNSETS: + return [] + pattern = re.compile(r"(? 1 or batch[0]["word"] == "bad": + raise ValueError("rejected") + return [{"enrichment": {}}], {} + + monkeypatch.setattr(generator, "generate", fake_generate) + monkeypatch.setattr(generator, "output_record", lambda row, item, args: row) + records, errors = generator.process_batch(rows, None) + assert [r["word"] for r in records] == ["good"] + assert [r["sense_id"] for r in errors] == ["bad"] + + +def test_known_noun_plurals_and_symbol_case(): + rows = [] + for i, word in enumerate(["have", "person", "Na"]): + rows.append( + { + "frequency_rank": i, + "pos": "n", + "definitions": [word], + "senses": [{"sense_id": word, "lemma": word, "matching_forms": [word]}], + } + ) + result = generator.expand_inventory(rows) + assert "haves" in result[0]["senses"][0]["matching_forms"] + assert "people" in result[1]["senses"][0]["matching_forms"] + assert result[2]["senses"][0]["matching_forms"] == ["Na"] + + +def test_response_type_and_duplicate_examples(row): + with pytest.raises(ValueError, match="must be an object"): + generator.validate([], row, 0) + with pytest.raises(ValueError, match="Duplicate examples"): + generator.validate( + { + "id": 0, + "enrichment": {"examples": ["Hello, my love.", "Hello, my love."]}, + }, + row, + 0, + ) + with pytest.raises(ValueError, match="Server response"): + generator.normalize_response([], "openai") + with pytest.raises(ValueError, match="Server choices"): + generator.normalize_response({"choices": []}, "openai") + + +def test_prompt_distractors_are_not_exported(): + entries = [ + { + "word": "bank", + "frequency_rank": 1, + "pos": "n", + "definitions": [definition], + "senses": [ + {"sense_id": str(i), "lemma": "bank", "matching_forms": ["bank"]} + ], + } + for i, definition in enumerate(["financial institution", "sloping land"]) + ] + output = generator.expand_inventory(entries) + assert output[0]["other_meanings"] == [{"pos": "n", "definition": "sloping land"}] + assert output[1]["other_meanings"] == [ + {"pos": "n", "definition": "financial institution"} + ] + + +def test_lowercase_word_allows_sentence_initial_capitalization(row): + generator.validate( + { + "id": 0, + "enrichment": { + "examples": [ + "Love, please come and sit beside me.", + "Good morning, my love.", + ] + }, + }, + row, + 0, + ) + + +def test_prompt_word_preserves_exact_lemma_case(): + row = { + "word": "in", + "pos": "n", + "definitions": ["a metallic element"], + "examples": [], + "relations": [], + "senses": [ + { + "lemma": "In", + "matching_forms": ["In"], + "forms": [], + "tags": [], + "syntactic_frames": [], + } + ], + } + assert generator.compact(row, 0)["word"] == "In" + row["examples"] = ["The label identified the sample as In."] + assert generator.compact(row, 0)["existing_examples"] == row["examples"] + previous_hash = generator.input_hash(row) + row["correction"] = { + "issues": [{"field": "examples", "reason": "Wrong part of speech."}] + } + assert generator.compact(row, 0)["correction"] == row["correction"] + assert generator.input_hash(row) != previous_hash + + +def test_malformed_openai_usage_is_rejected(): + with pytest.raises(ValueError, match="Server usage"): + generator.normalize_response( + { + "choices": [{"message": {"content": "{}"}, "finish_reason": "stop"}], + "usage": [1], + }, + "openai", + ) diff --git a/tests/learner_quality_test.py b/tests/learner_quality_test.py new file mode 100644 index 0000000..8c53d00 --- /dev/null +++ b/tests/learner_quality_test.py @@ -0,0 +1,449 @@ +"""A stale stamp, incomplete critic or failed calibration must not release text.""" + +import copy +import importlib.util +import json +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parents[1] / "extensions/english-learner" +SPEC = importlib.util.spec_from_file_location( + "learner_quality", ROOT / "quality_gate.py" +) +quality = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(quality) + + +@pytest.fixture +def source(): + records = [ + { + "sense_id": "love-1-n", + "word_id": "love-n", + "synset_id": "1-n", + "base_definition": "a beloved person", + "plain_language": "Someone you love.", + "examples": ["Hello, my love."], + "creator": "generator", + "source": "generated", + } + ] + inventory = [ + { + "synset_id": "1-n", + "definitions": ["a beloved person"], + "examples": [], + "synonyms": ["love", "beloved"], + "relations": [ + { + "type": "domain_topic", + "target_synset_id": "topic-1-n", + "forms": ["relationships"], + } + ], + "senses": [ + { + "sense_id": "love-1-n", + "entry_id": "love-n", + "lemma": "love", + "pos": "n", + "syntactic_frames": [], + } + ], + } + ] + return records, inventory + + +def decision(item, verdict="accept", run="test-run"): + return { + "sense_id": item["sense_id"], + "input_sha256": item["input_sha256"], + "reviewer": quality.REVIEWER, + "review_run": run, + "rubric_version": quality.RUBRIC_VERSION, + "decision": verdict, + "rationale": "The noun names a beloved person.", + "issues": [] + if verdict == "accept" + else [{"field": "examples", "reason": "Target is used as a verb."}], + } + + +def calibration(run="test-run"): + fixtures = json.loads( + (ROOT / "quality-calibration.json").read_text(encoding="utf-8") + ) + return fixtures, [ + decision(i, "accept" if i["expected_accept"] else "reject", run) + for i in fixtures + ] + + +def test_correction_author_requires_an_independent_reviewer(source): + records, inventory = source + records[0]["author_run"] = "author-run" + item = quality.review_items(records, inventory)[0] + with pytest.raises(ValueError, match="author cannot approve"): + quality.validate_decisions([item], [decision(item, run="author-run")]) + quality.validate_decisions([item], [decision(item, run="independent-run")]) + records[0]["author_run"] = "another-author" + changed = quality.review_items(records, inventory)[0] + with pytest.raises(ValueError, match="stale review content"): + quality.validate_decisions([changed], [decision(item, run="independent-run")]) + + +def test_paraphrase_scope_change_cannot_reuse_sense_review(source): + records, inventory = source + original = quality.review_items(records, inventory)[0] + records[0]["plain_language_scope"] = "synset" + changed = quality.review_items(records, inventory)[0] + assert changed["content"]["annotations"][0]["plain_language_scope"] == "synset" + with pytest.raises(ValueError, match="stale review content"): + quality.validate_decisions([changed], [decision(original)]) + # Paraphrases have their own scope, independent of the other learner fields. + records[0].pop("plain_language_scope") + records[0]["scope"] = "synset" + scoped = quality.review_items(records, inventory)[0] + assert scoped["content"]["annotations"][0]["plain_language_scope"] == "sense" + + +def test_release_requires_exact_content_approval_and_calibrated_run(source): + records, inventory = source + items = quality.review_items(records, inventory) + fixtures, controls = calibration() + result = quality.validate_release( + records, inventory, [decision(items[0])], fixtures, controls + ) + assert result["reviewed_senses"] == 1 + assert result["calibration"]["test-run"]["passed"] == 12 + with pytest.raises(ValueError, match="missing semantic reviews"): + quality.validate_release(records, inventory, [], fixtures, controls) + with pytest.raises(ValueError, match="missing semantic reviews"): + quality.validate_release(records, inventory, [decision(items[0])], fixtures, []) + + +@pytest.mark.parametrize( + "change", ["text", "scope", "definition", "pos", "lemma", "forms_scope"] +) +def test_content_or_context_change_invalidates_old_review(source, change): + records, inventory = source + previous = decision(quality.review_items(records, inventory)[0]) + if change == "text": + records[0]["examples"] = ["I love you."] + elif change == "scope": + records[0]["scope"] = "synset" + elif change == "forms_scope": + records[0]["irregular_forms_scope"] = "word" + elif change == "definition": + inventory[0]["definitions"] = ["have affection for"] + records[0]["base_definition"] = "have affection for" + else: + inventory[0]["senses"][0][change] = "v" if change == "pos" else "beloved" + with pytest.raises(ValueError, match="stale review content"): + quality.validate_decisions(quality.review_items(records, inventory), [previous]) + + +def test_rubric_change_invalidates_review(source, monkeypatch): + records, inventory = source + previous = decision(quality.review_items(records, inventory)[0]) + monkeypatch.setattr(quality, "RUBRIC", quality.RUBRIC + " Additional criterion.") + with pytest.raises(ValueError, match="stale review content"): + quality.validate_decisions(quality.review_items(records, inventory), [previous]) + + +@pytest.mark.parametrize("field", ["word_id", "synset_id", "base_definition"]) +def test_source_identity_must_match_review_context(source, field): + records, inventory = source + records[0][field] = "wrong" + with pytest.raises(ValueError, match="source identity differs"): + quality.review_items(records, inventory) + + +def test_removing_all_bad_fields_still_requires_review_of_the_correction(source): + records, inventory = source + records[0].pop("plain_language") + records[0].pop("examples") + assert quality.review_items(records, inventory) == [] + records[0]["requires_semantic_review"] = True + items = quality.review_items(records, inventory) + assert len(items) == 1 + with pytest.raises(ValueError, match="missing semantic reviews"): + quality.validate_decisions(items, []) + + +def test_correction_inputs_require_exact_negative_reviews(source): + spec = importlib.util.spec_from_file_location( + "learner_corrections", ROOT / "prepare_corrections.py" + ) + corrections = importlib.util.module_from_spec(spec) + spec.loader.exec_module(corrections) + records, inventory = source + inventory[0]["frequency_rank"] = 1 + item = quality.review_items(records, inventory)[0] + assert corrections.prepare(records, inventory, [decision(item)], []) == [] + result = corrections.prepare(records, inventory, [decision(item, "reject")], []) + assert result[0]["correction"]["issues"][0]["field"] == "examples" + records[0].update(reviewer="old-reviewer", review_status="ai-reviewed") + revised = corrections.remove_flagged_fields(records, [decision(item, "reject")]) + assert "examples" not in revised[0] + assert "reviewer" not in revised[0] + assert revised[0]["requires_semantic_review"] + assert revised[0]["plain_language"] == records[0]["plain_language"] + assert records[0]["examples"] == ["Hello, my love."] + assert ( + quality.review_items(revised, inventory)[0]["input_sha256"] + != item["input_sha256"] + ) + records[0]["examples"] = ["Different text."] + with pytest.raises(ValueError, match="stale review content"): + corrections.prepare(records, inventory, [decision(item, "reject")], []) + + +@pytest.mark.parametrize("verdict", ["reject", "uncertain"]) +def test_negative_or_uncertain_review_blocks_release(source, verdict): + items = quality.review_items(*source) + with pytest.raises(ValueError, match="unresolved semantic review"): + quality.validate_decisions(items, [decision(items[0], verdict)]) + + +def test_duplicate_unknown_and_contradictory_decisions_are_rejected(source): + items = quality.review_items(*source) + row = decision(items[0]) + for rows, message in [ + ([row, row], "duplicate or unknown"), + ([{**row, "sense_id": "wrong"}], "duplicate or unknown"), + ( + [{**row, "issues": [{"field": "examples", "reason": "bad"}]}], + "inconsistent review issues", + ), + ([{**row, "rationale": ""}], "missing review rationale"), + ([{**row, "review_run": ""}], "missing review run"), + ([{**row, "reviewer": "gpt-6-luna"}], "wrong reviewer"), + ]: + with pytest.raises(ValueError, match=message): + quality.validate_decisions(items, rows) + + +def test_calibration_rejects_blanket_pass_blanket_reject_and_stale_cases(): + fixtures, controls = calibration() + for rows in ( + [decision(i) for i in fixtures], + [decision(i, "reject") for i in fixtures], + ): + with pytest.raises(ValueError, match="failed calibration"): + quality.validate_calibration(fixtures, rows) + altered = copy.deepcopy(fixtures) + altered[0]["content"]["annotations"] = [] + with pytest.raises(ValueError, match="input hash does not match"): + quality.validate_calibration(altered, controls) + with pytest.raises(ValueError, match="incomplete calibration"): + quality.validate_calibration([], []) + + +def test_calibration_is_required_for_each_reviewer_run(source): + items = quality.review_items(*source) + fixtures, controls = calibration("different-run") + with pytest.raises(ValueError, match="missing semantic reviews"): + quality.validate_release(*source, [decision(items[0])], fixtures, controls) + + +def test_native_exemption_only_accepts_verified_topic_links(source): + _, inventory = source + row = { + "sense_id": "love-1-n", + "word_id": "love-n", + "synset_id": "1-n", + "base_definition": "a beloved person", + "creator": "WordNet", + "source": "omw-en:1.4", + "domains": ["relationships"], + "domain_evidence": [{"label": "relationships", "synset_id": "topic-1-n"}], + } + assert quality.review_items([row], inventory) == [] + with pytest.raises(ValueError, match="cannot bypass review"): + quality.review_items([{**row, "register": ["cakewalk"]}], inventory) + with pytest.raises(ValueError, match="cannot bypass review"): + quality.review_items([{**row, "examples": ["Bad example."]}], inventory) + with pytest.raises(ValueError, match="unverified native topic"): + quality.review_items([{**row, "domains": ["law"]}], inventory) + + +def test_known_failures_and_controls_remain_in_calibration(): + fixtures, _ = calibration() + assert sum(i["expected_accept"] for i in fixtures) == 4 + rejected = [i["content"] for i in fixtures if not i["expected_accept"]] + for sid in [ + "omw-en-run-00558883-n", + "omw-en-medical-00142361-n", + "omw-en-growth-13489037-n", + "omw-en-replacement-13547925-n", + ]: + assert any(i["sense_id"] == sid for i in rejected) + + +def test_reviewer_calibration_input_hides_answers(): + fixtures, controls = calibration() + inputs = [quality.blind_fixture(item) for item in fixtures] + assert all("expected_accept" not in item for item in inputs) + quality.validate_decisions(inputs, controls, require_accept=False) + + +def test_native_usage_examples_are_not_register_labels(): + spec = importlib.util.spec_from_file_location( + "learner_candidates", ROOT / "prepare_candidates.py" + ) + candidates = importlib.util.module_from_spec(spec) + spec.loader.exec_module(candidates) + row = {} + candidates.native_labels( + { + "relations": [ + { + "type": "exemplifies", + "forms": ["cakewalk"], + "target_synset_id": "term-1", + }, + { + "type": "domain_topic", + "forms": ["biology"], + "target_synset_id": "topic-1", + }, + ] + }, + row, + ) + assert row == { + "domains": ["biology"], + "domain_evidence": [{"label": "biology", "synset_id": "topic-1"}], + } + + +@pytest.fixture +def package(source, tmp_path): + import gzip + import hashlib + import lzma + + from wn import lmf + + spec = importlib.util.spec_from_file_location( + "learner_check", ROOT / "check_release.py" + ) + checker = importlib.util.module_from_spec(spec) + spec.loader.exec_module(checker) + records, inventory = source + records[0].update( + base_definition="a beloved person", + status="unreviewed", + reviewer="gpt-6-luna", + review_status="ai-reviewed", + ) + fixtures, controls = calibration() + items = quality.review_items(records, inventory) + reviews = [decision(items[0])] + data = { + "reviewed-records.jsonl.xz": records, + "quality-reviews.jsonl.xz": reviews, + "quality-context.jsonl.xz": quality.compact_inventory(inventory), + "quality-calibration-decisions.jsonl": controls, + "withheld.jsonl": [], + } + for name, rows in data.items(): + value = "".join(json.dumps(r) + "\n" for r in rows).encode() + (tmp_path / name).write_bytes( + lzma.compress(value) if name.endswith(".xz") else value + ) + (tmp_path / "quality-calibration.json").write_text(json.dumps(fixtures)) + resource, counts = checker.builder.build_resource( + checker.builder.mark_semantic_review(records), + {"1-n": []}, + {"love-1-n"}, + ) + xml = tmp_path / "extension.xml" + lmf.dump(resource, xml) + compiled = gzip.compress(xml.read_bytes(), mtime=0) + (tmp_path / "rylo-en-learner.xml.gz").write_bytes(compiled) + manifest = { + "inputs": { + name: hashlib.sha256((tmp_path / name).read_bytes()).hexdigest() + for name in [*data, "quality-calibration.json"] + }, + "extension_sha256": hashlib.sha256(compiled).hexdigest(), + "counts": counts, + "withheld_senses": [], + "quality_gate": quality.validate_release( + records, inventory, reviews, fixtures, controls + ), + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + return checker, tmp_path, manifest + + +def test_package_verifier_recompiles_approved_source(package): + checker, path, _ = package + assert checker.check_release(path)["status"] == "passed" + + +def test_updating_manifest_cannot_approve_changed_compiled_content(package): + import gzip + import hashlib + + checker, path, manifest = package + target = path / "rylo-en-learner.xml.gz" + xml = gzip.decompress(target.read_bytes()).replace( + b"Hello, my love.", b"I love this." + ) + target.write_bytes(gzip.compress(xml, mtime=0)) + manifest["extension_sha256"] = hashlib.sha256(target.read_bytes()).hexdigest() + (path / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="differs from approved source"): + checker.check_release(path) + + +def test_source_change_needs_new_review_even_with_updated_manifest(package): + import hashlib + import lzma + + checker, path, manifest = package + target = path / "reviewed-records.jsonl.xz" + rows = quality.read_jsonl(target) + rows[0]["examples"] = ["I love this."] + target.write_bytes( + lzma.compress("".join(json.dumps(r) + "\n" for r in rows).encode()) + ) + manifest["inputs"][target.name] = hashlib.sha256(target.read_bytes()).hexdigest() + (path / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="stale review content"): + checker.check_release(path) + + +def test_quarantine_preserves_bad_source_and_never_publishes_partial_review(source): + spec = importlib.util.spec_from_file_location( + "learner_apply", ROOT / "apply_reviews.py" + ) + apply = importlib.util.module_from_spec(spec) + spec.loader.exec_module(apply) + records, inventory = source + original = copy.deepcopy(records) + item = quality.review_items(records, inventory)[0] + fixtures, controls = calibration() + with pytest.raises(ValueError, match="missing semantic reviews"): + apply.select_reviewed(records, inventory, [], fixtures, controls, []) + result = apply.select_reviewed( + records, inventory, [decision(item, "reject")], fixtures, controls, [] + ) + assert result["accepted"] == [] + assert result["quarantined"] == original + assert result["reviews"] == [] + assert result["withheld"][0]["sense_id"] == "love-1-n" + assert "Target is used as a verb" in result["withheld"][0]["reason"] + assert records == original + revised = [{**records[0], "requires_semantic_review": True}] + approved = apply.select_reviewed( + revised, inventory, [decision(item)], fixtures, controls, [] + )["accepted"][0] + assert approved["reviewer"] == quality.REVIEWER + assert approved["review_status"] == "ai-reviewed" + assert "reviewer" not in revised[0] diff --git a/tests/learner_test.py b/tests/learner_test.py new file mode 100644 index 0000000..204739c --- /dev/null +++ b/tests/learner_test.py @@ -0,0 +1,300 @@ +import json + +import pytest +from starlette.testclient import TestClient + +import wn +from wn import lmf, web +from wn.learner import ANNOTATION_TYPE, validate_learner_fields + +SYNSET = "test-en-0001-n" +WORD = "test-en-information-n" +SENSE = "test-en-information-n-0001-01" + + +def annotation(scope, text="something that informs", **fields): + definition = { + "text": text, + "meta": { + "type": ANNOTATION_TYPE, + "description": json.dumps({"schema_version": 1, "scope": scope, **fields}), + "creator": "test-model", + "source": "test-generation", + "status": "unreviewed", + }, + } + if scope == "sense": + definition["sourceSense"] = SENSE + return definition + + +def resource(definitions=None): + return { + "lmf_version": "1.1", + "lexicons": [ + { + "id": "test-learner", + "version": "1", + "label": "Learner notes", + "language": "en", + "email": "test@example.com", + "license": "test", + "extends": {"id": "test-en", "version": "1"}, + "entries": [ + { + "external": True, + "id": WORD, + "senses": [{"external": True, "id": SENSE}], + } + ], + "synsets": [ + { + "external": True, + "id": SYNSET, + "definitions": definitions + if definitions is not None + else [ + annotation("synset", domains=["computing"]), + annotation( + "sense", + "Facts that tell you something.", + grammar_notes=["Usually uncountable."], + ), + annotation( + "word", + word_id=WORD, + usage_tips=["Ask for information."], + ), + ], + "examples": [ + { + "text": "The guide gave us useful information.", + "meta": { + "creator": "test-model", + "status": "unreviewed", + }, + } + ], + } + ], + } + ], + } + + +@pytest.fixture +def learner_db(monkeypatch, tmp_path, datadir): + with monkeypatch.context() as m: + m.setattr(wn.config, "data_directory", tmp_path / "database") + m.setattr(wn.config, "allow_multithreading", True) + m.setattr(wn._db, "pool", {}) + wn.add(datadir / "mini-lmf-1.0.xml", progress_handler=None) + yield + wn._db.clear_connections() + + +@pytest.mark.usefixtures("learner_db") +def test_native_extension_roundtrip_and_removal(tmp_path): + base = wn.Wordnet("test-en:1") + synset = base.synset(SYNSET) + original_examples = synset.examples() + original_counts = (len(base.words()), len(base.synsets()), len(base.senses())) + path = tmp_path / "learner.xml" + lmf.dump(resource(), path) + wn.add(path, progress_handler=None) + + assert synset.definition() == "something that informs" + assert synset.definitions() == ["something that informs"] + assert [definition.text for definition in synset.definitions(data=True)] == [ + "something that informs" + ] + assert synset.examples() == [ + *original_examples, + "The guide gave us useful information.", + ] + assert len(synset.learner_data()) == 3 + assert "plain_language" not in synset.learner_data()[0] + notes = base.sense(SENSE).learner_data() + assert len(notes) == 2 + assert notes[1]["plain_language"] == "Facts that tell you something." + assert notes[1]["source_sense_id"] == SENSE + assert notes[1]["metadata"]["creator"] == "test-model" + assert notes[1]["metadata"]["status"] == "unreviewed" + assert base.word(WORD).learner_data()[0]["usage_tips"] == ["Ask for information."] + assert original_counts == ( + len(base.words()), + len(base.synsets()), + len(base.senses()), + ) + assert wn.synset(SYNSET).learner_data() == synset.learner_data() + + exported = tmp_path / "export.xml" + wn.export(wn.lexicons(lexicon="test-learner:1"), exported, version="1.1") + wn.remove("test-learner:1") + assert synset.learner_data() == [] + assert synset.examples() == original_examples + assert synset.definition() == "something that informs" + wn.add(exported, progress_handler=None) + assert base.sense(SENSE).learner_data() == notes + + +@pytest.mark.usefixtures("learner_db") +def test_web_exposes_learner_notes_for_explicit_base(tmp_path): + path = tmp_path / "learner.xml" + lmf.dump(resource(), path) + wn.add(path, progress_handler=None) + client = TestClient(web.app) + response = client.get(f"/lexicons/test-en:1/synsets/{SYNSET}") + assert response.status_code == 200 + attrs = response.json()["data"]["attributes"] + assert attrs["definition"] == "something that informs" + assert len(attrs["learner_data"]) == 3 + assert "The guide gave us useful information." in attrs["examples"] + + +@pytest.mark.parametrize( + "fields", + [ + {}, + {"schema_version": True, "scope": "synset"}, + {"schema_version": 2, "scope": "synset"}, + {"schema_version": 1, "scope": "unknown"}, + {"schema_version": 1, "scope": "word"}, + {"schema_version": 1, "scope": "synset", "irregular_forms": {}}, + {"schema_version": 1, "scope": "sense", "domains": "medical"}, + {"schema_version": 1, "scope": "sense", "domains": [""]}, + {"schema_version": 1, "scope": "sense", "typo": []}, + ], +) +def test_reject_invalid_fields(fields): + with pytest.raises(wn.Error): + validate_learner_fields(fields) + + +def test_irregular_forms_are_word_scoped(): + fields = { + "schema_version": 1, + "scope": "word", + "word_id": "test-child-n", + "irregular_forms": {"plural": ["children"]}, + } + assert validate_learner_fields(fields) == fields + + +def test_irregular_forms_can_be_sense_specific(): + fields = { + "schema_version": 1, + "scope": "sense", + "irregular_forms": {"past": ["lay"], "past_participle": ["lain"]}, + } + assert validate_learner_fields(fields) == fields + + +@pytest.mark.usefixtures("learner_db") +def test_sense_notes_do_not_leak_to_synonyms(tmp_path): + entry = resource() + lexicon = entry["lexicons"][0] + lexicon["entries"][0]["id"] = "test-en-example-n" + sense = "test-en-example-n-0002-01" + lexicon["entries"][0]["senses"][0]["id"] = sense + note = annotation("sense", usage_tips=["An example of something."]) + note["sourceSense"] = sense + lexicon["synsets"][0]["id"] = "test-en-0002-n" + lexicon["synsets"][0]["definitions"] = [note] + path = tmp_path / "sense.xml" + lmf.dump(entry, path) + wn.add(path, progress_handler=None) + base = wn.Wordnet("test-en:1") + assert len(base.sense(sense).learner_data()) == 1 + assert base.sense("test-en-illustration-n-0002-01").learner_data() == [] + client = TestClient(web.app) + for word, expected in [("example", 1), ("illustration", 0)]: + response = client.get("/lexicons/test-en:1/words", params={"form": word}) + notes = response.json()["data"][0]["included"][0]["attributes"]["learner_data"] + assert len(notes) == expected + + +@pytest.mark.usefixtures("learner_db") +def test_reject_invalid_import_without_leaving_extension(tmp_path): + invalid = annotation("sense", grammar_notes=["Use as a noun."]) + invalid["meta"]["description"] = "not json" + path = tmp_path / "invalid.xml" + lmf.dump(resource([invalid]), path) + with pytest.raises(wn.Error, match="invalid learner annotation JSON"): + wn.add(path, progress_handler=None) + assert wn.lexicons(lexicon="test-learner:1") == [] + assert wn.synset(SYNSET).definition() == "something that informs" + + +@pytest.mark.usefixtures("learner_db") +@pytest.mark.parametrize( + ("scope", "target"), + [ + ("sense", "test-en-example-n-0002-01"), + ("sense", "missing-sense"), + ("word", "test-en-example-n"), + ("word", "missing-word"), + ], +) +def test_reject_annotation_target_outside_concept(tmp_path, scope, target): + item = annotation(scope) + if scope == "sense": + item["sourceSense"] = target + else: + item = annotation(scope, word_id=target) + path = tmp_path / "wrong-target.xml" + lmf.dump(resource([item]), path) + with pytest.raises(wn.Error, match="target is not a member"): + wn.add(path, progress_handler=None) + assert wn.lexicons(lexicon="test-learner:1") == [] + + +@pytest.mark.usefixtures("learner_db") +def test_web_keeps_generated_examples_on_their_exact_sense(tmp_path): + entry = resource() + lexicon = entry["lexicons"][0] + lexicon["entries"] = [] + example_text = "The teacher wrote an example on the board." + illustration_text = "The speaker offered an illustration of the problem." + lexicon["synsets"] = [ + { + "id": "test-en-0002-n", + "external": True, + "examples": [ + { + "text": text, + "meta": { + "type": "learner-example", + "identifier": sense, + "creator": "test-model", + "status": "unreviewed", + }, + } + for sense, text in [ + ("test-en-example-n-0002-01", example_text), + ("test-en-illustration-n-0002-01", illustration_text), + ] + ], + } + ] + path = tmp_path / "scoped-examples.xml" + lmf.dump(entry, path) + wn.add(path, progress_handler=None) + client = TestClient(web.app) + for word, own, other in [ + ("example", example_text, illustration_text), + ("illustration", illustration_text, example_text), + ]: + response = client.get("/lexicons/test-en:1/words", params={"form": word}) + assert response.status_code == 200 + attributes = response.json()["data"][0]["included"][0]["attributes"] + examples = attributes["examples"] + assert [item["text"] for item in attributes["example_details"]] == [own] + assert attributes["example_details"][0]["metadata"]["creator"] == "test-model" + assert own in examples + assert other not in examples + assert len(examples) == 3 # two original examples plus this sense's example + assert f"we need an {word} here" in examples # legacy personalization remains + concept_examples = wn.Wordnet("test-en:1").synset("test-en-0002-n").examples() + assert example_text in concept_examples + assert illustration_text in concept_examples diff --git a/wn/__init__.py b/wn/__init__.py index c6c2a87..132ee8c 100644 --- a/wn/__init__.py +++ b/wn/__init__.py @@ -37,7 +37,7 @@ "word", "words", ) -__version__ = "1.8.0" +__version__ = "1.9.0" from wn._add import add, add_lexical_resource, remove from wn._config import config # noqa: F401 diff --git a/wn/_add.py b/wn/_add.py index 180abbb..e5443f6 100644 --- a/wn/_add.py +++ b/wn/_add.py @@ -20,6 +20,7 @@ ) from wn._types import AnyPath from wn._util import format_lexicon_specifier, normalize_form +from wn.learner import ANNOTATION_TYPE, decode_learner_data from wn.project import iterpackages from wn.util import ProgressBar, ProgressHandler @@ -519,6 +520,9 @@ def _insert_synset_definitions( VALUES (null,?,({SYNSET_QUERY}),?,?,({SENSE_QUERY}),?) """ for batch in _batch(synsets): + for synset in batch: + for definition in synset.get("definitions", []): + _validate_learner_target(synset["id"], definition, lexid, lexidmap, cur) data = [ ( lexid, @@ -537,6 +541,41 @@ def _insert_synset_definitions( progress.update(len(data)) +def _validate_learner_target( + synset: str, + definition: lmf.Definition, + lexid: int, + lexidmap: _LexIdMap, + cur: sqlite3.Cursor, +) -> None: + meta = definition.get("meta") or {} + if meta.get("type") != ANNOTATION_TYPE: + return + fields = decode_learner_data( + definition["text"], definition.get("sourceSense"), "", meta, set() + ) + if fields["scope"] == "synset": + return + params: tuple[str | int, ...] + if fields["scope"] == "sense": + target = fields["source_sense_id"] + condition = "s.id = ? AND s.lexicon_rowid = ?" + params = (target, lexidmap.get(target, lexid)) + else: + condition = "e.id = ?" + params = (fields["word_id"],) + query = f""" + SELECT 1 FROM senses AS s + JOIN entries AS e ON e.rowid = s.entry_rowid + JOIN synsets AS ss ON ss.rowid = s.synset_rowid + WHERE ss.id = ? AND ss.lexicon_rowid = ? AND {condition} + """ + if not cur.execute( + query, (synset, lexidmap.get(synset, lexid), *params) + ).fetchone(): + raise Error("learner annotation target is not a member of its synset") + + def _insert_synset_relations( synsets: Sequence[_AnySynset], lexid: int, diff --git a/wn/_core.py b/wn/_core.py index 5fecaa3..d71d321 100644 --- a/wn/_core.py +++ b/wn/_core.py @@ -33,9 +33,11 @@ get_synset_relations, get_synsets_for_ilis, get_syntactic_behaviours, + get_word_definitions, resolve_lexicon_specifiers, ) from wn._util import unique_list +from wn.learner import ANNOTATION_TYPE, LearnerData, decode_learner_data if TYPE_CHECKING: from collections.abc import Iterator, Sequence @@ -107,6 +109,13 @@ def _get_lexicons(self) -> tuple[str, ...]: else: return self._lexconf.lexicons + def _get_content_lexicons(self) -> tuple[str, ...]: + """Include installed annotations even when the base is selected explicitly.""" + lexicons = self._get_lexicons() + if self._lexconf.default_mode: + return lexicons + return tuple(dict.fromkeys((*lexicons, *get_lexicon_extensions(lexicons)))) + @dataclass(frozen=True, slots=True) class Pronunciation(LexiconElement): @@ -275,6 +284,20 @@ def metadata(self) -> Metadata: """Return the word's metadata.""" return get_metadata(self.id, self._lexicon, "entries") + def learner_data(self) -> list[LearnerData]: + """Return word-scoped learner notes, including installed extensions.""" + definitions = get_word_definitions(self.id, self._get_content_lexicons()) + records = ( + decode_learner_data(text, sense, lex, meta, set()) + for text, _, sense, lex, meta in definitions + if meta is not None and meta.get("type") == ANNOTATION_TYPE + ) + return [ + record + for record in records + if record["scope"] == "word" and record["word_id"] == self.id + ] + def synsets(self) -> list[Synset]: """Return the list of synsets of the word. @@ -563,6 +586,9 @@ def definition(self, *, data: bool = False) -> str | Definition | None: """ lexicons = self._get_lexicons() if defns := get_definitions(self.id, lexicons): + # Learner notes are additive; never replace the reference definition. + defns = [d for d in defns if (d[4] or {}).get("type") != ANNOTATION_TYPE] + if defns: text, lang, sense_id, lex, meta = defns[0] if data: return Definition( @@ -603,6 +629,9 @@ def definitions(self, *, data: bool = False) -> list[str] | list[Definition]: """ lexicons = self._get_lexicons() defns = get_definitions(self.id, lexicons) + # Learner annotations are exposed separately and must not appear as + # reference definitions in either the singular or plural API. + defns = [d for d in defns if (d[4] or {}).get("type") != ANNOTATION_TYPE] if data: return [ Definition( @@ -639,7 +668,7 @@ def examples(self, *, data: bool = False) -> list[str] | list[Example]: ['"orbital revolution"', '"orbital velocity"'] """ - lexicons = self._get_lexicons() + lexicons = self._get_content_lexicons() exs = get_examples(self.id, "synsets", lexicons) if data: return [ @@ -647,7 +676,26 @@ def examples(self, *, data: bool = False) -> list[str] | list[Example]: for text, lang, lex, meta in exs ] else: - return [text for text, *_ in exs] + return unique_list(text for text, *_ in exs) + + def learner_data(self) -> list[LearnerData]: + """Return scoped learner annotations attached to this existing concept. + + Records retain their source sense/word, extension lexicon, and native + provenance metadata. A repeated reference definition is omitted from + ``plain_language``. Word-specific records must not be applied to synonyms. + """ + definitions = get_definitions(self.id, self._get_content_lexicons()) + originals = { + text + for text, _, _, _, meta in definitions + if (meta or {}).get("type") != ANNOTATION_TYPE + } + return [ + decode_learner_data(text, sense, lex, meta, originals) + for text, _, sense, lex, meta in definitions + if meta is not None and meta.get("type") == ANNOTATION_TYPE + ] def senses(self) -> list[Sense]: """Return the list of sense members of the synset. @@ -1015,6 +1063,15 @@ def synset(self) -> Synset: id, pos, ili, lex = next(find_synsets(id=self._synset_id, lexicons=lexicons)) return Synset(id, pos, ili=ili, _lexicon=lex, _lexconf=self._lexconf) + def learner_data(self) -> list[LearnerData]: + """Return concept-wide and this sense's learner notes, not its synonyms'.""" + return [ + record + for record in self.synset().learner_data() + if record["scope"] == "synset" + or (record["scope"] == "sense" and record["source_sense_id"] == self.id) + ] + @overload def examples(self, *, data: Literal[False] = False) -> list[str]: ... @overload @@ -1031,7 +1088,7 @@ def examples(self, *, data: bool = False) -> list[str] | list[Example]: examples are returned as :class:`str` types. If it is :python:`True`, :class:`wn.Example` objects are used instead. """ - lexicons = self._get_lexicons() + lexicons = self._get_content_lexicons() exs = get_examples(self.id, "senses", lexicons) if data: return [ @@ -1039,7 +1096,7 @@ def examples(self, *, data: bool = False) -> list[str] | list[Example]: for text, lang, lex, meta in exs ] else: - return [text for text, *_ in exs] + return unique_list(text for text, *_ in exs) def lexicalized(self) -> bool: """Return True if the sense is lexicalized.""" diff --git a/wn/_export.py b/wn/_export.py index 3324a5b..5788027 100644 --- a/wn/_export.py +++ b/wn/_export.py @@ -15,6 +15,7 @@ find_synsets, find_syntactic_behaviours, get_adjposition, + get_definition_source_sense_ids, get_definitions, get_entry_forms, get_entry_index, @@ -569,7 +570,7 @@ def _get_external_sense_ids(lexspecs: _LexSpecs) -> set[str]: """Get ids of external senses needed for an extension.""" return get_relation_targets( "sense_relations", "senses", (lexspecs.primary,), lexspecs - ) + ) | get_definition_source_sense_ids((lexspecs.primary,)) def _get_external_synset_ids(lexspecs: _LexSpecs) -> set[str]: diff --git a/wn/_queries.py b/wn/_queries.py index abd991d..e0a9cf8 100644 --- a/wn/_queries.py +++ b/wn/_queries.py @@ -187,23 +187,24 @@ def get_lexicon_extension_bases(lexicon: str, depth: int = -1) -> list[str]: return [row[0] for row in rows] -def get_lexicon_extensions(lexicon: str, depth: int = -1) -> list[str]: - query = """ +def get_lexicon_extensions(lexicon: str | Sequence[str], depth: int = -1) -> list[str]: + lexicons = (lexicon,) if isinstance(lexicon, str) else lexicon + query = f""" WITH RECURSIVE ext(x, d) AS (SELECT extension_rowid, 1 FROM lexicon_extensions JOIN lexicons AS lex ON lex.rowid = base_rowid - WHERE lex.specifier = :specifier + WHERE lex.specifier IN ({_qs(lexicons)}) UNION SELECT extension_rowid, d+1 FROM lexicon_extensions JOIN ext ON base_rowid = x) SELECT extlex.specifier FROM ext JOIN lexicons AS extlex ON extlex.rowid = ext.x - WHERE :depth < 0 OR d <= :depth + WHERE ? < 0 OR d <= ? ORDER BY d """ - rows = connect().execute(query, {"specifier": lexicon, "depth": depth}) + rows = connect().execute(query, (*lexicons, depth, depth)) return [row[0] for row in rows] @@ -681,6 +682,34 @@ def get_expanded_synset_relations( yield from result_rows +def get_definition_source_sense_ids(lexicons: Sequence[str]) -> set[str]: + """Return senses referenced by definitions when exporting extensions.""" + query = f""" + SELECT DISTINCT s.id + FROM definitions AS d + JOIN senses AS s ON s.rowid = d.sense_rowid + JOIN lexicons AS lex ON lex.rowid = d.lexicon_rowid + WHERE lex.specifier IN ({_qs(lexicons)}) + """ + return {row[0] for row in connect().execute(query, lexicons)} + + +def get_word_definitions(entry_id: str, lexicons: Sequence[str]) -> list[_Definition]: + """Get annotations for a word in one query rather than visiting each sense.""" + query = f""" + SELECT DISTINCT d.definition, d.language, source.id, lex.specifier, + d.metadata, d.rowid + FROM entries AS e + JOIN senses AS s ON s.entry_rowid = e.rowid + JOIN definitions AS d ON d.synset_rowid = s.synset_rowid + LEFT JOIN senses AS source ON source.rowid = d.sense_rowid + JOIN lexicons AS lex ON lex.rowid = d.lexicon_rowid + WHERE e.id = ? AND lex.specifier IN ({_qs(lexicons)}) + ORDER BY d.rowid + """ + return [row[:5] for row in connect().execute(query, (entry_id, *lexicons))] + + def get_definitions( synset_id: str, lexicons: Sequence[str], @@ -697,6 +726,7 @@ def get_definitions( JOIN lexicons AS lex ON lex.rowid = d.lexicon_rowid WHERE ss.id = ? AND lex.specifier IN ({_qs(lexicons)}) + ORDER BY d.rowid """ return conn.execute(query, (synset_id, *lexicons)).fetchall() @@ -723,6 +753,7 @@ def get_examples( JOIN lexicons AS lex ON lex.rowid = ex.lexicon_rowid WHERE tbl.id = ? AND lex.specifier IN ({_qs(lexicons)}) + ORDER BY ex.rowid """ return conn.execute(query, (id, *lexicons)).fetchall() diff --git a/wn/learner.py b/wn/learner.py new file mode 100644 index 0000000..8d6fee6 --- /dev/null +++ b/wn/learner.py @@ -0,0 +1,103 @@ +"""Typed learner annotations carried by ordinary WN-LMF definitions. + +An additive definition with ``dc:type="learner-enrichment"`` carries a +plain-language explanation as its text and a versioned JSON object in +``dc:description``. Native metadata records provenance and review status. +This uses standard extension import/export and removal, without new entities. +""" + +import json +from typing import Literal, TypedDict, cast + +from wn._exceptions import Error +from wn._metadata import Metadata + +ANNOTATION_TYPE = "learner-enrichment" + + +class LearnerFields(TypedDict, total=False): + schema_version: Literal[1] + scope: Literal["word", "sense", "synset"] + word_id: str + domains: list[str] + register: list[str] + usage: list[str] + grammar_notes: list[str] + irregular_forms: dict[str, list[str]] + usage_tips: list[str] + + +class LearnerData(LearnerFields, total=False): + plain_language: str + source_sense_id: str + lexicon: str + metadata: Metadata + + +_LIST_FIELDS = {"domains", "register", "usage", "grammar_notes", "usage_tips"} +_FIELDS = _LIST_FIELDS | {"schema_version", "scope", "word_id", "irregular_forms"} + + +def _strings(value: object) -> bool: + return isinstance(value, list) and all( + isinstance(item, str) and bool(item.strip()) for item in value + ) + + +def _validate_irregular_forms(forms: object, scope: str) -> None: + if scope == "synset": + raise Error("irregular_forms require word or sense scope") + if not isinstance(forms, dict) or not all( + isinstance(key, str) and key.strip() and _strings(forms[key]) for key in forms + ): + raise Error("invalid learner annotation irregular_forms") + + +def validate_learner_fields(value: object) -> LearnerFields: + """Validate a version-1 annotation, rejecting misspelled or mistyped fields. + + Empty optional fields should be omitted by producers. Irregular forms + may be sense-specific (lie/lay versus lie/lied), but never concept-wide. + """ + if not isinstance(value, dict) or value.keys() - _FIELDS: + raise Error("invalid learner annotation fields") + if type(value.get("schema_version")) is not int or value["schema_version"] != 1: + raise Error("unsupported learner annotation schema_version") + scope = value.get("scope") + if scope not in ("word", "sense", "synset"): + raise Error("invalid learner annotation scope") + if scope == "word": + if not isinstance(value.get("word_id"), str) or not value["word_id"].strip(): + raise Error("word learner annotation requires word_id") + elif "word_id" in value: + raise Error("word_id requires word scope") + for key in _LIST_FIELDS & value.keys(): + if not _strings(value[key]): + raise Error(f"learner annotation {key} must be a list of strings") + if "irregular_forms" in value: + _validate_irregular_forms(value["irregular_forms"], scope) + return cast("LearnerFields", value) + + +def decode_learner_data( + text: str, + source_sense_id: str | None, + lexicon: str, + metadata: Metadata, + original_definitions: set[str], +) -> LearnerData: + """Decode a marked definition; fail clearly on malformed annotations.""" + try: + fields = validate_learner_fields(json.loads(metadata.get("description", ""))) + except (ValueError, TypeError) as exc: + raise Error("invalid learner annotation JSON") from exc + if (fields["scope"] == "sense") != bool(source_sense_id): + raise Error("only sense learner annotations require sourceSense") + result = cast("LearnerData", dict(fields)) + result["lexicon"] = lexicon + result["metadata"] = metadata + if source_sense_id: + result["source_sense_id"] = source_sense_id + if text and text not in original_definitions and fields["scope"] != "word": + result["plain_language"] = text + return result diff --git a/wn/web.py b/wn/web.py index 5c7380b..ecdcac5 100644 --- a/wn/web.py +++ b/wn/web.py @@ -171,6 +171,34 @@ def _first_lemma(ss: wn.Synset) -> str | None: return senses[0].word().lemma() if senses else None +def _word_examples( + ss: wn.Synset, + word: wn.Word, + sense_ids: set[str], + members: list[str], + is_english: bool, +) -> tuple[list[str], list[dict]]: + """Keep authored examples on their source sense and preserve their wording.""" + examples = [] + details = [] + lemma, forms = word.lemma(), word.forms() + for example in ss.examples(data=True): + meta = example.metadata() + if meta.get("type") == "learner-example": + if meta.get("identifier") not in sense_ids: + continue + text = example.text + details.append({"text": text, "metadata": meta}) + elif is_english: + text = personalize_example( + example.text, lemma, forms, members, ss.pos or "" + ) + else: + text = example.text + examples.append(text) + return list(dict.fromkeys(examples)), details + + def make_word(w: wn.Word, request: Request, basic: bool = False) -> dict: lex_spec = w.lexicon().specifier() d: dict = { @@ -180,6 +208,7 @@ def make_word(w: wn.Word, request: Request, basic: bool = False) -> dict: 'pos': w.pos, 'lemma': w.lemma(), 'forms': [_make_form_data(f) for f in w.forms(data=True)], + 'learner_data': w.learner_data(), }, 'links': { 'self': _url_for_obj(request, 'word', w, lexicon=lex_spec) @@ -190,21 +219,36 @@ def make_word(w: wn.Word, request: Request, basic: bool = False) -> dict: lex_link = str(request.url_for('lexicon', lexicon=lex_spec)) senses_link = str(request.url_for('senses', word=w.id, lexicon=lex_spec)) - sense_counts = {s.synset().id: sum(s.counts()) for s in w.senses()} + senses = w.senses() + sense_ids = {s.id for s in senses} + sense_counts = {s.synset().id: sum(s.counts()) for s in senses} is_english = w.lexicon().language == 'en' included = [] for ss in synsets: - ss_data = make_synset(ss, request, basic=True) + members = ss.lemmas() + examples, example_details = _word_examples( + ss, + w, + sense_ids, + members, + is_english, + ) + ss_data = make_synset( + ss, + request, + basic=True, + examples=examples, + example_details=example_details, + ) attrs = ss_data['attributes'] + attrs['learner_data'] = [ + note for note in attrs['learner_data'] + if note['scope'] == 'synset' + or (note['scope'] == 'sense' and note['source_sense_id'] in sense_ids) + or (note['scope'] == 'word' and note['word_id'] == w.id) + ] attrs['count'] = sense_counts.get(ss.id, 0) - attrs['members'] = ss.lemmas() - if is_english: - attrs['examples'] = [ - personalize_example( - ex, w.lemma(), w.forms(), attrs['members'], ss.pos or '' - ) - for ex in attrs['examples'] - ] + attrs["members"] = members # One lazily-computed hypernym path (hypernym_paths() enumerates # every path exhaustively), nearest ancestors only, emitted # root-most first — exactly the breadcrumb the word page renders. @@ -247,6 +291,7 @@ def make_sense(s: wn.Sense, request: Request, basic: bool = False) -> dict: d: dict = { 'id': s.id, 'type': 'sense', + 'attributes': {'learner_data': s.learner_data()}, 'links': { 'self': _url_for_obj(request, 'sense', s, lexicon=lex_spec) } @@ -272,8 +317,23 @@ def make_sense(s: wn.Sense, request: Request, basic: bool = False) -> dict: return d -def make_synset(ss: wn.Synset, request: Request, basic: bool = False) -> dict: +def make_synset( + ss: wn.Synset, + request: Request, + basic: bool = False, + *, + examples: list[str] | None = None, + example_details: list[dict] | None = None, +) -> dict: lex_spec = ss.lexicon().specifier() + if examples is None: + source_examples = ss.examples(data=True) + examples = list(dict.fromkeys(ex.text for ex in source_examples)) + example_details = [ + {"text": ex.text, "metadata": ex.metadata()} + for ex in source_examples + if ex.metadata().get("type") == "learner-example" + ] d: dict = { 'id': ss.id, 'type': 'synset', @@ -281,7 +341,9 @@ def make_synset(ss: wn.Synset, request: Request, basic: bool = False) -> dict: 'pos': ss.pos, 'ili': ss._ili, 'definition': ss.definition(), - 'examples': ss.examples(), + 'examples': examples, + 'example_details': example_details or [], + 'learner_data': ss.learner_data(), }, 'links': { 'self': _url_for_obj(request, 'synset', ss, lexicon=lex_spec)