diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1d16f60..28ee0cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - mongodb-version: ['4.2', '4.4', '5.0', '6.0'] + python-version: ['3.10', '3.11', '3.12', '3.13'] + mongodb-version: ['5.0', '6.0', '7.0', '8.0'] dservercore-version: [ main ] dserver-retrieve-plugin-mongo-version: [ main ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af66fac --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +.idea +app.db +TODO.rst +.coverage + +*.swp +*.pyc +*.egg-info +*.pdf + +test_rsa* +migrations/* +data/* +env* +.cache/* +.pytest_cache/* +.tox/* +build/* +dist/* +venv/* +old-provision/* +jwt-spike/* + +dserver_search_plugin_mongo/version.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a4f8b98..37e0cdd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,23 @@ CHANGELOG This project uses `semantic versioning `_. This change log uses principles from `keep a changelog `_. +[Unreleased] +------------ + +Changed +^^^^^^^ + +- Dropped support for Python 3.7–3.9; minimum is now Python 3.10 +- Updated CI Python matrix to 3.10–3.13; updated MongoDB matrix to 5.0–8.0 + +Fixed +^^^^^ + +- Test fixtures: hardcoded MongoDB URI replaced by ``TEST_MONGO_URI`` environment variable (default: ``mongodb://localhost:27017/``) +- Test fixtures: added ``client.close()`` after ``drop_database()`` in teardown to prevent connection pool exhaustion +- Test fixtures: removed stale ``FLASK_ENV`` key from app config (removed in Flask 3.0) +- Config route test: URI assertion now reads from ``TEST_MONGO_URI`` environment variable + [0.4.2] ------- diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..cc0f31f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,137 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +A search plugin for **dserver** (the dtool lookup server, package `dservercore`). It implements +MongoDB-backed dataset registration and search. The plugin is discovered by dserver at runtime +through the `dservercore.search` entry point declared in `pyproject.toml`: + +``` +[project.entry-points."dservercore.search"] +"MongoSearch" = "dserver_search_plugin_mongo.utils_search:MongoSearch" +``` + +This package is a *companion* to `dserver-retrieve-plugin-mongo`; search and retrieve are separate +plugins that, in the common deployment, point at the *same* MongoDB collection (see test config +where `SEARCH_MONGO_*` and `RETRIEVE_MONGO_*` use the same db name). + +## Architecture + +The entire plugin is one class, `MongoSearch(SearchABC)` in `dserver_search_plugin_mongo/utils_search.py`, +implementing the abstract interface from `dservercore.SearchABC`: + +- `init_app(app)` — pulls `SEARCH_MONGO_URI`/`_DB`/`_COLLECTION` from Flask `app.config`, opens the + `MongoClient`, and creates a wildcard text index (`[("$**", pymongo.TEXT)]`) so free-text search + works across all fields. Re-running is safe; Mongo skips index recreation. +- `register_dataset(dataset_info)` — upserts a dataset record keyed on `(uuid, uri)`. Converts + `frozen_at`/`created_at` to datetimes via `dservercore.date_utils`. Wraps + `pymongo.errors.DocumentTooLarge` into `dservercore.ValidationError`. Also stores a parsed copy of + the README under `readme_parsed` (see below). +- `set_tags(uri, tags)` / `set_annotations(uri, annotations)` / `set_readme(uri, readme)` — + in-place mutators that `$set` the respective field on the matching record, raising + `dservercore.UnknownURIError` when no record matches the `uri`. `set_readme` also re-parses and + updates `readme_parsed`. +- `search(query, pagination_parameters, sort_parameters)` — translates the query dict to a Mongo + query, runs `find` with a fixed projection that *excludes* `_id`, `readme`, `manifest`, + `annotations`. Returns `[]` early when `query["base_uris"]` is empty (a user with access to no + base URIs). Pagination sets `pagination_parameters.item_count` from `count_documents` and applies + skip/limit; sorting maps `sort_parameters.order` items directly to pymongo sort tuples. +- `get_config` / `get_config_secrets_to_obfuscate` — expose `config.Config` and the secret list to + dserver's `/config` route. + +### Query translation (the core logic) + +`_dict_to_mongo_query(query_dict)` is the heart of the plugin. Only keys in `VALID_MONGO_QUERY_KEYS` +(`free_text`, `creator_usernames`, `base_uris`, `uuids`, `tags`, `uploaded_by`) survive sanitisation; +everything else is dropped, and empty list-valued keys are removed. Semantics: + +- `free_text` → `{"$text": {"$search": ...}}` +- `creator_usernames`, `base_uris`, `uuids`, `uploaded_by` → single value matches directly; multiple + values become an `$or`. +- `tags` → single tag matches directly; multiple tags become `$all` (AND semantics). +- Multiple sub-queries are joined with `$and`; zero sub-queries returns `{}` (match all). + +When changing query behavior, update both `_dict_to_mongo_query` and the corresponding standalone +tests in `tests/test_utils_search_standalone.py` (the `test_*` functions at the bottom assert exact +Mongo query dicts). + +### README parsing (`readme_parsed`) + +The README is stored verbatim as a string under `readme`. `_parse_readme(readme)` additionally +YAML-parses it (returning the dict, or `None` on failure / non-dict) and the result is stored under +`readme_parsed`. This enables structured queries over README content by *co-installed* plugins — +e.g. the dependency-graph plugin's `readme_parsed.derived_from.uuid` dependency key. `readme_parsed` +is written by `register_dataset` and refreshed by `set_readme`. Requires `PyYAML`. + +### Configuration + +`config.py` defines `Config` (env-var-backed defaults) and `CONFIG_SECRETS_TO_OBFUSCATE`. Note the +default `SEARCH_MONGO_DB` in `Config` is `"dtool_info"`, while the README example uses `"dserver"`. + +## Commands + +```bash +# Install for development (with test deps) +pip install .[test] + +# The test extras also need the server core + retrieve plugin: +pip install dservercore dserver-retrieve-plugin-mongo + +# Run the full test suite (pytest config + coverage live in pyproject.toml) +pytest + +# Run a single test file / single test +pytest tests/test_utils_search_standalone.py +pytest tests/test_utils_search_standalone.py::test_combinations + +# Lint (matches CI) +flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics +flake8 . # full run +``` + +### A running MongoDB is required for most tests + +Tests connect to `mongodb://localhost:27017` by default, overridable via the `TEST_MONGO_URI` env var +(used in `tests/test_utils_search_standalone.py`, `tests/conftest.py` and `tests/test_config_route.py`, +e.g. for an authenticated MongoDB). They create a randomly-named temp database per test and drop it +(and close the client) on teardown. +The pure `_dict_to_mongo_query` unit tests (`test_empty_dict`, `test_free_text`, etc.) do *not* need +Mongo, but the registration/search tests do. + +The test app config also sets bare `MONGO_URI`/`MONGO_DB`/`MONGO_COLLECTION` keys (in addition to the +`SEARCH_MONGO_*` / `RETRIEVE_MONGO_*` ones) so co-installed plugins like the dependency-graph plugin +can initialise against the same temp database. + +## Testing structure + +- `tests/test_utils_search_standalone.py` — exercises `MongoSearch` directly via a `_MockApp` holding + a `config` dict (no Flask). Builds real dtool datasets with `DataSetCreator` and registers them. + This is the file to extend when changing search/registration logic. +- `tests/test_config_route.py` and `tests/conftest.py` — full Flask-app integration via + `dservercore.create_app`. `tmp_app_with_users` builds an in-memory SQLite app with JWT users and + permissions, wiring both retrieve and search plugins to the same temp Mongo db. Use this for + route-level behavior (e.g. `/config/info`, `/config/versions`). +- `tests/utils.py` — `compare_nested` does partial/marked dict comparison; `make_marker` builds the + comparison mask. + +## Versioning & packaging + +The build backend is **flit** (`flit_scm:buildapi`, configured in `pyproject.toml`); there is no +`setup.cfg`/`setup.py`. pytest and coverage config also live in `pyproject.toml` under +`[tool.pytest.ini_options]`. + +Version is managed by `setuptools_scm` (via `flit_scm`) from git tags (`guess-next-dev`, +`no-local-version`), written to `dserver_search_plugin_mongo/version.py` (generated, not committed). +`__init__.py` resolves the version at runtime first via `importlib.metadata`, falling back to the +generated `version.py`. +Releases are tag-driven: pushing a tag triggers `.github/workflows/publish.yml` (trusted publishing +to PyPI + GitHub release + Zenodo). Update `CHANGELOG.rst` (keep-a-changelog format, semver) when +preparing a release. + +## CI + +`.github/workflows/test.yml` runs a matrix of Python 3.10–3.13 × MongoDB 5.0/6.0/7.0/8.0, installing +`dservercore` and `dserver-retrieve-plugin-mongo` from their `main` branches. Keep the plugin +compatible with that whole range; `pyproject.toml` declares `requires-python = ">=3.10"` to match. diff --git a/dserver_search_plugin_mongo/utils_search.py b/dserver_search_plugin_mongo/utils_search.py index 31ca686..d962a92 100644 --- a/dserver_search_plugin_mongo/utils_search.py +++ b/dserver_search_plugin_mongo/utils_search.py @@ -3,6 +3,8 @@ import os import dtoolcore.utils +import yaml + import pymongo.errors from pymongo import MongoClient @@ -22,12 +24,33 @@ Config, CONFIG_SECRETS_TO_OBFUSCATE) +def _parse_readme(readme): + """Return the README parsed into a dict, or None. + + The README is stored verbatim as a string under 'readme'. The parsed + representation stored under 'readme_parsed' enables structured queries + over README content, e.g. the dependency graph plugin's + 'readme_parsed.derived_from.uuid' dependency key. + """ + if isinstance(readme, dict): + return readme + if isinstance(readme, str): + try: + parsed = yaml.safe_load(readme) + except yaml.YAMLError: + return None + if isinstance(parsed, dict): + return parsed + return None + + VALID_MONGO_QUERY_KEYS = ( "free_text", "creator_usernames", "base_uris", "uuids", "tags", + "uploaded_by", ) MONGO_QUERY_LIST_KEYS = ( @@ -35,6 +58,7 @@ "base_uris", "uuids", "tags", + "uploaded_by", ) @@ -52,6 +76,10 @@ def _register_dataset_descriptive_metadata(collection, dataset_info): # get mangled by the datetime replacements. dataset_info = dataset_info.copy() + # Store a parsed representation of the README alongside the verbatim + # string to enable structured queries over README content. + dataset_info["readme_parsed"] = _parse_readme(dataset_info.get("readme")) + frozen_at = extract_frozen_at_as_datetime(dataset_info) created_at = extract_created_at_as_datetime(dataset_info) @@ -116,6 +144,10 @@ def _deal_with_possible_and_statement(a_list, key): ) if "uuids" in query_dict: sub_queries.append(_deal_with_possible_or_statment(query_dict["uuids"], "uuid")) # NOQA + if "uploaded_by" in query_dict: + sub_queries.append( + _deal_with_possible_or_statment( + query_dict["uploaded_by"], "uploaded_by")) if "tags" in query_dict: sub_queries.append( _deal_with_possible_and_statement(query_dict["tags"], "tags") @@ -205,6 +237,40 @@ def search(self, query: SearchDatasetSchema, datasets = [ds for ds in cx] return datasets + def set_tags(self, uri, tags): + """Set a dataset's tags (replaces existing tags).""" + from dservercore import UnknownURIError + result = self.collection.update_one( + {"uri": uri}, + {"$set": {"tags": tags}} + ) + if result.matched_count == 0: + raise (UnknownURIError()) + return tags + + def set_annotations(self, uri, annotations): + """Set a dataset's annotations (replaces existing annotations).""" + from dservercore import UnknownURIError + result = self.collection.update_one( + {"uri": uri}, + {"$set": {"annotations": annotations}} + ) + if result.matched_count == 0: + raise (UnknownURIError()) + return annotations + + def set_readme(self, uri, readme): + """Set a dataset's readme content.""" + from dservercore import UnknownURIError + result = self.collection.update_one( + {"uri": uri}, + {"$set": {"readme": readme, + "readme_parsed": _parse_readme(readme)}} + ) + if result.matched_count == 0: + raise (UnknownURIError()) + return readme + def get_config(self): """Return initial Config object, available app-instance independent.""" return Config diff --git a/pyproject.toml b/pyproject.toml index 77886e3..19d4f31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,24 @@ [build-system] -requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"] -build-backend = "setuptools.build_meta" +requires = ["flit_scm"] +build-backend = "flit_scm:buildapi" [project] name = "dserver-search-plugin-mongo" description = "Search plugin for dserver using mongodb" readme = "README.rst" -license = {file = "LICENSE"} +license = {text = "MIT"} authors = [ {name = "Tjelvar Olsson", email = "tjelvar.olsson@gmail.com"}, {name = "Johannes L. Hörmann", email = "johannes.laurin@gmail.com"}, ] dynamic = ["version"] +requires-python = ">=3.10" dependencies = [ - "pymongo", - "dtoolcore>=3.18.0", - "dservercore" - ] + "pymongo", + "PyYAML", + "dtoolcore>=3.18.0", + "dservercore" +] [project.optional-dependencies] test = [ @@ -31,13 +33,17 @@ Documentation = "https://dserver-search-plugin-mongo.readthedocs.io" Repository = "https://github.com/jic-dtool/dserver-search-plugin-mongo" Changelog = "https://github.com/jic-dtool/dserver-search-plugin-mongo/blob/main/CHANGELOG.rst" +[project.entry-points."dservercore.search"] +MongoSearch = "dserver_search_plugin_mongo.utils_search:MongoSearch" + +[tool.flit.module] +name = "dserver_search_plugin_mongo" + [tool.setuptools_scm] version_scheme = "guess-next-dev" local_scheme = "no-local-version" write_to = "dserver_search_plugin_mongo/version.py" -[tool.setuptools] -packages = ["dserver_search_plugin_mongo"] - -[project.entry-points."dservercore.search"] -"MongoSearch" = "dserver_search_plugin_mongo.utils_search:MongoSearch" +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--cov=dserver_search_plugin_mongo --cov-report=term-missing" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6b5a90d..0000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -exclude=env*,.tox,.git,*.egg,build - -[tool:pytest] -testpaths = tests -addopts = --cov=dserver_search_plugin_mongo -#addopts = -x --pdb - -[cov:run] -source = dserver_search_plugin_mongo diff --git a/tests/conftest.py b/tests/conftest.py index 9fa27cb..ccc2797 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Reusable fixtures""" +import os import random import string @@ -7,6 +8,7 @@ JWT_PUBLIC_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8LrEp0Q6l1WPsY32uOPqEjaisQScnzO/XvlhQTzj5w+hFObjiNgIaHRceYh3hZZwsRsHIkCxOY0JgUPeFP9IVXso0VptIjCPRF5yrV/+dF1rtl4eyYj/XOBvSDzbQQwqdjhHffw0TXW0f/yjGGJCYM+tw/9dmj9VilAMNTx1H76uPKUo4M3vLBQLo2tj7z1jlh4Jlw5hKBRcWQWbpWP95p71Db6gSpqReDYbx57BW19APMVketUYsXfXTztM/HWz35J9HDya3ID0Dl+pE22Wo8SZo2+ULKu/4OYVcD8DjF15WwXrcuFDypX132j+LUWOVWxCs5hdMybSDwF3ZhVBH ec2-user@ip-172-31-41-191.eu-west-1.compute.internal" # NOQA +MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/") def random_string( @@ -41,14 +43,18 @@ def tmp_app_with_users(request): "CONFIG_SECRETS_TO_OBFUSCATE": [], "OPENAPI_VERSION": '3.0.2', "SECRET_KEY": "secret", - "FLASK_ENV": "development", "SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:", - "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/", + "RETRIEVE_MONGO_URI": MONGO_URI, "RETRIEVE_MONGO_DB": tmp_mongo_db_name, "RETRIEVE_MONGO_COLLECTION": "datasets", - "SEARCH_MONGO_URI": "mongodb://localhost:27017/", + "SEARCH_MONGO_URI": MONGO_URI, "SEARCH_MONGO_DB": tmp_mongo_db_name, "SEARCH_MONGO_COLLECTION": "datasets", + # Required by extensions that may be co-installed in the test + # environment (e.g. the dependency graph plugin). + "MONGO_URI": MONGO_URI, + "MONGO_DB": tmp_mongo_db_name, + "MONGO_COLLECTION": "datasets", "SQLALCHEMY_TRACK_MODIFICATIONS": False, "JWT_ALGORITHM": "RS256", "JWT_PUBLIC_KEY": JWT_PUBLIC_KEY, @@ -84,7 +90,9 @@ def tmp_app_with_users(request): @request.addfinalizer def teardown(): current_app.retrieve.client.drop_database(tmp_mongo_db_name) + current_app.retrieve.client.close() current_app.search.client.drop_database(tmp_mongo_db_name) + current_app.search.client.close() sql_db.session.remove() - return app.test_client() \ No newline at end of file + return app.test_client() diff --git a/tests/test_config_route.py b/tests/test_config_route.py index 50203a7..aa91c4f 100644 --- a/tests/test_config_route.py +++ b/tests/test_config_route.py @@ -1,6 +1,7 @@ """Test the /config blueprint route.""" import json +import os import dserver_search_plugin_mongo from .utils import compare_nested @@ -17,7 +18,7 @@ def test_config_info_route(tmp_app_with_users, snowwhite_token): # NOQA expected_content = { 'search_mongo_collection': 'datasets', - 'search_mongo_uri': 'mongodb://localhost:27017/', + 'search_mongo_uri': os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"), } response = json.loads(r.data.decode("utf-8")) diff --git a/tests/test_utils_search_standalone.py b/tests/test_utils_search_standalone.py index 460a505..170c4bd 100644 --- a/tests/test_utils_search_standalone.py +++ b/tests/test_utils_search_standalone.py @@ -13,6 +13,8 @@ import pytest +import os + from pymongo import MongoClient from dtoolcore import DataSetCreator, DataSet @@ -23,8 +25,7 @@ from dserver_search_plugin_mongo.utils_search import MongoSearch from dserver_search_plugin_mongo.utils_search import _dict_to_mongo_query - -MONGO_URI = "mongodb://localhost:27017" +MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/") def random_string( @@ -43,6 +44,7 @@ def tmp_mongo_db(request): @request.addfinalizer def teardown(): client.drop_database(tmp_mongo_db_name) + client.close() return tmp_mongo_db_name @@ -108,7 +110,12 @@ def test_register_basic(tmp_mongo_db): # NOQA app.config = { "SEARCH_MONGO_URI": MONGO_URI, "SEARCH_MONGO_DB": tmp_mongo_db, - "SEARCH_MONGO_COLLECTION": "datasets" + "SEARCH_MONGO_COLLECTION": "datasets", + # Required by extensions that may be co-installed in the test + # environment (e.g. the dependency graph plugin). + "MONGO_URI": MONGO_URI, + "MONGO_DB": tmp_mongo_db, + "MONGO_COLLECTION": "datasets", } mongo_search.init_app(app) @@ -148,7 +155,12 @@ def test_register_raises_when_metadata_too_large(tmp_mongo_db): # NOQA app.config = { "SEARCH_MONGO_URI": MONGO_URI, "SEARCH_MONGO_DB": tmp_mongo_db, - "SEARCH_MONGO_COLLECTION": "datasets" + "SEARCH_MONGO_COLLECTION": "datasets", + # Required by extensions that may be co-installed in the test + # environment (e.g. the dependency graph plugin). + "MONGO_URI": MONGO_URI, + "MONGO_DB": tmp_mongo_db, + "MONGO_COLLECTION": "datasets", } mongo_search.init_app(app) @@ -163,7 +175,12 @@ def test_search_free_text(tmp_mongo_db): app.config = { "SEARCH_MONGO_URI": MONGO_URI, "SEARCH_MONGO_DB": tmp_mongo_db, - "SEARCH_MONGO_COLLECTION": "datasets" + "SEARCH_MONGO_COLLECTION": "datasets", + # Required by extensions that may be co-installed in the test + # environment (e.g. the dependency graph plugin). + "MONGO_URI": MONGO_URI, + "MONGO_DB": tmp_mongo_db, + "MONGO_COLLECTION": "datasets", } mongo_search.init_app(app)