diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1d16f60..28ee0cd 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,8 +14,8 @@ jobs:
strategy:
matrix:
- python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
- mongodb-version: ['4.2', '4.4', '5.0', '6.0']
+ python-version: ['3.10', '3.11', '3.12', '3.13']
+ mongodb-version: ['5.0', '6.0', '7.0', '8.0']
dservercore-version: [ main ]
dserver-retrieve-plugin-mongo-version: [ main ]
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..af66fac
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,24 @@
+.idea
+app.db
+TODO.rst
+.coverage
+
+*.swp
+*.pyc
+*.egg-info
+*.pdf
+
+test_rsa*
+migrations/*
+data/*
+env*
+.cache/*
+.pytest_cache/*
+.tox/*
+build/*
+dist/*
+venv/*
+old-provision/*
+jwt-spike/*
+
+dserver_search_plugin_mongo/version.py
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index a4f8b98..37e0cdd 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -4,6 +4,23 @@ CHANGELOG
This project uses `semantic versioning `_.
This change log uses principles from `keep a changelog `_.
+[Unreleased]
+------------
+
+Changed
+^^^^^^^
+
+- Dropped support for Python 3.7–3.9; minimum is now Python 3.10
+- Updated CI Python matrix to 3.10–3.13; updated MongoDB matrix to 5.0–8.0
+
+Fixed
+^^^^^
+
+- Test fixtures: hardcoded MongoDB URI replaced by ``TEST_MONGO_URI`` environment variable (default: ``mongodb://localhost:27017/``)
+- Test fixtures: added ``client.close()`` after ``drop_database()`` in teardown to prevent connection pool exhaustion
+- Test fixtures: removed stale ``FLASK_ENV`` key from app config (removed in Flask 3.0)
+- Config route test: URI assertion now reads from ``TEST_MONGO_URI`` environment variable
+
[0.4.2]
-------
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..cc0f31f
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,137 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+A search plugin for **dserver** (the dtool lookup server, package `dservercore`). It implements
+MongoDB-backed dataset registration and search. The plugin is discovered by dserver at runtime
+through the `dservercore.search` entry point declared in `pyproject.toml`:
+
+```
+[project.entry-points."dservercore.search"]
+"MongoSearch" = "dserver_search_plugin_mongo.utils_search:MongoSearch"
+```
+
+This package is a *companion* to `dserver-retrieve-plugin-mongo`; search and retrieve are separate
+plugins that, in the common deployment, point at the *same* MongoDB collection (see test config
+where `SEARCH_MONGO_*` and `RETRIEVE_MONGO_*` use the same db name).
+
+## Architecture
+
+The entire plugin is one class, `MongoSearch(SearchABC)` in `dserver_search_plugin_mongo/utils_search.py`,
+implementing the abstract interface from `dservercore.SearchABC`:
+
+- `init_app(app)` — pulls `SEARCH_MONGO_URI`/`_DB`/`_COLLECTION` from Flask `app.config`, opens the
+ `MongoClient`, and creates a wildcard text index (`[("$**", pymongo.TEXT)]`) so free-text search
+ works across all fields. Re-running is safe; Mongo skips index recreation.
+- `register_dataset(dataset_info)` — upserts a dataset record keyed on `(uuid, uri)`. Converts
+ `frozen_at`/`created_at` to datetimes via `dservercore.date_utils`. Wraps
+ `pymongo.errors.DocumentTooLarge` into `dservercore.ValidationError`. Also stores a parsed copy of
+ the README under `readme_parsed` (see below).
+- `set_tags(uri, tags)` / `set_annotations(uri, annotations)` / `set_readme(uri, readme)` —
+ in-place mutators that `$set` the respective field on the matching record, raising
+ `dservercore.UnknownURIError` when no record matches the `uri`. `set_readme` also re-parses and
+ updates `readme_parsed`.
+- `search(query, pagination_parameters, sort_parameters)` — translates the query dict to a Mongo
+ query, runs `find` with a fixed projection that *excludes* `_id`, `readme`, `manifest`,
+ `annotations`. Returns `[]` early when `query["base_uris"]` is empty (a user with access to no
+ base URIs). Pagination sets `pagination_parameters.item_count` from `count_documents` and applies
+ skip/limit; sorting maps `sort_parameters.order` items directly to pymongo sort tuples.
+- `get_config` / `get_config_secrets_to_obfuscate` — expose `config.Config` and the secret list to
+ dserver's `/config` route.
+
+### Query translation (the core logic)
+
+`_dict_to_mongo_query(query_dict)` is the heart of the plugin. Only keys in `VALID_MONGO_QUERY_KEYS`
+(`free_text`, `creator_usernames`, `base_uris`, `uuids`, `tags`, `uploaded_by`) survive sanitisation;
+everything else is dropped, and empty list-valued keys are removed. Semantics:
+
+- `free_text` → `{"$text": {"$search": ...}}`
+- `creator_usernames`, `base_uris`, `uuids`, `uploaded_by` → single value matches directly; multiple
+ values become an `$or`.
+- `tags` → single tag matches directly; multiple tags become `$all` (AND semantics).
+- Multiple sub-queries are joined with `$and`; zero sub-queries returns `{}` (match all).
+
+When changing query behavior, update both `_dict_to_mongo_query` and the corresponding standalone
+tests in `tests/test_utils_search_standalone.py` (the `test_*` functions at the bottom assert exact
+Mongo query dicts).
+
+### README parsing (`readme_parsed`)
+
+The README is stored verbatim as a string under `readme`. `_parse_readme(readme)` additionally
+YAML-parses it (returning the dict, or `None` on failure / non-dict) and the result is stored under
+`readme_parsed`. This enables structured queries over README content by *co-installed* plugins —
+e.g. the dependency-graph plugin's `readme_parsed.derived_from.uuid` dependency key. `readme_parsed`
+is written by `register_dataset` and refreshed by `set_readme`. Requires `PyYAML`.
+
+### Configuration
+
+`config.py` defines `Config` (env-var-backed defaults) and `CONFIG_SECRETS_TO_OBFUSCATE`. Note the
+default `SEARCH_MONGO_DB` in `Config` is `"dtool_info"`, while the README example uses `"dserver"`.
+
+## Commands
+
+```bash
+# Install for development (with test deps)
+pip install .[test]
+
+# The test extras also need the server core + retrieve plugin:
+pip install dservercore dserver-retrieve-plugin-mongo
+
+# Run the full test suite (pytest config + coverage live in pyproject.toml)
+pytest
+
+# Run a single test file / single test
+pytest tests/test_utils_search_standalone.py
+pytest tests/test_utils_search_standalone.py::test_combinations
+
+# Lint (matches CI)
+flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+flake8 . # full run
+```
+
+### A running MongoDB is required for most tests
+
+Tests connect to `mongodb://localhost:27017` by default, overridable via the `TEST_MONGO_URI` env var
+(used in `tests/test_utils_search_standalone.py`, `tests/conftest.py` and `tests/test_config_route.py`,
+e.g. for an authenticated MongoDB). They create a randomly-named temp database per test and drop it
+(and close the client) on teardown.
+The pure `_dict_to_mongo_query` unit tests (`test_empty_dict`, `test_free_text`, etc.) do *not* need
+Mongo, but the registration/search tests do.
+
+The test app config also sets bare `MONGO_URI`/`MONGO_DB`/`MONGO_COLLECTION` keys (in addition to the
+`SEARCH_MONGO_*` / `RETRIEVE_MONGO_*` ones) so co-installed plugins like the dependency-graph plugin
+can initialise against the same temp database.
+
+## Testing structure
+
+- `tests/test_utils_search_standalone.py` — exercises `MongoSearch` directly via a `_MockApp` holding
+ a `config` dict (no Flask). Builds real dtool datasets with `DataSetCreator` and registers them.
+ This is the file to extend when changing search/registration logic.
+- `tests/test_config_route.py` and `tests/conftest.py` — full Flask-app integration via
+ `dservercore.create_app`. `tmp_app_with_users` builds an in-memory SQLite app with JWT users and
+ permissions, wiring both retrieve and search plugins to the same temp Mongo db. Use this for
+ route-level behavior (e.g. `/config/info`, `/config/versions`).
+- `tests/utils.py` — `compare_nested` does partial/marked dict comparison; `make_marker` builds the
+ comparison mask.
+
+## Versioning & packaging
+
+The build backend is **flit** (`flit_scm:buildapi`, configured in `pyproject.toml`); there is no
+`setup.cfg`/`setup.py`. pytest and coverage config also live in `pyproject.toml` under
+`[tool.pytest.ini_options]`.
+
+Version is managed by `setuptools_scm` (via `flit_scm`) from git tags (`guess-next-dev`,
+`no-local-version`), written to `dserver_search_plugin_mongo/version.py` (generated, not committed).
+`__init__.py` resolves the version at runtime first via `importlib.metadata`, falling back to the
+generated `version.py`.
+Releases are tag-driven: pushing a tag triggers `.github/workflows/publish.yml` (trusted publishing
+to PyPI + GitHub release + Zenodo). Update `CHANGELOG.rst` (keep-a-changelog format, semver) when
+preparing a release.
+
+## CI
+
+`.github/workflows/test.yml` runs a matrix of Python 3.10–3.13 × MongoDB 5.0/6.0/7.0/8.0, installing
+`dservercore` and `dserver-retrieve-plugin-mongo` from their `main` branches. Keep the plugin
+compatible with that whole range; `pyproject.toml` declares `requires-python = ">=3.10"` to match.
diff --git a/dserver_search_plugin_mongo/utils_search.py b/dserver_search_plugin_mongo/utils_search.py
index 31ca686..d962a92 100644
--- a/dserver_search_plugin_mongo/utils_search.py
+++ b/dserver_search_plugin_mongo/utils_search.py
@@ -3,6 +3,8 @@
import os
import dtoolcore.utils
+import yaml
+
import pymongo.errors
from pymongo import MongoClient
@@ -22,12 +24,33 @@
Config, CONFIG_SECRETS_TO_OBFUSCATE)
+def _parse_readme(readme):
+ """Return the README parsed into a dict, or None.
+
+ The README is stored verbatim as a string under 'readme'. The parsed
+ representation stored under 'readme_parsed' enables structured queries
+ over README content, e.g. the dependency graph plugin's
+ 'readme_parsed.derived_from.uuid' dependency key.
+ """
+ if isinstance(readme, dict):
+ return readme
+ if isinstance(readme, str):
+ try:
+ parsed = yaml.safe_load(readme)
+ except yaml.YAMLError:
+ return None
+ if isinstance(parsed, dict):
+ return parsed
+ return None
+
+
VALID_MONGO_QUERY_KEYS = (
"free_text",
"creator_usernames",
"base_uris",
"uuids",
"tags",
+ "uploaded_by",
)
MONGO_QUERY_LIST_KEYS = (
@@ -35,6 +58,7 @@
"base_uris",
"uuids",
"tags",
+ "uploaded_by",
)
@@ -52,6 +76,10 @@ def _register_dataset_descriptive_metadata(collection, dataset_info):
# get mangled by the datetime replacements.
dataset_info = dataset_info.copy()
+ # Store a parsed representation of the README alongside the verbatim
+ # string to enable structured queries over README content.
+ dataset_info["readme_parsed"] = _parse_readme(dataset_info.get("readme"))
+
frozen_at = extract_frozen_at_as_datetime(dataset_info)
created_at = extract_created_at_as_datetime(dataset_info)
@@ -116,6 +144,10 @@ def _deal_with_possible_and_statement(a_list, key):
)
if "uuids" in query_dict:
sub_queries.append(_deal_with_possible_or_statment(query_dict["uuids"], "uuid")) # NOQA
+ if "uploaded_by" in query_dict:
+ sub_queries.append(
+ _deal_with_possible_or_statment(
+ query_dict["uploaded_by"], "uploaded_by"))
if "tags" in query_dict:
sub_queries.append(
_deal_with_possible_and_statement(query_dict["tags"], "tags")
@@ -205,6 +237,40 @@ def search(self, query: SearchDatasetSchema,
datasets = [ds for ds in cx]
return datasets
+ def set_tags(self, uri, tags):
+ """Set a dataset's tags (replaces existing tags)."""
+ from dservercore import UnknownURIError
+ result = self.collection.update_one(
+ {"uri": uri},
+ {"$set": {"tags": tags}}
+ )
+ if result.matched_count == 0:
+ raise (UnknownURIError())
+ return tags
+
+ def set_annotations(self, uri, annotations):
+ """Set a dataset's annotations (replaces existing annotations)."""
+ from dservercore import UnknownURIError
+ result = self.collection.update_one(
+ {"uri": uri},
+ {"$set": {"annotations": annotations}}
+ )
+ if result.matched_count == 0:
+ raise (UnknownURIError())
+ return annotations
+
+ def set_readme(self, uri, readme):
+ """Set a dataset's readme content."""
+ from dservercore import UnknownURIError
+ result = self.collection.update_one(
+ {"uri": uri},
+ {"$set": {"readme": readme,
+ "readme_parsed": _parse_readme(readme)}}
+ )
+ if result.matched_count == 0:
+ raise (UnknownURIError())
+ return readme
+
def get_config(self):
"""Return initial Config object, available app-instance independent."""
return Config
diff --git a/pyproject.toml b/pyproject.toml
index 77886e3..19d4f31 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,22 +1,24 @@
[build-system]
-requires = ["setuptools>=42", "setuptools_scm[toml]>=6.3"]
-build-backend = "setuptools.build_meta"
+requires = ["flit_scm"]
+build-backend = "flit_scm:buildapi"
[project]
name = "dserver-search-plugin-mongo"
description = "Search plugin for dserver using mongodb"
readme = "README.rst"
-license = {file = "LICENSE"}
+license = {text = "MIT"}
authors = [
{name = "Tjelvar Olsson", email = "tjelvar.olsson@gmail.com"},
{name = "Johannes L. Hörmann", email = "johannes.laurin@gmail.com"},
]
dynamic = ["version"]
+requires-python = ">=3.10"
dependencies = [
- "pymongo",
- "dtoolcore>=3.18.0",
- "dservercore"
- ]
+ "pymongo",
+ "PyYAML",
+ "dtoolcore>=3.18.0",
+ "dservercore"
+]
[project.optional-dependencies]
test = [
@@ -31,13 +33,17 @@ Documentation = "https://dserver-search-plugin-mongo.readthedocs.io"
Repository = "https://github.com/jic-dtool/dserver-search-plugin-mongo"
Changelog = "https://github.com/jic-dtool/dserver-search-plugin-mongo/blob/main/CHANGELOG.rst"
+[project.entry-points."dservercore.search"]
+MongoSearch = "dserver_search_plugin_mongo.utils_search:MongoSearch"
+
+[tool.flit.module]
+name = "dserver_search_plugin_mongo"
+
[tool.setuptools_scm]
version_scheme = "guess-next-dev"
local_scheme = "no-local-version"
write_to = "dserver_search_plugin_mongo/version.py"
-[tool.setuptools]
-packages = ["dserver_search_plugin_mongo"]
-
-[project.entry-points."dservercore.search"]
-"MongoSearch" = "dserver_search_plugin_mongo.utils_search:MongoSearch"
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=dserver_search_plugin_mongo --cov-report=term-missing"
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 6b5a90d..0000000
--- a/setup.cfg
+++ /dev/null
@@ -1,10 +0,0 @@
-[flake8]
-exclude=env*,.tox,.git,*.egg,build
-
-[tool:pytest]
-testpaths = tests
-addopts = --cov=dserver_search_plugin_mongo
-#addopts = -x --pdb
-
-[cov:run]
-source = dserver_search_plugin_mongo
diff --git a/tests/conftest.py b/tests/conftest.py
index 9fa27cb..ccc2797 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,5 +1,6 @@
"""Reusable fixtures"""
+import os
import random
import string
@@ -7,6 +8,7 @@
JWT_PUBLIC_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8LrEp0Q6l1WPsY32uOPqEjaisQScnzO/XvlhQTzj5w+hFObjiNgIaHRceYh3hZZwsRsHIkCxOY0JgUPeFP9IVXso0VptIjCPRF5yrV/+dF1rtl4eyYj/XOBvSDzbQQwqdjhHffw0TXW0f/yjGGJCYM+tw/9dmj9VilAMNTx1H76uPKUo4M3vLBQLo2tj7z1jlh4Jlw5hKBRcWQWbpWP95p71Db6gSpqReDYbx57BW19APMVketUYsXfXTztM/HWz35J9HDya3ID0Dl+pE22Wo8SZo2+ULKu/4OYVcD8DjF15WwXrcuFDypX132j+LUWOVWxCs5hdMybSDwF3ZhVBH ec2-user@ip-172-31-41-191.eu-west-1.compute.internal" # NOQA
+MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/")
def random_string(
@@ -41,14 +43,18 @@ def tmp_app_with_users(request):
"CONFIG_SECRETS_TO_OBFUSCATE": [],
"OPENAPI_VERSION": '3.0.2',
"SECRET_KEY": "secret",
- "FLASK_ENV": "development",
"SQLALCHEMY_DATABASE_URI": "sqlite:///:memory:",
- "RETRIEVE_MONGO_URI": "mongodb://localhost:27017/",
+ "RETRIEVE_MONGO_URI": MONGO_URI,
"RETRIEVE_MONGO_DB": tmp_mongo_db_name,
"RETRIEVE_MONGO_COLLECTION": "datasets",
- "SEARCH_MONGO_URI": "mongodb://localhost:27017/",
+ "SEARCH_MONGO_URI": MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db_name,
"SEARCH_MONGO_COLLECTION": "datasets",
+ # Required by extensions that may be co-installed in the test
+ # environment (e.g. the dependency graph plugin).
+ "MONGO_URI": MONGO_URI,
+ "MONGO_DB": tmp_mongo_db_name,
+ "MONGO_COLLECTION": "datasets",
"SQLALCHEMY_TRACK_MODIFICATIONS": False,
"JWT_ALGORITHM": "RS256",
"JWT_PUBLIC_KEY": JWT_PUBLIC_KEY,
@@ -84,7 +90,9 @@ def tmp_app_with_users(request):
@request.addfinalizer
def teardown():
current_app.retrieve.client.drop_database(tmp_mongo_db_name)
+ current_app.retrieve.client.close()
current_app.search.client.drop_database(tmp_mongo_db_name)
+ current_app.search.client.close()
sql_db.session.remove()
- return app.test_client()
\ No newline at end of file
+ return app.test_client()
diff --git a/tests/test_config_route.py b/tests/test_config_route.py
index 50203a7..aa91c4f 100644
--- a/tests/test_config_route.py
+++ b/tests/test_config_route.py
@@ -1,6 +1,7 @@
"""Test the /config blueprint route."""
import json
+import os
import dserver_search_plugin_mongo
from .utils import compare_nested
@@ -17,7 +18,7 @@ def test_config_info_route(tmp_app_with_users, snowwhite_token): # NOQA
expected_content = {
'search_mongo_collection': 'datasets',
- 'search_mongo_uri': 'mongodb://localhost:27017/',
+ 'search_mongo_uri': os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/"),
}
response = json.loads(r.data.decode("utf-8"))
diff --git a/tests/test_utils_search_standalone.py b/tests/test_utils_search_standalone.py
index 460a505..170c4bd 100644
--- a/tests/test_utils_search_standalone.py
+++ b/tests/test_utils_search_standalone.py
@@ -13,6 +13,8 @@
import pytest
+import os
+
from pymongo import MongoClient
from dtoolcore import DataSetCreator, DataSet
@@ -23,8 +25,7 @@
from dserver_search_plugin_mongo.utils_search import MongoSearch
from dserver_search_plugin_mongo.utils_search import _dict_to_mongo_query
-
-MONGO_URI = "mongodb://localhost:27017"
+MONGO_URI = os.environ.get("TEST_MONGO_URI", "mongodb://localhost:27017/")
def random_string(
@@ -43,6 +44,7 @@ def tmp_mongo_db(request):
@request.addfinalizer
def teardown():
client.drop_database(tmp_mongo_db_name)
+ client.close()
return tmp_mongo_db_name
@@ -108,7 +110,12 @@ def test_register_basic(tmp_mongo_db): # NOQA
app.config = {
"SEARCH_MONGO_URI": MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db,
- "SEARCH_MONGO_COLLECTION": "datasets"
+ "SEARCH_MONGO_COLLECTION": "datasets",
+ # Required by extensions that may be co-installed in the test
+ # environment (e.g. the dependency graph plugin).
+ "MONGO_URI": MONGO_URI,
+ "MONGO_DB": tmp_mongo_db,
+ "MONGO_COLLECTION": "datasets",
}
mongo_search.init_app(app)
@@ -148,7 +155,12 @@ def test_register_raises_when_metadata_too_large(tmp_mongo_db): # NOQA
app.config = {
"SEARCH_MONGO_URI": MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db,
- "SEARCH_MONGO_COLLECTION": "datasets"
+ "SEARCH_MONGO_COLLECTION": "datasets",
+ # Required by extensions that may be co-installed in the test
+ # environment (e.g. the dependency graph plugin).
+ "MONGO_URI": MONGO_URI,
+ "MONGO_DB": tmp_mongo_db,
+ "MONGO_COLLECTION": "datasets",
}
mongo_search.init_app(app)
@@ -163,7 +175,12 @@ def test_search_free_text(tmp_mongo_db):
app.config = {
"SEARCH_MONGO_URI": MONGO_URI,
"SEARCH_MONGO_DB": tmp_mongo_db,
- "SEARCH_MONGO_COLLECTION": "datasets"
+ "SEARCH_MONGO_COLLECTION": "datasets",
+ # Required by extensions that may be co-installed in the test
+ # environment (e.g. the dependency graph plugin).
+ "MONGO_URI": MONGO_URI,
+ "MONGO_DB": tmp_mongo_db,
+ "MONGO_COLLECTION": "datasets",
}
mongo_search.init_app(app)