feat: split Studio search into separate course and library indexes

openedx/core/djangoapps/content/search/docs/decisions/0002-separate-course-and-library-indexes.rst

@@ -0,0 +1,84 @@
+Separate Meilisearch Indexes for Course and Library Content
+############################################################
+
+Status
+******
+
+Accepted
+
+
+Context
+*******
+
+`0001-meilisearch.rst`_ put all Studio content into one Meilisearch index,
+``studio_content``: course blocks (``type = course_block``) plus Libraries V2
+components, containers and collections.
+
+Library authoring updates that index synchronously so the frontend sees the
+change on its next query. Meilisearch's per-document indexing cost grows with
+index size, and course blocks dominate the index on large instances. One
+instance reported 1.3 million documents (11.9 GiB) of which 99.93% were course
+blocks, and saw Studio time out when creating library components
+(`openedx-platform#38993`_).
+
+Course content is indexed asynchronously by Celery tasks, so it does not need to
+share an index with library content.
+
+.. _0001-meilisearch.rst: ./0001-meilisearch.rst
+.. _openedx-platform#38993: https://github.com/openedx/openedx-platform/issues/38993
+
+
+Decision
+********
+
+Studio content is stored in two indexes:
+
+* ``<MEILISEARCH_INDEX_PREFIX>studio_content``: course blocks
+  (``DocType.course_block``). This is the original index name, so upgrading does
+  not require reindexing course content.
+* ``<MEILISEARCH_INDEX_PREFIX>studio_library_content``: library blocks, library
+  containers and collections.
+
+Both indexes use the same settings. Every write is routed by document type or by
+the learning context of its key. Rebuild locks and temporary ``_new`` indexes
+are per index.
+
+Every modulestore block is indexed as a ``course_block`` document, including
+course blocks that link to upstream library content, so those stay in the
+course index.
+
+``GET /api/content_search/v2/studio/`` returns ``course_index_name`` and
+``library_index_name``, and one tenant token whose search rules cover both
+indexes with the same access filter. Each Studio search surface already
+searches either course content or library content, never both, so each one
+picks the matching index. ``index_name`` is still returned (equal to
+``course_index_name``) for one release, for frontends that predate the split.
+
+
+Upgrading
+*********
+
+On an instance that already has a populated ``studio_content`` index:
+
+#. ``./manage.py cms migrate``. The ``post_migrate`` reconciliation creates and
+   configures the empty library index.
+#. ``./manage.py cms reindex_studio --libraries-only``. This enqueues a Celery
+   task that rebuilds the library index, then deletes every document with
+   ``type != "course_block"`` from the course index. Courses are not reindexed.
+
+Deploy a frontend that reads ``library_index_name`` at the same time. A frontend
+that only knows ``index_name`` searches the course index, so it stops finding
+library content once step 2 has run.
+
+A full ``./manage.py cms reindex_studio`` also populates both indexes and runs
+the same cleanup, but it reindexes every course.
+
+
+Consequences
+************
+
+* Library writes no longer pay for the size of the course index.
+* A future search across courses and libraries together would need a
+  multi-index search request.
+* The Meilisearch API key used by Studio must be allowed to manage both index
+  names (and their ``_new`` temporary indexes).

openedx/core/djangoapps/content/search/handlers.py

@@ -49,7 +49,7 @@
 from .api import (
     is_meilisearch_enabled,
     only_if_meilisearch_enabled,
-    reconcile_index,
+    reconcile_indexes,
     upsert_content_object_tags_index_doc,
     upsert_item_collections_index_docs,
     upsert_item_containers_index_docs,
@@ -86,7 +86,7 @@ def handle_post_migrate(sender, **kwargs):
         return
 
     try:
-        reconcile_index(status_cb=log.info, warn_cb=log.warning)
+        reconcile_indexes(status_cb=log.info, warn_cb=log.warning)
     except ConnectionError as exc:
         log.warning(
             "Meilisearch reconciliation skipped during post_migrate: %s. "

openedx/core/djangoapps/content/search/management/commands/reindex_studio.py

@@ -15,7 +15,7 @@
 from django.core.management import BaseCommand, CommandError
 
 from ... import api
-from ...tasks import rebuild_index_incremental
+from ...tasks import rebuild_index_incremental, rebuild_library_index
 
 log = logging.getLogger(__name__)
 
@@ -37,11 +37,24 @@ class Command(BaseCommand):
     ./manage.py cms shell -c 'IncrementalIndexCompleted.objects.all().delete()'
 
     This will delete all the IncrementalIndexCompleted records and will help in restarting the index population.
+
+    When upgrading from the single shared Studio index to separate course and library indexes, run
+    `./manage.py cms reindex_studio --libraries-only` once instead. It rebuilds only the library index and
+    deletes library documents from the course index, without reindexing courses.
     """
 
     help = "Add all course and library content to the Studio search index."
 
     def add_arguments(self, parser):
+        parser.add_argument(
+            "--libraries-only",
+            action="store_true",
+            default=False,
+            help=(
+                "Rebuild only the library index, then delete library documents from the course index. "
+                "Run this once when upgrading from the single shared Studio index; courses are not reindexed."
+            ),
+        )
         # Removed flags — provide clear error messages for operators with old automation.
         parser.add_argument(
             "--experimental",
@@ -97,6 +110,17 @@ def handle(self, *args, **options):
                 "reindex_studio is now a stable command, so the flag is no longer necessary."
             )
 
+        if options["libraries_only"]:
+            result = rebuild_library_index.delay()
+            if settings.CELERY_ALWAYS_EAGER:
+                self.stdout.write("Library indexing complete!")
+            else:
+                self.stdout.write(
+                    f"Studio library index rebuild has been queued (task_id={result.id}). "
+                    "Monitor progress in Celery worker logs."
+                )
+            return
+
         result = rebuild_index_incremental.delay()
 
         if settings.CELERY_ALWAYS_EAGER:

openedx/core/djangoapps/content/search/migrations/0003_clear_library_incremental_index_checkpoints.py

@@ -0,0 +1,26 @@
+from django.db import migrations
+from opaque_keys.edx.locator import LibraryLocatorV2
+
+
+def clear_library_checkpoints(apps, schema_editor):
+    """
+    Checkpoints written before the course/library index split record libraries indexed into the course index.
+    An incremental rebuild would skip those libraries and never add them to the new library index.
+    """
+    IncrementalIndexCompleted = apps.get_model("search", "IncrementalIndexCompleted")
+    library_ids = [
+        checkpoint.id
+        for checkpoint in IncrementalIndexCompleted.objects.all()
+        if isinstance(checkpoint.context_key, LibraryLocatorV2)
+    ]
+    IncrementalIndexCompleted.objects.filter(id__in=library_ids).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("search", "0002_incrementalindexcompleted"),
+    ]
+
+    operations = [
+        migrations.RunPython(clear_library_checkpoints, migrations.RunPython.noop),
+    ]

openedx/core/djangoapps/content/search/tasks.py

@@ -203,3 +203,28 @@ def rebuild_index_incremental() -> None:
         raise
 
     log.info("Incremental Studio search index population complete.")
+
+
+@shared_task(
+    base=LoggedTask,
+    autoretry_for=(MeilisearchError, ConnectionError),
+    max_retries=3,
+    retry_backoff=True,
+)
+def rebuild_library_index() -> None:
+    """
+    Celery task to rebuild the Studio library index and remove library documents from the course index.
+
+    Run once when upgrading from the single shared index. Courses are not reindexed.
+    """
+    log.info("Starting Studio library index rebuild...")
+
+    try:
+        api.rebuild_index(status_cb=log.info, include_courses=False)
+    except RuntimeError as exc:
+        if "already in progress" in str(exc).lower():
+            log.warning("Studio library index rebuild skipped: a rebuild is already in progress.")
+            return
+        raise
+
+    log.info("Studio library index rebuild complete.")