Apply and record schema migrations with a runner

.github/workflows/test.yml

@@ -13,6 +13,9 @@ on:
       - "api/**"
       - "mcp/**"
       - "mysql/schema/**"
+      - "mysql/migrations/**"
+      - "mysql/expected_schema.tsv"
+      - "scripts/**"
       - "tests/**"
       - "DESCRIPTION"
       - "install_r_packages.R"
@@ -180,6 +183,15 @@ jobs:
       - name: Seed test fixtures
         run: mysql -h"$DB_HOST" -uroot -psigrepo_root_pw sigrepo_test < tests/testthat/fixtures/seed.sql
 
+      # The schema was loaded from mysql/schema/ directly, not by /init_db, so
+      # nothing has recorded the migrations yet and the API's boot check below
+      # would refuse this database. On a current schema every migration takes
+      # its no-op branch, which this step also proves.
+      - name: Record the schema migrations
+        env:
+          MYSQL_PWD: sigrepo_root_pw
+        run: bash scripts/migrate.sh --host "$DB_HOST" --user root --database sigrepo_test
+
       - name: Run testthat
         run: testthat::test_dir("tests/testthat", stop_on_failure = TRUE)
         shell: Rscript {0}
@@ -229,3 +241,39 @@ jobs:
       - name: Show MCP server log on failure
         if: failure()
         run: cat /tmp/mcp.log || true
+
+  # The migrations, which need no R. On the runner, not in the image: the
+  # history test needs git and the full history.
+  shell-tests:
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_DATABASE: sigrepo_test
+          MYSQL_ROOT_PASSWORD: sigrepo_root_pw
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping -uroot -psigrepo_root_pw"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=10
+    env:
+      MYSQL_PWD: sigrepo_root_pw
+    steps:
+      - name: Checkout SigRepo_Server with its history
+        uses: actions/checkout@v4
+        with:
+          # test_history_upgrade.sh rebuilds every schema this repository has
+          # shipped, so it needs every commit that touched mysql/schema/.
+          fetch-depth: 0
+
+      - name: The migration runner
+        run: bash tests/shell/test_migrate.sh --host 127.0.0.1 --user root
+
+      - name: Every historical schema migrates to the current one
+        run: bash tests/shell/test_history_upgrade.sh --host 127.0.0.1 --user root
+
+      - name: mysql/expected_schema.tsv is current
+        run: bash tests/shell/test_expected_schema.sh --host 127.0.0.1 --user root

api/api.R

@@ -167,6 +167,13 @@ for (lib_file in base::sort(base::list.files(base::file.path(sigrepo_server_path
   base::source(lib_file, local = TRUE)
 }
 
+# Refuse to serve a database that is missing a schema migration this code
+# ships (api/lib/schema_migrations.R). Same idea as
+# assert_omic_signature_version() above: fail at startup, where the cause is
+# named, not per request with "Unknown column". It reads and never applies;
+# scripts/migrate.sh and update_sigrepo.sh are what apply.
+assert_schema_migrated(db_connect_local, sigrepo_server_path)
+
 # Resolved once at boot (default_msigdb_cache_dir lives in api/lib/msigdb_cache.R)
 # so /init_db_genesets, /geneset_resources/ensure, /init_db's combined
 # bootstrap, and every /annotate/* request all share the same cache lookup.

api/lib/database_admin.R

@@ -98,6 +98,11 @@ generate_db_schema <- function(sigrepo_server_path = base::Sys.getenv("SIGREPO_S
   run_schema_file("mysql/schema/signature_feature_set_ambiguity.sql", "signature_feature_set_ambiguity")
   run_schema_file("mysql/schema/genetic_variants_features.sql", "genetic_variants_features")
 
+  # The schema just built is the current one, so it already contains every
+  # change in mysql/migrations/. Record them as applied (api/lib/schema_migrations.R),
+  # or the API's boot check would treat this database as never migrated.
+  record_schema_migrations(conn, sigrepo_server_path)
+
   # Disconnect from database ####
   base::suppressWarnings(DBI::dbDisconnect(conn))
 

api/lib/schema_migrations.R

@@ -0,0 +1,111 @@
+# Which schema migrations a database has had.
+#
+# The files in mysql/migrations/ are applied by scripts/migrate.sh, which
+# records each one in the schema_migrations table. Two things in the API need
+# to agree with that record:
+#
+#   * generate_db_schema() builds the CURRENT schema from mysql/schema/, so a
+#     database it builds already contains every change the migrations make. It
+#     records them all as applied; otherwise a fresh instance would look like
+#     one that has never been migrated.
+#   * At boot the API refuses a database that is missing a migration this code
+#     ships. New code against an old schema does not fail at startup by
+#     itself; it fails per request, with "Unknown column", on whichever route
+#     touches the changed table. That is how the type/platform rename broke
+#     /signatures/compare on a database nobody had migrated.
+
+# The forward migrations this code ships, in the order they apply. Rollback
+# scripts live in mysql/migrations/rollback/ and are not listed.
+shipped_schema_migrations <- function(sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) {
+  files <- base::list.files(
+    base::file.path(sigrepo_server_path, "mysql", "migrations"),
+    pattern = "\\.sql$",
+    recursive = FALSE
+  )
+  # Same rule as scripts/migrate.sh: a *-rollback.sql file is never a forward
+  # migration, wherever it sits.
+  base::sort(files[!base::grepl("-rollback\\.sql$", files)])
+}
+
+# Mark every shipped migration as applied. Only correct straight after the
+# schema has been built from mysql/schema/.
+record_schema_migrations <- function(conn, sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) {
+  base::suppressWarnings(DBI::dbGetQuery(conn = conn, statement = "
+    CREATE TABLE IF NOT EXISTS `schema_migrations` (
+      `name` VARCHAR(255) NOT NULL,
+      `applied_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY (`name`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;"))
+
+  shipped <- shipped_schema_migrations(sigrepo_server_path)
+
+  # The names go into SQL text, so hold them to what migrate.sh accepts.
+  unexpected <- shipped[!base::grepl("^[A-Za-z0-9._-]+$", shipped)]
+  if (base::length(unexpected) > 0) {
+    base::stop("Unexpected migration file name: ", base::paste(unexpected, collapse = ", "))
+  }
+
+  for (name in shipped) {
+    base::suppressWarnings(DBI::dbGetQuery(
+      conn = conn,
+      statement = base::sprintf("INSERT IGNORE INTO `schema_migrations` (`name`) VALUES ('%s');", name)
+    ))
+  }
+
+  base::invisible(shipped)
+}
+
+# The shipped migrations this database has no record of. A database with no
+# signatures table has not been through /init_db yet and is not behind
+# anything: the API has to be able to start on it, because /init_db is an API
+# route.
+pending_schema_migrations <- function(conn, sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) {
+  tables <- DBI::dbGetQuery(
+    conn = conn,
+    statement = "SELECT TABLE_NAME AS table_name FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE();"
+  )$table_name
+
+  if (!("signatures" %in% tables)) {
+    return(base::character(0))
+  }
+
+  recorded <- base::character(0)
+  if ("schema_migrations" %in% tables) {
+    recorded <- DBI::dbGetQuery(conn = conn, statement = "SELECT `name` FROM `schema_migrations`;")$name
+  }
+
+  base::setdiff(shipped_schema_migrations(sigrepo_server_path), recorded)
+}
+
+# Stop unless the database has every shipped migration. `connect` is a
+# function returning a connection, so that a database which cannot be reached
+# at boot is reported and tolerated: the API has always been able to start
+# before MySQL is up, and a schema that cannot be read cannot be judged.
+assert_schema_migrated <- function(connect, sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) {
+  pending <- base::tryCatch(
+    pending_schema_migrations(connect(), sigrepo_server_path),
+    error = function(err) {
+      base::message("Could not check schema migrations, continuing: ", base::conditionMessage(err))
+      NULL
+    }
+  )
+
+  if (base::is.null(pending)) {
+    return(base::invisible(FALSE))
+  }
+
+  if (base::length(pending) > 0) {
+    base::stop(base::sprintf(
+      base::paste(
+        "The database is behind this version of SigRepo: %d schema migration(s) have not been applied (%s).",
+        "On an installed instance, run update_sigrepo.sh.",
+        "Anywhere else, run scripts/migrate.sh; see mysql/migrations/README.md.",
+        "The API will not start until then, because its queries assume the migrated schema."
+      ),
+      base::length(pending),
+      base::paste(pending, collapse = ", ")
+    ))
+  }
+
+  base::invisible(TRUE)
+}

api/lib/signature.R

@@ -436,7 +436,7 @@ delete_signature <- function(auth, signature_hashkey) {
 #
 # The reverse of the Rummagene lookup, and the direction that lets someone
 # arrive with a gene list and discover signatures without knowing any of them by
-# name. Needs the indexes from scripts/migrate_gene_search_indexes.R to be fast;
+# name. Needs the indexes from mysql/migrations/2026-08-19-gene-search-indexes.sql to be fast;
 # it is correct without them, just slow.
 #
 # The join is keyed on BOTH feature_id and assay_type, which is load-bearing: