From e38c71929f9d110e6712c87b598e901c424e9b49 Mon Sep 17 00:00:00 2001 From: Cvicnaire Date: Wed, 30 Sep 2026 13:57:58 -0400 Subject: [PATCH 1/6] Add scripts/migrate.sh: apply the migrations a database has not had and record them Co-Authored-By: Claude Fable 5.1 --- scripts/lib/mysql_conn.sh | 67 +++++++++++++++++++ scripts/migrate.sh | 110 +++++++++++++++++++++++++++++++ tests/shell/test_migrate.sh | 125 ++++++++++++++++++++++++++++++++++++ tests/shell/testlib.sh | 98 ++++++++++++++++++++++++++++ 4 files changed, 400 insertions(+) create mode 100644 scripts/lib/mysql_conn.sh create mode 100755 scripts/migrate.sh create mode 100644 tests/shell/test_migrate.sh create mode 100644 tests/shell/testlib.sh diff --git a/scripts/lib/mysql_conn.sh b/scripts/lib/mysql_conn.sh new file mode 100644 index 0000000..290bb0d --- /dev/null +++ b/scripts/lib/mysql_conn.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# How migrate.sh and schema_fingerprint.sh reach MySQL. Sourced, not run. +# +# Two ways in, because the two callers live in different places: +# +# --container NAME run the mysql client that is already inside the MySQL +# container. This is what an installed instance, the local +# stack, montilab and sigrepo.org use: the host needs no +# MySQL client, and the root password never leaves the +# container, which already holds it as MYSQL_ROOT_PASSWORD. +# --host HOST run a mysql client on this machine. This is what CI uses, +# where MySQL is a service container and the password comes +# from MYSQL_PWD in the environment. +# +# The password is never put on a command line in either mode. + +CONN_CONTAINER="" +CONN_HOST="" +CONN_PORT=3306 +CONN_USER=root +CONN_DB="" +CONN_SHIFT=0 +# "docker" or "sudo docker"; deliberately unquoted where it is used. +DOCKER=${DOCKER:-docker} + +# conn_parse_arg "$@": consume one connection option if $1 is one. Returns 0 +# and sets CONN_SHIFT to the number of words used, or returns 1. +conn_parse_arg() { + case "${1:-}" in + --container) CONN_CONTAINER=$2; CONN_SHIFT=2 ;; + --host) CONN_HOST=$2; CONN_SHIFT=2 ;; + --port) CONN_PORT=$2; CONN_SHIFT=2 ;; + --user) CONN_USER=$2; CONN_SHIFT=2 ;; + --database) CONN_DB=$2; CONN_SHIFT=2 ;; + *) return 1 ;; + esac + return 0 +} + +conn_require() { + if [ -n "$CONN_CONTAINER" ] && [ -n "$CONN_HOST" ]; then + echo "give --container or --host, not both" >&2; return 2 + fi + if [ -z "$CONN_CONTAINER" ] && [ -z "$CONN_HOST" ]; then + echo "give --container NAME or --host HOST" >&2; return 2 + fi + if [ -n "$CONN_HOST" ] && [ -z "$CONN_DB" ]; then + echo "--host needs --database" >&2; return 2 + fi +} + +# conn_mysql [mysql options]: run the client against the chosen database, SQL +# on stdin. In container mode the database defaults to the container's own +# MYSQL_DATABASE. +conn_mysql() { + if [ -n "$CONN_CONTAINER" ]; then + $DOCKER exec -i -e SIGREPO_DB="$CONN_DB" "$CONN_CONTAINER" sh -c \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysql -uroot "$@" "${SIGREPO_DB:-$MYSQL_DATABASE}"' sh "$@" + else + mysql -h"$CONN_HOST" -P"$CONN_PORT" -u"$CONN_USER" "$@" "$CONN_DB" + fi +} + +# conn_query "SQL": one statement, tab separated rows, no header. +conn_query() { + conn_mysql -N -B -e "$1" &2; exit 2 ;; + esac +done + +conn_require || exit 2 + +if [ ! -d "$DIR" ]; then + echo "no migrations directory at $DIR" >&2 + exit 2 +fi + +if ! conn_query "SELECT 1" >/dev/null; then + echo "cannot reach the database; nothing was changed" >&2 + exit 1 +fi + +if [ "$STATUS_ONLY" -eq 0 ]; then + conn_query "CREATE TABLE IF NOT EXISTS schema_migrations ( + name VARCHAR(255) NOT NULL, + applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (name) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci" || exit 1 +fi + +# An absent table (only possible with --status) means nothing is recorded. +RECORDED=$(conn_query "SELECT name FROM schema_migrations" 2>/dev/null || true) + +applied=0 +skipped=0 +pending=0 + +while IFS= read -r file; do + [ -n "$file" ] || continue + name=$(basename "$file") + + # The name goes into an SQL string below, so refuse anything unusual. + case "$name" in + *[!A-Za-z0-9._-]*) echo "refusing migration with an unexpected name: $name" >&2; exit 1 ;; + esac + + if printf '%s\n' "$RECORDED" | grep -qxF "$name"; then + skipped=$((skipped + 1)) + continue + fi + + if [ "$STATUS_ONLY" -eq 1 ]; then + echo "pending $name" + pending=$((pending + 1)) + continue + fi + + if ! conn_mysql < "$file"; then + echo "FAILED $name" >&2 + echo "Nothing was recorded for it. Fix the cause and run this again:" >&2 + echo "migrations already applied are skipped, and this one is safe to repeat." >&2 + exit 1 + fi + conn_query "INSERT INTO schema_migrations (name) VALUES ('$name')" || exit 1 + echo "applied $name" + applied=$((applied + 1)) +done </dev/null 2>&1' EXIT + +script_args "$DB" + +recorded() { db_sql "$DB" "SELECT name FROM schema_migrations ORDER BY name" | tr '\n' ' '; } + +echo "--- a first run applies everything, in filename order ---" + +fresh_db "$DB" +echo "CREATE TABLE t (id INT);" > "$DIR/2026-01-01-create.sql" +echo "ALTER TABLE t ADD COLUMN added INT;" > "$DIR/2026-01-02-alter.sql" +mkdir "$DIR/rollback" +echo "DROP TABLE t;" > "$DIR/rollback/2026-01-01-create-rollback.sql" +# A rollback script left beside the forward migrations, where it sorts last. +echo "DROP TABLE t;" > "$DIR/2026-01-02-alter-rollback.sql" + +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" 2>&1); RC=$? +if [ "$RC" -eq 0 ] && [ "$(recorded)" = "2026-01-01-create.sql 2026-01-02-alter.sql " ]; then + ok "both migrations applied and recorded" +else + bad "first run: rc=$RC recorded='$(recorded)'"; printf '%s\n' "$OUT" +fi +if [ "$(db_sql "$DB" "SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA='$DB' AND TABLE_NAME='t'")" = "2" ]; then + ok "the ALTER ran after the CREATE, and no rollback script was run" +else + bad "table t does not have its two columns" +fi + +echo "--- a second run changes nothing ---" + +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" 2>&1); RC=$? +if [ "$RC" -eq 0 ] && printf '%s' "$OUT" | grep -q "0 applied, 2 already recorded"; then + ok "nothing is applied twice" +else + bad "second run: rc=$RC"; printf '%s\n' "$OUT" +fi + +echo "--- a failing migration stops the run and is not recorded ---" + +echo "ALTER TABLE no_such_table ADD COLUMN x INT;" > "$DIR/2026-01-03-broken.sql" +echo "ALTER TABLE t ADD COLUMN later INT;" > "$DIR/2026-01-04-after.sql" + +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" 2>&1); RC=$? +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "FAILED 2026-01-03-broken.sql"; then + ok "the failure is reported by name with exit 1" +else + bad "failing run: rc=$RC"; printf '%s\n' "$OUT" +fi +if [ "$(recorded)" = "2026-01-01-create.sql 2026-01-02-alter.sql " ]; then + ok "neither the failed migration nor the one after it is recorded" +else + bad "recorded after failure: '$(recorded)'" +fi + +echo "--- --status lists what is pending and changes nothing ---" + +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" --status 2>&1); RC=$? +if [ "$RC" -eq 0 ] && printf '%s' "$OUT" | grep -q "pending 2026-01-03-broken.sql" \ + && printf '%s' "$OUT" | grep -q "2 pending, 2 already recorded"; then + ok "--status names the pending migrations" +else + bad "--status: rc=$RC"; printf '%s\n' "$OUT" +fi + +echo "--- after the cause is fixed, a re-run finishes the job ---" + +echo "ALTER TABLE t ADD COLUMN x INT;" > "$DIR/2026-01-03-broken.sql" +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" 2>&1); RC=$? +if [ "$RC" -eq 0 ] && printf '%s' "$OUT" | grep -q "2 applied, 2 already recorded"; then + ok "the re-run applies only what was left" +else + bad "re-run: rc=$RC"; printf '%s\n' "$OUT" +fi + +echo "--- --status on a database with no schema_migrations table ---" + +fresh_db "$DB" +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" --status 2>&1); RC=$? +TABLES=$(db_sql "$DB" "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='$DB'") +if [ "$RC" -eq 0 ] && printf '%s' "$OUT" | grep -q "4 pending, 0 already recorded" && [ "$TABLES" = "0" ]; then + ok "everything is pending, and --status created no table" +else + bad "--status on an untracked database: rc=$RC tables=$TABLES"; printf '%s\n' "$OUT" +fi + +echo "--- a migration file with an unexpected name is refused ---" + +echo "SELECT 1;" > "$DIR/2026-01-05-it's.sql" +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" 2>&1); RC=$? +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "unexpected name"; then + ok "a quote in a file name never reaches SQL" +else + bad "odd file name: rc=$RC"; printf '%s\n' "$OUT" +fi +rm -f "$DIR/2026-01-05-it's.sql" + +echo "--- an unreachable database is a clean failure ---" + +script_args sigrepo_test_does_not_exist +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" --dir "$DIR" 2>&1); RC=$? +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "cannot reach the database"; then + ok "a missing database is reported, not half-run" +else + bad "missing database: rc=$RC"; printf '%s\n' "$OUT" +fi + +finish diff --git a/tests/shell/testlib.sh b/tests/shell/testlib.sh new file mode 100644 index 0000000..4bfc4dd --- /dev/null +++ b/tests/shell/testlib.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Shared by the shell tests in this directory. Sourced, not run. +# +# The database tests create and drop scratch databases whose names start with +# sigrepo_test_, on whatever server the connection options name. Point them at +# a disposable MySQL: CI's service container, or a throwaway one started with +# +# docker run -d --name sigrepo-test-mysql \ +# -e MYSQL_ROOT_PASSWORD=scratch_pw_123 -e MYSQL_DATABASE=sigrepo_test mysql:8.0 +# +# and then `bash tests/shell/test_migrate.sh --container sigrepo-test-mysql`. + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +FAILURES=0 + +ok() { echo "PASS $1"; } +bad() { echo "FAIL $1"; FAILURES=$((FAILURES + 1)); } + +finish() { + echo + if [ "$FAILURES" -eq 0 ]; then echo "all passed"; exit 0; fi + echo "$FAILURES failed"; exit 1 +} + +# shellcheck source=scripts/lib/mysql_conn.sh +. "$ROOT/scripts/lib/mysql_conn.sh" + +# The options the test was started with, passed on to the scripts under test. +CONN_ARGS=() + +parse_conn_args() { + while [ $# -gt 0 ]; do + if conn_parse_arg "$@"; then + CONN_ARGS+=("${@:1:$CONN_SHIFT}") + shift "$CONN_SHIFT" + else + echo "unknown argument: $1" >&2; exit 2 + fi + done + if [ -z "$CONN_CONTAINER" ] && [ -z "$CONN_HOST" ]; then + echo "give --container NAME or --host HOST (see tests/shell/testlib.sh)" >&2; exit 2 + fi + # The server's default database is only used to issue CREATE/DROP DATABASE. + ADMIN_DB=${CONN_DB:-} + if [ -n "$CONN_HOST" ] && [ -z "$ADMIN_DB" ]; then ADMIN_DB=mysql; fi +} + +# admin_sql "SQL": run against the server, not a scratch database. +admin_sql() { + CONN_DB=$ADMIN_DB conn_query "$1" +} + +# fresh_db NAME: drop and recreate an empty scratch database. +fresh_db() { + case "$1" in + sigrepo_test_*) ;; + *) echo "refusing to drop '$1': scratch databases are named sigrepo_test_*" >&2; exit 2 ;; + esac + admin_sql "DROP DATABASE IF EXISTS \`$1\`" && admin_sql "CREATE DATABASE \`$1\`" +} + +# db_sql NAME "SQL": one statement against a scratch database. +db_sql() { + CONN_DB=$1 conn_query "$2" +} + +# db_load NAME: SQL on stdin into a scratch database. +db_load() { + CONN_DB=$1 conn_mysql +} + +# script_args NAME: the connection options for a script under test, aimed at +# a scratch database. +script_args() { + local i skip=0 + SCRIPT_ARGS=() + for i in "${CONN_ARGS[@]}"; do + if [ "$skip" -eq 1 ]; then skip=0; continue; fi + if [ "$i" = "--database" ]; then skip=1; continue; fi + SCRIPT_ARGS+=("$i") + done + SCRIPT_ARGS+=(--database "$1") +} + +# schema_at COMMIT: the CREATE TABLE statements mysql/schema/ held at that +# commit, as one script. WORKTREE means the files on disk now. +# snps_features.sql is skipped: generate_db_schema() never created that table. +schema_at() { + local f + echo "SET FOREIGN_KEY_CHECKS=0;" + if [ "$1" = "WORKTREE" ]; then + for f in "$ROOT"/mysql/schema/*.sql; do cat "$f"; echo ";"; done + else + git -C "$ROOT" ls-tree --name-only "$1" mysql/schema/ | grep -v 'snps_features\.sql$' | + while IFS= read -r f; do git -C "$ROOT" show "$1:$f"; echo ";"; done + fi + echo "SET FOREIGN_KEY_CHECKS=1;" +} From 9d3a8cdf3d489d978583fcc998a8fb50e2383d39 Mon Sep 17 00:00:00 2001 From: Cvicnaire Date: Wed, 30 Sep 2026 13:58:21 -0400 Subject: [PATCH 2/6] Add schema_fingerprint.sh and the expected schema a migrated database is compared with Co-Authored-By: Claude Fable 5.1 --- mysql/expected_schema.tsv | 233 ++++++++++++++++++++++++++++ scripts/schema_fingerprint.sh | 96 ++++++++++++ tests/shell/test_expected_schema.sh | 62 ++++++++ 3 files changed, 391 insertions(+) create mode 100644 mysql/expected_schema.tsv create mode 100755 scripts/schema_fingerprint.sh create mode 100644 tests/shell/test_expected_schema.sh diff --git a/mysql/expected_schema.tsv b/mysql/expected_schema.tsv new file mode 100644 index 0000000..6ed716b --- /dev/null +++ b/mysql/expected_schema.tsv @@ -0,0 +1,233 @@ +column collection collection_hashkey varchar(32) NO NULL +column collection collection_id int unsigned NO NULL auto_increment +column collection collection_name varchar(255) NO NULL +column collection date_created datetime YES CURRENT_TIMESTAMP DEFAULT_GENERATED +column collection description text YES NULL +column collection user_name varchar(255) NO NULL +column collection visibility tinyint(1) YES 0 +column collection_access access_collection_hashkey varchar(32) NO NULL +column collection_access access_type set('owner','editor','viewer') NO NULL +column collection_access collection_id int unsigned NO NULL +column collection_access user_name varchar(255) NO NULL +column geneset_entries description text YES NULL +column geneset_entries geneset_entry_hashkey varchar(32) NO NULL +column geneset_entries geneset_entry_id int unsigned NO NULL auto_increment +column geneset_entries geneset_name varchar(255) NO NULL +column geneset_entries geneset_resource_id int unsigned NO NULL +column geneset_entries n_features int unsigned YES NULL +column geneset_resources checksum varchar(128) YES NULL +column geneset_resources collection varchar(128) NO NULL +column geneset_resources created_at datetime NO CURRENT_TIMESTAMP DEFAULT_GENERATED +column geneset_resources format varchar(32) NO rds +column geneset_resources geneset_resource_hashkey varchar(32) NO NULL +column geneset_resources geneset_resource_id int unsigned NO NULL auto_increment +column geneset_resources is_current tinyint(1) YES 1 +column geneset_resources n_features int unsigned YES NULL +column geneset_resources n_genesets int unsigned YES NULL +column geneset_resources notes text YES NULL +column geneset_resources source varchar(64) NO NULL +column geneset_resources source_version varchar(64) YES NULL +column geneset_resources species varchar(128) NO NULL +column geneset_resources storage_path varchar(512) NO NULL +column geneset_resources subcollection varchar(128) YES NULL +column geneset_resources updated_at datetime NO CURRENT_TIMESTAMP DEFAULT_GENERATED on update CURRENT_TIMESTAMP +column geneset_resources version varchar(64) NO NULL +column genetic_variants_features annotation varchar(50) YES NULL +column genetic_variants_features chromosome varchar(10) YES NULL +column genetic_variants_features feature_hashkey char(32) YES NULL +column genetic_variants_features feature_id int unsigned NO NULL auto_increment +column genetic_variants_features feature_name varchar(255) NO NULL +column genetic_variants_features is_current tinyint(1) NO 1 +column genetic_variants_features organism_id int unsigned NO NULL +column genetic_variants_features position int unsigned YES NULL +column genetic_variants_features version int YES NULL +column keywords keyword varchar(255) NO NULL +column keywords keyword_id int unsigned NO NULL auto_increment +column metabolite_reference hmdb_id varchar(255) YES NULL +column metabolite_reference inchikey varchar(64) YES NULL +column metabolite_reference is_current tinyint(1) NO 1 +column metabolite_reference metabolite_hashkey varchar(32) NO NULL +column metabolite_reference metabolite_id int unsigned NO NULL auto_increment +column metabolite_reference refmet_id varchar(255) YES NULL +column metabolite_reference refmet_name varchar(255) YES NULL +column metabolite_reference smiles text YES NULL +column metabolite_reference version int NO NULL +column metabolite_xref is_primary tinyint(1) NO 0 +column metabolite_xref metabolite_id int unsigned NO NULL +column metabolite_xref source_db varchar(32) NO NULL +column metabolite_xref source_value varchar(255) NO NULL +column metabolite_xref xref_hashkey varchar(32) NO NULL +column metabolite_xref xref_id int unsigned NO NULL auto_increment +column organisms biomart_dataset varchar(255) YES NULL +column organisms biomart_db varchar(255) YES NULL +column organisms biomart_description varchar(255) YES NULL +column organisms biomart_updated_date varchar(255) YES NULL +column organisms biomart_version varchar(255) YES NULL +column organisms organism varchar(255) NO NULL +column organisms organism_id int unsigned NO NULL auto_increment +column organisms prot_organism_code varchar(255) YES NULL +column organisms prot_organism_taxid varchar(255) YES NULL +column organisms prot_updated_date varchar(255) YES NULL +column phenotypes phenotype varchar(255) NO NULL +column phenotypes phenotype_id int unsigned NO NULL auto_increment +column platforms platform varchar(255) NO NULL +column platforms platform_id int unsigned NO NULL auto_increment +column proteomics_features feature_hashkey varchar(32) NO NULL +column proteomics_features feature_id int unsigned NO NULL auto_increment +column proteomics_features feature_name varchar(255) NO NULL +column proteomics_features gene_symbol text YES NULL +column proteomics_features is_current tinyint(1) YES 1 +column proteomics_features organism_id int unsigned NO NULL +column proteomics_features version datetime YES CURRENT_TIMESTAMP DEFAULT_GENERATED +column sample_types brenda_accession varchar(255) YES NULL +column sample_types sample_type varchar(255) NO NULL +column sample_types sample_type_id int unsigned NO NULL auto_increment +column signature_access access_sig_hashkey varchar(32) NO NULL +column signature_access access_type set('owner','editor','viewer') NO NULL +column signature_access signature_id int unsigned NO NULL +column signature_access user_name varchar(255) NO NULL +column signature_collection_access collection_id int unsigned NO NULL +column signature_collection_access signature_collection_hashkey varchar(32) NO NULL +column signature_collection_access signature_id int unsigned NO NULL +column signature_feature_set assay_type set('transcriptomics','proteomics','metabolomics','methylomics','genetic_variants') NO NULL +column signature_feature_set feature_id int unsigned YES NULL +column signature_feature_set group_label varchar(255) YES NULL +column signature_feature_set match_status varchar(32) YES NULL +column signature_feature_set nomenclature_type varchar(64) YES NULL +column signature_feature_set probe_id varchar(255) YES NULL +column signature_feature_set score decimal(12,8) YES NULL +column signature_feature_set sig_feature_hashkey varchar(32) NO NULL +column signature_feature_set signature_id int unsigned NO NULL +column signature_feature_set_ambiguity ambiguity_id int unsigned NO NULL auto_increment +column signature_feature_set_ambiguity candidate_metabolite_id int unsigned NO NULL +column signature_feature_set_ambiguity sig_feature_hashkey varchar(32) NO NULL +column signatures PMID int YES NULL +column signatures adj_p_cutoff decimal(12,8) YES NULL +column signatures assay_type set('transcriptomics','proteomics','metabolomics','methylomics','genetic_variants') NO NULL +column signatures covariates text YES NULL +column signatures cutoff_description text YES NULL +column signatures date_created datetime YES CURRENT_TIMESTAMP DEFAULT_GENERATED +column signatures description text YES NULL +column signatures has_difexp tinyint(1) YES 0 +column signatures keywords text YES NULL +column signatures logfc_cutoff decimal(12,8) YES NULL +column signatures num_down_regulated int YES NULL +column signatures num_of_difexp int YES NULL +column signatures num_up_regulated int YES NULL +column signatures organism_id int unsigned NO NULL +column signatures others text YES NULL +column signatures p_value_cutoff decimal(12,8) YES NULL +column signatures phenotype_id int unsigned NO NULL +column signatures platform_id int unsigned NO NULL +column signatures sample_type_id int unsigned NO NULL +column signatures score_cutoff decimal(12,8) YES NULL +column signatures signature_hashkey varchar(32) NO NULL +column signatures signature_id int unsigned NO NULL auto_increment +column signatures signature_name varchar(255) NO NULL +column signatures type set('uni-directional','bi-directional','categorical') NO NULL +column signatures user_name varchar(255) NO NULL +column signatures visibility tinyint(1) YES 0 +column signatures year int YES NULL +column transcriptomics_features feature_hashkey varchar(32) NO NULL +column transcriptomics_features feature_id int unsigned NO NULL auto_increment +column transcriptomics_features feature_name varchar(255) NO NULL +column transcriptomics_features gene_symbol text YES NULL +column transcriptomics_features is_current tinyint(1) YES 1 +column transcriptomics_features organism_id int unsigned NO NULL +column transcriptomics_features version int NO NULL +column users active tinyint(1) YES 0 +column users api_key varchar(32) NO NULL +column users date_created datetime YES CURRENT_TIMESTAMP DEFAULT_GENERATED +column users user_affiliation text YES NULL +column users user_email varchar(255) NO NULL +column users user_first varchar(255) YES NULL +column users user_hashkey varchar(32) NO NULL +column users user_last varchar(255) YES NULL +column users user_name varchar(255) NO NULL +column users user_password_hashkey varchar(255) NO NULL +column users user_role set('admin','editor','viewer') NO NULL +fk collection user_name users user_name +fk collection_access collection_id collection collection_id +fk collection_access user_name users user_name +fk geneset_entries geneset_resource_id geneset_resources geneset_resource_id +fk genetic_variants_features organism_id organisms organism_id +fk metabolite_xref metabolite_id metabolite_reference metabolite_id +fk proteomics_features organism_id organisms organism_id +fk signature_access signature_id signatures signature_id +fk signature_access user_name users user_name +fk signature_collection_access collection_id collection collection_id +fk signature_collection_access signature_id signatures signature_id +fk signature_feature_set signature_id signatures signature_id +fk signature_feature_set_ambiguity candidate_metabolite_id metabolite_reference metabolite_id +fk signatures organism_id organisms organism_id +fk signatures phenotype_id phenotypes phenotype_id +fk signatures platform_id platforms platform_id +fk signatures sample_type_id sample_types sample_type_id +fk signatures user_name users user_name +fk transcriptomics_features organism_id organisms organism_id +index collection PRIMARY 0 collection_id +index collection collection_name 0 collection_name,user_name +index collection user_name 1 user_name +index collection_access PRIMARY 0 collection_id,user_name +index collection_access collection_id 0 collection_id,user_name +index collection_access user_name 1 user_name +index geneset_entries PRIMARY 0 geneset_entry_id +index geneset_entries geneset_entry_hashkey 0 geneset_entry_hashkey +index geneset_entries geneset_resource_id 0 geneset_resource_id,geneset_name +index geneset_resources PRIMARY 0 geneset_resource_id +index geneset_resources geneset_resource_hashkey 0 geneset_resource_hashkey +index geneset_resources source 0 source,species,collection,subcollection,version +index genetic_variants_features PRIMARY 0 feature_id +index genetic_variants_features feature_organism_ukey 0 feature_name,organism_id +index genetic_variants_features organism_id_idx 1 organism_id +index keywords PRIMARY 0 keyword_id +index keywords keyword 0 keyword +index metabolite_reference PRIMARY 0 metabolite_id +index metabolite_reference idx_hmdb_id 1 hmdb_id +index metabolite_reference idx_inchikey 1 inchikey +index metabolite_reference idx_refmet_id 1 refmet_id +index metabolite_reference idx_refmet_name 1 refmet_name +index metabolite_reference idx_smiles 1 smiles(255) +index metabolite_reference uq_metabolite_hash 0 metabolite_hashkey +index metabolite_xref PRIMARY 0 xref_id +index metabolite_xref idx_metabolite_id 1 metabolite_id +index metabolite_xref idx_source_lookup 1 source_db,source_value +index metabolite_xref uq_source_value_metabolite 0 source_db,source_value,metabolite_id +index metabolite_xref uq_xref_hash 0 xref_hashkey +index organisms PRIMARY 0 organism_id +index organisms organism 0 organism +index phenotypes PRIMARY 0 phenotype_id +index phenotypes phenotype 0 phenotype +index platforms PRIMARY 0 platform_id +index platforms platform 0 platform +index proteomics_features PRIMARY 0 feature_id +index proteomics_features feature_organism 0 feature_name,organism_id +index proteomics_features organism_id 1 organism_id +index proteomics_features pf_gene_symbol 1 gene_symbol(64) +index sample_types PRIMARY 0 sample_type_id +index sample_types sample_type 0 sample_type +index signature_access PRIMARY 0 signature_id,user_name +index signature_access signature_id 0 signature_id,user_name +index signature_access user_name 1 user_name +index signature_collection_access PRIMARY 0 collection_id,signature_id +index signature_collection_access collection_id 0 collection_id,signature_id +index signature_collection_access signature_id 1 signature_id +index signature_feature_set idx_signature_nomenclature 1 assay_type,nomenclature_type +index signature_feature_set sfs_feature_id 1 feature_id +index signature_feature_set sig_feature_assay_probe 0 signature_id,feature_id,assay_type,probe_id +index signature_feature_set_ambiguity PRIMARY 0 ambiguity_id +index signature_feature_set_ambiguity idx_candidate_metabolite_id 1 candidate_metabolite_id +index signature_feature_set_ambiguity idx_sig_feature_hashkey 1 sig_feature_hashkey +index signatures PRIMARY 0 signature_id +index signatures organism_id 1 organism_id +index signatures phenotype_id 1 phenotype_id +index signatures platform_id 1 platform_id +index signatures sample_type_id 1 sample_type_id +index signatures signature_name 0 signature_name,user_name +index signatures user_name 1 user_name +index transcriptomics_features PRIMARY 0 feature_id +index transcriptomics_features feature_name 0 feature_name,organism_id +index transcriptomics_features organism_id 1 organism_id +index transcriptomics_features tf_gene_symbol 1 gene_symbol(64) +index users PRIMARY 0 user_name +index users user_name 0 user_name diff --git a/scripts/schema_fingerprint.sh b/scripts/schema_fingerprint.sh new file mode 100755 index 0000000..b6fe981 --- /dev/null +++ b/scripts/schema_fingerprint.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Print a database's schema as sorted, tab separated lines, or compare it with +# a saved copy. +# +# scripts/schema_fingerprint.sh --container sigrepo-mysql > mysql/expected_schema.tsv +# scripts/schema_fingerprint.sh --container sigrepo-mysql --verify mysql/expected_schema.tsv +# +# Connection options are the same as scripts/migrate.sh. +# +# What is recorded, one line each: +# column table column type nullable default extra +# index table index non_unique columns +# fk table column referenced_table referenced_column +# +# What is deliberately left out: column order, character sets, CHECK +# constraints, and foreign key names. None of them change what a query sees, +# and all of them differ between a database that was migrated and one that was +# built fresh. The schema_migrations table is left out as well. +# +# --verify compares only the tables the saved copy names. A table the saved +# copy does not know about is reported and is not a failure: an instance may +# hold tables of its own. Exit 0 when they match, 1 when they differ. + +set -uo pipefail + +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=scripts/lib/mysql_conn.sh +. "$HERE/lib/mysql_conn.sh" + +VERIFY="" + +while [ $# -gt 0 ]; do + if conn_parse_arg "$@"; then shift "$CONN_SHIFT"; continue; fi + case "$1" in + --verify) VERIFY=$2; shift 2 ;; + -h|--help) sed -n '2,22p' "$0"; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +conn_require || exit 2 + +fingerprint() { + { + conn_query "SELECT 'column', TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, + IFNULL(COLUMN_DEFAULT, 'NULL'), EXTRA + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME <> 'schema_migrations'" && + conn_query "SELECT 'index', TABLE_NAME, INDEX_NAME, NON_UNIQUE, + GROUP_CONCAT(CONCAT(COLUMN_NAME, IFNULL(CONCAT('(', SUB_PART, ')'), '')) + ORDER BY SEQ_IN_INDEX SEPARATOR ',') + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME <> 'schema_migrations' + GROUP BY TABLE_NAME, INDEX_NAME, NON_UNIQUE" && + conn_query "SELECT 'fk', TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME + FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = DATABASE() AND REFERENCED_TABLE_NAME IS NOT NULL" + } | LC_ALL=C sort +} + +if [ -z "$VERIFY" ]; then + fingerprint + exit $? +fi + +if [ ! -s "$VERIFY" ]; then + echo "no saved schema at $VERIFY" >&2 + exit 2 +fi + +LIVE=$(mktemp) +KNOWN=$(mktemp) +trap 'rm -f "$LIVE" "$KNOWN" "$LIVE.same"' EXIT + +if ! fingerprint > "$LIVE" || [ ! -s "$LIVE" ]; then + echo "could not read the schema" >&2 + exit 1 +fi + +# Column 2 is the table name on every kind of line. +cut -f2 "$VERIFY" | LC_ALL=C sort -u > "$KNOWN" + +awk -F'\t' 'NR == FNR { known[$1] = 1; next } ($2 in known)' "$KNOWN" "$LIVE" > "$LIVE.same" +awk -F'\t' 'NR == FNR { known[$1] = 1; next } !($2 in known) { print $2 }' "$KNOWN" "$LIVE" \ + | LC_ALL=C sort -u | while IFS= read -r extra; do + echo "note: table '$extra' is not part of SigRepo's schema; left alone" + done + +if diff "$VERIFY" "$LIVE.same" > /dev/null; then + echo "schema matches" + exit 0 +fi + +echo "schema differs from what this version expects (< expected, > found):" >&2 +diff "$VERIFY" "$LIVE.same" >&2 +exit 1 diff --git a/tests/shell/test_expected_schema.sh b/tests/shell/test_expected_schema.sh new file mode 100644 index 0000000..cb60742 --- /dev/null +++ b/tests/shell/test_expected_schema.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Check that mysql/expected_schema.tsv is the schema mysql/schema/ builds. +# +# bash tests/shell/test_expected_schema.sh --container NAME +# MYSQL_PWD=... bash tests/shell/test_expected_schema.sh --host HOST --user root +# +# Needs a disposable MySQL; see tests/shell/testlib.sh. +# +# Add --write to regenerate the file instead of checking it. +# +# update_sigrepo.sh compares a migrated database with that file and stops the +# update when they differ, so a stale file would stop every update. When this +# fails after a schema change, regenerate the file with the command it prints +# and commit it with the change. +set -uo pipefail + +. "$(dirname "$0")/testlib.sh" + +# --write regenerates the file from mysql/schema/ in place of checking it. +WRITE=0 +ARGS=() +for arg in "$@"; do + if [ "$arg" = "--write" ]; then WRITE=1; else ARGS+=("$arg"); fi +done +parse_conn_args "${ARGS[@]}" + +DB=sigrepo_test_expected +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"; admin_sql "DROP DATABASE IF EXISTS \`$DB\`" >/dev/null 2>&1' EXIT + +fresh_db "$DB" +schema_at WORKTREE > "$WORK/schema.sql" +db_load "$DB" < "$WORK/schema.sql" || { bad "mysql/schema/ does not load"; finish; } +script_args "$DB" + +if [ "$WRITE" -eq 1 ]; then + bash "$ROOT/scripts/schema_fingerprint.sh" "${SCRIPT_ARGS[@]}" > "$ROOT/mysql/expected_schema.tsv" + echo "wrote mysql/expected_schema.tsv ($(wc -l < "$ROOT/mysql/expected_schema.tsv" | tr -d ' ') lines)" + exit 0 +fi + +if bash "$ROOT/scripts/schema_fingerprint.sh" "${SCRIPT_ARGS[@]}" --verify "$ROOT/mysql/expected_schema.tsv" > "$WORK/out.txt" 2>&1; then + ok "mysql/expected_schema.tsv matches mysql/schema/" +else + bad "mysql/expected_schema.tsv is stale" + cat "$WORK/out.txt" + echo + echo "Regenerate it against a database built from mysql/schema/, for example:" + echo " bash tests/shell/test_expected_schema.sh ${CONN_ARGS[*]} --write" +fi + +# A table added to mysql/schema/ but missing from the saved copy is only a +# "note" to --verify, which is right for an instance and wrong here. +bash "$ROOT/scripts/schema_fingerprint.sh" "${SCRIPT_ARGS[@]}" > "$WORK/live.tsv" +if diff -q "$ROOT/mysql/expected_schema.tsv" "$WORK/live.tsv" >/dev/null; then + ok "and it names every table mysql/schema/ creates" +else + bad "mysql/schema/ creates something mysql/expected_schema.tsv does not list" + diff "$ROOT/mysql/expected_schema.tsv" "$WORK/live.tsv" | head -20 +fi + +finish From 16e26f341d116d732c292541b1aa7b95acaa5fbd Mon Sep 17 00:00:00 2001 From: Cvicnaire Date: Wed, 30 Sep 2026 14:00:38 -0400 Subject: [PATCH 3/6] Add the migrations for ten schema changes that shipped without one, and the test that every historical schema converges Co-Authored-By: Claude Fable 5.1 --- api/lib/signature.R | 2 +- ...2025-12-11-organisms-reference-columns.sql | 140 ++++++++++++ ...025-12-11-proteomics-unique-index-name.sql | 38 ++++ ...01-29-widen-score-and-cutoff-precision.sql | 76 +++++++ ...02-25-create-genetic-variants-features.sql | 39 ++++ .../2026-06-23-create-geneset-tables.sql | 55 +++++ .../2026-07-06-create-metabolite-tables.sql | 76 +++++++ .../2026-08-19-gene-search-indexes.sql | 56 +++++ ...026-08-25-metabolite-reference-columns.sql | 92 ++++++++ ...2026-08-26-assay-type-genetic-variants.sql | 116 ++++++++++ .../2026-09-13-signature-feature-set-keys.sql | 158 +++++++++++++ ...26-09-25-rename-type-platform-rollback.sql | 0 scripts/migrate_gene_search_indexes.R | 81 ------- scripts/migrate_geneset_schema.R | 99 -------- tests/shell/test_history_upgrade.sh | 212 ++++++++++++++++++ 15 files changed, 1059 insertions(+), 181 deletions(-) create mode 100644 mysql/migrations/2025-12-11-organisms-reference-columns.sql create mode 100644 mysql/migrations/2025-12-11-proteomics-unique-index-name.sql create mode 100644 mysql/migrations/2026-01-29-widen-score-and-cutoff-precision.sql create mode 100644 mysql/migrations/2026-02-25-create-genetic-variants-features.sql create mode 100644 mysql/migrations/2026-06-23-create-geneset-tables.sql create mode 100644 mysql/migrations/2026-07-06-create-metabolite-tables.sql create mode 100644 mysql/migrations/2026-08-19-gene-search-indexes.sql create mode 100644 mysql/migrations/2026-08-25-metabolite-reference-columns.sql create mode 100644 mysql/migrations/2026-08-26-assay-type-genetic-variants.sql create mode 100644 mysql/migrations/2026-09-13-signature-feature-set-keys.sql rename mysql/migrations/{ => rollback}/2026-09-25-rename-type-platform-rollback.sql (100%) delete mode 100755 scripts/migrate_gene_search_indexes.R delete mode 100644 scripts/migrate_geneset_schema.R create mode 100644 tests/shell/test_history_upgrade.sh diff --git a/api/lib/signature.R b/api/lib/signature.R index adba4d9..82c9337 100644 --- a/api/lib/signature.R +++ b/api/lib/signature.R @@ -436,7 +436,7 @@ delete_signature <- function(auth, signature_hashkey) { # # The reverse of the Rummagene lookup, and the direction that lets someone # arrive with a gene list and discover signatures without knowing any of them by -# name. Needs the indexes from scripts/migrate_gene_search_indexes.R to be fast; +# name. Needs the indexes from mysql/migrations/2026-08-19-gene-search-indexes.sql to be fast; # it is correct without them, just slow. # # The join is keyed on BOTH feature_id and assay_type, which is load-bearing: diff --git a/mysql/migrations/2025-12-11-organisms-reference-columns.sql b/mysql/migrations/2025-12-11-organisms-reference-columns.sql new file mode 100644 index 0000000..e7cbd44 --- /dev/null +++ b/mysql/migrations/2025-12-11-organisms-reference-columns.sql @@ -0,0 +1,140 @@ +-- Add the BioMart and proteomics reference columns to `organisms`. +-- +-- Schema change of 2025-12-11 (commits 5a22eacb, abd4611d, 12ae811c), which +-- shipped without a migration. A database built before it has only +-- organism_id and organism. Four of the columns were briefly INT or DATETIME +-- before settling on VARCHAR(255); a database caught in between is converted. +-- +-- Idempotent: each column is added only if absent and converted only if it is +-- not already a VARCHAR. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_db'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_db' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `biomart_db` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `biomart_db` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_dataset'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_dataset' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `biomart_dataset` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `biomart_dataset` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_description'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_description' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `biomart_description` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `biomart_description` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_version'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_version' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `biomart_version` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `biomart_version` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_updated_date'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'biomart_updated_date' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `biomart_updated_date` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `biomart_updated_date` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'prot_organism_code'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'prot_organism_code' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `prot_organism_code` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `prot_organism_code` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'prot_organism_taxid'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'prot_organism_taxid' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `prot_organism_taxid` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `prot_organism_taxid` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'prot_updated_date'); +SET @is_varchar := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_NAME = 'prot_updated_date' + AND COLUMN_TYPE = 'varchar(255)'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `organisms` ADD COLUMN `prot_updated_date` VARCHAR(255) DEFAULT NULL', + IF(@is_varchar = 0, + 'ALTER TABLE `organisms` MODIFY COLUMN `prot_updated_date` VARCHAR(255) DEFAULT NULL', + 'DO 0')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @done := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'organisms' AND COLUMN_TYPE = 'varchar(255)' + AND COLUMN_NAME IN ('biomart_db', 'biomart_dataset', 'biomart_description', 'biomart_version', + 'biomart_updated_date', 'prot_organism_code', 'prot_organism_taxid', 'prot_updated_date')); +SET @postcondition_sql := IF(@done = 8, 'DO 0', + 'SELECT `organisms is missing a BioMart or proteomics column` FROM `migration_postcondition_failed`'); +PREPARE postcondition_stmt FROM @postcondition_sql; +EXECUTE postcondition_stmt; +DEALLOCATE PREPARE postcondition_stmt; diff --git a/mysql/migrations/2025-12-11-proteomics-unique-index-name.sql b/mysql/migrations/2025-12-11-proteomics-unique-index-name.sql new file mode 100644 index 0000000..667f277 --- /dev/null +++ b/mysql/migrations/2025-12-11-proteomics-unique-index-name.sql @@ -0,0 +1,38 @@ +-- Name the unique index on proteomics_features (feature_name, organism_id) +-- `feature_organism`. +-- +-- The schema first declared it as an unnamed UNIQUE, which MySQL names after +-- its first column: `feature_name`. Since 2025-12-11 (commit 756844a6, made +-- valid SQL in 05bebc6d) it is CONSTRAINT `feature_organism`. Same columns, +-- same guarantee; only the name differs, and a migrated database should carry +-- the name a fresh one does. +-- +-- Idempotent: renames only when the old name exists and the new one does not. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +SET @old := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'proteomics_features' AND INDEX_NAME = 'feature_name'); +SET @new := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'proteomics_features' AND INDEX_NAME = 'feature_organism'); +SET @sql := IF(@old > 0 AND @new = 0, + 'ALTER TABLE `proteomics_features` RENAME INDEX `feature_name` TO `feature_organism`', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @new := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'proteomics_features' AND INDEX_NAME = 'feature_organism'); +SET @postcondition_sql := IF(@new > 0, 'DO 0', + 'SELECT `proteomics_features has no feature_organism unique index` FROM `migration_postcondition_failed`'); +PREPARE postcondition_stmt FROM @postcondition_sql; +EXECUTE postcondition_stmt; +DEALLOCATE PREPARE postcondition_stmt; diff --git a/mysql/migrations/2026-01-29-widen-score-and-cutoff-precision.sql b/mysql/migrations/2026-01-29-widen-score-and-cutoff-precision.sql new file mode 100644 index 0000000..63d7924 --- /dev/null +++ b/mysql/migrations/2026-01-29-widen-score-and-cutoff-precision.sql @@ -0,0 +1,76 @@ +-- Widen score and cutoff columns from NUMERIC(10,8) to NUMERIC(12,8). +-- +-- Schema change of 2026-01-29 (commits 7bb9077e, 4fb3dc28), which shipped +-- without a migration. NUMERIC(10,8) holds two digits before the point, so a +-- score of 100 or more was rejected. Widening keeps every stored value. +-- +-- Idempotent: a column already NUMERIC(12,8) is left alone. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +SET @wide := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signatures' AND COLUMN_NAME = 'score_cutoff' + AND COLUMN_TYPE = 'decimal(12,8)'); +SET @sql := IF(@wide = 0, + 'ALTER TABLE `signatures` MODIFY COLUMN `score_cutoff` NUMERIC(12, 8) DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @wide := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signatures' AND COLUMN_NAME = 'logfc_cutoff' + AND COLUMN_TYPE = 'decimal(12,8)'); +SET @sql := IF(@wide = 0, + 'ALTER TABLE `signatures` MODIFY COLUMN `logfc_cutoff` NUMERIC(12, 8) DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @wide := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signatures' AND COLUMN_NAME = 'p_value_cutoff' + AND COLUMN_TYPE = 'decimal(12,8)'); +SET @sql := IF(@wide = 0, + 'ALTER TABLE `signatures` MODIFY COLUMN `p_value_cutoff` NUMERIC(12, 8) DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @wide := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signatures' AND COLUMN_NAME = 'adj_p_cutoff' + AND COLUMN_TYPE = 'decimal(12,8)'); +SET @sql := IF(@wide = 0, + 'ALTER TABLE `signatures` MODIFY COLUMN `adj_p_cutoff` NUMERIC(12, 8) DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @wide := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' AND COLUMN_NAME = 'score' + AND COLUMN_TYPE = 'decimal(12,8)'); +SET @sql := IF(@wide = 0, + 'ALTER TABLE `signature_feature_set` MODIFY COLUMN `score` NUMERIC(12, 8) DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @done := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND COLUMN_TYPE = 'decimal(12,8)' + AND ((TABLE_NAME = 'signatures' AND COLUMN_NAME IN ('score_cutoff', 'logfc_cutoff', 'p_value_cutoff', 'adj_p_cutoff')) + OR (TABLE_NAME = 'signature_feature_set' AND COLUMN_NAME = 'score'))); +SET @postcondition_sql := IF(@done = 5, 'DO 0', + 'SELECT `a score or cutoff column is not NUMERIC(12,8)` FROM `migration_postcondition_failed`'); +PREPARE postcondition_stmt FROM @postcondition_sql; +EXECUTE postcondition_stmt; +DEALLOCATE PREPARE postcondition_stmt; diff --git a/mysql/migrations/2026-02-25-create-genetic-variants-features.sql b/mysql/migrations/2026-02-25-create-genetic-variants-features.sql new file mode 100644 index 0000000..ad40c93 --- /dev/null +++ b/mysql/migrations/2026-02-25-create-genetic-variants-features.sql @@ -0,0 +1,39 @@ +-- Create genetic_variants_features on a database that predates it. +-- +-- The table joined the schema on 2026-02-25 (commit ff6e1bab) without a +-- migration. The definition is the one in mysql/schema/ on 2026-09-30. +-- +-- Idempotent: CREATE TABLE IF NOT EXISTS. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + + +CREATE TABLE IF NOT EXISTS genetic_variants_features ( + feature_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + feature_name VARCHAR(255) NOT NULL, + chromosome VARCHAR(10) DEFAULT NULL, + position INT UNSIGNED DEFAULT NULL, + annotation VARCHAR(50) DEFAULT NULL, + organism_id INT UNSIGNED NOT NULL, + is_current TINYINT(1) NOT NULL DEFAULT 1, + version INT DEFAULT NULL, + feature_hashkey CHAR(32) DEFAULT NULL, + PRIMARY KEY (feature_id), + UNIQUE KEY feature_organism_ukey (feature_name, organism_id), + KEY organism_id_idx (organism_id), + CONSTRAINT feature_organism_fkey + FOREIGN KEY (organism_id) + REFERENCES organisms (organism_id), + CONSTRAINT genetic_variants_features_chk_1 + CHECK (is_current IN (0,1)) +) +ENGINE=InnoDB +DEFAULT CHARSET=utf8mb4 +COLLATE=utf8mb4_unicode_ci; diff --git a/mysql/migrations/2026-06-23-create-geneset-tables.sql b/mysql/migrations/2026-06-23-create-geneset-tables.sql new file mode 100644 index 0000000..02e37e8 --- /dev/null +++ b/mysql/migrations/2026-06-23-create-geneset-tables.sql @@ -0,0 +1,55 @@ +-- Create geneset_resources and geneset_entries on a database that predates +-- them. +-- +-- The tables joined the schema on 2026-06-23 (commit 3c88d943). This replaces +-- scripts/migrate_geneset_schema.R, which did the same from R and recorded +-- nothing. geneset_resources comes first: geneset_entries has a foreign key +-- to it. The definitions are the ones in mysql/schema/ on 2026-09-30. +-- +-- Idempotent: CREATE TABLE IF NOT EXISTS. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +CREATE TABLE IF NOT EXISTS `geneset_resources` ( + `geneset_resource_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `source` VARCHAR(64) NOT NULL, + `species` VARCHAR(128) NOT NULL, + `collection` VARCHAR(128) NOT NULL, + `subcollection` VARCHAR(128) DEFAULT NULL, + `version` VARCHAR(64) NOT NULL, + `source_version` VARCHAR(64) DEFAULT NULL, + `format` VARCHAR(32) NOT NULL DEFAULT 'rds', + `storage_path` VARCHAR(512) NOT NULL, + `checksum` VARCHAR(128) DEFAULT NULL, + `n_genesets` INT UNSIGNED DEFAULT NULL, + `n_features` INT UNSIGNED DEFAULT NULL, + `is_current` BOOL DEFAULT 1, + `notes` TEXT DEFAULT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `geneset_resource_hashkey` VARCHAR(32) NOT NULL, + PRIMARY KEY (`geneset_resource_id`), + UNIQUE (`source`, `species`, `collection`, `subcollection`, `version`), + UNIQUE (`geneset_resource_hashkey`), + CHECK (`is_current` IN (0,1)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + +CREATE TABLE IF NOT EXISTS `geneset_entries` ( + `geneset_entry_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, + `geneset_resource_id` INT UNSIGNED NOT NULL, + `geneset_name` VARCHAR(255) NOT NULL, + `description` TEXT DEFAULT NULL, + `n_features` INT UNSIGNED DEFAULT NULL, + `geneset_entry_hashkey` VARCHAR(32) NOT NULL, + PRIMARY KEY (`geneset_entry_id`), + UNIQUE (`geneset_resource_id`, `geneset_name`), + UNIQUE (`geneset_entry_hashkey`), + FOREIGN KEY (`geneset_resource_id`) REFERENCES `geneset_resources` (`geneset_resource_id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; diff --git a/mysql/migrations/2026-07-06-create-metabolite-tables.sql b/mysql/migrations/2026-07-06-create-metabolite-tables.sql new file mode 100644 index 0000000..c6a6832 --- /dev/null +++ b/mysql/migrations/2026-07-06-create-metabolite-tables.sql @@ -0,0 +1,76 @@ +-- Create metabolite_reference, metabolite_xref and +-- signature_feature_set_ambiguity on a database that predates them. +-- +-- The files joined the repository on 2026-07-06 (commit ea7c3522). +-- metabolite_reference comes first: the other two have foreign keys to it. +-- The definitions are the ones in mysql/schema/ on 2026-09-30, so a database +-- that gets metabolite_reference here already has the columns the +-- 2026-08-25 migration adds. +-- +-- Idempotent: CREATE TABLE IF NOT EXISTS. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + + +CREATE TABLE IF NOT EXISTS metabolite_reference ( + metabolite_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + refmet_id VARCHAR(255) NULL, + refmet_name VARCHAR(255) NULL, + hmdb_id VARCHAR(255) NULL, + smiles TEXT NULL, + inchikey VARCHAR(64) NULL, + is_current BOOL NOT NULL DEFAULT 1, + version INT NOT NULL, + metabolite_hashkey VARCHAR(32) NOT NULL, + PRIMARY KEY (metabolite_id), + UNIQUE KEY uq_metabolite_hash (metabolite_hashkey), + KEY idx_refmet_id (refmet_id), + KEY idx_refmet_name (refmet_name), + KEY idx_hmdb_id (hmdb_id), + KEY idx_smiles (smiles(255)), + KEY idx_inchikey (inchikey), + CHECK (is_current IN (0,1)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + + + + +CREATE TABLE IF NOT EXISTS metabolite_xref ( + xref_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + metabolite_id INT UNSIGNED NOT NULL, + source_db VARCHAR(32) NOT NULL, + source_value VARCHAR(255) NOT NULL, + is_primary BOOL NOT NULL DEFAULT 0, + xref_hashkey VARCHAR(32) NOT NULL, + PRIMARY KEY (xref_id), + UNIQUE KEY uq_xref_hash (xref_hashkey), + UNIQUE KEY uq_source_value_metabolite (source_db, source_value, metabolite_id), + KEY idx_source_lookup (source_db, source_value), + KEY idx_metabolite_id (metabolite_id), + CONSTRAINT fk_metabolite_xref_metabolite + FOREIGN KEY (metabolite_id) REFERENCES metabolite_reference(metabolite_id) + ON DELETE CASCADE, + CHECK (is_primary IN (0,1)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + + + + +CREATE TABLE IF NOT EXISTS signature_feature_set_ambiguity ( + ambiguity_id INT UNSIGNED NOT NULL AUTO_INCREMENT, + sig_feature_hashkey VARCHAR(32) NOT NULL, + candidate_metabolite_id INT UNSIGNED NOT NULL, + PRIMARY KEY (ambiguity_id), + KEY idx_sig_feature_hashkey (sig_feature_hashkey), + KEY idx_candidate_metabolite_id (candidate_metabolite_id), + CONSTRAINT fk_ambiguity_metabolite + FOREIGN KEY (candidate_metabolite_id) REFERENCES metabolite_reference(metabolite_id) + ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; diff --git a/mysql/migrations/2026-08-19-gene-search-indexes.sql b/mysql/migrations/2026-08-19-gene-search-indexes.sql new file mode 100644 index 0000000..fae0d73 --- /dev/null +++ b/mysql/migrations/2026-08-19-gene-search-indexes.sql @@ -0,0 +1,56 @@ +-- Add the indexes that make "which signatures contain this gene?" answerable. +-- +-- Schema change of 2026-08-19 (commit fab22a06). This replaces +-- scripts/migrate_gene_search_indexes.R, which did the same from R and +-- recorded nothing. Additive only. gene_symbol is TEXT, which MySQL cannot +-- index without a prefix length; 64 characters covers every real symbol. +-- +-- The build takes a metadata lock on each table for its duration, about two +-- seconds at production scale. +-- +-- Idempotent: each index is created only if no index of that name exists. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'proteomics_features' AND INDEX_NAME = 'pf_gene_symbol'); +SET @sql := IF(@has = 0, + 'CREATE INDEX `pf_gene_symbol` ON `proteomics_features` (`gene_symbol`(64))', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'transcriptomics_features' AND INDEX_NAME = 'tf_gene_symbol'); +SET @sql := IF(@has = 0, + 'CREATE INDEX `tf_gene_symbol` ON `transcriptomics_features` (`gene_symbol`(64))', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' AND INDEX_NAME = 'sfs_feature_id'); +SET @sql := IF(@has = 0, + 'CREATE INDEX `sfs_feature_id` ON `signature_feature_set` (`feature_id`)', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @done := (SELECT COUNT(DISTINCT INDEX_NAME) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND INDEX_NAME IN ('pf_gene_symbol', 'tf_gene_symbol', 'sfs_feature_id')); +SET @postcondition_sql := IF(@done = 3, 'DO 0', + 'SELECT `a gene search index is missing` FROM `migration_postcondition_failed`'); +PREPARE postcondition_stmt FROM @postcondition_sql; +EXECUTE postcondition_stmt; +DEALLOCATE PREPARE postcondition_stmt; diff --git a/mysql/migrations/2026-08-25-metabolite-reference-columns.sql b/mysql/migrations/2026-08-25-metabolite-reference-columns.sql new file mode 100644 index 0000000..7a90204 --- /dev/null +++ b/mysql/migrations/2026-08-25-metabolite-reference-columns.sql @@ -0,0 +1,92 @@ +-- Bring metabolite_reference to the shape the client and the MCP server read. +-- +-- Schema change of 2026-08-25 (commit 0707ebb3), which shipped without a +-- migration. A database built from the file as first committed has a +-- `chemical_name` column nothing reads, and lacks `refmet_id` and `hmdb_id`, +-- which createOmicSignature() and the metabolomics feature search select. On +-- such a database every metabolomics signature fails with +-- "Unknown column 'refmet_id' in 'field list'". +-- +-- `chemical_name` is dropped. It is a reference column no code has ever read +-- or written, and keeping it would leave a migrated database different from a +-- fresh one. +-- +-- Idempotent: each column and index is added only if absent, and +-- `chemical_name` is dropped only if present. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND COLUMN_NAME = 'refmet_id'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `metabolite_reference` ADD COLUMN `refmet_id` VARCHAR(255) NULL AFTER `metabolite_id`', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND COLUMN_NAME = 'hmdb_id'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `metabolite_reference` ADD COLUMN `hmdb_id` VARCHAR(255) NULL AFTER `refmet_name`', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND INDEX_NAME = 'idx_refmet_id'); +SET @sql := IF(@has = 0, + 'CREATE INDEX `idx_refmet_id` ON `metabolite_reference` (`refmet_id`)', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND INDEX_NAME = 'idx_hmdb_id'); +SET @sql := IF(@has = 0, + 'CREATE INDEX `idx_hmdb_id` ON `metabolite_reference` (`hmdb_id`)', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- smiles is TEXT, so the index needs an explicit prefix length. +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND INDEX_NAME = 'idx_smiles'); +SET @sql := IF(@has = 0, + 'CREATE INDEX `idx_smiles` ON `metabolite_reference` (`smiles`(255))', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- Dropping the column drops idx_chemical_name with it: MySQL removes an index +-- when its only column goes. +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND COLUMN_NAME = 'chemical_name'); +SET @sql := IF(@has > 0, + 'ALTER TABLE `metabolite_reference` DROP COLUMN `chemical_name`', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @added := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' + AND COLUMN_NAME IN ('refmet_id', 'hmdb_id')); +SET @left := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND COLUMN_NAME = 'chemical_name'); +SET @postcondition_sql := IF(@added = 2 AND @left = 0, 'DO 0', + 'SELECT `metabolite_reference lacks refmet_id or hmdb_id` FROM `migration_postcondition_failed`'); +PREPARE postcondition_stmt FROM @postcondition_sql; +EXECUTE postcondition_stmt; +DEALLOCATE PREPARE postcondition_stmt; diff --git a/mysql/migrations/2026-08-26-assay-type-genetic-variants.sql b/mysql/migrations/2026-08-26-assay-type-genetic-variants.sql new file mode 100644 index 0000000..93ee88d --- /dev/null +++ b/mysql/migrations/2026-08-26-assay-type-genetic-variants.sql @@ -0,0 +1,116 @@ +-- Make 'genetic_variants' the fifth assay_type on `signatures` and +-- `signature_feature_set`. +-- +-- Schema change of 2026-08-26 (commit 37d92a64), which shipped without a +-- migration. The member list has had three shapes: +-- +-- until 2025-12-11 ..., 'genetic_variations', 'dna_binding_sites' +-- until 2026-08-26 ..., 'snps' +-- since ..., 'genetic_variants' +-- +-- The first four members never changed. 'snps' and 'genetic_variations' both +-- mean what is now 'genetic_variants', so rows holding either are rewritten. +-- 'dna_binding_sites' has no equivalent: if any row holds it, this migration +-- stops before changing the member list and says so, rather than let MySQL +-- truncate the value to an empty string. +-- +-- mysql/migrations/README.md warns against restating a SET definition by +-- hand, because the repository and a deployed table have disagreed on one +-- before and data was truncated. This migration has to change a SET, so it +-- does it in three steps that never drop a member a row is using: +-- +-- 1. append 'genetic_variants' to whatever list the table has now, built +-- from information_schema, not from this file; +-- 2. rewrite the rows; +-- 3. count rows outside the final list, and narrow only if there are none. +-- +-- Idempotent: a column already at the final definition is left alone. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +SET @final := 'set(''transcriptomics'',''proteomics'',''metabolomics'',''methylomics'',''genetic_variants'')'; + +-- signatures ---------------------------------------------------------------- + +SET @cur := (SELECT COLUMN_TYPE FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signatures' AND COLUMN_NAME = 'assay_type'); + +SET @sql := IF(@cur = @final OR LOCATE('''genetic_variants''', @cur) > 0, + 'DO 0', + CONCAT('ALTER TABLE `signatures` MODIFY COLUMN `assay_type` ', + LEFT(@cur, CHAR_LENGTH(@cur) - 1), ',''genetic_variants'') NOT NULL')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @sql := IF(@cur = @final, + 'DO 0', + 'UPDATE `signatures` SET `assay_type` = ''genetic_variants'' + WHERE FIND_IN_SET(''snps'', `assay_type`) > 0 OR FIND_IN_SET(''genetic_variations'', `assay_type`) > 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stranded := (SELECT COUNT(*) FROM `signatures` + WHERE `assay_type` NOT IN ('transcriptomics', 'proteomics', 'metabolomics', 'methylomics', 'genetic_variants')); +SET @sql := IF(@stranded > 0, + 'SELECT `signatures holds an assay_type with no current equivalent` FROM `migration_blocked`', + IF(@cur = @final, + 'DO 0', + 'ALTER TABLE `signatures` MODIFY COLUMN `assay_type` + SET(''transcriptomics'', ''proteomics'', ''metabolomics'', ''methylomics'', ''genetic_variants'') NOT NULL')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- signature_feature_set ----------------------------------------------------- + +SET @cur := (SELECT COLUMN_TYPE FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' AND COLUMN_NAME = 'assay_type'); + +SET @sql := IF(@cur = @final OR LOCATE('''genetic_variants''', @cur) > 0, + 'DO 0', + CONCAT('ALTER TABLE `signature_feature_set` MODIFY COLUMN `assay_type` ', + LEFT(@cur, CHAR_LENGTH(@cur) - 1), ',''genetic_variants'') NOT NULL')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @sql := IF(@cur = @final, + 'DO 0', + 'UPDATE `signature_feature_set` SET `assay_type` = ''genetic_variants'' + WHERE FIND_IN_SET(''snps'', `assay_type`) > 0 OR FIND_IN_SET(''genetic_variations'', `assay_type`) > 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stranded := (SELECT COUNT(*) FROM `signature_feature_set` + WHERE `assay_type` NOT IN ('transcriptomics', 'proteomics', 'metabolomics', 'methylomics', 'genetic_variants')); +SET @sql := IF(@stranded > 0, + 'SELECT `signature_feature_set holds an assay_type with no current equivalent` FROM `migration_blocked`', + IF(@cur = @final, + 'DO 0', + 'ALTER TABLE `signature_feature_set` MODIFY COLUMN `assay_type` + SET(''transcriptomics'', ''proteomics'', ''metabolomics'', ''methylomics'', ''genetic_variants'') NOT NULL')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- Postcondition ------------------------------------------------------------- + +SET @done := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND COLUMN_NAME = 'assay_type' + AND TABLE_NAME IN ('signatures', 'signature_feature_set') + AND COLUMN_TYPE = @final); +SET @postcondition_sql := IF(@done = 2, 'DO 0', + 'SELECT `assay_type does not have its final member list` FROM `migration_postcondition_failed`'); +PREPARE postcondition_stmt FROM @postcondition_sql; +EXECUTE postcondition_stmt; +DEALLOCATE PREPARE postcondition_stmt; diff --git a/mysql/migrations/2026-09-13-signature-feature-set-keys.sql b/mysql/migrations/2026-09-13-signature-feature-set-keys.sql new file mode 100644 index 0000000..74a0741 --- /dev/null +++ b/mysql/migrations/2026-09-13-signature-feature-set-keys.sql @@ -0,0 +1,158 @@ +-- Give signature_feature_set the keys and columns of the production table. +-- +-- Schema change of 2026-09-13 (commit 1210f74d, issue 78), which shipped +-- without a migration. The table has had three shapes: +-- +-- until 2026-07-07 PRIMARY KEY (signature_id, feature_id) +-- until 2026-09-13 PRIMARY KEY (signature_id, group_label, probe_id), +-- probe_id and group_label NOT NULL +-- since no primary key; UNIQUE sig_feature_assay_probe +-- (signature_id, feature_id, assay_type, probe_id); +-- feature_id, probe_id, group_label nullable; two new +-- columns, nomenclature_type and match_status +-- +-- Both older shapes reject real signatures: array probes and aptamers that +-- map to several features, and signatures whose probe_id repeats. +-- +-- Order matters. The foreign key on signature_id needs an index that leads +-- with signature_id at every moment, so the new unique key is added before +-- the old primary key and the old unnamed unique key (which MySQL named +-- `signature_id`) are dropped. feature_id cannot become nullable while it is +-- part of a primary key, so the nullability changes come last. +-- +-- The new unique key can be refused by the data: the 2026-07-07 shape allowed +-- one feature under one probe in two groups of the same signature, which the +-- production key does not. If such rows exist this migration stops before +-- changing any key and says so. See "signature_feature_set duplicates" in +-- mysql/migrations/README.md. +-- +-- Idempotent: every step checks information_schema first. + +SET @precondition_sql := IF( + DATABASE() IS NULL, + 'SELECT `Select a database first: pass the schema name to the mysql client` FROM `migration_precondition_failed`', + 'DO 0' +); +PREPARE precondition_stmt FROM @precondition_sql; +EXECUTE precondition_stmt; +DEALLOCATE PREPARE precondition_stmt; + +-- New columns --------------------------------------------------------------- + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' AND COLUMN_NAME = 'nomenclature_type'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `signature_feature_set` ADD COLUMN `nomenclature_type` VARCHAR(64) DEFAULT NULL AFTER `assay_type`', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' AND COLUMN_NAME = 'match_status'); +SET @sql := IF(@has = 0, + 'ALTER TABLE `signature_feature_set` ADD COLUMN `match_status` VARCHAR(32) DEFAULT NULL AFTER `sig_feature_hashkey`', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- The production unique key, added before anything is dropped ----------------- + +SET @has_new := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND INDEX_NAME = 'sig_feature_assay_probe'); +SET @duplicates := (SELECT COUNT(*) FROM ( + SELECT 1 FROM `signature_feature_set` + WHERE `feature_id` IS NOT NULL AND `probe_id` IS NOT NULL + GROUP BY `signature_id`, `feature_id`, `assay_type`, `probe_id` + HAVING COUNT(*) > 1) AS d); +SET @sql := IF(@has_new > 0, + 'DO 0', + IF(@duplicates > 0, + 'SELECT `signature_feature_set repeats a feature under one probe` FROM `migration_blocked`', + 'ALTER TABLE `signature_feature_set` + ADD UNIQUE KEY `sig_feature_assay_probe` (`signature_id`, `feature_id`, `assay_type`, `probe_id`)')); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND INDEX_NAME = 'idx_signature_nomenclature'); +SET @sql := IF(@has = 0, + 'CREATE INDEX `idx_signature_nomenclature` ON `signature_feature_set` (`assay_type`, `nomenclature_type`)', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- The old keys ---------------------------------------------------------------- + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' AND INDEX_NAME = 'PRIMARY'); +SET @sql := IF(@has > 0, + 'ALTER TABLE `signature_feature_set` DROP PRIMARY KEY', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @has := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' AND INDEX_NAME = 'signature_id'); +SET @sql := IF(@has > 0, + 'ALTER TABLE `signature_feature_set` DROP INDEX `signature_id`', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- Nullability ----------------------------------------------------------------- + +SET @strict := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND COLUMN_NAME = 'feature_id' AND IS_NULLABLE = 'NO'); +SET @sql := IF(@strict > 0, + 'ALTER TABLE `signature_feature_set` MODIFY COLUMN `feature_id` INT UNSIGNED DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @strict := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND COLUMN_NAME = 'probe_id' AND IS_NULLABLE = 'NO'); +SET @sql := IF(@strict > 0, + 'ALTER TABLE `signature_feature_set` MODIFY COLUMN `probe_id` VARCHAR(255) DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- group_label also loses its 'All Features' default. Rows keep their value. +SET @strict := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND COLUMN_NAME = 'group_label' AND (IS_NULLABLE = 'NO' OR COLUMN_DEFAULT IS NOT NULL)); +SET @sql := IF(@strict > 0, + 'ALTER TABLE `signature_feature_set` MODIFY COLUMN `group_label` VARCHAR(255) DEFAULT NULL', + 'DO 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- Postcondition --------------------------------------------------------------- + +SET @has_new := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND INDEX_NAME = 'sig_feature_assay_probe'); +SET @old_keys := (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND INDEX_NAME IN ('PRIMARY', 'signature_id')); +SET @nullable := (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'signature_feature_set' + AND COLUMN_NAME IN ('feature_id', 'probe_id', 'group_label') AND IS_NULLABLE = 'YES'); +SET @postcondition_sql := IF(@has_new > 0 AND @old_keys = 0 AND @nullable = 3, 'DO 0', + 'SELECT `signature_feature_set does not have the production keys` FROM `migration_postcondition_failed`'); +PREPARE postcondition_stmt FROM @postcondition_sql; +EXECUTE postcondition_stmt; +DEALLOCATE PREPARE postcondition_stmt; diff --git a/mysql/migrations/2026-09-25-rename-type-platform-rollback.sql b/mysql/migrations/rollback/2026-09-25-rename-type-platform-rollback.sql similarity index 100% rename from mysql/migrations/2026-09-25-rename-type-platform-rollback.sql rename to mysql/migrations/rollback/2026-09-25-rename-type-platform-rollback.sql diff --git a/scripts/migrate_gene_search_indexes.R b/scripts/migrate_gene_search_indexes.R deleted file mode 100755 index b541c82..0000000 --- a/scripts/migrate_gene_search_indexes.R +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env Rscript -# Migration: add the indexes that make "which signatures contain this gene?" -# answerable, on a database created before mysql/schema/ carried them. -# -# Additive only -- three CREATE INDEX statements, no ALTER of any column, no -# DROP, no data touched. Safe to re-run: each index is skipped if it already -# exists, so running this twice (or against an already-migrated database) is -# harmless. -# -# Why: every index on signature_feature_set leads with signature_id, which -# answers "what is in this signature?" but gives no way in from the feature -# side. And gene_symbol is TEXT, which MySQL cannot index without a prefix -# length. Together that forced the gene-to-signature query to walk every -# signature's features and filter on the gene last -- a full pass over -# signature_feature_set. -# -# Measured on a 1.35M-row copy matching production scale: -# query 1.21s -> 0.11s -# build 1.81s (signature_feature_set) + 0.20s (feature tables) -# -# The build takes a metadata lock on each table for its duration. Two seconds -# at production scale, but run it in a quiet window rather than mid-upload. -# -# Usage: -# DB_NAME=sigrepo DB_HOST= DB_PORT=3306 \ -# DB_USER= DB_PASSWORD= \ -# Rscript scripts/migrate_gene_search_indexes.R - -library(DBI) -library(RMySQL) - -conn <- dbConnect( - RMySQL::MySQL(), - dbname = Sys.getenv("DB_NAME"), - host = Sys.getenv("DB_HOST"), - port = as.integer(Sys.getenv("DB_PORT", "3306")), - user = Sys.getenv("DB_USER"), - password = Sys.getenv("DB_PASSWORD") -) -on.exit(dbDisconnect(conn), add = TRUE) - -table_exists <- function(table_name) { - table_name %in% DBI::dbListTables(conn) -} - -index_exists <- function(table_name, index_name) { - n <- DBI::dbGetQuery(conn, sprintf( - "SELECT COUNT(*) AS n FROM information_schema.STATISTICS - WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '%s' AND INDEX_NAME = '%s'", - table_name, index_name - )) - isTRUE(as.integer(n$n[1]) > 0) -} - -add_index <- function(table_name, index_name, definition) { - if (!table_exists(table_name)) { - cat(sprintf(" skip %-26s table not present\n", table_name)) - return(invisible(FALSE)) - } - if (index_exists(table_name, index_name)) { - cat(sprintf(" skip %-26s %s already exists\n", table_name, index_name)) - return(invisible(FALSE)) - } - started <- Sys.time() - DBI::dbExecute(conn, sprintf("CREATE INDEX `%s` ON `%s` (%s)", index_name, table_name, definition)) - cat(sprintf(" added %-26s %s (%.2fs)\n", table_name, index_name, - as.numeric(difftime(Sys.time(), started, units = "secs")))) - invisible(TRUE) -} - -cat("Adding gene-content search indexes\n") - -# The way in from the feature side. -add_index("signature_feature_set", "sfs_feature_id", "`feature_id`") - -# gene_symbol is TEXT -> needs a prefix length. Both feature tables that carry -# one get the same treatment so the lookup behaves the same across assay types. -add_index("transcriptomics_features", "tf_gene_symbol", "`gene_symbol`(64)") -add_index("proteomics_features", "pf_gene_symbol", "`gene_symbol`(64)") - -cat("Done.\n") diff --git a/scripts/migrate_geneset_schema.R b/scripts/migrate_geneset_schema.R deleted file mode 100644 index 707b22f..0000000 --- a/scripts/migrate_geneset_schema.R +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env Rscript -# Migration: create geneset_resources + geneset_entries on a database that -# predates them (mysql/schema/geneset_resources.sql, geneset_entries.sql). -# Additive only -- CREATE TABLE, no ALTER/DROP, nothing else on the database -# is touched. Safe to re-run: skips any table that already exists instead of -# erroring, so running this twice (or against a DB that's already been -# migrated) is harmless. -# -# Usage: -# DB_NAME=sigrepo DB_HOST= DB_PORT=3306 \ -# DB_USER= DB_PASSWORD= \ -# Rscript scripts/migrate_geneset_schema.R - -library(DBI) -library(RMySQL) - -conn <- dbConnect( - RMySQL::MySQL(), - dbname = Sys.getenv("DB_NAME"), - host = Sys.getenv("DB_HOST"), - port = as.integer(Sys.getenv("DB_PORT", "3306")), - user = Sys.getenv("DB_USER"), - password = Sys.getenv("DB_PASSWORD") -) -on.exit(dbDisconnect(conn), add = TRUE) - -table_exists <- function(conn, table_name) { - table_name %in% DBI::dbListTables(conn) -} - -# geneset_resources first -- geneset_entries has a foreign key to it, so -# creating them in the other order would fail. -create_geneset_resources <- " -CREATE TABLE `geneset_resources` ( - `geneset_resource_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, - `source` VARCHAR(64) NOT NULL, - `species` VARCHAR(128) NOT NULL, - `collection` VARCHAR(128) NOT NULL, - `subcollection` VARCHAR(128) DEFAULT NULL, - `version` VARCHAR(64) NOT NULL, - `source_version` VARCHAR(64) DEFAULT NULL, - `format` VARCHAR(32) NOT NULL DEFAULT 'rds', - `storage_path` VARCHAR(512) NOT NULL, - `checksum` VARCHAR(128) DEFAULT NULL, - `n_genesets` INT UNSIGNED DEFAULT NULL, - `n_features` INT UNSIGNED DEFAULT NULL, - `is_current` BOOL DEFAULT 1, - `notes` TEXT DEFAULT NULL, - `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `geneset_resource_hashkey` VARCHAR(32) NOT NULL, - PRIMARY KEY (`geneset_resource_id`), - UNIQUE (`source`, `species`, `collection`, `subcollection`, `version`), - UNIQUE (`geneset_resource_hashkey`), - CHECK (`is_current` IN (0,1)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; -" - -create_geneset_entries <- " -CREATE TABLE `geneset_entries` ( - `geneset_entry_id` INT UNSIGNED NOT NULL AUTO_INCREMENT, - `geneset_resource_id` INT UNSIGNED NOT NULL, - `geneset_name` VARCHAR(255) NOT NULL, - `description` TEXT DEFAULT NULL, - `n_features` INT UNSIGNED DEFAULT NULL, - `geneset_entry_hashkey` VARCHAR(32) NOT NULL, - PRIMARY KEY (`geneset_entry_id`), - UNIQUE (`geneset_resource_id`, `geneset_name`), - UNIQUE (`geneset_entry_hashkey`), - FOREIGN KEY (`geneset_resource_id`) REFERENCES `geneset_resources` (`geneset_resource_id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; -" - -cat("geneset_* tables before migration:\n") -before <- DBI::dbListTables(conn) -print(before[grepl("^geneset_", before)]) -cat("\n") - -if (table_exists(conn, "geneset_resources")) { - cat("[skip] geneset_resources already exists, not touching it.\n") -} else { - dbExecute(conn, create_geneset_resources) - cat("[created] geneset_resources\n") -} - -if (table_exists(conn, "geneset_entries")) { - cat("[skip] geneset_entries already exists, not touching it.\n") -} else { - dbExecute(conn, create_geneset_entries) - cat("[created] geneset_entries\n") -} - -cat("\ngeneset_* tables after migration:\n") -after <- DBI::dbListTables(conn) -print(after[grepl("^geneset_", after)]) - -cat("\nCurrent row counts:\n") -cat("geneset_resources:", dbGetQuery(conn, "SELECT COUNT(*) AS n FROM geneset_resources")$n, "\n") -cat("geneset_entries: ", dbGetQuery(conn, "SELECT COUNT(*) AS n FROM geneset_entries")$n, "\n") diff --git a/tests/shell/test_history_upgrade.sh b/tests/shell/test_history_upgrade.sh new file mode 100644 index 0000000..5fe88b8 --- /dev/null +++ b/tests/shell/test_history_upgrade.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Prove that a database built from any schema this repository has ever shipped +# ends up, after the migrations, with the schema a fresh install has today. +# +# bash tests/shell/test_history_upgrade.sh --container NAME +# MYSQL_PWD=... bash tests/shell/test_history_upgrade.sh --host HOST --user root +# +# Needs a disposable MySQL (see tests/shell/testlib.sh) and full git history: +# in CI, check out with fetch-depth 0. +# +# This is the test that stands behind updating an install nobody has a record +# of. For every commit that changed mysql/schema/, it builds the schema as it +# was, runs every migration, and compares the result with a database built +# from mysql/schema/ as it is now. A schema change that ships without a +# migration fails here, because the commit before it no longer converges. +set -uo pipefail + +. "$(dirname "$0")/testlib.sh" +parse_conn_args "$@" + +OLD=sigrepo_test_history +NEW=sigrepo_test_fresh +MIGRATE="$ROOT/scripts/migrate.sh" +FINGERPRINT="$ROOT/scripts/schema_fingerprint.sh" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"; admin_sql "DROP DATABASE IF EXISTS \`$OLD\`" >/dev/null 2>&1; admin_sql "DROP DATABASE IF EXISTS \`$NEW\`" >/dev/null 2>&1' EXIT + +# Commits whose mysql/schema/ cannot be loaded at all, so no install can have +# that schema. Each needs a reason. A commit that fails to load and is NOT +# listed here fails the test. +# +# 5a22eacb organisms.sql says "DATE DEFAULT CURRENT_DATE", which +# MySQL 8.0 rejects. Replaced the same day by abd4611d. +# 756844a6 .. 3c88d943 proteomics_features.sql says "ADD CONSTRAINT" inside +# CREATE TABLE, which is a syntax error. Fixed by +# 05bebc6d on 2026-06-23. d85a25d2 is a merge commit +# inside that range. +UNBUILDABLE="5a22eacb 756844a6 7bb9077e 4fb3dc28 ff6e1bab d85a25d2 3c88d943" + +echo "--- the schema a fresh install has today ---" + +fresh_db "$NEW" +schema_at WORKTREE > "$WORK/schema.sql" +if db_load "$NEW" < "$WORK/schema.sql"; then + ok "mysql/schema/ loads" +else + bad "mysql/schema/ as it is now does not load"; finish +fi +script_args "$NEW" +bash "$FINGERPRINT" "${SCRIPT_ARGS[@]}" > "$WORK/fresh.tsv" +if [ -s "$WORK/fresh.tsv" ]; then ok "fingerprinted"; else bad "empty fingerprint"; finish; fi + +echo "--- on that schema, every migration is a no-op ---" + +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +bash "$FINGERPRINT" "${SCRIPT_ARGS[@]}" > "$WORK/fresh_after.tsv" +if [ "$RC" -eq 0 ] && diff -q "$WORK/fresh.tsv" "$WORK/fresh_after.tsv" >/dev/null; then + ok "migrations leave a current schema unchanged" +else + bad "migrations changed a current schema (rc=$RC)"; printf '%s\n' "$OUT" + diff "$WORK/fresh.tsv" "$WORK/fresh_after.tsv" +fi + +echo "--- every schema this repository has shipped ---" + +COMMITS=$(git -C "$ROOT" log --reverse --format=%h --abbrev=8 -- mysql/schema) +if [ -z "$COMMITS" ]; then bad "no schema history found; is this a shallow clone?"; finish; fi + +script_args "$OLD" +for c in $COMMITS; do + when=$(git -C "$ROOT" log -1 --format=%ad --date=short "$c") + fresh_db "$OLD" + + schema_at "$c" > "$WORK/schema.sql" + if ! db_load "$OLD" < "$WORK/schema.sql" 2>"$WORK/load.err"; then + if printf '%s\n' $UNBUILDABLE | grep -qx "$c"; then + echo "SKIP $c $when (schema files are not valid SQL; listed in UNBUILDABLE)" + else + bad "$c $when schema does not load and is not listed in UNBUILDABLE" + head -3 "$WORK/load.err" + fi + continue + fi + if printf '%s\n' $UNBUILDABLE | grep -qx "$c"; then + bad "$c $when is listed in UNBUILDABLE but loads; remove it from the list" + continue + fi + + if ! OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); then + bad "$c $when a migration failed"; printf '%s\n' "$OUT" | tail -5 + continue + fi + bash "$FINGERPRINT" "${SCRIPT_ARGS[@]}" > "$WORK/old.tsv" + if diff "$WORK/fresh.tsv" "$WORK/old.tsv" > "$WORK/diff.txt"; then + ok "$c $when converges" + else + bad "$c $when differs from a fresh schema (< fresh, > migrated)"; cat "$WORK/diff.txt" + fi +done + +echo "--- rows survive, and old assay types are rewritten ---" + +# The oldest schema: PRIMARY KEY (signature_id, feature_id), NUMERIC(10,8) +# scores, and an assay_type list ending in genetic_variations and +# dna_binding_sites. +FIRST=$(printf '%s\n' $COMMITS | head -1) +fresh_db "$OLD" +schema_at "$FIRST" > "$WORK/schema.sql" +db_load "$OLD" < "$WORK/schema.sql" +db_load "$OLD" <<'SQL' +SET FOREIGN_KEY_CHECKS=0; +INSERT INTO signature_feature_set + (signature_id, feature_id, probe_id, score, group_label, assay_type, sig_feature_hashkey) +VALUES + (1, 10, 'p10', 1.50000000, NULL, 'transcriptomics', 'h1'), + (1, 11, 'p11', -2.25000000, NULL, 'proteomics', 'h2'), + (2, 20, 'p20', 0.12345678, NULL, 'genetic_variations', 'h3'); +SQL +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +ROWS=$(db_sql "$OLD" "SELECT CONCAT_WS('|', signature_id, feature_id, probe_id, score, assay_type) FROM signature_feature_set ORDER BY sig_feature_hashkey" | tr '\n' ' ') +WANT="1|10|p10|1.50000000|transcriptomics 1|11|p11|-2.25000000|proteomics 2|20|p20|0.12345678|genetic_variants " +if [ "$RC" -eq 0 ] && [ "$ROWS" = "$WANT" ]; then + ok "three rows kept their values; genetic_variations became genetic_variants" +else + bad "rows after migrating (rc=$RC): $ROWS"; printf '%s\n' "$OUT" | tail -5 +fi + +echo "--- an assay type with no equivalent stops the migration ---" + +fresh_db "$OLD" +schema_at "$FIRST" > "$WORK/schema.sql" +db_load "$OLD" < "$WORK/schema.sql" +db_load "$OLD" <<'SQL' +SET FOREIGN_KEY_CHECKS=0; +INSERT INTO signature_feature_set + (signature_id, feature_id, probe_id, score, group_label, assay_type, sig_feature_hashkey) +VALUES (1, 10, 'p10', 1.0, NULL, 'dna_binding_sites', 'h1'); +SQL +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +KEPT=$(db_sql "$OLD" "SELECT assay_type FROM signature_feature_set") +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "FAILED 2026-08-26-assay-type-genetic-variants.sql" \ + && [ "$KEPT" = "dna_binding_sites" ]; then + ok "the run stops and the row still holds its value" +else + bad "dna_binding_sites row (rc=$RC, value now '$KEPT')"; printf '%s\n' "$OUT" | tail -5 +fi + +echo "--- a row holding two assay types stops the migration ---" + +# SET columns can hold several members. Nothing in SigRepo writes more than +# one, so such a row is not something to guess a meaning for. +fresh_db "$OLD" +schema_at "$FIRST" > "$WORK/schema.sql" +db_load "$OLD" < "$WORK/schema.sql" +db_load "$OLD" <<'SQL' +SET FOREIGN_KEY_CHECKS=0; +INSERT INTO signature_feature_set + (signature_id, feature_id, probe_id, score, group_label, assay_type, sig_feature_hashkey) +VALUES (1, 10, 'p10', 1.0, NULL, 'transcriptomics,proteomics', 'h1'); +SQL +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +KEPT=$(db_sql "$OLD" "SELECT assay_type FROM signature_feature_set") +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "FAILED 2026-08-26-assay-type-genetic-variants.sql" \ + && [ "$KEPT" = "transcriptomics,proteomics" ]; then + ok "the run stops and the row still holds both" +else + bad "two-member row (rc=$RC, value now '$KEPT')"; printf '%s\n' "$OUT" | tail -5 +fi + +echo "--- a key change interrupted half way finishes on a re-run ---" + +# DDL commits statement by statement, so a dropped connection can leave the +# new unique key added and the old primary key still there. +fresh_db "$OLD" +schema_at "$FIRST" > "$WORK/schema.sql" +db_load "$OLD" < "$WORK/schema.sql" +db_sql "$OLD" "ALTER TABLE signature_feature_set ADD UNIQUE KEY sig_feature_assay_probe (signature_id, feature_id, assay_type, probe_id)" +db_sql "$OLD" "ALTER TABLE signature_feature_set ADD COLUMN nomenclature_type VARCHAR(64) DEFAULT NULL" +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +bash "$FINGERPRINT" "${SCRIPT_ARGS[@]}" > "$WORK/old.tsv" +if [ "$RC" -eq 0 ] && diff "$WORK/fresh.tsv" "$WORK/old.tsv" > "$WORK/diff.txt"; then + ok "the half-applied database converges" +else + bad "half-applied key change (rc=$RC)"; printf '%s\n' "$OUT" | tail -5; cat "$WORK/diff.txt" +fi + +echo "--- a feature repeated under one probe stops the key change ---" + +# The 2026-07-07 shape: PRIMARY KEY (signature_id, group_label, probe_id). +GROUPED=$(git -C "$ROOT" log --format=%h --abbrev=8 -1 12ed4be1 2>/dev/null) +fresh_db "$OLD" +schema_at "$GROUPED" > "$WORK/schema.sql" +db_load "$OLD" < "$WORK/schema.sql" +db_load "$OLD" <<'SQL' +SET FOREIGN_KEY_CHECKS=0; +INSERT INTO signature_feature_set + (signature_id, feature_id, probe_id, score, group_label, assay_type, sig_feature_hashkey) +VALUES + (1, 10, 'p10', 1.0, 'group A', 'transcriptomics', 'h1'), + (1, 10, 'p10', 2.0, 'group B', 'transcriptomics', 'h2'); +SQL +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +COUNT=$(db_sql "$OLD" "SELECT COUNT(*) FROM signature_feature_set") +PK=$(db_sql "$OLD" "SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA='$OLD' AND TABLE_NAME='signature_feature_set' AND INDEX_NAME='PRIMARY'") +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "FAILED 2026-09-13-signature-feature-set-keys.sql" \ + && [ "$COUNT" = "2" ] && [ "$PK" != "0" ]; then + ok "the run stops with both rows and the old primary key intact" +else + bad "duplicate rows (rc=$RC, rows=$COUNT, pk columns=$PK)"; printf '%s\n' "$OUT" | tail -5 +fi + +finish From 60a8837997105f01a870cfd2a9c2289f5e587920 Mon Sep 17 00:00:00 2001 From: Cvicnaire Date: Wed, 30 Sep 2026 14:05:04 -0400 Subject: [PATCH 4/6] Record migrations when /init_db builds the schema, and refuse to start the API on a database that is behind Co-Authored-By: Claude Fable 5.1 --- api/api.R | 7 ++ api/lib/database_admin.R | 5 ++ api/lib/schema_migrations.R | 111 ++++++++++++++++++++++++ tests/testthat/helper-db.R | 20 +++++ tests/testthat/test-database_admin.R | 22 +---- tests/testthat/test-schema-migrations.R | 92 ++++++++++++++++++++ 6 files changed, 237 insertions(+), 20 deletions(-) create mode 100644 api/lib/schema_migrations.R create mode 100644 tests/testthat/test-schema-migrations.R diff --git a/api/api.R b/api/api.R index 9c1d674..7fcc608 100644 --- a/api/api.R +++ b/api/api.R @@ -167,6 +167,13 @@ for (lib_file in base::sort(base::list.files(base::file.path(sigrepo_server_path base::source(lib_file, local = TRUE) } +# Refuse to serve a database that is missing a schema migration this code +# ships (api/lib/schema_migrations.R). Same idea as +# assert_omic_signature_version() above: fail at startup, where the cause is +# named, not per request with "Unknown column". It reads and never applies; +# scripts/migrate.sh and update_sigrepo.sh are what apply. +assert_schema_migrated(db_connect_local, sigrepo_server_path) + # Resolved once at boot (default_msigdb_cache_dir lives in api/lib/msigdb_cache.R) # so /init_db_genesets, /geneset_resources/ensure, /init_db's combined # bootstrap, and every /annotate/* request all share the same cache lookup. diff --git a/api/lib/database_admin.R b/api/lib/database_admin.R index fca8c0d..196c1a1 100644 --- a/api/lib/database_admin.R +++ b/api/lib/database_admin.R @@ -98,6 +98,11 @@ generate_db_schema <- function(sigrepo_server_path = base::Sys.getenv("SIGREPO_S run_schema_file("mysql/schema/signature_feature_set_ambiguity.sql", "signature_feature_set_ambiguity") run_schema_file("mysql/schema/genetic_variants_features.sql", "genetic_variants_features") + # The schema just built is the current one, so it already contains every + # change in mysql/migrations/. Record them as applied (api/lib/schema_migrations.R), + # or the API's boot check would treat this database as never migrated. + record_schema_migrations(conn, sigrepo_server_path) + # Disconnect from database #### base::suppressWarnings(DBI::dbDisconnect(conn)) diff --git a/api/lib/schema_migrations.R b/api/lib/schema_migrations.R new file mode 100644 index 0000000..3dd73e8 --- /dev/null +++ b/api/lib/schema_migrations.R @@ -0,0 +1,111 @@ +# Which schema migrations a database has had. +# +# The files in mysql/migrations/ are applied by scripts/migrate.sh, which +# records each one in the schema_migrations table. Two things in the API need +# to agree with that record: +# +# * generate_db_schema() builds the CURRENT schema from mysql/schema/, so a +# database it builds already contains every change the migrations make. It +# records them all as applied; otherwise a fresh instance would look like +# one that has never been migrated. +# * At boot the API refuses a database that is missing a migration this code +# ships. New code against an old schema does not fail at startup by +# itself; it fails per request, with "Unknown column", on whichever route +# touches the changed table. That is how the type/platform rename broke +# /signatures/compare on a database nobody had migrated. + +# The forward migrations this code ships, in the order they apply. Rollback +# scripts live in mysql/migrations/rollback/ and are not listed. +shipped_schema_migrations <- function(sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) { + files <- base::list.files( + base::file.path(sigrepo_server_path, "mysql", "migrations"), + pattern = "\\.sql$", + recursive = FALSE + ) + # Same rule as scripts/migrate.sh: a *-rollback.sql file is never a forward + # migration, wherever it sits. + base::sort(files[!base::grepl("-rollback\\.sql$", files)]) +} + +# Mark every shipped migration as applied. Only correct straight after the +# schema has been built from mysql/schema/. +record_schema_migrations <- function(conn, sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) { + base::suppressWarnings(DBI::dbGetQuery(conn = conn, statement = " + CREATE TABLE IF NOT EXISTS `schema_migrations` ( + `name` VARCHAR(255) NOT NULL, + `applied_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;")) + + shipped <- shipped_schema_migrations(sigrepo_server_path) + + # The names go into SQL text, so hold them to what migrate.sh accepts. + unexpected <- shipped[!base::grepl("^[A-Za-z0-9._-]+$", shipped)] + if (base::length(unexpected) > 0) { + base::stop("Unexpected migration file name: ", base::paste(unexpected, collapse = ", ")) + } + + for (name in shipped) { + base::suppressWarnings(DBI::dbGetQuery( + conn = conn, + statement = base::sprintf("INSERT IGNORE INTO `schema_migrations` (`name`) VALUES ('%s');", name) + )) + } + + base::invisible(shipped) +} + +# The shipped migrations this database has no record of. A database with no +# signatures table has not been through /init_db yet and is not behind +# anything: the API has to be able to start on it, because /init_db is an API +# route. +pending_schema_migrations <- function(conn, sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) { + tables <- DBI::dbGetQuery( + conn = conn, + statement = "SELECT TABLE_NAME AS table_name FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE();" + )$table_name + + if (!("signatures" %in% tables)) { + return(base::character(0)) + } + + recorded <- base::character(0) + if ("schema_migrations" %in% tables) { + recorded <- DBI::dbGetQuery(conn = conn, statement = "SELECT `name` FROM `schema_migrations`;")$name + } + + base::setdiff(shipped_schema_migrations(sigrepo_server_path), recorded) +} + +# Stop unless the database has every shipped migration. `connect` is a +# function returning a connection, so that a database which cannot be reached +# at boot is reported and tolerated: the API has always been able to start +# before MySQL is up, and a schema that cannot be read cannot be judged. +assert_schema_migrated <- function(connect, sigrepo_server_path = base::Sys.getenv("SIGREPO_SERVER_DIR")) { + pending <- base::tryCatch( + pending_schema_migrations(connect(), sigrepo_server_path), + error = function(err) { + base::message("Could not check schema migrations, continuing: ", base::conditionMessage(err)) + NULL + } + ) + + if (base::is.null(pending)) { + return(base::invisible(FALSE)) + } + + if (base::length(pending) > 0) { + base::stop(base::sprintf( + base::paste( + "The database is behind this version of SigRepo: %d schema migration(s) have not been applied (%s).", + "On an installed instance, run update_sigrepo.sh.", + "Anywhere else, run scripts/migrate.sh; see mysql/migrations/README.md.", + "The API will not start until then, because its queries assume the migrated schema." + ), + base::length(pending), + base::paste(pending, collapse = ", ") + )) + } + + base::invisible(TRUE) +} diff --git a/tests/testthat/helper-db.R b/tests/testthat/helper-db.R index 9c5c55d..a22a927 100644 --- a/tests/testthat/helper-db.R +++ b/tests/testthat/helper-db.R @@ -49,6 +49,26 @@ skip_if_no_test_db <- function() { ) } +# Tests that call reset_db_tables() or generate_db_schema() tear down every +# table in the test database, which would break the other test-*.R files that +# depend on tests/testthat/fixtures/seed.sql already being loaded (test files +# run alphabetically in the same session). Reseed after each destructive +# assertion so the database is left as the test file found it. The caller must +# have sourced api/lib/schema_migrations.R and api/lib/database_admin.R. +reseed_test_db <- function() { + sigrepo_server_path <- Sys.getenv("SIGREPO_SERVER_DIR", unset = testthat::test_path("../..")) + generate_db_schema(sigrepo_server_path) + + conn <- db_connect_local() + on.exit(DBI::dbDisconnect(conn)) + lines <- readLines(testthat::test_path("fixtures/seed.sql")) + lines <- lines[!grepl("^\\s*--", lines)] + seed_sql <- paste(lines, collapse = "\n") + for (stmt in Filter(nzchar, trimws(strsplit(seed_sql, ";")[[1]]))) { + DBI::dbGetQuery(conn, stmt) + } +} + # api/lib/common.R closes the pool it replaces each time a test file sources it # again, which leaves the last file's pool open. Close that one when the run # ends, so its maintenance task does not outlive the suite (issue #81). diff --git a/tests/testthat/test-database_admin.R b/tests/testthat/test-database_admin.R index feedb9c..e69b607 100644 --- a/tests/testthat/test-database_admin.R +++ b/tests/testthat/test-database_admin.R @@ -1,26 +1,8 @@ source(testthat::test_path("../../api/lib/common.R"), local = FALSE) +source(testthat::test_path("../../api/lib/schema_migrations.R"), local = FALSE) source(testthat::test_path("../../api/lib/database_admin.R"), local = FALSE) source(testthat::test_path("helper-db.R"), local = FALSE) -# Both tests below tear down every table in the test database, which would -# break other test-*.R files that depend on tests/testthat/fixtures/seed.sql -# already being loaded (test files run alphabetically in the same session). -# Reseed after each destructive assertion so the DB is left as this test -# file found it. -reseed_test_db <- function() { - sigrepo_server_path <- Sys.getenv("SIGREPO_SERVER_DIR", unset = testthat::test_path("../..")) - generate_db_schema(sigrepo_server_path) - - conn <- db_connect_local() - on.exit(DBI::dbDisconnect(conn)) - lines <- readLines(testthat::test_path("fixtures/seed.sql")) - lines <- lines[!grepl("^\\s*--", lines)] - seed_sql <- paste(lines, collapse = "\n") - for (stmt in Filter(nzchar, trimws(strsplit(seed_sql, ";")[[1]]))) { - DBI::dbGetQuery(conn, stmt) - } -} - test_that("generate_db_schema (re)creates every expected table regardless of the configured DB name", { skip_if_no_test_db() on.exit(reseed_test_db()) @@ -37,7 +19,7 @@ test_that("generate_db_schema (re)creates every expected table regardless of the "organisms", "phenotypes", "platforms", "proteomics_features", "sample_types", "signature_access", "signature_collection_access", "signature_feature_set", "signatures", "transcriptomics_features", "users", "metabolite_reference", "metabolite_xref", - "signature_feature_set_ambiguity", "genetic_variants_features" + "signature_feature_set_ambiguity", "genetic_variants_features", "schema_migrations" ) expect_true(all(expected %in% tables)) }) diff --git a/tests/testthat/test-schema-migrations.R b/tests/testthat/test-schema-migrations.R new file mode 100644 index 0000000..66a9df3 --- /dev/null +++ b/tests/testthat/test-schema-migrations.R @@ -0,0 +1,92 @@ +# Guards the link between mysql/migrations/ and the schema_migrations table: +# a database built by generate_db_schema() must count as fully migrated, and +# the API's boot check must refuse one that is not. +source(testthat::test_path("../../api/lib/common.R"), local = FALSE) +source(testthat::test_path("../../api/lib/schema_migrations.R"), local = FALSE) +source(testthat::test_path("../../api/lib/database_admin.R"), local = FALSE) +source(testthat::test_path("helper-db.R"), local = FALSE) + +server_path <- function() { + Sys.getenv("SIGREPO_SERVER_DIR", unset = testthat::test_path("../..")) +} + +test_that("the shipped list is the forward migrations, in order, without rollbacks", { + shipped <- shipped_schema_migrations(server_path()) + + expect_gt(length(shipped), 0) + expect_true("2026-09-25-rename-type-platform.sql" %in% shipped) + expect_false(any(grepl("rollback", shipped))) + expect_equal(shipped, sort(shipped)) +}) + +test_that("a database built by generate_db_schema has nothing pending", { + skip_if_no_test_db() + on.exit(reseed_test_db()) + + generate_db_schema(server_path()) + + conn <- db_connect_local() + expect_equal(pending_schema_migrations(conn, server_path()), character(0)) + recorded <- DBI::dbGetQuery(conn, "SELECT name FROM schema_migrations")$name + expect_setequal(recorded, shipped_schema_migrations(server_path())) + expect_true(assert_schema_migrated(db_connect_local, server_path())) +}) + +test_that("recording twice neither fails nor duplicates", { + skip_if_no_test_db() + on.exit(reseed_test_db()) + + generate_db_schema(server_path()) + conn <- db_connect_local() + record_schema_migrations(conn, server_path()) + + n <- DBI::dbGetQuery(conn, "SELECT COUNT(*) AS n FROM schema_migrations")$n + expect_equal(as.integer(n), length(shipped_schema_migrations(server_path()))) +}) + +test_that("a missing record is pending, and the boot check names it", { + skip_if_no_test_db() + on.exit(reseed_test_db()) + + generate_db_schema(server_path()) + conn <- db_connect_local() + DBI::dbGetQuery(conn, "DELETE FROM schema_migrations WHERE name = '2026-09-25-rename-type-platform.sql'") + + expect_equal(pending_schema_migrations(conn, server_path()), "2026-09-25-rename-type-platform.sql") + expect_error( + assert_schema_migrated(db_connect_local, server_path()), + "2026-09-25-rename-type-platform.sql" + ) + expect_error(assert_schema_migrated(db_connect_local, server_path()), "update_sigrepo.sh") +}) + +test_that("a database with tables but no schema_migrations table is entirely pending", { + skip_if_no_test_db() + on.exit(reseed_test_db()) + + generate_db_schema(server_path()) + conn <- db_connect_local() + DBI::dbGetQuery(conn, "DROP TABLE schema_migrations") + + expect_setequal(pending_schema_migrations(conn, server_path()), shipped_schema_migrations(server_path())) +}) + +test_that("an empty database is not behind: /init_db has to be able to run", { + skip_if_no_test_db() + on.exit(reseed_test_db()) + + reset_db_tables(conn_handler = NULL) + + expect_equal(pending_schema_migrations(db_connect_local(), server_path()), character(0)) + expect_true(assert_schema_migrated(db_connect_local, server_path())) +}) + +test_that("a database that cannot be reached is tolerated, with a message", { + unreachable <- function() stop("Can't connect to MySQL server on 'sigrepo-mysql:3306' (111)") + + expect_message( + result <- assert_schema_migrated(unreachable, server_path()), + "Could not check schema migrations" + ) + expect_false(result) +}) From 9f228dd6be9ec6b52c1e26c324cfab28f70789ee Mon Sep 17 00:00:00 2001 From: Cvicnaire Date: Wed, 30 Sep 2026 14:08:26 -0400 Subject: [PATCH 5/6] Run the migration tests in CI and document the runner Co-Authored-By: Claude Fable 5.1 --- .github/workflows/test.yml | 48 ++++++ NEWS.md | 1 + mysql/migrations/README.md | 342 +++++++++++++++++-------------------- 3 files changed, 208 insertions(+), 183 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fa15817..7795498 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,9 @@ on: - "api/**" - "mcp/**" - "mysql/schema/**" + - "mysql/migrations/**" + - "mysql/expected_schema.tsv" + - "scripts/**" - "tests/**" - "DESCRIPTION" - "install_r_packages.R" @@ -180,6 +183,15 @@ jobs: - name: Seed test fixtures run: mysql -h"$DB_HOST" -uroot -psigrepo_root_pw sigrepo_test < tests/testthat/fixtures/seed.sql + # The schema was loaded from mysql/schema/ directly, not by /init_db, so + # nothing has recorded the migrations yet and the API's boot check below + # would refuse this database. On a current schema every migration takes + # its no-op branch, which this step also proves. + - name: Record the schema migrations + env: + MYSQL_PWD: sigrepo_root_pw + run: bash scripts/migrate.sh --host "$DB_HOST" --user root --database sigrepo_test + - name: Run testthat run: testthat::test_dir("tests/testthat", stop_on_failure = TRUE) shell: Rscript {0} @@ -229,3 +241,39 @@ jobs: - name: Show MCP server log on failure if: failure() run: cat /tmp/mcp.log || true + + # The migrations, which need no R. On the runner, not in the image: the + # history test needs git and the full history. + shell-tests: + runs-on: ubuntu-latest + services: + mysql: + image: mysql:8.0 + env: + MYSQL_DATABASE: sigrepo_test + MYSQL_ROOT_PASSWORD: sigrepo_root_pw + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping -uroot -psigrepo_root_pw" + --health-interval=10s + --health-timeout=5s + --health-retries=10 + env: + MYSQL_PWD: sigrepo_root_pw + steps: + - name: Checkout SigRepo_Server with its history + uses: actions/checkout@v4 + with: + # test_history_upgrade.sh rebuilds every schema this repository has + # shipped, so it needs every commit that touched mysql/schema/. + fetch-depth: 0 + + - name: The migration runner + run: bash tests/shell/test_migrate.sh --host 127.0.0.1 --user root + + - name: Every historical schema migrates to the current one + run: bash tests/shell/test_history_upgrade.sh --host 127.0.0.1 --user root + + - name: mysql/expected_schema.tsv is current + run: bash tests/shell/test_expected_schema.sh --host 127.0.0.1 --user root diff --git a/NEWS.md b/NEWS.md index 7d1371b..fc1e082 100755 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ # SigRepo_Server 1.0.0 * Documentation: https://montilab.github.io/SigRepo_Server/ * GitHub: https://github.com/montilab/SigRepo_Server/ +* Schema migrations are applied by `scripts/migrate.sh` and recorded in a `schema_migrations` table. Ten migrations were added for schema changes since 2025 that had none, so a database built from any earlier version of `mysql/schema/` can be brought to the current schema. The API now refuses to start on a database that is missing a migration it ships: run `scripts/migrate.sh` before deploying this version to an existing database. `scripts/migrate_geneset_schema.R` and `scripts/migrate_gene_search_indexes.R` are replaced by migrations (#150). * `docker-compose-local.yml` takes the host address for the Shiny interface from `SHINY_BIND`, default `127.0.0.1`. montilab.bu.edu sets it to `0.0.0.0` so lab members on the BU network open http://montilab.bu.edu:8051 without an SSH tunnel; the database, API and MCP proxy ports stay on loopback, and CONTRIBUTING describes both routes (#142). * The MCP `run_enrichment` tool is withdrawn until it is rebuilt on the new `SigRepo::runHypeR()` from SigRepo #223, which requires `organism` and `test` and returns hypeR's own objects; the old wrapper failed on every call against the new client. `rummagene_enrich` and the other eight tools are unchanged (#137, rebuild tracked in #138). * The repository now carries only code, docs and the testthat suite. Removed: a 2024 scratch script and a one-off table builder under `tests/`, a stray `Rplots.pdf`, two unreferenced knitted copies of the client tutorial and an old nav in `legacy_app/www/`, the three type-rename planning documents in `spec/`, and the montilab-specific staging tooling (`scripts/deploy_montilab.sh`, `scripts/reseed_staging.sh`, their shell tests, the fake API fixture and the validate-and-ship skill), which now lives outside the repository. `scripts/smoke_test.sh`, `scripts/smoke_test_ui.R`, the cache builders and the migrations stay (#135). diff --git a/mysql/migrations/README.md b/mysql/migrations/README.md index 085e0cc..7a5effa 100644 --- a/mysql/migrations/README.md +++ b/mysql/migrations/README.md @@ -1,203 +1,179 @@ # mysql/migrations -Hand-applied schema migrations for the SigRepo database. There is no migration -runner and no migration-tracking table in this project: each file is a plain -SQL script, applied in order, by hand, once per environment, and the -`information_schema` query in "Verifying a run" below is the only way to know -whether a given environment has been migrated. +Schema migrations for the SigRepo database: the changes that take a database +built by an older version to the schema the current code expects, keeping its +data. -Every migration here is idempotent (guarded on `information_schema`), so -re-running one that has already been applied is a safe no-op. Each script -also checks, as its first action, that a default database was actually -selected on the connection, and, as its last action, that the columns it was -supposed to produce actually exist; either check aborts the script loudly -rather than let it exit 0 having silently changed nothing. See the comments -at the top of each `.sql` file for why that matters. +Every `.sql` file directly in this directory is a forward migration, applied in +filename order. `scripts/migrate.sh` applies the ones a database has not had +and records each in the `schema_migrations` table. Rollback scripts live in +`rollback/` and are never applied by the runner. -Requires MySQL 8.0 or later. These migrations use `RENAME COLUMN`, which does -not exist before 8.0. Do not run them against an older server. +Requires MySQL 8.0 or later. -## Applying a migration to the local stack +## Applying migrations -From the repo root, with the local stack running: +An installed instance does not need this section: `update_sigrepo.sh` takes a +backup, runs the migrations that ship in the release it is moving to, and +checks the result. + +Everywhere else (the local stack, montilab.bu.edu, sigrepo.org), from the +repository root: ```bash -set -a; . /path/to/sigrepo-local/.env.local; set +a -docker exec -i sigrepo-local-mysql mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" sigrepo \ - < mysql/migrations/2026-09-25-rename-type-platform.sql +scripts/migrate.sh --container sigrepo-mysql --status # what is pending +scripts/migrate.sh --container sigrepo-mysql # apply it ``` -Substitute the actual path to the local stack's env file (it defines -`MYSQL_USER`, `MYSQL_PASSWORD`, `MYSQL_DATABASE`, and the root credentials). -Never hardcode or print these values; always source them from the env file. - -To roll a migration back, run its `-rollback.sql` counterpart the same way. +Name the MySQL container of the stack in question (`sigrepo-local-mysql` for +the local stack). The runner uses the mysql client inside that container and +the root password the container already holds, so no credential is typed or +printed. For a MySQL that is not in a container, give `--host`, `--user` and +`--database`, and put the password in `MYSQL_PWD`. + +The run stops at the first migration that fails and records nothing for it. +Migrations before it stay applied and recorded. + +### Before running on montilab.bu.edu or sigrepo.org + +These two hosts hold data that cannot be regenerated. Before applying anything +here to either one: + +1. Take a `mysqldump` of the whole database and store it outside the database + container. +2. Confirm the dump completed and is not empty. + +### A database with no `schema_migrations` table + +The first run on such a database applies every file. That is safe and +intended: each migration checks `information_schema` and does only what is +missing, so one whose change is already present does nothing and is then +recorded. This is how a database of unknown age is brought forward without +anyone having to know which changes it has. + +`generate_db_schema()`, which `/init_db` calls, builds the current schema from +`mysql/schema/` and records every migration as applied, so a fresh database is +never pending. + +## Code and schema move together + +The API checks at startup that every migration it ships is recorded, and +refuses to start if one is not (`assert_schema_migrated()` in +`api/lib/schema_migrations.R`). So on a host that bind mounts this checkout, +the order of a deploy that includes a migration is: + +1. Back up. +2. `git pull`, which brings the new migration file. +3. `scripts/migrate.sh --container `. +4. Recreate the API, Shiny and MCP containers. + +Between steps 2 and 4 the running API still holds the old code and keeps +serving; a restart in that window, before step 3, is what the check refuses. + +If the migration needs code that lives in the image and not in this checkout +(the `SigRepo` client, `OmicSignature`), pull the rebuilt image before step 4. +A `git pull` and a restart do not update either package: both are installed +into the image at build time, and nothing loads them from a bind mount. + +## Writing a migration + +Add a file named `YYYY-MM-DD-what-it-does.sql` and change `mysql/schema/` in +the same commit. The rules, each of which a test enforces: + +- **Idempotent.** Guard every statement on `information_schema`, so the file + can run against a database that already has the change, or has half of it. + DDL in MySQL commits statement by statement and cannot be rolled back as a + group; re-running the same file is the recovery from an interrupted run. +- **Starts by checking a database is selected.** The guards filter on + `TABLE_SCHEMA = DATABASE()`. With no default database, `DATABASE()` is NULL, + every guard matches nothing, and the file would exit 0 having done nothing. +- **Ends by checking its own result**, and fails if the intended state is not + there. +- **Never drops a value a row is using.** To change a `SET` or `ENUM`, add the + new member to the list the table has now (read it from + `information_schema`; do not restate it from memory), rewrite the rows, then + narrow. `2026-08-26-assay-type-genetic-variants.sql` is the pattern. A + restated `SET` that disagreed with the deployed table has truncated data in + this project before. +- **Stops, without changing anything, when the data cannot be carried over.** + The way to stop from a plain SQL script is to select from a table that does + not exist, with the message as the column name; every file here does that + for its pre- and postconditions. + +Then regenerate the expected schema and run the tests: -## Before running on montilab.bu.edu or sigrepo.org - -These two hosts hold data that cannot be regenerated locally. Before applying -any migration in this directory to either one: +```bash +bash tests/shell/test_expected_schema.sh --container --write +bash tests/shell/test_history_upgrade.sh --container +``` -1. Take a `mysqldump` of at least the `signatures` and `platforms` tables - (schema and data) and store it somewhere outside the database itself. -2. Confirm the dump completed and is non-empty before proceeding. +`tests/shell/testlib.sh` says how to start a disposable MySQL. +`test_history_upgrade.sh` builds every schema this repository has ever shipped, +runs all the migrations, and compares the result with a database built from +`mysql/schema/` today. A schema change with no migration fails there, because +the commit before it no longer converges. + +## When a migration stops on the data + +Two migrations can refuse a database because of what is stored in it. In both +cases nothing has been changed by that migration, and the fix is in the data. + +**signature_feature_set duplicates.** +`2026-09-13-signature-feature-set-keys.sql` stops with "signature_feature_set +repeats a feature under one probe" when one signature holds the same feature +under the same probe more than once, in different groups. The production key +does not allow that. Find the rows: + +```sql +SELECT signature_id, feature_id, assay_type, probe_id, COUNT(*) AS copies + FROM signature_feature_set + WHERE feature_id IS NOT NULL AND probe_id IS NOT NULL + GROUP BY signature_id, feature_id, assay_type, probe_id +HAVING COUNT(*) > 1; +``` -Only after that backup exists should the migration be applied, using the same -`mysql < file` invocation shown above, adjusted for that host's container -name and credentials. +Decide per signature which row is right, remove the others, and run the +migrations again. -## Verifying a run +**An assay type with no equivalent.** +`2026-08-26-assay-type-genetic-variants.sql` stops with "holds an assay_type +with no current equivalent" when a row's `assay_type` is `dna_binding_sites`, +which the first schema allowed and no later one does. Those signatures cannot +be represented in the current schema; they have to be removed or re-uploaded +under a current assay type before the migration can finish. -After applying a migration to any environment, before doing anything else -(in particular, before restarting the API), confirm it actually took effect. -The migration's own postcondition check already aborts the script if it did -not, but an operator relying on visual confirmation, or checking a run that -happened earlier, should run this directly: +## Rolling back -```bash -docker exec -i sigrepo-local-mysql mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" sigrepo -e \ - "SELECT COLUMN_NAME FROM information_schema.COLUMNS - WHERE TABLE_SCHEMA='sigrepo' AND TABLE_NAME IN ('signatures','platforms') - AND COLUMN_NAME IN ('type','platform','direction_type','platform_name');" -``` +`rollback/` holds the inverse of a migration where one was written. Apply it +by hand with the mysql client, then delete the migration's row from +`schema_migrations`, and run the code that matches the rolled back schema. The +runner never applies a file named `*-rollback.sql`, wherever it is. -(Substitute the container name and database name for BUMC or production.) - -Expected output after the forward migration is exactly two rows, `type` and -`platform`, with neither `direction_type` nor `platform_name` present. -Expected output after the rollback is exactly two rows, `direction_type` and -`platform_name`, with neither `type` nor `platform` present. Anything else, -including only one of the two renamed columns present, means the database is -in a mixed state; see "Recovering from a failure partway through" below -before doing anything further. - -## Deploying this migration: rebuild the image, don't just restart it - -Renaming these columns immediately breaks any running code that still -queries the old column names: the Plumber API, the MCP server, and the -legacy Shiny app in this repo (`SigRepo_Server`), plus the R client package -in the separate `SigRepo` repo, which now also requires `OmicSignature (>= -1.4.0)` (see `SigRepo/DESCRIPTION`) because building an `OmicSignature` -object with `type =` metadata is a hard validation error on any older -`OmicSignature`. The database migration and the code changes in both repos -must be deployed together -- but "deploy the code changes" is **not** the -same as "restart the containers", and getting that distinction wrong is -exactly what breaks `/signatures/compare`, the GEM enrichment route, and -Annotate on this rename. - -**What actually determines which client code runs**, verified directly -against `docker-compose-vm.yml` and the current `montilab/sigrepo:latest` -image (do not take this on faith -- re-check both if either changes): - -- `sigrepo-api`, `sigrepo-mcp`, and `sigrepo-shiny` all run - `image: montilab/sigrepo:latest`. That image's `Dockerfile` installs - `SigRepo` and `OmicSignature` with `remotes::install_github()` at build - time -- they are ordinary installed R packages baked into the image, not - loaded from a working copy by default. -- `docker-compose-vm.yml` bind-mounts `/SigRepo` (the `sigrepo-volume`) into - all three of those services, and `/OmicSignature` - (`omic-signature-volume`) into **only `sigrepo-shiny`** -- it appears - exactly once in the compose file, at the `sigrepo-shiny` service. Neither - `sigrepo-api` nor `sigrepo-mcp` mount `/OmicSignature` at all. -- A bind mount only changes which code runs if something actually loads - from it. `legacy_app/app_src/bootstrap.R` (Shiny) calls - `load_repo_package("OMICSIG_DIR", "OmicSignature")`, so Shiny at least - attempts to load OmicSignature from the mounted source. `api/api.R` and - `mcp/run_sigrepo_mcp.R` never call `load_repo_package` for OmicSignature - at all -- only for `SigRepo` -- so the API and MCP always run the - image-installed OmicSignature regardless of any mount. -- `load_repo_package()` itself only loads from a mounted source path if - `pkgload` or `devtools` is importable; otherwise it silently falls back to - `library(package_name)`, i.e. the image-installed version, with no - warning. Confirmed directly against the running `montilab/sigrepo:latest` - image: **neither `pkgload` nor `devtools` is installed.** - `SigRepo_Server/DESCRIPTION`'s `Imports` (what `install_r_packages.R` - installs) lists neither, `devtools` is `Suggests`-only, and the - `Dockerfile`'s explicit `remotes::install_github()` calls for - `SigRepo`/`OmicSignature`/`hypeR`/`hypeR-GEM` all pass - `dependencies = c('Depends','Imports','LinkingTo')`, which excludes - `Suggests`. -- The practical consequence: **the `/SigRepo` bind mount is inert in this - image for all three services, and the `/OmicSignature` bind mount is - inert too, including on `sigrepo-shiny`, which is the one service that - actually tries to use it.** A `git pull` on the host followed by only - restarting containers changes none of the R package code that - `sigrepo-api`, `sigrepo-mcp`, or `sigrepo-shiny` execute. (This is - different from `legacy_app/`, `api/`, and `mcp/` themselves, which - `shiny-server.sh`/`api-server.sh`/`mcp-server.sh` run directly off the - bind-mounted `/SigRepo_Server` tree as plain scripts, not as an installed - package -- `git pull` plus restart genuinely does update those.) -- Checked directly: the `montilab/sigrepo:latest` image pulled for this - repo's local stack currently has **OmicSignature 1.3.0** installed -- - already below the 1.4.0 floor this rename requires. Restarting that image - against a migrated database, with no rebuild, reproduces the exact - production incident recorded in the comment at the top of - `api/lib/omic_signature.R`: `build_omic_signature()` passes `type =` into - `OmicSignature$new()`, which is a hard validation error against - OmicSignature < 1.4.0, so `/signatures/compare` and the GEM enrichment - route fail on every call, and Annotate breaks wherever it builds a - signature the same way. - -**So the deploy step is:** - -1. Apply the migration and verify it per "Verifying a run" above. -2. Rebuild the `montilab/sigrepo` image from this branch's `Dockerfile` (or - re-pull it once CI has published a rebuilt image), so the image's - installed `SigRepo` and `OmicSignature` are both current -- `SigRepo` at - a release containing this rename's client changes, `OmicSignature` at - `>= 1.4.0`. Recreate `sigrepo-api`, `sigrepo-mcp`, and `sigrepo-shiny` - from that image (`docker compose up -d --force-recreate` or equivalent); - a plain `restart` reuses the already-running container's already-loaded - packages and, on `sigrepo-shiny`, its already-started R process, so it is - not sufficient even after the image is rebuilt. -3. As a backstop, not a substitute for the above: this branch adds a - boot-time check (`assert_omic_signature_version()` in `api/api.R` and - `mcp/run_sigrepo_mcp.R`) that refuses to start `sigrepo-api` or - `sigrepo-mcp` at all if the installed `OmicSignature` is older than - 1.4.0. This turns a silent per-request 500 into an immediate, loud - startup failure, but it does not make the mismatched image work -- the - fix is still to rebuild it. - -Do not leave a migrated database pointed at old code, and do not deploy new -code against an unmigrated database: the two must move together, per -environment. A given environment (local, BUMC, production) is migrated and -running a rebuilt image with matching code, or it is unmigrated and running -the old image; it should never sit in between. - -## Recovering from a failure partway through - -DDL statements in MySQL auto-commit individually and cannot be rolled back -as a group. If a script fails between its two `ALTER TABLE` statements (for -example, the `signatures` rename succeeds but something interrupts the -connection before the `platforms` rename runs), the database is left in a -genuine mixed state: one column renamed, the other not. - -This is expected to be recoverable, and the correct recovery is simply to -re-run the same script. Each of the two guarded blocks checks the current -state of its own column independently, so a block whose rename already -happened takes the `DO 0` no-op branch and a block whose rename did not -happen still performs it. Re-running is safe and idempotent even from a -partial failure; it does not need to be run from a fresh database or undone -first. - -Do **not** reach for a hand-written `CHANGE COLUMN` to patch up a mixed -state. `signatures.direction_type` (mid-migration, still under either name) -is a `SET(...)` column, and `CHANGE COLUMN` requires restating that -definition. Restating it by hand from memory or from the schema file is -exactly the mistake that has already truncated data once in this project's -history (see the comment in `mysql/schema/signatures.sql`), which is the -whole reason these migrations use `RENAME COLUMN` in the first place. -Re-running the provided script carries none of that risk. +For an installed instance the way back is `update_sigrepo.sh --restore`, which +reloads the dump taken before the update. ## Migrations in this directory -- `2026-09-25-rename-type-platform.sql` / - `2026-09-25-rename-type-platform-rollback.sql`: renames - `signatures.direction_type` to `signatures.type` and - `platforms.platform_name` to `platforms.platform`, and also renames the - `platforms` unique index from `platform_name` to `platform` so a migrated - database's index name matches what a fresh install from - `mysql/schema/platforms.sql` produces. Metadata-only (`RENAME COLUMN` and - `RENAME INDEX`, not `CHANGE COLUMN`); see the comments in the forward - script for why `CHANGE COLUMN` is avoided for the `SET(...)` column, and - for why the index needs its own explicit rename. +| File | Change | +| --- | --- | +| `2025-12-11-organisms-reference-columns.sql` | Adds the BioMart and proteomics reference columns to `organisms` | +| `2025-12-11-proteomics-unique-index-name.sql` | Names the unique index on `proteomics_features` `feature_organism` | +| `2026-01-29-widen-score-and-cutoff-precision.sql` | `NUMERIC(10,8)` to `NUMERIC(12,8)` for scores and cutoffs | +| `2026-02-25-create-genetic-variants-features.sql` | Creates `genetic_variants_features` | +| `2026-06-23-create-geneset-tables.sql` | Creates `geneset_resources` and `geneset_entries` | +| `2026-07-06-create-metabolite-tables.sql` | Creates `metabolite_reference`, `metabolite_xref`, `signature_feature_set_ambiguity` | +| `2026-08-19-gene-search-indexes.sql` | Adds the three indexes gene-content search needs | +| `2026-08-25-metabolite-reference-columns.sql` | Adds `refmet_id` and `hmdb_id`, drops the unused `chemical_name` | +| `2026-08-26-assay-type-genetic-variants.sql` | Makes `genetic_variants` the fifth `assay_type`, rewriting `snps` and `genetic_variations` | +| `2026-09-13-signature-feature-set-keys.sql` | Gives `signature_feature_set` the production keys, nullability and two new columns | +| `2026-09-25-rename-type-platform.sql` | Renames `signatures.direction_type` to `type` and `platforms.platform_name` to `platform` | + +All but the last were written on 2026-09-30, after the fact, for schema +changes that had shipped without a migration. Each is dated by the commit that +changed `mysql/schema/`, so they apply in the order the changes were made. + +Schema history in this repository starts on 2025-09-18. A database built +before that, or from one of the commits between 2025-12-11 and 2026-06-23 +whose schema files were not valid SQL, has a schema there is no record of. +`update_sigrepo.sh` detects that by comparing the migrated schema with +`mysql/expected_schema.tsv` and stops. From cc59665d2994f4815f9717751deb093da4668532 Mon Sep 17 00:00:00 2001 From: Cvicnaire Date: Wed, 30 Sep 2026 14:42:25 -0400 Subject: [PATCH 6/6] Stop the assay_type and chemical_name migrations on data they would otherwise lose A row whose assay_type held a current member beside snps was rewritten to genetic_variants alone, and chemical_name was dropped whether or not it held anything. Both now stop and say so, as every other migration does. Co-Authored-By: Claude Fable 5.1 --- ...026-08-25-metabolite-reference-columns.sql | 18 ++++++-- ...2026-08-26-assay-type-genetic-variants.sql | 15 +++--- mysql/migrations/README.md | 23 ++++++++-- tests/shell/test_history_upgrade.sh | 46 ++++++++++++++++++- 4 files changed, 86 insertions(+), 16 deletions(-) diff --git a/mysql/migrations/2026-08-25-metabolite-reference-columns.sql b/mysql/migrations/2026-08-25-metabolite-reference-columns.sql index 7a90204..ea885cb 100644 --- a/mysql/migrations/2026-08-25-metabolite-reference-columns.sql +++ b/mysql/migrations/2026-08-25-metabolite-reference-columns.sql @@ -9,10 +9,12 @@ -- -- `chemical_name` is dropped. It is a reference column no code has ever read -- or written, and keeping it would leave a migrated database different from a --- fresh one. +-- fresh one. It is dropped only while it holds nothing: a value someone put +-- there by hand is still a value, so if any row has one this migration stops +-- and says so instead of losing it. -- -- Idempotent: each column and index is added only if absent, and --- `chemical_name` is dropped only if present. +-- `chemical_name` is dropped only if present and empty. SET @precondition_sql := IF( DATABASE() IS NULL, @@ -73,8 +75,18 @@ DEALLOCATE PREPARE stmt; -- when its only column goes. SET @has := (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'metabolite_reference' AND COLUMN_NAME = 'chemical_name'); +-- Counted through a prepared statement, because a plain SELECT on the column +-- is an error on a database where it is already gone. SET @sql := IF(@has > 0, - 'ALTER TABLE `metabolite_reference` DROP COLUMN `chemical_name`', + 'SELECT COUNT(*) INTO @held FROM `metabolite_reference` WHERE `chemical_name` IS NOT NULL AND `chemical_name` <> ''''', + 'SET @held := 0'); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +SET @sql := IF(@has > 0, + IF(@held > 0, + 'SELECT `metabolite_reference.chemical_name holds values; move them before this column can go` FROM `migration_blocked`', + 'ALTER TABLE `metabolite_reference` DROP COLUMN `chemical_name`'), 'DO 0'); PREPARE stmt FROM @sql; EXECUTE stmt; diff --git a/mysql/migrations/2026-08-26-assay-type-genetic-variants.sql b/mysql/migrations/2026-08-26-assay-type-genetic-variants.sql index 93ee88d..1679ef5 100644 --- a/mysql/migrations/2026-08-26-assay-type-genetic-variants.sql +++ b/mysql/migrations/2026-08-26-assay-type-genetic-variants.sql @@ -9,10 +9,13 @@ -- since ..., 'genetic_variants' -- -- The first four members never changed. 'snps' and 'genetic_variations' both --- mean what is now 'genetic_variants', so rows holding either are rewritten. --- 'dna_binding_sites' has no equivalent: if any row holds it, this migration --- stops before changing the member list and says so, rather than let MySQL --- truncate the value to an empty string. +-- mean what is now 'genetic_variants', so a row holding exactly one of them +-- is rewritten. Anything else outside the final list stops the migration +-- before the member list changes: 'dna_binding_sites', which has no +-- equivalent, and a value with more than one member (SET columns allow it; +-- nothing in SigRepo writes one), which is not something to guess a meaning +-- for. Stopping is what keeps MySQL from truncating such a value to an +-- empty string, and this file from quietly dropping one of its members. -- -- mysql/migrations/README.md warns against restating a SET definition by -- hand, because the repository and a deployed table have disagreed on one @@ -53,7 +56,7 @@ DEALLOCATE PREPARE stmt; SET @sql := IF(@cur = @final, 'DO 0', 'UPDATE `signatures` SET `assay_type` = ''genetic_variants'' - WHERE FIND_IN_SET(''snps'', `assay_type`) > 0 OR FIND_IN_SET(''genetic_variations'', `assay_type`) > 0'); + WHERE `assay_type` IN (''snps'', ''genetic_variations'')'); PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; @@ -86,7 +89,7 @@ DEALLOCATE PREPARE stmt; SET @sql := IF(@cur = @final, 'DO 0', 'UPDATE `signature_feature_set` SET `assay_type` = ''genetic_variants'' - WHERE FIND_IN_SET(''snps'', `assay_type`) > 0 OR FIND_IN_SET(''genetic_variations'', `assay_type`) > 0'); + WHERE `assay_type` IN (''snps'', ''genetic_variations'')'); PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mysql/migrations/README.md b/mysql/migrations/README.md index 7a5effa..96bdcae 100644 --- a/mysql/migrations/README.md +++ b/mysql/migrations/README.md @@ -115,8 +115,9 @@ the commit before it no longer converges. ## When a migration stops on the data -Two migrations can refuse a database because of what is stored in it. In both -cases nothing has been changed by that migration, and the fix is in the data. +Three migrations can refuse a database because of what is stored in it. In +every case nothing has been changed by that migration, and the fix is in the +data. **signature_feature_set duplicates.** `2026-09-13-signature-feature-set-keys.sql` stops with "signature_feature_set @@ -138,9 +139,21 @@ migrations again. **An assay type with no equivalent.** `2026-08-26-assay-type-genetic-variants.sql` stops with "holds an assay_type with no current equivalent" when a row's `assay_type` is `dna_binding_sites`, -which the first schema allowed and no later one does. Those signatures cannot -be represented in the current schema; they have to be removed or re-uploaded -under a current assay type before the migration can finish. +which the first schema allowed and no later one does, or holds more than one +member (a `SET` allows `transcriptomics,snps`; nothing in SigRepo writes +that). Those signatures cannot be represented in the current schema; they +have to be removed or re-uploaded under a current assay type before the +migration can finish. Find them with +`SELECT signature_id, assay_type FROM signatures WHERE assay_type NOT IN +('transcriptomics','proteomics','metabolomics','methylomics','genetic_variants')`, +and the same on `signature_feature_set`. + +**A `chemical_name` that holds values.** +`2026-08-25-metabolite-reference-columns.sql` drops that column, which no +code has ever read, but only while it is empty. If it stops with +"chemical_name holds values", someone filled the column by hand; copy the +values wherever they belong (`refmet_name`, most likely), set the column to +NULL, and run the migrations again. ## Rolling back diff --git a/tests/shell/test_history_upgrade.sh b/tests/shell/test_history_upgrade.sh index 5fe88b8..9cb3c44 100644 --- a/tests/shell/test_history_upgrade.sh +++ b/tests/shell/test_history_upgrade.sh @@ -145,6 +145,9 @@ else bad "dna_binding_sites row (rc=$RC, value now '$KEPT')"; printf '%s\n' "$OUT" | tail -5 fi +# The 2026-07-07 shape: PRIMARY KEY (signature_id, group_label, probe_id). +GROUPED=$(git -C "$ROOT" log --format=%h --abbrev=8 -1 12ed4be1 2>/dev/null) + echo "--- a row holding two assay types stops the migration ---" # SET columns can hold several members. Nothing in SigRepo writes more than @@ -167,6 +170,47 @@ else bad "two-member row (rc=$RC, value now '$KEPT')"; printf '%s\n' "$OUT" | tail -5 fi +echo "--- a row holding an old member beside another stops the migration too ---" + +# 'transcriptomics,snps' contains a member that has a current equivalent and +# one that does not need one. Rewriting the whole value to genetic_variants +# would silently drop transcriptomics; the migration must stop instead. +fresh_db "$OLD" +schema_at "$GROUPED" > "$WORK/schema.sql" +db_load "$OLD" < "$WORK/schema.sql" +db_load "$OLD" <<'SQL' +SET FOREIGN_KEY_CHECKS=0; +INSERT INTO signature_feature_set + (signature_id, feature_id, probe_id, score, group_label, assay_type, sig_feature_hashkey) +VALUES (1, 10, 'p10', 1.0, 'All Features', 'transcriptomics,snps', 'h1'); +SQL +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +KEPT=$(db_sql "$OLD" "SELECT assay_type FROM signature_feature_set") +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "FAILED 2026-08-26-assay-type-genetic-variants.sql" \ + && [ "$KEPT" = "transcriptomics,snps" ]; then + ok "the run stops and the row still holds both members" +else + bad "member beside snps (rc=$RC, value now '$KEPT')"; printf '%s\n' "$OUT" | tail -5 +fi + +echo "--- a chemical_name that holds data stops the metabolite migration ---" + +# The column is dropped because no code ever read it; a value someone put +# there by hand is still a value, and dropping it would lose it silently. +METABOLITE=$(git -C "$ROOT" log --format=%h --abbrev=8 -1 ea7c3522 2>/dev/null) +fresh_db "$OLD" +schema_at "$METABOLITE" > "$WORK/schema.sql" +db_load "$OLD" < "$WORK/schema.sql" +db_sql "$OLD" "INSERT INTO metabolite_reference (chemical_name, version, metabolite_hashkey) VALUES ('glucose', 1, 'm1')" +OUT=$(bash "$MIGRATE" "${SCRIPT_ARGS[@]}" 2>&1); RC=$? +KEPT=$(db_sql "$OLD" "SELECT chemical_name FROM metabolite_reference") +if [ "$RC" -eq 1 ] && printf '%s' "$OUT" | grep -q "FAILED 2026-08-25-metabolite-reference-columns.sql" \ + && [ "$KEPT" = "glucose" ]; then + ok "the run stops and the column and its value are still there" +else + bad "chemical_name with data (rc=$RC, value now '$KEPT')"; printf '%s\n' "$OUT" | tail -5 +fi + echo "--- a key change interrupted half way finishes on a re-run ---" # DDL commits statement by statement, so a dropped connection can leave the @@ -186,8 +230,6 @@ fi echo "--- a feature repeated under one probe stops the key change ---" -# The 2026-07-07 shape: PRIMARY KEY (signature_id, group_label, probe_id). -GROUPED=$(git -C "$ROOT" log --format=%h --abbrev=8 -1 12ed4be1 2>/dev/null) fresh_db "$OLD" schema_at "$GROUPED" > "$WORK/schema.sql" db_load "$OLD" < "$WORK/schema.sql"