diff --git a/.formatter.exs b/.formatter.exs index d2cda26..70c39b3 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,5 @@ # Used by "mix format" [ + import_deps: [:ecto, :ecto_sql, :phoenix], inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9b2afb2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,70 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + mix_test: + runs-on: ubuntu-latest + env: + MIX_ENV: test + services: + pg: + image: postgres:18 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: data_migration_test + TZ: UTC + ports: + - 15435:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + mysql: + image: mysql:8 + env: + MYSQL_DATABASE: data_migration_test + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + MYSQL_ROOT_PASSWORD: mysql + ports: + - 13306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + mssql: + image: mcr.microsoft.com/mssql/server:2022-latest + env: + ACCEPT_EULA: Y + MSSQL_PID: Developer + MSSQL_SA_PASSWORD: StrongPassword! + ports: + - 11433:1433 + options: >- + --health-cmd="/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'StrongPassword!' -C -Q 'SELECT 1'" + --health-interval 10s + --health-retries 10 + --health-timeout 5s + --health-start-period 10s + + steps: + - uses: actions/checkout@v4 + - uses: erlef/setup-beam@v1 + with: + elixir-version: '1.18' + otp-version: '27' + - run: rm mix.lock && mix deps.get --only test + - run: mix compile --force --warnings-as-errors + - run: mix format --check-formatted + - run: ECTO_ADAPTER=sqlite mix test --warnings-as-errors + - run: ECTO_ADAPTER=pg mix test --warnings-as-errors + - run: ECTO_ADAPTER=myxql mix test --warnings-as-errors + - run: ECTO_ADAPTER=tds mix test --warnings-as-errors diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..816075c --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "tidewave": { + "type": "http", + "url": "http://localhost:4011/tidewave/mcp" + } + } +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2ecd541 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,133 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is an Elixir library that provides a Phoenix LiveDashboard page for managing Ecto data migrations. It allows viewing, running, and monitoring data migrations with real-time log streaming through Phoenix PubSub. + +## Multi-Database Testing Architecture + +The project supports multiple database adapters (PostgreSQL, MySQL, MSSQL, SQLite) and uses environment variables to switch between them during testing: + +- **ECTO_ADAPTER environment variable** controls which database adapter tests run against + - `pg` - PostgreSQL (Test.PGRepo) + - `myxql` - MySQL (Test.MyXQLRepo) + - `tds` - SQL Server/MSSQL (Test.TDSRepo) + - `sqlite` or unset - SQLite (Test.SQLiteRepo, default) + +- **Database connection details** for local testing (via docker-compose): + - PostgreSQL: localhost:15435 + - MySQL: localhost:13306 + - SQL Server: localhost:11433 + +## Common Commands + +### Testing + +Run tests for a specific database adapter: +```bash +ECTO_ADAPTER=sqlite mix test +ECTO_ADAPTER=pg mix test +ECTO_ADAPTER=myxql mix test +ECTO_ADAPTER=tds mix test +``` + +Run tests for all adapters sequentially: +```bash +mix test.all +``` + +Run a single test file: +```bash +ECTO_ADAPTER=pg mix test test/data_migration/live_dashboard/page_test.exs +``` + +### Development + +Start database containers (required before running tests): +```bash +docker-compose up -d +``` + +Stop database containers: +```bash +docker-compose down +``` + +Format code: +```bash +mix format +``` + +Compile with warnings as errors: +```bash +mix compile --force --warnings-as-errors +``` + +Start the development dashboard (uses Tidewave at port 4011): +```bash +mix tidewave +``` + +### Dependencies + +Get dependencies: +```bash +mix deps.get +``` + +## Architecture + +### Core Components + +1. **DataMigration.LiveDashboard.Page** (`lib/data_migration/live_dashboard/page.ex`) + - Phoenix LiveView page that integrates with LiveDashboard + - Lists all data migrations from configured Ecto repos and folders + - Provides UI to run migrations up/down with confirmation + - Streams logs in real-time via PubSub as migrations execute + - Uses `Ecto.Migrator` for migration discovery and execution + - Caches migration list in `:persistent_term` for performance + - In dev mode, automatically recompiles migrations on each page load + +2. **DataMigration.Logger** (`lib/data_migration/logger.ex`) + - Custom logger backend that captures Ecto migration logs + - Implements both `:gen_event` (legacy) and `:logger_handler` (OTP 21+) behaviors + - Filters logs by MFA (module/function/arity) patterns + - Broadcasts captured logs to Phoenix PubSub topic + - Automatically captures logs from: `Ecto.Adapters.SQL`, `Ecto.Migration.Runner`, `Ecto.Migrator` + +### Configuration Pattern + +The LiveDashboard page is configured in the Phoenix router with a 3-tuple: +```elixir +{PubSubServer, %{Repo => [migration_folders]}, options} +``` + +Example: +```elixir +{MyApp.PubSub, %{MyApp.Repo => ["data_migrations"]}, [topic: "custom-topic"]} +``` + +### Migration Discovery + +- Uses `Ecto.Migrator.migrations/3` to get migration status +- Compiles migration files dynamically with `Code.require_file/2` +- Extracts metadata: id, name, file path, status (:up or :down) +- Matches migration files by glob pattern: `*.exs` in configured folders + +## Test Structure + +- **test/support/** contains test repos for each database adapter +- **test/support/conn_case.ex** provides test helpers +- **test/support/endpoint.ex** is a minimal Phoenix endpoint for testing +- Tests are adapter-specific and selected at runtime via environment variable +- CI runs the full test suite against all adapters sequentially + +## Important Development Notes + +- When adding Ecto-related features, ensure compatibility with all four adapters (pg, myxql, tds, sqlite) +- The logger captures logs based on MFA patterns - be mindful of what gets captured +- Migration status is queried on each page load but cached in persistent_term +- The page requires `allow_destructive_actions: true` in LiveDashboard config to enable migration execution +- ANSI color codes are stripped from logs before displaying in the UI diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..44c649c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## unreleased + +- Initial release diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index bc043ba..55377de 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,55 @@ -# DataMigration + -**TODO: Add description** +[![Hex.pm Version](http://img.shields.io/hexpm/v/data_migration.svg?style=flat&logo=elixir)](https://hex.pm/packages/data_migration) +[![Hex docs](http://img.shields.io/badge/hexdocs.pm/data_migration-blue.svg?logo=elixir)](https://hexdocs.pm/data_migration) +[![License](http://img.shields.io/hexpm/l/data_migration.svg?style=flat)](./LICENSE) -## Installation +# Data Migration -If [available in Hex](https://hex.pm/docs/publish), the package can be installed -by adding `data_migration_live_dashboard` to your list of dependencies in `mix.exs`: +You're reading the main branch's readme. Please visit +[hexdocs](https://hexdocs.pm/data_migration) for the latest published documentation. + + + +View [Ecto](https://hexdocs.pm/ecto_sql) Data Migrations and run them from a [Phoenix LiveDashboard](https://hexdocs.pm/phoenix_live_dashboard) page. Streams logs as +the data migrations runs to the dashboard. + +For example, in your Phoenix router: ```elixir -def deps do - [ - {:data_migration_live_dashboard, "~> 0.1.0"} +live_dashboard "/my/admin/dashboard", + # must have `allow_destructive_actions: true` in order to run data migrations + # otherwise it will be view-only to see the status + allow_destructive_actions: true, + # Provide the page with Repo and migration folders config + additional_pages: [ + # so the route becomes "/my/admin/dashboard/data_migrations" + data_migrations: { + DataMigration.LiveDashboard.Page, + {MyApp.PubSub, %{MyApp.Repo => ["data_migrations"]}, options} + # These paths will be passed into `Ecto.Migrator.migrations_path(repo, path)` + # `options` is optional; you may supply 2 item tuple instead to omit options + } ] -end ``` -Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) -and published on [HexDocs](https://hexdocs.pm). Once published, the docs can -be found at . +Options you may supply to the page: + +- `:topic` a different PubSub topic to listen to for capturing migration logs. +- `:listen_for_logs` A list of MFAs (tuple of length 1, 2, or 3) for which the page to listen for logs. + You can also supply a module namespace, eg, `MyApp.DataMigration` and any module under that namespace + will have its logs listened to, eg `MyApp.DataMigration.FooBar`. By default, + the app will listen to `Ecto.Adapters.SQL`, Ecto.Migration.Runner, and `Ecto.Migrator` for logs. + + +Requires OTP 27+ + +### Screenshots + +![Migration List](./assets/migration-list.png) + +![Migration Show](./assets/migration-show.png) + +![Migration Ran with Logs](./assets/logs.png) +![Migration Ran with Logs and errors](./assets/logs-with-error.png) diff --git a/assets/logs-with-error.png b/assets/logs-with-error.png new file mode 100644 index 0000000..484d978 Binary files /dev/null and b/assets/logs-with-error.png differ diff --git a/assets/logs.png b/assets/logs.png new file mode 100644 index 0000000..a18eeec Binary files /dev/null and b/assets/logs.png differ diff --git a/assets/migration-list.png b/assets/migration-list.png new file mode 100644 index 0000000..a89019f Binary files /dev/null and b/assets/migration-list.png differ diff --git a/assets/migration-show.png b/assets/migration-show.png new file mode 100644 index 0000000..94e99c0 Binary files /dev/null and b/assets/migration-show.png differ diff --git a/bin/release b/bin/release new file mode 100755 index 0000000..2b4d9cb --- /dev/null +++ b/bin/release @@ -0,0 +1,22 @@ +#!/bin/sh +# Usage: ./bin/release {old_version} {new_version} +set -e + +previous_version="${1}" +release_version="${2}" + +mix test + +sed -i "" "s/$previous_version/$release_version/" README.md +sed -i "" "s/$previous_version/$release_version/" mix.exs +sed -i "" "s/unreleased/$release_version ($(date +%F))/" CHANGELOG.md +git add mix.exs +git add README.md +git add CHANGELOG.md + +git commit +git tag -a "$release_version" -m "Release version $release_version" +git push origin "$release_version" + +mix hex.build +mix hex.publish diff --git a/docker-compose.yml b/docker-compose.yml index 2238c96..a5f07e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,10 +38,10 @@ services: sqlserver: container_name: data_migration_mssql - image: mcr.microsoft.com/mssql/server:2019-latest + image: mcr.microsoft.com/mssql/server:2022-latest environment: ACCEPT_EULA: Y - SA_PASSWORD: StrongPassword! + MSSQL_SA_PASSWORD: StrongPassword! MSSQL_PID: Developer ports: - 11433:1433 @@ -49,7 +49,7 @@ services: - sqlserver_data:/var/opt/mssql restart: always healthcheck: - test: ["CMD-SHELL", "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'StrongPassword!' -Q 'SELECT 1' || exit 1"] + test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'StrongPassword!' -C -Q 'SELECT 1' || exit 1"] interval: 10s retries: 10 start_period: 10s diff --git a/lib/data_migration.ex b/lib/data_migration.ex index f6e13c1..2ef02f0 100644 --- a/lib/data_migration.ex +++ b/lib/data_migration.ex @@ -1,5 +1,5 @@ defmodule DataMigration do @moduledoc """ - Documentation for `DataMigration`. + See `DataMigration.LiveDashboard.Page` """ end diff --git a/lib/data_migration/live_dashboard/page.ex b/lib/data_migration/live_dashboard/page.ex index 0037385..33401d0 100644 --- a/lib/data_migration/live_dashboard/page.ex +++ b/lib/data_migration/live_dashboard/page.ex @@ -1,31 +1,9 @@ defmodule DataMigration.LiveDashboard.Page do - @moduledoc """ - The page to view data migrations. - - For example, in your Phoenix router: - - live_dashboard "/my/admin/dashboard", - # must have `allow_destructive_actions: true` in order to run data migrations - # otherwise it will be view-only to see the status - allow_destructive_actions: true, - # Provide the page with Repo and migration folders config - additional_pages: [ - # so the route becomes "/my/admin/dashboard/data_migrations" - data_migrations: { - DataMigration.LiveDashboard.Page, - {MyApp.PubSub, %{MyApp.Repo => ["data_migrations"]}, options} - # These paths will be passed into `Ecto.Migrator.migrations_path(repo, path)` - # `options` is optional; you may supply 2 item tuple instead to omit options - } - ] - - Options you may supply to the page: - - - `:topic` a different PubSub topic to listen to for capturing migration logs. - - `:listen_for_logs` A list of MFAs (tuple of length 1, 2, or 3) for which the page to listen for logs. - You can also supply a module namespace, eg, `MyApp.DataMigration` and any module under that namespace - will have its logs listened to, eg `MyApp.DataMigration.FooBar` - """ + @external_resource "README.md" + @moduledoc "README.md" + |> File.read!() + |> String.split("") + |> Enum.fetch!(1) use Phoenix.LiveDashboard.PageBuilder @@ -238,7 +216,7 @@ defmodule DataMigration.LiveDashboard.Page do attr(:stream, :any, required: true) attr(:logs_present, :boolean, default: false) - def event_logs(assigns) do + defp event_logs(assigns) do ~H"""