diff --git a/README.md b/README.md index ce6dc434..dfa209dd 100644 --- a/README.md +++ b/README.md @@ -469,9 +469,9 @@ The JDBC driver and materialized views work on **free/basic Elasticsearch cluste |---------------------------------------------|-----------------------------------| | Transforms (continuous data sync) | Free / Basic (ES 7.5+) | | Enrich Policies (JOIN enrichment) | Free / Basic (ES 7.5+) | -| **Watchers** (auto-refresh enrich policies) | **Platinum / Enterprise / Trial** | +| **Watcher** (auto-refresh enrich policies) | **Trial, or a [subscription that includes it](https://www.elastic.co/subscriptions)** | -Materialized views with JOINs rely on **Elasticsearch Watchers** to automatically re-execute enrich policies when lookup table data changes. Without a Platinum ES license, this automation is unavailable — but an external scheduler (cron, Kubernetes CronJob, Airflow) can be used as a workaround. See the [Materialized Views documentation](documentation/sql/materialized_views.md#watcher-dependency-and-elasticsearch-licensing) for details. +Materialized views with JOINs rely on **Elasticsearch Watcher** to automatically re-execute enrich policies when lookup table data changes. On a Basic cluster, `CREATE MATERIALIZED VIEW` **still succeeds** and returns a warning: the view is created and immediately queryable, and `REFRESH MATERIALIZED VIEW ` — which re-executes exactly what the watcher would have — works on every license. Only the *scheduled* refresh is unavailable; run it from an external scheduler (cron, Kubernetes CronJob, Airflow) instead. See the [Materialized Views documentation](documentation/sql/materialized_views.md#watcher-dependency-and-elasticsearch-licensing) for details. --- diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala index b6e603d3..77200593 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/JsonFormatter.scala @@ -53,10 +53,16 @@ object JsonFormatter { ) pretty(render(json)) - case DdlResult(success) => + case DdlResult(success, warnings) => + // `warnings` is omitted entirely when empty so existing consumers see no new key. val json = JObject( - "success" -> JBool(success), - "execution_time_ms" -> JInt(executionTime.toMillis) + List[(String, JValue)]( + "success" -> JBool(success), + "execution_time_ms" -> JInt(executionTime.toMillis) + ) ++ ( + if (warnings.isEmpty) Nil + else List("warnings" -> JArray(warnings.map(JString(_)).toList)) + ) ) pretty(render(json)) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala index b6ff6966..874eb243 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/ResultRenderer.scala @@ -57,8 +57,8 @@ object ResultRenderer { case DmlResult(inserted, updated, deleted, rejected) => renderDml(inserted, updated, deleted, rejected, executionTime) - case DdlResult(success) => - renderDdl(success, executionTime) + case DdlResult(success, warnings) => + renderDdl(success, warnings, executionTime) case TableResult(table) => renderTableDefinition(table) @@ -195,12 +195,19 @@ object ResultRenderer { // ==================== DDL Rendering ==================== - private def renderDdl(success: Boolean, executionTime: Duration): String = { - if (success) { - s"${emoji("✅")} ${green("Success")} ${gray(s"(${executionTime.toMillis}ms)")}" - } else { - s"${emoji("ℹ️")} ${gray("No changes")} ${gray(s"(${executionTime.toMillis}ms)")}" - } + private def renderDdl( + success: Boolean, + warnings: Seq[String], + executionTime: Duration + ): String = { + val head = + if (success) { + s"${emoji("✅")} ${green("Success")} ${gray(s"(${executionTime.toMillis}ms)")}" + } else { + s"${emoji("ℹ️")} ${gray("No changes")} ${gray(s"(${executionTime.toMillis}ms)")}" + } + // One line per warning, below the outcome: the statement SUCCEEDED, the caveat is secondary. + head + warnings.map(warning => s"\n${emoji("⚠️")} ${yellow(warning)}").mkString } private def renderEmpty(): String = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index e9c4d8a6..d35b02d0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -467,7 +467,34 @@ package object result { // -------------------- // DDL (CREATE / ALTER / DROP / TRUNCATE) // -------------------- - case class DdlResult(success: Boolean) extends QueryResult + + /** The outcome of a DDL statement. + * + * `warnings` carries caveats about a statement that did '''not''' fail — an operation that + * completed, but in a degraded mode. The motivating case (story R1FIX.8) is a materialized view + * created without its auto-refresh watcher because the cluster's licence does not include + * Watcher: the view exists and is queryable, only the scheduled refresh is missing, so failing + * the statement would discard real work. Renderers surface each entry as its own `⚠️` line. + * + * Warnings are rendered for `success = false` too. That combination is unusual — a no-op has + * normally nothing to caveat — but it is not a contract violation, and dropping the text there + * would silently lose information; renderers therefore do not special-case it. A genuine + * '''failure''' is an `ElasticFailure`, never a `DdlResult`, so nothing here is ever an error + * message. + * + * Follows the same additive, default-valued shape as `QueryRows.truncation` (story P0.5): every + * existing `DdlResult(ok)` '''constructor''' call keeps compiling untouched. Positional + * '''pattern''' matches (`case DdlResult(ok)`) must add a trailing `_` (`case DdlResult(ok, _)`) + * — the compiler's synthetic extractor now binds both fields. Matches that only need "is this a + * DDL result" are better written `case _: DdlResult`, which no future field can break. + * + * @param success + * `false` means the statement was a no-op (e.g. `IF NOT EXISTS` on an existing object), not a + * failure — a failure is an `ElasticFailure`, never a `DdlResult` + * @param warnings + * user-facing caveats about a successful statement; empty for the overwhelmingly common case + */ + case class DdlResult(success: Boolean, warnings: Seq[String] = Nil) extends QueryResult case class TableResult(table: Table) extends QueryResult diff --git a/core/src/test/scala/app/softnetwork/elastic/client/result/JsonFormatterSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/result/JsonFormatterSpec.scala new file mode 100644 index 00000000..82b954dd --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/result/JsonFormatterSpec.scala @@ -0,0 +1,32 @@ +package app.softnetwork.elastic.client.result + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.concurrent.duration._ + +class JsonFormatterSpec extends AnyFlatSpec with Matchers { + + behavior of "JsonFormatter" + + it should "emit no warnings key for a plain DDL result" in { + val output = JsonFormatter.format(DdlResult(success = true), 20.millis) + println(output) + + output should include("\"success\"") + // Story R1FIX.8 — existing consumers must see exactly the keys they saw before. + output should not include "warnings" + } + + it should "emit DDL warnings as a JSON array when present" in { + val output = JsonFormatter.format( + DdlResult(success = true, warnings = Seq("first caveat", "second caveat")), + 20.millis + ) + println(output) + + output should include("\"warnings\"") + output should include("first caveat") + output should include("second caveat") + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/result/ResultRendererSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/result/ResultRendererSpec.scala index c33cd2c8..0043f7fc 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/result/ResultRendererSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/result/ResultRendererSpec.scala @@ -53,6 +53,60 @@ class ResultRendererSpec extends AsyncFlatSpec with Matchers { output should include("Success") } + // Story R1FIX.8 — a DDL statement may succeed in a degraded mode and say so. + // NB: the renderer is fansi-coloured unconditionally, so an escape sequence sits between the + // emoji and the word. Assert on words, never on "✅ Success" / "⚠️ …" as a single literal. + + it should "render DDL success without a warning line when there are no warnings" in { + val output = ResultRenderer.render(DdlResult(success = true), 20.millis) + println(output) + + output should include("Success") + output should not include "⚠️" + output.linesIterator.size shouldBe 1 + } + + it should "render a DDL warning below the success line" in { + val warning = + "Materialized view 'orders_mv' was created, but automatic refresh is unavailable: " + + "run 'REFRESH MATERIALIZED VIEW orders_mv' whenever the joined tables change." + val output = + ResultRenderer.render(DdlResult(success = true, warnings = Seq(warning)), 20.millis) + println(output) + + // The statement still reports success — a caveat must not read as a failure. + output should include("Success") + output should include("⚠️") + output should include("REFRESH MATERIALIZED VIEW orders_mv") + + val lines = output.linesIterator.toSeq + lines.size shouldBe 2 + lines.head should include("Success") + lines(1) should include("automatic refresh is unavailable") + } + + it should "render one line per DDL warning" in { + val output = ResultRenderer.render( + DdlResult(success = true, warnings = Seq("first caveat", "second caveat")), + 20.millis + ) + println(output) + + val lines = output.linesIterator.toSeq + lines.size shouldBe 3 + lines(1) should include("first caveat") + lines(2) should include("second caveat") + } + + it should "render a DDL warning on a no-op result too" in { + val output = + ResultRenderer.render(DdlResult(success = false, warnings = Seq("a caveat")), 20.millis) + println(output) + + output should include("No changes") + output should include("a caveat") + } + it should "format values correctly" in { val rows = Seq( ListMap( diff --git a/documentation/sql/materialized_views.md b/documentation/sql/materialized_views.md index 29208b69..3440190e 100644 --- a/documentation/sql/materialized_views.md +++ b/documentation/sql/materialized_views.md @@ -128,8 +128,9 @@ WHERE status = 'active'; This creates: - One transform (source → view) — no changelog transform, no enrich policy, no ingest pipeline -- **No watcher** — a single-table view is therefore the one materialized-view shape that runs on a - **basic** Elasticsearch licence (see [Watcher Dependency and Elasticsearch Licensing](#watcher-dependency-and-elasticsearch-licensing)) +- **No watcher** — nothing needs re-executing on a schedule, so a single-table view never touches + Watcher at all and needs no automatic refresh (see + [Watcher Dependency and Elasticsearch Licensing](#watcher-dependency-and-elasticsearch-licensing)) - The view index `active_orders_mv` A single-table view whose `SELECT` has no `WHERE`, no `GROUP BY` and no aggregation is also accepted — @@ -507,7 +508,7 @@ DROP MATERIALIZED VIEW IF EXISTS orders_with_customers_mv; | **UNNEST JOIN** | Not supported in materialized views | | **`RIGHT JOIN` / `FULL OUTER JOIN`** | Not supported (see below). Use `LEFT JOIN` with swapped table order. | | **Quota limits** | Community: 1 view · Pro: 50 · Enterprise: unlimited | -| **Watcher dependency (ES license)** | Automatic enrich policy re-execution relies on Elasticsearch Watchers, which require an Elasticsearch Platinum or Enterprise license (see below) | +| **Watcher dependency (ES license)** | Automatic enrich policy re-execution relies on Elasticsearch Watcher, which the free Basic license does not include. The view is still created and `REFRESH MATERIALIZED VIEW` still works (see below) | | **Eventual consistency** | Data is eventually consistent based on refresh frequency and delay | | **Join cardinality** | JOINs use enrich policies which match on a single field | @@ -526,13 +527,26 @@ Attempting to create a materialized view with `RIGHT JOIN` or `FULL OUTER JOIN` Materialized views with JOINs rely on **enrich policies** to denormalize data from lookup tables into the view. When data in a lookup table (e.g. `customers`) changes, the corresponding enrich policy must be **re-executed** so that new documents flowing through the ingest pipeline pick up the updated values. -To automate this re-execution, the engine creates an **Elasticsearch Watcher** that periodically triggers `EXECUTE ENRICH POLICY` calls. However, **Watchers require an Elasticsearch Platinum or Enterprise license** (or an active Trial license). This is an Elasticsearch-side requirement, independent of the JDBC driver license. +To automate this re-execution, the engine creates an **Elasticsearch Watcher** that periodically triggers `EXECUTE ENRICH POLICY` calls. However, **Watcher is not included in the free Basic license** — it requires a subscription that includes it, or an active Trial license. See [Elastic's subscription matrix](https://www.elastic.co/subscriptions) for the current tier that first offers Watcher. This is an Elasticsearch-side requirement, independent of the JDBC driver license. **Impact:** -- **With Elasticsearch Platinum/Enterprise/Trial license**: Fully automatic — the watcher handles enrich policy re-execution transparently -- **Without Elasticsearch Platinum license**: The watcher cannot be created. Changes to lookup tables will **not** be reflected in the materialized view until the enrich policies are manually re-executed +- **With a license that includes Watcher (Trial, or a paid subscription)**: fully automatic — the watcher re-executes the enrich policies transparently. +- **Without it (the free Basic license)**: `CREATE MATERIALIZED VIEW` **still succeeds** and returns a warning. The view is created, its metadata is persisted and it is immediately queryable — only the *automatic* refresh is unavailable. `SHOW MATERIALIZED VIEW ` then reports `auto_refresh` as `unavailable: …` and `watcher_id` as `N/A`. Changes to lookup tables are **not** reflected until the enrich policies are re-executed, which is exactly what `REFRESH MATERIALIZED VIEW ` does — it always works, on every license. -**Workaround for clusters without Watcher support:** +The warning looks like this: + +``` +✅ Success (14203ms) +⚠️ Materialized view 'orders_with_customers_mv' was created, but automatic refresh is + unavailable: this Elasticsearch cluster's licence does not include Watcher, so the joined + data cannot be refreshed on a schedule. Run 'REFRESH MATERIALIZED VIEW + orders_with_customers_mv' whenever the joined tables change, or schedule that statement + externally (cron, Kubernetes CronJob, Airflow). Elasticsearch reported: … +``` + +A view created this way keeps `auto_refresh: unavailable` even if the cluster is later upgraded to a license that includes Watcher — re-run `CREATE OR REPLACE MATERIALIZED VIEW` with a changed definition to redeploy it with a watcher. + +**Refreshing a view on a cluster without Watcher:** Use an external scheduled job (cron, Kubernetes CronJob, Airflow, etc.) to periodically re-execute the enrich policies via SQL: @@ -544,7 +558,7 @@ EXECUTE ENRICH POLICY orders_with_customers_mv_customers_enrich_policy; REFRESH MATERIALIZED VIEW orders_with_customers_mv; ``` -Note that **transforms** (which power the continuous data sync) and **enrich policies** themselves are available in the free/basic Elasticsearch license starting from ES 7.5+. Only the **Watcher** component requires a paid Elasticsearch license. +Note that **transforms** (which power the continuous data sync), **enrich policies** and **ingest pipelines** are all available in the free/basic Elasticsearch license starting from ES 7.5+. The **Watcher** component is the only part of a materialized-view deployment that a Basic license refuses — which is why the view itself is still created. --- diff --git a/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala b/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala index ce29f385..3c2391a1 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala @@ -47,6 +47,16 @@ trait ElasticDockerTestKit extends ElasticTestKit { _: Suite => lazy val xpackGraphEnabled: Boolean = false + /** Self-generated licence type forced into `elasticsearch.yml`. + * + * Defaults to `"trial"`, which enables every X-Pack feature. Override to `"basic"` to exercise + * the paths a licence without Watcher refuses — `CREATE MATERIALIZED VIEW` on a joined view + * being the motivating one (softclient4es-extensions story R1FIX.8). Until this knob existed the + * licence type was hard-coded here, so no test in any repository had ever run against a cluster + * that declines a feature. + */ + lazy val xpackLicenseType: String = "trial" + lazy val elasticContainer: ElasticsearchContainer = { val tmpDir = if (localExecution) { @@ -101,8 +111,8 @@ trait ElasticDockerTestKit extends ElasticTestKit { _: Suite => |# Discovery |discovery.type: single-node | - |# X-Pack License (force Trial license) - |xpack.license.self_generated.type: trial + |# X-Pack License (forced; see xpackLicenseType) + |xpack.license.self_generated.type: $xpackLicenseType | |# X-Pack Security (disabled for tests) |xpack.security.enabled: $xpackSecurityEnabled