feat: replace Elasticsearch with PostgreSQL full-text search (PPT-2644, 2/2)

docker-compose.yml

@@ -3,10 +3,6 @@ x-deployment-env: &deployment-env
   ENV: ${ENV:-development}
   SG_ENV: ${SG_ENV:-development}
 
-x-elastic-client-env: &elastic-client-env
-  ELASTIC_HOST: ${ELASTIC_HOST:-elastic}
-  ELASTIC_PORT: ${ELASTIC_PORT:-9200}
-
 x-redis-client-env: &redis-client-env
   REDIS_URL: ${REDIS_URL:-redis://redis:6379}
 
@@ -19,9 +15,6 @@ x-postgresdb-client-env: &postgresdb-client-env
   PG_DATABASE: ${PG_DATABASE:-place_development}
   PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/place_development}
 
-x-search-ingest-client-env: &search-ingest-client-env
-  SEARCH_INGEST_URI: ${SEARCH_INGEST_URI:-http://search-ingest:3000}
-
 x-s3-client-env: &s3-client-env
   AWS_REGION: ${AWS_REGION:-"us-east-1"}
   AWS_KEY: ${AWS_KEY:-"root"}
@@ -47,11 +40,9 @@ services:
       - ${PWD}/coverage:/app/coverage
     depends_on:
       - core
-      - elastic
       - redis
       - postgres
       - migrator
-      - search-ingest
     security_opt:
       - seccomp:unconfined
     environment:
@@ -60,25 +51,10 @@ services:
       <<: [
         *deployment-env,
         # Service Hosts
-        *elastic-client-env,
         *redis-client-env,
         *postgresdb-client-env,
       ]
 
-  elastic:
-    image: elasticsearch:${ELASTIC_VERSION:-7.17.6}
-    restart: always
-    hostname: elastic
-    healthcheck:
-      test: wget -q --no-verbose --tries=1 --spider http://localhost:9200/_cat/health
-    environment:
-      discovery.type: single-node
-      ES_JAVA_OPTS: -Xms1g -Xmx1g
-      http.cors.enabled: "true"
-      http.cors.allow-origin: http://localhost:8080
-    ports:
-      - 9200:9200
-
   redis:
     image: eqalpha/keydb
     restart: always
@@ -114,24 +90,6 @@ services:
       PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/place_development}
 
 
-  search-ingest: # PostgreSQL to Elasticsearch Service
-    image: placeos/search-ingest:nightly
-    restart: always
-    hostname: search-ingest
-    depends_on:
-      - elastic
-      - migrator
-      - postgres
-    environment:
-      LOG_LEVEL: trace
-      # Service Hosts
-      <<: [
-        *postgresdb-client-env,
-        *elastic-client-env,
-        # Environment
-        *deployment-env,
-      ]
-
   core: # Module coordinator
     image: placeos/core:nightly
     restart: always

shard.lock

@@ -223,7 +223,7 @@ shards:
 
   placeos-models:
     git: https://github.com/placeos/models.git
-    version: 9.105.2
+    version: 9.107.1
 
   placeos-resource:
     git: https://github.com/place-labs/resource.git
@@ -273,10 +273,6 @@ shards:
     git: https://github.com/straight-shoota/sanitize.git
     version: 0.1.0+git.commit.c17d933fed22c6d3d615cca91a895166414389af
 
-  search-ingest:
-    git: https://github.com/placeos/search-ingest.git
-    version: 2.11.3+git.commit.fc43c1f8ad8733b6a6aa829e6def8a4e6756cf18
-
   secrets-env: # Overridden
     git: https://github.com/spider-gazelle/secrets-env.git
     version: 1.3.1

shard.yml

@@ -31,11 +31,6 @@ dependencies:
     github: crystal-community/jwt
     version: ~> 1
 
-  # Elasticsearch extension for PgORM
-  neuroplastic:
-    github: spider-gazelle/neuroplastic
-    version: ~> 1.14
-
   # Performs ping requests
   pinger:
     github: spider-gazelle/pinger
@@ -74,11 +69,6 @@ dependencies:
   pg-orm:
     github: spider-gazelle/pg-orm
 
-  # Client for Search Ingest PostgreSQL Elasticsearch indexer
-  search-ingest:
-    github: placeos/search-ingest
-    branch: master
-
   # Scheduling
   tasker:
     github: spider-gazelle/tasker

spec/controllers/alert_dashboards_spec.cr

@@ -6,6 +6,61 @@ module PlaceOS::Api
 
     describe "index", tags: "search" do
       Spec.test_base_index(Model::AlertDashboard, AlertDashboards)
+
+      it "filters by authority_id" do
+        other_authority = PlaceOS::Model::Generator.authority("other-#{random_name}.example.com")
+        other_authority.save!
+
+        other_dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Other Authority Dashboard", description: "Test Description", authority_id: other_authority.id)
+        other_dashboard.save!
+
+        local_dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Local Dashboard", description: "Test Description")
+        local_dashboard.save!
+
+        params = HTTP::Params.encode({"authority_id" => other_authority.id.as(String)})
+        result = client.get(
+          "#{AlertDashboards.base_route}?#{params}",
+          headers: Spec::Authentication.headers
+        )
+
+        result.success?.should be_true
+        dashboards = Array(Hash(String, JSON::Any)).from_json(result.body)
+        ids = dashboards.map(&.["id"].as_s)
+        ids.should contain(other_dashboard.id)
+        ids.should_not contain(local_dashboard.id)
+        dashboards.each &.["authority_id"].as_s.should eq other_authority.id
+
+        other_dashboard.destroy
+        local_dashboard.destroy
+        other_authority.destroy
+      end
+
+      it "limits non-support users to their own authority's dashboards" do
+        other_authority = PlaceOS::Model::Generator.authority("other-#{random_name}.example.com")
+        other_authority.save!
+
+        foreign_dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Foreign Dashboard", description: "Test Description", authority_id: other_authority.id)
+        foreign_dashboard.save!
+
+        # generator defaults to the localhost authority the test user belongs to
+        own_dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Own Dashboard", description: "Test Description")
+        own_dashboard.save!
+
+        params = HTTP::Params.encode({"limit" => "1000"})
+        result = client.get(
+          "#{AlertDashboards.base_route}?#{params}",
+          headers: Spec::Authentication.headers(sys_admin: false, support: false)
+        )
+
+        result.success?.should be_true
+        ids = Array(Hash(String, JSON::Any)).from_json(result.body).map(&.["id"].as_s)
+        ids.should contain(own_dashboard.id)
+        ids.should_not contain(foreign_dashboard.id)
+
+        foreign_dashboard.destroy
+        own_dashboard.destroy
+        other_authority.destroy
+      end
     end
 
     describe "CRUD operations", tags: "crud" do

spec/controllers/alerts_spec.cr

@@ -51,10 +51,6 @@ module PlaceOS::Api
 
     describe "filtering", tags: "search" do
       it "filters by severity" do
-        # ensure the enum columns are mapped before search-ingest indexes the new
-        # alerts (the pinned search-ingest image doesn't map enums; see helper)
-        ensure_keyword_mapping(Model::Alert.table_name, ["severity", "alert_type"])
-
         dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Test Dashboard", description: "Test Description")
         dashboard.save!
 
@@ -64,11 +60,8 @@ module PlaceOS::Api
         low_alert = PlaceOS::Model::Generator.alert(name: "Low Alert", description: "Low Priority Alert", alert_dashboard_id: dashboard.id, severity: PlaceOS::Model::Alert::Severity::LOW)
         low_alert.save!
 
-        # make the new alerts searchable, then scope to this test's dashboard so the
-        # count is deterministic regardless of other alerts in the shared ES index
-        sleep 1.second
-        refresh_elastic(Model::Alert.table_name)
-
+        # scope to this test's dashboard so the count is deterministic
+        # regardless of alerts created by other tests
         params = HTTP::Params.encode({"severity" => "HIGH", "alert_dashboard_id" => dashboard.id.as(String)})
         result = client.get(
           "#{Alerts.base_route}?#{params}",
@@ -78,15 +71,12 @@ module PlaceOS::Api
         result.success?.should be_true
         alerts = Array(Hash(String, JSON::Any)).from_json(result.body)
         alerts.size.should eq 1
+        alerts.first["id"].as_s.should eq high_alert.id
         # the API serializes the enum as its lower-cased member name
         alerts.first["severity"].as_s.should eq "high"
       end
 
       it "filters by alert type" do
-        # ensure the enum columns are mapped before search-ingest indexes the new
-        # alerts (the pinned search-ingest image doesn't map enums; see helper)
-        ensure_keyword_mapping(Model::Alert.table_name, ["severity", "alert_type"])
-
         dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Test Dashboard", description: "Test Description")
         dashboard.save!
 
@@ -96,11 +86,6 @@ module PlaceOS::Api
         status_alert = PlaceOS::Model::Generator.alert(name: "Status Alert", description: "Status Alert", alert_dashboard_id: dashboard.id, alert_type: PlaceOS::Model::Alert::AlertType::STATUS)
         status_alert.save!
 
-        # make the new alerts searchable, then scope to this test's dashboard so the
-        # count is deterministic regardless of other alerts in the shared ES index
-        sleep 1.second
-        refresh_elastic(Model::Alert.table_name)
-
         params = HTTP::Params.encode({"alert_type" => "THRESHOLD", "alert_dashboard_id" => dashboard.id.as(String)})
         result = client.get(
           "#{Alerts.base_route}?#{params}",
@@ -110,9 +95,104 @@ module PlaceOS::Api
         result.success?.should be_true
         alerts = Array(Hash(String, JSON::Any)).from_json(result.body)
         alerts.size.should eq 1
+        alerts.first["id"].as_s.should eq threshold_alert.id
         # the API serializes the enum as its lower-cased member name
         alerts.first["alert_type"].as_s.should eq "threshold"
       end
+
+      it "filters by enabled status, including enabled=false" do
+        dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Test Dashboard", description: "Test Description")
+        dashboard.save!
+
+        enabled_alert = PlaceOS::Model::Generator.alert(name: "Enabled Alert", description: "Enabled Alert", alert_dashboard_id: dashboard.id, enabled: true)
+        enabled_alert.save!
+
+        disabled_alert = PlaceOS::Model::Generator.alert(name: "Disabled Alert", description: "Disabled Alert", alert_dashboard_id: dashboard.id, enabled: false)
+        disabled_alert.save!
+
+        # enabled=false previously matched everything (the Elasticsearch-era
+        # filter was skipped for falsy values) — it must now filter
+        params = HTTP::Params.encode({"enabled" => "false", "alert_dashboard_id" => dashboard.id.as(String)})
+        result = client.get(
+          "#{Alerts.base_route}?#{params}",
+          headers: Spec::Authentication.headers
+        )
+
+        result.success?.should be_true
+        alerts = Array(Hash(String, JSON::Any)).from_json(result.body)
+        alerts.size.should eq 1
+        alerts.first["id"].as_s.should eq disabled_alert.id
+        alerts.first["enabled"].as_bool.should be_false
+
+        params = HTTP::Params.encode({"enabled" => "true", "alert_dashboard_id" => dashboard.id.as(String)})
+        result = client.get(
+          "#{Alerts.base_route}?#{params}",
+          headers: Spec::Authentication.headers
+        )
+
+        result.success?.should be_true
+        alerts = Array(Hash(String, JSON::Any)).from_json(result.body)
+        alerts.size.should eq 1
+        alerts.first["id"].as_s.should eq enabled_alert.id
+        alerts.first["enabled"].as_bool.should be_true
+      end
+
+      it "combines q with filters" do
+        dashboard = PlaceOS::Model::Generator.alert_dashboard(name: "Test Dashboard", description: "Test Description")
+        dashboard.save!
+
+        name = random_name
+        named_alert = PlaceOS::Model::Generator.alert(name: name, description: "Named Alert", alert_dashboard_id: dashboard.id, severity: PlaceOS::Model::Alert::Severity::CRITICAL)
+        named_alert.save!
+
+        other_alert = PlaceOS::Model::Generator.alert(name: "Other Alert", description: "Other Alert", alert_dashboard_id: dashboard.id, severity: PlaceOS::Model::Alert::Severity::CRITICAL)
+        other_alert.save!
+
+        params = HTTP::Params.encode({
+          "q"                  => name,
+          "severity"           => "critical",
+          "alert_dashboard_id" => dashboard.id.as(String),
+        })
+        result = client.get(
+          "#{Alerts.base_route}?#{params}",
+          headers: Spec::Authentication.headers
+        )
+
+        result.success?.should be_true
+        alerts = Array(Hash(String, JSON::Any)).from_json(result.body)
+        alerts.size.should eq 1
+        alerts.first["id"].as_s.should eq named_alert.id
+      end
+    end
+
+    describe "non-support scoping", tags: "search" do
+      it "returns alerts across all of the authority's dashboards" do
+        # two dashboards in the caller's authority (generator defaults to the
+        # localhost authority the test user belongs to), one alert on each —
+        # the Elasticsearch version ANDed the dashboard ids and returned
+        # nothing for authorities with more than one dashboard; this pins the
+        # IN() semantics
+        dashboard_one = PlaceOS::Model::Generator.alert_dashboard(name: "Scoped Dashboard One", description: "Test Description")
+        dashboard_one.save!
+        dashboard_two = PlaceOS::Model::Generator.alert_dashboard(name: "Scoped Dashboard Two", description: "Test Description")
+        dashboard_two.save!
+
+        alert_one = PlaceOS::Model::Generator.alert(name: "Scoped Alert One", description: "Test", alert_dashboard_id: dashboard_one.id)
+        alert_one.save!
+        alert_two = PlaceOS::Model::Generator.alert(name: "Scoped Alert Two", description: "Test", alert_dashboard_id: dashboard_two.id)
+        alert_two.save!
+
+        params = HTTP::Params.encode({"limit" => "1000"})
+        result = client.get(
+          "#{Alerts.base_route}?#{params}",
+          headers: Spec::Authentication.headers(sys_admin: false, support: false)
+        )
+
+        result.success?.should be_true
+        ids = Array(Hash(String, JSON::Any)).from_json(result.body).map(&.["id"].as_s)
+        ids.should contain(alert_one.id)
+        ids.should contain(alert_two.id)
+      end
     end
   end
 end

spec/controllers/api_key_spec.cr

@@ -9,6 +9,35 @@ module PlaceOS::Api
 
     describe "index", tags: "search" do
       Spec.test_base_index(Model::ApiKey, ApiKeys)
+
+      it "filters by authority_id" do
+        other_authority = PlaceOS::Model::Generator.authority("other-#{random_name}.example.com")
+        other_authority.save!
+
+        other_key = PlaceOS::Model::Generator.api_key(other_authority)
+        other_key.save!
+
+        # a key under the default (localhost) authority must be excluded
+        local_key = PlaceOS::Model::Generator.api_key
+        local_key.save!
+
+        params = HTTP::Params.encode({"authority_id" => other_authority.id.as(String)})
+        result = client.get(
+          "#{ApiKeys.base_route}?#{params}",
+          headers: Spec::Authentication.headers
+        )
+
+        result.status_code.should eq 200
+        keys = Array(Hash(String, JSON::Any)).from_json(result.body)
+        ids = keys.map(&.["id"].as_s)
+        ids.should contain(other_key.id)
+        ids.should_not contain(local_key.id)
+        keys.each &.["authority_id"].as_s.should eq other_authority.id
+
+        other_key.destroy
+        local_key.destroy
+        other_authority.destroy
+      end
     end
 
     describe "CRUD operations", tags: "crud" do