Run it the way a deployment runs it, and fix what that found

.github/workflows/ci.yml

@@ -78,7 +78,17 @@ jobs:
           --config=/repo/.gitleaks.toml
 
   images:
-    if: ${{ github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop') }}
+    # Pull requests build and publish too, so the e2e job below can run on them.
+    #
+    # It used not to, which made a pull request the *weaker* path: it ran build, web and
+    # secret-scan, while a push to main additionally stood the whole stack up and drove it.
+    # The browser suite is what caught the seeder losing its identity and the employee page
+    # disagreeing with itself - so gating merges on everything except that suite would have
+    # let both through. Tags are by commit SHA, so a PR's images are identifiable and
+    # disposable.
+    if: >-
+      ${{ github.event_name == 'pull_request' ||
+          (github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')) }}
     needs: [build, web]
     runs-on: ubuntu-latest
     permissions:
@@ -129,7 +139,7 @@ jobs:
           cache-to: type=gha,scope=${{ matrix.service }},mode=max
           tags: |
             ghcr.io/${{ env.OWNER_LC }}/asset-tracker-${{ matrix.service }}:${{ github.sha }}
-            ghcr.io/${{ env.OWNER_LC }}/asset-tracker-${{ matrix.service }}:${{ github.ref_name }}
+            ghcr.io/${{ env.OWNER_LC }}/asset-tracker-${{ matrix.service }}:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || github.ref_name }}
 
   # Stand the ten images that were just published up as a real stack and drive it
   # through the gateway. The suite self-skips when no stack is reachable, so this

deploy/k8s/base/config.yaml

@@ -47,5 +47,10 @@ stringData:
   # runs before any person has asked for anything. A cluster that seeds no demo data
   # can leave these out entirely - blank means the seeding call goes unauthenticated
   # and is refused, rather than being quietly privileged.
+  # The first admin of a fresh cluster. The seeders are all @Profile("!prod") and this
+  # overlay runs prod, so without these the user table is empty and every sign-in is
+  # refused. Used once, when there are no accounts; change the password after signing in.
+  SECURITY_BOOTSTRAP_EMAIL: "ops@example.invalid"
+  SECURITY_BOOTSTRAP_PASSWORD: "change-me-after-first-sign-in"
   SERVICE_ACCOUNT_EMAIL: "admin@platform.example"
   SERVICE_ACCOUNT_PASSWORD: "Passw0rd!"

infra/RUNBOOK.md

@@ -52,6 +52,23 @@ since that exchange also ends in a locally signed token - and two replicas rejec
 each other's tokens. In the cloud overlay the key comes from the
 `asset-tracker-jwt` secret, which is what lets auth-service run two pods.
 
+**Running it the way a deployment runs it.** The demo stack is H2 with seeders; the
+`prod` profile is Postgres with Flyway and no seed data at all, because every seeder is
+`@Profile("!prod")`. Bring that up and prove the journey with:
+
+```bash
+SECURITY_BOOTSTRAP_EMAIL=ops@acme.example SECURITY_BOOTSTRAP_PASSWORD='Boot5trap!' \
+  docker compose -f docker-compose.yml -f docker-compose.postgres.yml up -d
+node ../qa/prod-smoke.cjs
+```
+
+Without `SECURITY_BOOTSTRAP_*` a fresh prod database has no accounts and every sign-in
+is refused - auth-service says so in the log rather than leaving you with a bare 401.
+The account is created once, when the user table is empty; change its password after
+the first sign-in. It can act only on the tenants in `SECURITY_BOOTSTRAP_CLIENT_IDS`
+(default `1`), and there is no user administration yet, so a tenant created later is
+not reachable by that admin - see the gaps list.
+
 **Messaging.** `assignment-service` publishes custody events to the RabbitMQ
 `asset-tracker.events` exchange; `notification-service` consumes them off a
 durable queue. Management UI at `http://localhost:15672` (guest / guest). A

infra/compose/docker-compose.yml

@@ -202,6 +202,11 @@ services:
       # (see secrets/README.md) to keep sessions across restarts, as the cloud
       # overlay does with the asset-tracker-jwt secret.
       JWT_PRIVATE_KEY_FILE: ${JWT_PRIVATE_KEY_FILE:-}
+      # The first account, for a stack that seeds no demo data - the prod profile, where
+      # every seeder is switched off. Blank here because this stack does seed its own.
+      SECURITY_BOOTSTRAP_EMAIL: ${SECURITY_BOOTSTRAP_EMAIL:-}
+      SECURITY_BOOTSTRAP_PASSWORD: ${SECURITY_BOOTSTRAP_PASSWORD:-}
+      SECURITY_BOOTSTRAP_CLIENT_IDS: ${SECURITY_BOOTSTRAP_CLIENT_IDS:-1}
       ENTRA_ISSUER_URI: ${ENTRA_ISSUER_URI:-}
       ENTRA_CLIENT_ID: ${ENTRA_CLIENT_ID:-}
       ENTRA_DEFAULT_CLIENT_IDS: ${ENTRA_DEFAULT_CLIENT_IDS:-1,2,3}
@@ -313,6 +318,7 @@ services:
       - --storage.tsdb.retention.time=6h
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
 
   grafana:
     image: grafana/grafana:11.4.0

infra/compose/prometheus/alerts.yml

@@ -0,0 +1,30 @@
+# Alert rules for the demo stack. Kept next to the scrape config so what the stack
+# watches is reviewable in the same place as what it collects.
+groups:
+  - name: asset-tracker
+    rules:
+      # The brake failing open is deliberate - a Redis outage must not take sign-in
+      # with it - but it means credential stuffing is unthrottled while it lasts, and
+      # until now the only trace was a log line nobody reads.
+      - alert: AuthRateLimitStoreUnavailable
+        expr: increase(assettracker_auth_rate_limit_store_unavailable_total[5m]) > 0
+        for: 1m
+        labels: { severity: warning }
+        annotations:
+          summary: "The auth rate limiter cannot reach its store"
+          description: >-
+            {{ $value | printf "%.0f" }} sign-in attempts in the last five minutes were
+            let through without being counted, because Redis was unreachable. Logins keep
+            working; brute-force protection does not.
+
+      # Sustained refusals are the other half of the same story: either somebody is
+      # trying passwords, or a client is retrying a broken sign-in in a loop.
+      - alert: AuthRateLimitRejectingSteadily
+        expr: rate(assettracker_auth_rate_limited_total[5m]) > 0.2
+        for: 10m
+        labels: { severity: info }
+        annotations:
+          summary: "Sign-ins are being rate limited continuously"
+          description: >-
+            More than one refusal every five seconds for ten minutes. Worth a look at
+            which address it is coming from before raising the budget.

infra/compose/prometheus/prometheus.yml

@@ -2,6 +2,9 @@ global:
   scrape_interval: 10s
   evaluation_interval: 10s
 
+rule_files:
+  - /etc/prometheus/alerts.yml
+
 scrape_configs:
   - job_name: asset-tracker
     metrics_path: /actuator/prometheus

infra/qa/prod-smoke.cjs

@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+/**
+ * The core journey against a stack running the `prod` profile on PostgreSQL.
+ *
+ * The other suites all run against the demo stack, which is H2 with seeders. That leaves the
+ * configuration a real deployment actually uses - Flyway migrations, ddl-auto=validate, Postgres
+ * types, no seed data - proven only per service by Testcontainers, and never end to end. The first
+ * time this ran it found that nobody could sign in at all: every seeder is @Profile("!prod"), so
+ * the user table was empty and there was no way to make an account.
+ *
+ * Start the stack with:
+ *   SECURITY_BOOTSTRAP_EMAIL=ops@acme.example SECURITY_BOOTSTRAP_PASSWORD='Boot5trap!' \
+ *   docker compose -f docker-compose.yml -f docker-compose.postgres.yml up -d
+ *
+ * It creates its own tenant and records, so it needs no seed data - which is the point.
+ */
+const BASE = process.env.E2E_BASE_URL ?? "http://localhost:8080";
+const EMAIL = process.env.SECURITY_BOOTSTRAP_EMAIL ?? "ops@acme.example";
+const PASSWORD = process.env.SECURITY_BOOTSTRAP_PASSWORD ?? "Boot5trap!";
+
+let token = "";
+let passed = 0;
+const failures = [];
+
+function check(name, got, want) {
+  const ok = got === want;
+  console.log(`  ${ok ? "PASS" : "FAIL"}  ${name.padEnd(44)} got=${got} want=${want}`);
+  ok ? passed++ : failures.push(name);
+  return ok;
+}
+
+async function call(method, path, body) {
+  const res = await fetch(`${BASE}${path}`, {
+    method,
+    headers: {
+      ...(token ? { Authorization: `Bearer ${token}` } : {}),
+      ...(body ? { "Content-Type": "application/json" } : {}),
+    },
+    ...(body ? { body: JSON.stringify(body) } : {}),
+  });
+  const text = await res.text();
+  let json = null;
+  try {
+    json = text ? JSON.parse(text) : null;
+  } catch {
+    /* a non-JSON body is itself the answer */
+  }
+  return { status: res.status, body: json };
+}
+
+(async () => {
+  const stamp = Date.now().toString(36).slice(-5);
+
+  console.log("SIGN IN");
+  const login = await call("POST", "/api/auth/login", { email: EMAIL, password: PASSWORD });
+  if (!check("the bootstrap admin can sign in", login.status, 200)) {
+    console.error("\nNo way in - is SECURITY_BOOTSTRAP_* set on auth-service?");
+    process.exit(1);
+  }
+  token = login.body.token;
+
+  console.log("\nA TENANT AND ITS RECORDS");
+  // Reused if one is already there, because the bootstrap admin is scoped to the tenants named
+  // in SECURITY_BOOTSTRAP_CLIENT_IDS and nothing can widen that afterwards - there is no user
+  // administration yet. A second run that made a second tenant locked itself out of it: every
+  // call after the create came back 403, which is the system being right and the script being
+  // wrong.
+  const existing = await call("GET", "/api/clients");
+  let clientId = existing.body?.[0]?.id;
+  if (clientId === undefined) {
+    const created = await call("POST", "/api/clients", {
+      name: `Smoke ${stamp}`,
+      slug: `smoke-${stamp}`,
+    });
+    check("a client can be created", created.status, 201);
+    clientId = created.body?.id;
+  } else {
+    check("a tenant exists to work in", typeof clientId, "number");
+  }
+
+  const person = await call("POST", "/api/people", {
+    clientId,
+    fullName: `Smoke Person ${stamp}`,
+    email: `smoke.${stamp}@example.invalid`,
+    department: "QA",
+  });
+  check("a person can be created", person.status, 201);
+
+  const desk = await call("POST", "/api/locations", {
+    clientId,
+    kind: "DESK",
+    label: `Smoke Desk ${stamp}`,
+    qrTag: `SMOKE-${stamp}`,
+  });
+  check("a desk can be created", desk.status, 201);
+
+  const asset = await call("POST", "/api/assets", {
+    clientId,
+    assetTag: `SMOKE-A-${stamp}`,
+    type: "Laptop",
+    make: "Smoke",
+    model: "Test",
+  });
+  check("an asset can be created", asset.status, 201);
+
+  console.log("\nCUSTODY");
+  const out = await call("POST", "/api/assignments", {
+    clientId,
+    assetId: asset.body?.id,
+    holderType: "PERSON",
+    holderId: person.body?.id,
+  });
+  check("it can be checked out", out.status, 201);
+
+  const held = await call(
+    "GET",
+    `/api/assets?clientId=${clientId}&holderType=PERSON&holderId=${person.body?.id}`,
+  );
+  check("it shows on the person", held.body?.length ?? 0, 1);
+
+  // an idempotent retry of the same intent must replay, not check out twice
+  const back = await call("POST", `/api/assignments/return?assetId=${asset.body?.id}`);
+  check("it can be returned", back.status, 200);
+
+  console.log("\nCORRECTIONS");
+  const renamed = await call("PATCH", `/api/people/${person.body?.id}`, {
+    fullName: `Smoke Person ${stamp} (renamed)`,
+  });
+  check("a person can be corrected", renamed.status, 200);
+
+  // A desk with gear on it must refuse to disappear, and that answer comes from asset-service -
+  // so this also proves one service asking another while carrying the caller's token.
+  await call("POST", "/api/assignments", {
+    clientId,
+    assetId: asset.body?.id,
+    holderType: "LOCATION",
+    holderId: desk.body?.id,
+  });
+  const occupied = await call("DELETE", `/api/locations/${desk.body?.id}`);
+  check("a desk with gear on it is not deleted", occupied.status, 409);
+  await call("POST", `/api/assignments/return?assetId=${asset.body?.id}`);
+
+  const deleted = await call("DELETE", `/api/locations/${desk.body?.id}`);
+  check("an empty desk can be deleted", deleted.status, 204);
+
+  console.log(`\n${passed}/${passed + failures.length} passed`);
+  if (failures.length) {
+    console.error(`failed: ${failures.join(", ")}`);
+    process.exit(1);
+  }
+})();

services/api-gateway/src/main/java/com/assettracker/apigateway/ratelimit/RateLimitStoreConfig.java

@@ -1,5 +1,6 @@
 package com.assettracker.apigateway.ratelimit;
 
+import io.micrometer.core.instrument.MeterRegistry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -25,9 +26,10 @@ public class RateLimitStoreConfig {
   @ConditionalOnProperty(name = "spring.data.redis.host")
   public RateLimitStore redisRateLimitStore(
       StringRedisTemplate redis,
+      MeterRegistry meters,
       @Value("${security.rate-limit.key-prefix:asset-tracker:auth-rate:}") String prefix) {
     log.info("auth rate limit is cluster-wide, counted in Redis under '{}'", prefix);
-    return new RedisRateLimitStore(redis, prefix);
+    return new RedisRateLimitStore(redis, prefix, meters);
   }
 
   @Bean

services/api-gateway/src/main/java/com/assettracker/apigateway/ratelimit/RedisRateLimitStore.java

@@ -1,5 +1,7 @@
 package com.assettracker.apigateway.ratelimit;
 
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
 import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -37,10 +39,14 @@ public class RedisRateLimitStore implements RateLimitStore {
 
   private final StringRedisTemplate redis;
   private final String prefix;
+  private final Counter unavailable;
 
-  public RedisRateLimitStore(StringRedisTemplate redis, String prefix) {
+  public RedisRateLimitStore(StringRedisTemplate redis, String prefix, MeterRegistry meters) {
     this.redis = redis;
     this.prefix = prefix;
+    // Counted, not only logged. Failing open is the right trade against an authentication
+    // outage, but it removes the brake - and a trade nobody can see is one nobody revisits.
+    this.unavailable = meters.counter("assettracker.auth.rate_limit.store_unavailable");
   }
 
   @Override
@@ -50,6 +56,7 @@ public boolean tryAcquire(String key, int maxPerWindow, long windowMs) {
           redis.execute(INCREMENT_AND_EXPIRE, List.of(prefix + key), String.valueOf(windowMs));
       return hits == null || hits <= maxPerWindow;
     } catch (DataAccessException unreachable) {
+      this.unavailable.increment();
       log.warn("rate limit store unavailable, allowing the request: {}", unreachable.getMessage());
       return true;
     }

services/api-gateway/src/test/java/com/assettracker/apigateway/ratelimit/RedisRateLimitStoreTest.java

@@ -5,6 +5,7 @@
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.when;
 
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
@@ -21,7 +22,7 @@ class RedisRateLimitStoreTest {
   @Test
   @SuppressWarnings("unchecked")
   void allowsUntilTheSharedCounterPassesTheBudget() {
-    RedisRateLimitStore store = new RedisRateLimitStore(redis, "p:");
+    RedisRateLimitStore store = new RedisRateLimitStore(redis, "p:", new SimpleMeterRegistry());
     when(redis.execute(any(RedisScript.class), any(), anyString())).thenReturn(10L, 11L);
 
     // the tenth attempt across the whole cluster is still within a budget of ten
@@ -36,7 +37,7 @@ void allowsUntilTheSharedCounterPassesTheBudget() {
   @Test
   @SuppressWarnings("unchecked")
   void anUnreachableStoreLetsTheRequestThrough() {
-    RedisRateLimitStore store = new RedisRateLimitStore(redis, "p:");
+    RedisRateLimitStore store = new RedisRateLimitStore(redis, "p:", new SimpleMeterRegistry());
     when(redis.execute(any(RedisScript.class), any(), anyString()))
         .thenThrow(new RedisConnectionFailureException("down"));