diff --git a/.env b/.env
deleted file mode 100644
index b8aa34f..0000000
--- a/.env
+++ /dev/null
@@ -1,41 +0,0 @@
-# Server
-PORT=8080
-LOG_LEVEL=info
-ENVIRONMENT=development
-
-# Database
-DATABASE_URL=postgres://notifyhub:notifyhub@localhost:5432/notifyhub?sslmode=disable
-DATABASE_MAX_CONNS=25
-DATABASE_MIN_CONNS=5
-
-# Redis
-REDIS_URL=redis://localhost:6379
-REDIS_PASSWORD=
-
-# Kafka
-KAFKA_BROKERS=localhost:9092
-KAFKA_PARTITION_COUNT=6
-KAFKA_REPLICATION_FACTOR=1
-
-# Workers
-WORKER_EMAIL_CONCURRENCY=5
-WORKER_PUSH_CONCURRENCY=3
-WORKER_SMS_CONCURRENCY=2
-WORKER_INAPP_CONCURRENCY=5
-WORKER_RETRY_MAX_ATTEMPTS=3
-WORKER_RETRY_BASE_DELAY_MS=1000
-
-# Providers
-SES_REGION=ap-south-1
-SES_FROM_EMAIL=notifications@yourdomain.com
-FCM_CREDENTIALS_FILE=/etc/notifyhub/fcm-credentials.json
-TWILIO_ACCOUNT_SID=
-TWILIO_AUTH_TOKEN=
-TWILIO_FROM_NUMBER=
-
-# Rate Limiting
-RATELIMIT_API_RPM=100
-RATELIMIT_EMAIL_PER_HOUR=10
-RATELIMIT_PUSH_PER_HOUR=5
-RATELIMIT_SMS_PER_HOUR=3
-RATELIMIT_INAPP_PER_HOUR=50
diff --git a/.env.example b/.env.example
index b8aa34f..003eb63 100644
--- a/.env.example
+++ b/.env.example
@@ -39,3 +39,33 @@ RATELIMIT_EMAIL_PER_HOUR=10
RATELIMIT_PUSH_PER_HOUR=5
RATELIMIT_SMS_PER_HOUR=3
RATELIMIT_INAPP_PER_HOUR=50
+
+# In-app WebSocket
+INAPP_WS_HEARTBEAT_SECONDS=25
+INAPP_WS_BUFFER_SIZE=32
+INAPP_WS_READ_TIMEOUT_SECONDS=60
+
+# WebSocket JWT
+WS_JWT_SECRET=change-me-in-production
+WS_JWT_TTL_SECONDS=60
+
+# Admin API
+ADMIN_TOKEN=change-me-in-production
+
+# DLQ
+DLQ_ENABLED=true
+DLQ_CONSUMER_ENABLED=true
+DLQ_GROUP_ID=notifyhub-dlq-consumer
+
+# Webhooks
+WEBHOOK_ENABLED=true
+WEBHOOK_GROUP_ID=notifyhub-webhook-worker
+WEBHOOK_CONCURRENCY=3
+WEBHOOK_MAX_ATTEMPTS=5
+WEBHOOK_RETRY_BASE_DELAY_MS=1000
+
+# Observability
+OTEL_EXPORTER_OTLP_ENDPOINT=
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_TRACES_SAMPLER_ARG=0.1
+METRICS_PORT=9091
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..965dfe9
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,71 @@
+name: CI
+
+on:
+ pull_request:
+ push:
+ branches:
+ - main
+
+permissions:
+ contents: read
+
+jobs:
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version: "1.25"
+ cache: true
+
+ - name: golangci-lint
+ uses: golangci/golangci-lint-action@v6
+ with:
+ version: latest
+ args: --go=1.24
+
+ test:
+ name: Unit Tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version: "1.25"
+ cache: true
+
+ - name: Run unit tests
+ run: make test
+
+ test-integration:
+ name: Integration Tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version: "1.25"
+ cache: true
+
+ - name: Run integration tests
+ run: make test-integration
+
+ build:
+ name: Build
+ runs-on: ubuntu-latest
+ needs: [lint, test]
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version: "1.25"
+ cache: true
+
+ - name: Build binaries
+ run: make build
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 0000000..f8da19b
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,60 @@
+name: Docker
+
+on:
+ push:
+ branches:
+ - main
+ tags:
+ - "v*"
+
+permissions:
+ contents: read
+ packages: write
+
+jobs:
+ build-push:
+ name: Build & Push (${{ matrix.target }})
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ target: [api, worker]
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Login to GHCR
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Docker metadata
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ghcr.io/${{ github.repository_owner }}/notifyhub-${{ matrix.target }}
+ tags: |
+ type=sha,prefix=sha-,format=short
+ type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+ type=semver,pattern={{version}}
+ type=semver,pattern={{major}}.{{minor}}
+
+ - name: Build and push
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: deployments/Dockerfile
+ target: ${{ matrix.target }}
+ platforms: linux/amd64,linux/arm64
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ cache-from: type=gha,scope=${{ matrix.target }}
+ cache-to: type=gha,mode=max,scope=${{ matrix.target }}
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..3730ceb
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,55 @@
+# Binaries
+bin/
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test artifacts
+*.test
+*.out
+coverage.html
+coverage.txt
+
+# Build cache
+/tmp/
+
+# Environment files
+.env
+.env.local
+.env.*.local
+*.env
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# sqlc
+# internal/db/ is generated — keep in source control but ignore any local overrides
+
+# Secrets / keys
+*.pem
+*.key
+*.p12
+*.crt
+
+# Logs
+*.log
+
+# Docker volumes (if mounted locally)
+postgres-data/
+redis-data/
+
+# k6 results
+scripts/loadtest/results/
+
+# md files
+docs/*.md
+docs/adr/
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..14ea12a
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,182 @@
+# AGENTS.md
+
+This file provides guidance to Codex (Codex.ai/code) when working with code in this repository.
+
+# NotifyHub
+
+Production-grade, multi-channel notification system for general-purpose use cases (user alerts, status updates, reminders, real-time events). Supports both **single-tenant** and **multi-tenant** deployments from the same codebase.
+
+Module: `github.com/amitrajitdas31/notifyhub`
+
+## Tech Stack
+
+- **Language:** Go 1.25+ (use stdlib where possible)
+- **Router:** `go-chi/chi/v5`
+- **Database:** PostgreSQL 16 via `jackc/pgx/v5` — NO ORM. Use `sqlc` for type-safe queries, `golang-migrate` for migrations.
+- **Cache / Rate Limiting:** Redis 7 via `redis/go-redis/v9`
+- **Message Queue:** Apache Kafka (KRaft mode, no Zookeeper) via `segmentio/kafka-go`
+- **Observability:** `log/slog` (logging), OpenTelemetry (tracing), Prometheus (metrics)
+- **Circuit Breaker:** `sony/gobreaker`
+- **Testing:** `go test` + `testcontainers-go` for integration tests
+- **Validation:** `go-playground/validator/v10`
+
+## Common Commands
+
+```bash
+make build # Build api + worker binaries to bin/
+make test # Unit tests with race detector + coverage
+make test-integration # Integration tests (testcontainers, 120s timeout)
+make lint # golangci-lint
+make docker-up # Start full local stack (postgres, redis, kafka, api, worker)
+make docker-up-infra # Start only infra (postgres, redis, kafka) — run api/worker locally
+make docker-down # Stop local stack
+make migrate-up # Apply database migrations
+make migrate-down # Roll back one migration
+make generate # Regenerate sqlc queries (internal/db/)
+make run-api # go run ./cmd/api
+make run-worker # go run ./cmd/worker
+make seed # Seed local DB with a test tenant + API key
+
+# Run a single test
+go test ./internal/service/... -run TestNotificationService_Send -v -race
+
+# Run a single integration test
+go test ./internal/repository/... -run TestNotificationRepo -v -race -tags=integration
+```
+
+## Architecture
+
+### Layering
+
+The codebase enforces strict one-way dependencies:
+
+```
+domain ← service ← handler (HTTP)
+ ↑ ↓
+ └──── repository ←── db (sqlc-generated)
+ ↓
+ queue (Kafka)
+```
+
+- `internal/domain/` — pure Go structs and `AppError`; **no external imports**
+- `internal/service/` — business logic; depends on repository and queue interfaces, never on `net/http`
+- `internal/repository/` — wraps sqlc; converts between `db.*` DB models and `domain.*` types
+- `internal/api/` — HTTP handlers and middleware; translates HTTP ↔ service calls
+- `internal/db/` — **do not edit manually**; regenerated by `make generate` from `queries/*.sql`
+
+### End-to-End: Sending a Notification
+
+1. `POST /api/v1/notifications` → `NotificationHandler.Send()` parses body, extracts `tenantID` from `middleware.ClientFromContext(ctx).TenantID`
+2. `NotificationService.Send()` validates, checks idempotency key, creates row in DB (status=`pending`), then calls `s.enqueue()`
+3. `enqueue()` marshals a `queue.Message` and calls `publisher.Publish(ctx, topic, recipientID, payload)`
+ - Topic: `notifyhub.notifications.{channel}` (e.g. `notifyhub.notifications.email`)
+ - Key: `recipientID` — ensures per-recipient ordering within a Kafka partition
+4. On successful publish, status updated to `queued`; handler returns 202 Accepted
+5. If `ScheduledAt` is in the future, step 3 is skipped — the scheduler (worker process) enqueues it later
+
+### End-to-End: Delivering a Notification (Worker)
+
+Each channel runs an independent `worker.Pool` with configurable goroutine concurrency. The pool's fetch loop calls `consumer.Fetch()` (blocks), then fans work out to goroutines.
+
+`Processor.Process()` runs a 7-step pipeline per message:
+
+1. Fetch notification from DB
+2. Mark status `processing`
+3. **Preference check** — `PreferenceService.IsAllowed()` evaluates enabled flag, quiet hours (with IANA timezone), frequency caps; drop if denied
+4. **Rate limit check** — `RateLimitService.Allow()` enforces per-recipient per-channel hourly cap via Redis sliding-window; drop if exceeded
+5. **Template render** — if `TemplateID` is set, `TemplateService.Render()` executes Go `text/template` with payload; exposes `payload["_subject"]` and `payload["_body"]`; permanent failure → fail immediately
+6. **Provider send with retry** — `sendWithRetry()` calls `provider.Send()`; `ErrDeliveryTemporary` retries with exponential backoff (`retryDelay * 2^attempt`); `ErrDeliveryPermanent` fails immediately
+7. Write `DeliveryLog`, update final status (`delivered` or `failed`)
+
+Offset is committed **only** after `Process()` returns nil — infrastructure errors (DB, Redis) leave the message unconsumed for reprocessing.
+
+### Multi-Tenancy
+
+Every resource is scoped by `tenant_id`. The auth middleware resolves `X-API-Key` → `APIClient` (carries `TenantID`) and stores it in context. Handlers extract tenantID via `middleware.ClientFromContext(ctx).TenantID` and pass it down through every service and repository call.
+
+- Redis rate-limit key: `ratelimit:{userID}:{channel}` (worker), `ratelimit:api:{api_client_id}` (HTTP middleware)
+- Kafka `queue.Message` carries `TenantID` for worker-side isolation
+- Tenant provisioning is not in the public API; use `make seed` or `POST /internal/tenants` (protected by `X-Admin-Token` header)
+
+### Error Handling
+
+All errors flow through `domain.AppError{Code, Message, Details, Err}`. Use the constructors:
+
+```go
+domain.NewNotFoundError("notification not found")
+domain.NewValidationError("invalid request", map[string]any{"field": "reason"})
+domain.NewRateLimitedError("API rate limit exceeded")
+domain.NewInternalError("db query failed", err)
+```
+
+`response.JSONError()` maps `Code` to HTTP status automatically. Wrap underlying errors with `fmt.Errorf("op: %w", err)` for context; do not log at the repository layer — log at the handler or middleware level.
+
+### Adding a New Provider
+
+1. Create `internal/provider/{name}/{name}.go` implementing `provider.Provider` interface (`Send`, `Channel`)
+2. Return `provider.ErrDeliveryTemporary` for transient failures (network, rate limit), `provider.ErrDeliveryPermanent` for terminal failures (invalid address, account suspended)
+3. Register in `cmd/worker/main.go` via `registry.Register(provider)`
+
+### Modifying DB Queries
+
+1. Edit or add SQL in `queries/*.sql`
+2. Run `make generate` to regenerate `internal/db/`
+3. Update the repository wrapper in `internal/repository/` to call the new generated method and convert types using the helpers in `internal/repository/convert.go`
+
+## Design Principles
+
+1. **Dependency injection everywhere** — no global state, all deps passed via constructors
+2. **Interface-based design** — repositories, providers, queue ops behind interfaces
+3. **Context propagation** — pass `context.Context` through every layer
+4. **Graceful shutdown** — handle SIGINT/SIGTERM, drain queues, wait for in-flight work
+5. **Idempotency** — every notification safely retryable without duplicates (scoped per tenant)
+6. **Structured errors** — wrap with `fmt.Errorf("...: %w", err)`
+7. **No magic** — explicit config, no auto-wiring frameworks
+8. **Test at boundaries** — unit test business logic, integration test with real infra via testcontainers
+
+## Reference
+
+Full specification: `NOTIFICATION_SYSTEM_PLAN.md`
+
+---
+
+## Implementation Progress
+
+### Done
+
+- [x] Domain types — `notification`, `template`, `preference`, `delivery`, `api_client`, `errors`
+- [x] Config — `internal/config/config.go`
+- [x] Database — migrations, sqlc-generated queries, repository wrappers (notification, template, preference, delivery, api_client)
+- [x] Queue — Kafka producer & consumer (`internal/queue/`)
+- [x] Middleware — auth, logging, recovery, request-id, rate-limit
+- [x] Handlers — notification, template, preference, health
+- [x] Router — `internal/api/router.go`
+- [x] Services — notification, template, preference, rate-limit, scheduler, validate
+- [x] Worker — pool + processor (`internal/worker/`)
+- [x] Provider — mock (`internal/provider/mock/`)
+- [x] Providers — SES (`internal/provider/ses/`), FCM (`internal/provider/fcm/`), Twilio (`internal/provider/twilio/`); config-based selection in worker (falls back to mock when creds absent)
+- [x] Internal tenant API — `POST /internal/tenants`, `POST /internal/tenants/:id/api-keys`
+- [x] Seed script — `scripts/seed.go` (provisions tenant + API key + starter templates)
+- [x] Notification templates — `templates/email/` (welcome, password_reset, alert) and `templates/sms/` (otp, alert); file loader at `internal/template/loader.go`
+- [x] Deployments — Dockerfile, docker-compose, Prometheus config, Alloy, Loki, Tempo, OTel Collector configs
+- [x] Observability — `internal/observability/` (metrics, tracing, health, logging, span); metrics middleware at `internal/api/middleware/metrics.go`
+- [x] Docs — db-schema, deployment, production-grade
+- [x] **DLQ / poison-message handling** — per-channel dead-letter topics `notifyhub.notifications.{channel}.dlq`; processor routes to DLQ after max retry attempts or on permanent provider failure with full message + error context; DLQ consumer (`internal/worker/dlq_consumer.go`) persists to `dead_letter_messages` table; admin endpoints `GET /internal/dlq`, `GET /internal/dlq/:id`, `POST /internal/dlq/:id/replay`, `DELETE /internal/dlq/:id`; Prometheus metrics on DLQ depth, published, persisted, replayed per channel
+- [x] **In-app channel** — `inapp_messages` table (migration 000004); `internal/provider/inapp/` persists to DB + publishes to Redis pub/sub; fan-out via `internal/realtime/Hub` (WebSocket, `coder/websocket`); `GET /api/v1/inbox`, `GET /api/v1/inbox/unread-count`, `POST /api/v1/inbox/{id}/read`, `POST /api/v1/inbox/read-all`, `GET /api/v1/inbox/stream` (WebSocket); recipient identity via `X-Recipient-ID` header; presence registry keyed by `{tenant_id}:{recipient_id}`; Prometheus metrics for persisted/ws-connections/ws-dropped/pubsub-published
+
+### Remaining
+
+- [x] **CI/CD** — `.github/workflows/ci.yml` (lint + test + build), `.github/workflows/docker.yml` (build + push)
+- [x] **Load tests** — `scripts/loadtest/send_notification.js` (k6)
+- [x] **ADRs** — `docs/adr/` (Go over Node, Kafka over RabbitMQ, pgx over GORM, sqlc for queries)
+- [x] **OpenAPI spec** — `docs/api.yaml`
+- [ ] **Kubernetes / Helm** — `charts/notifyhub/` (Helm chart for `api` + `worker`); `deploy/kind-config.yaml` (local); `deploy/eksctl-config.yaml` (EKS reference); infra (Kafka, Redis, Postgres) via Bitnami charts locally / managed services on EKS; HPA per service; Ingress via nginx; GHA `deploy.yml` (`helm upgrade --install` on merge to main). See `docs/k8s-learning.md` for prerequisite concepts.
+
+
+
+# Memory Context
+
+# [Notification-System] recent context, 2026-05-14 11:55pm GMT+5:30
+
+No previous sessions found.
+
diff --git a/CLAUDE.md b/CLAUDE.md
index 884015b..a61392b 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -166,8 +166,8 @@ Full specification: `NOTIFICATION_SYSTEM_PLAN.md`
### Remaining
-- [ ] **CI/CD** — `.github/workflows/ci.yml` (lint + test + build), `.github/workflows/docker.yml` (build + push)
-- [ ] **Load tests** — `scripts/loadtest/send_notification.js` (k6)
-- [ ] **ADRs** — `docs/adr/` (Go over Node, Kafka over RabbitMQ, pgx over GORM, sqlc for queries)
-- [ ] **OpenAPI spec** — `docs/api.yaml`
+- [x] **CI/CD** — `.github/workflows/ci.yml` (lint + test + build), `.github/workflows/docker.yml` (build + push)
+- [x] **Load tests** — `scripts/loadtest/send_notification.js` (k6)
+- [x] **ADRs** — `docs/adr/` (Go over Node, Kafka over RabbitMQ, pgx over GORM, sqlc for queries)
+- [x] **OpenAPI spec** — `docs/api.yaml`
- [ ] **Kubernetes / Helm** — `charts/notifyhub/` (Helm chart for `api` + `worker`); `deploy/kind-config.yaml` (local); `deploy/eksctl-config.yaml` (EKS reference); infra (Kafka, Redis, Postgres) via Bitnami charts locally / managed services on EKS; HPA per service; Ingress via nginx; GHA `deploy.yml` (`helm upgrade --install` on merge to main). See `docs/k8s-learning.md` for prerequisite concepts.
diff --git a/Makefile b/Makefile
index 90d1169..77d69b1 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,8 @@
.PHONY: all build test test-integration lint run-api run-worker \
- docker-up docker-down migrate-up migrate-down generate seed
+ docker-up docker-down migrate-up migrate-down generate seed \
+ loadtest-k6 loadtest-k6-smoke loadtest-k6-docker loadtest-k6-docker-smoke \
+ loadtest-inapp loadtest-inapp-smoke loadtest-inapp-prod1000 \
+ loadtest-inapp-docker loadtest-inapp-docker-smoke loadtest-inapp-docker-prod1000
# Load .env if it exists, then export all vars to subprocesses
-include .env
@@ -72,6 +75,38 @@ docker-logs:
seed:
go run ./scripts/seed.go
+# ── Load Testing ───────────────────────────────────────────────────────────────
+
+loadtest-k6:
+ k6 run scripts/loadtest/send_notification.js
+
+loadtest-k6-smoke:
+ K6_SCENARIO=smoke k6 run scripts/loadtest/send_notification.js
+
+loadtest-k6-docker:
+ docker run --rm -v "$(PWD)/scripts/loadtest:/scripts" -e NOTIFYHUB_BASE_URL=$${NOTIFYHUB_BASE_URL:-http://host.docker.internal:8080} -e NOTIFYHUB_API_KEY=$${NOTIFYHUB_API_KEY:-loadtest-api-key} -e NOTIFYHUB_CHANNEL=$${NOTIFYHUB_CHANNEL:-inapp} -e NOTIFYHUB_RECIPIENT_PREFIX=$${NOTIFYHUB_RECIPIENT_PREFIX:-k6-user} -e NOTIFYHUB_IDEMPOTENCY=$${NOTIFYHUB_IDEMPOTENCY:-false} -e K6_SCENARIO=$${K6_SCENARIO:-baseline} grafana/k6:latest run /scripts/send_notification.js
+
+loadtest-k6-docker-smoke:
+ K6_SCENARIO=smoke $(MAKE) loadtest-k6-docker
+
+loadtest-inapp:
+ k6 run scripts/loadtest/inapp_realtime.js
+
+loadtest-inapp-smoke:
+ K6_SCENARIO=smoke k6 run scripts/loadtest/inapp_realtime.js
+
+loadtest-inapp-prod1000:
+ K6_SCENARIO=prod_1000 k6 run scripts/loadtest/inapp_realtime.js
+
+loadtest-inapp-docker:
+ docker run --rm -v "$(PWD)/scripts/loadtest:/scripts" -e NOTIFYHUB_BASE_URL=$${NOTIFYHUB_BASE_URL:-http://host.docker.internal:8080} -e NOTIFYHUB_WS_URL=$${NOTIFYHUB_WS_URL:-ws://host.docker.internal:8080} -e NOTIFYHUB_API_KEY=$${NOTIFYHUB_API_KEY:-loadtest-api-key} -e NOTIFYHUB_RECIPIENT_PREFIX=$${NOTIFYHUB_RECIPIENT_PREFIX:-k6-inapp-user} -e NOTIFYHUB_CONNECT_JITTER_MS=$${NOTIFYHUB_CONNECT_JITTER_MS:-1000} -e NOTIFYHUB_FIRST_SEND_DELAY_MS=$${NOTIFYHUB_FIRST_SEND_DELAY_MS:-1000} -e NOTIFYHUB_SEND_INTERVAL_MS=$${NOTIFYHUB_SEND_INTERVAL_MS:-5000} -e NOTIFYHUB_SENDS_PER_CLIENT=$${NOTIFYHUB_SENDS_PER_CLIENT:-1} -e K6_SCENARIO=$${K6_SCENARIO:-smoke} grafana/k6:latest run /scripts/inapp_realtime.js
+
+loadtest-inapp-docker-smoke:
+ K6_SCENARIO=smoke $(MAKE) loadtest-inapp-docker
+
+loadtest-inapp-docker-prod1000:
+ K6_SCENARIO=prod_1000 $(MAKE) loadtest-inapp-docker
+
# ── Helpers ────────────────────────────────────────────────────────────────────
tidy:
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..73321c0
--- /dev/null
+++ b/README.md
@@ -0,0 +1,196 @@
+# NotifyHub
+
+Production-grade, multi-channel notification system built in Go. Supports **email**, **SMS**, **push** (FCM), and **in-app** channels with real-time WebSocket delivery. Designed for both single-tenant and multi-tenant deployments from the same codebase.
+
+## Features
+
+- **Multi-channel delivery** — Email (AWS SES), SMS (Twilio), Push (FCM), In-app (WebSocket + Redis pub/sub)
+- **Multi-tenancy** — every resource scoped by `tenant_id`; API key auth per tenant
+- **Kafka-backed queue** — per-channel topics with per-recipient ordering; worker pool with configurable concurrency
+- **Idempotency** — duplicate suppression per tenant via idempotency key
+- **Scheduled notifications** — `scheduled_at` support; scheduler worker enqueues at the right time
+- **Templates** — Go `text/template` rendering with per-channel file-based templates
+- **User preferences** — enabled/disabled per channel, quiet hours (IANA timezone), frequency caps
+- **Rate limiting** — Redis sliding-window; per-recipient per-channel + per-API-client HTTP limits
+- **Dead-letter queue** — poison messages routed to per-channel DLQ topics; persisted to DB; admin replay/delete endpoints
+- **Observability** — structured logging (`slog`), distributed tracing (OTel → Tempo), metrics (Prometheus → Grafana)
+- **Webhook delivery** — outbound webhooks for notification events
+- **Circuit breaker** — `sony/gobreaker` on provider calls
+
+## Tech Stack
+
+| Concern | Choice |
+|---|---|
+| Language | Go 1.25 |
+| Router | go-chi/chi v5 |
+| Database | PostgreSQL 16 (pgx/v5 + sqlc, no ORM) |
+| Cache / Rate limit | Redis 7 |
+| Message queue | Apache Kafka (KRaft, no Zookeeper) |
+| Observability | slog + OpenTelemetry + Prometheus |
+| Email | AWS SES v2 |
+| SMS | Twilio |
+| Push | Firebase FCM |
+| WebSocket | coder/websocket |
+
+## Architecture
+
+```
+domain ← service ← handler (HTTP)
+ ↑ ↓
+ └──── repository ←── db (sqlc-generated)
+ ↓
+ queue (Kafka)
+```
+
+The worker side runs independent per-channel `worker.Pool`s. Each message goes through a 7-step pipeline: fetch → mark processing → preference check → rate-limit check → template render → provider send (with retry/backoff) → write delivery log.
+
+Offsets are committed only after successful processing — infrastructure errors leave messages unconsumed for reprocessing.
+
+## Quick Start
+
+### Prerequisites
+
+- Docker + Docker Compose
+- Go 1.25+
+- `make`
+
+### Run full local stack
+
+```bash
+make docker-up
+```
+
+Starts PostgreSQL, Redis, Kafka, the API server, and the worker. Grafana/Loki/Tempo/Prometheus observability stack is also included.
+
+### Run infra only (develop locally)
+
+```bash
+make docker-up-infra # start postgres, redis, kafka
+make migrate-up # apply DB migrations
+make seed # provision test tenant + API key + starter templates
+make run-api # start API server (default :8080)
+make run-worker # start worker
+```
+
+The seed script prints the `X-API-Key` value to use in requests.
+
+## API
+
+### Send a notification
+
+```bash
+curl -X POST http://localhost:8080/api/v1/notifications \
+ -H "X-API-Key: " \
+ -H "Content-Type: application/json" \
+ -d '{
+ "recipient_id": "user-123",
+ "channel": "email",
+ "template_id": "welcome",
+ "payload": {"name": "Alice"}
+ }'
+```
+
+Returns `202 Accepted`. Delivery is async.
+
+### In-app inbox
+
+```bash
+# List inbox messages
+GET /api/v1/inbox
+X-API-Key:
+X-Recipient-ID: user-123
+
+# WebSocket stream (real-time)
+GET /api/v1/inbox/stream
+Authorization: Bearer
+```
+
+### Admin / internal
+
+| Method | Path | Description |
+|---|---|---|
+| `POST` | `/internal/tenants` | Provision tenant |
+| `POST` | `/internal/tenants/:id/api-keys` | Issue API key |
+| `GET` | `/internal/dlq` | List DLQ messages |
+| `POST` | `/internal/dlq/:id/replay` | Replay DLQ message |
+| `DELETE` | `/internal/dlq/:id` | Delete DLQ message |
+
+Protected by `X-Admin-Token` header.
+
+Full OpenAPI spec: [`docs/api.yaml`](docs/api.yaml)
+
+## Development
+
+```bash
+make build # build api + worker → bin/
+make test # unit tests with race detector + coverage
+make test-integration # integration tests (testcontainers)
+make lint # golangci-lint
+make generate # regenerate sqlc queries (internal/db/)
+make migrate-up # apply migrations
+make migrate-down # roll back one migration
+```
+
+### Adding a channel provider
+
+1. Create `internal/provider/{name}/{name}.go` implementing the `provider.Provider` interface (`Send`, `Channel`)
+2. Return `provider.ErrDeliveryTemporary` for transient failures, `provider.ErrDeliveryPermanent` for terminal failures
+3. Register in `cmd/worker/main.go` via `registry.Register(provider)`
+
+### Modifying DB queries
+
+1. Edit SQL in `queries/*.sql`
+2. Run `make generate`
+3. Update the repository wrapper in `internal/repository/`
+
+## Load Testing
+
+k6 script at [`scripts/loadtest/send_notification.js`](scripts/loadtest/send_notification.js). See [`docs/load-testing.md`](docs/load-testing.md) for results and methodology.
+
+## Observability
+
+| Signal | Stack |
+|---|---|
+| Logs | slog → Grafana Alloy → Loki |
+| Traces | OTel SDK → OTel Collector → Tempo |
+| Metrics | Prometheus → Grafana |
+
+Grafana dashboards provisioned automatically via `deployments/grafana/provisioning/`.
+
+## Docs
+
+- [`docs/api.yaml`](docs/api.yaml) — OpenAPI 3.0 spec
+- [`docs/db-schema.md`](docs/db-schema.md) — database schema reference
+- [`docs/deployment.md`](docs/deployment.md) — deployment guide
+- [`docs/production-grade.md`](docs/production-grade.md) — production readiness notes
+- [`docs/adr/`](docs/adr/) — architecture decision records (Go, Kafka, pgx, sqlc)
+- [`docs/incident-story.md`](docs/incident-story.md) — fictional on-call incident walkthrough
+
+## Project Structure
+
+```
+cmd/
+ api/ # API server entrypoint
+ worker/ # Worker entrypoint
+internal/
+ api/ # HTTP handlers, middleware, router
+ auth/ # JWT utilities
+ config/ # Config loading
+ db/ # sqlc-generated queries (do not edit)
+ domain/ # Pure Go types and errors (no external imports)
+ observability/ # Metrics, tracing, logging, health
+ provider/ # Channel providers (ses, twilio, fcm, inapp, mock)
+ queue/ # Kafka producer + consumer
+ realtime/ # WebSocket hub + Redis pub/sub fan-out
+ repository/ # DB access wrappers
+ service/ # Business logic
+ template/ # Template file loader
+ worker/ # Pool, processor, DLQ consumer, scheduler
+scripts/
+ seed.go # Provision local tenant + API key
+ loadtest/ # k6 load test scripts
+deployments/ # Dockerfile, docker-compose, OTel/Loki/Tempo/Prometheus configs
+docs/ # API spec, ADRs, deployment docs
+queries/ # Raw SQL (input to sqlc)
+migrations/ # golang-migrate SQL migrations
+```
diff --git a/bin/api b/bin/api
index 1d6ad48..5b7020e 100755
Binary files a/bin/api and b/bin/api differ
diff --git a/bin/worker b/bin/worker
index 2473153..7bbf24d 100755
Binary files a/bin/worker and b/bin/worker differ
diff --git a/cmd/api/main.go b/cmd/api/main.go
index a1f34b8..87678fa 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -17,6 +17,7 @@ import (
"github.com/amitrajitdas31/notifyhub/internal/api"
"github.com/amitrajitdas31/notifyhub/internal/api/handler"
+ "github.com/amitrajitdas31/notifyhub/internal/auth"
"github.com/amitrajitdas31/notifyhub/internal/config"
"github.com/amitrajitdas31/notifyhub/internal/db"
"github.com/amitrajitdas31/notifyhub/internal/observability"
@@ -103,6 +104,8 @@ func main() {
preferenceRepo := repository.NewPreferenceRepository(queries)
dlqRepo := repository.NewDeadLetterRepository(queries)
inappRepo := repository.NewInAppRepository(queries)
+ deviceTokenRepo := repository.NewDeviceTokenRepository(queries)
+ webhookRepo := repository.NewWebhookRepository(queries)
// 9. Kafka producer
publisher := queue.NewProducer(cfg.KafkaBrokers, logger)
@@ -119,6 +122,9 @@ func main() {
preferenceSvc := service.NewPreferenceService(preferenceRepo, validate)
notifSvc := service.NewNotificationService(notifRepo, publisher, validate, metrics, cfg.WorkerRetryMaxAttempts)
rateLimitSvc := service.NewRateLimitService(redisClient)
+ deviceTokenSvc := service.NewDeviceTokenService(deviceTokenRepo, validate)
+ wsTokenSvc := auth.NewWSTokenService(cfg.WSJWTSecret, cfg.WSJWTTTLSeconds)
+ webhookSvc := service.NewWebhookService(webhookRepo, publisher, cfg.WebhookMaxAttempts, validate, logger)
// 10b. In-app realtime hub — subscribe to Redis pub/sub
hub := realtime.NewHub(redisClient, logger, cfg.InAppWSBufferSize)
@@ -141,7 +147,10 @@ func main() {
prefHandler := handler.NewPreferenceHandler(preferenceSvc)
tenantHandler := handler.NewTenantHandler(tenantSvc)
dlqHandler := handler.NewDLQHandler(dlqRepo, publisher, cfg.WorkerRetryMaxAttempts)
- inboxHandler := handler.NewInboxHandler(inappRepo, hub, metrics, logger, handler.InboxHandlerConfig{
+ deviceTokenHandler := handler.NewDeviceTokenHandler(deviceTokenSvc)
+ webhookHandler := handler.NewWebhookHandler(webhookSvc)
+ wsTokenHandler := handler.NewWSTokenHandler(wsTokenSvc)
+ inboxHandler := handler.NewInboxHandler(inappRepo, hub, wsTokenSvc, metrics, logger, handler.InboxHandlerConfig{
HeartbeatSeconds: cfg.InAppWSHeartbeatSeconds,
ReadTimeoutSeconds: cfg.InAppWSReadTimeoutSeconds,
})
@@ -161,6 +170,9 @@ func main() {
Tenant: tenantHandler,
DLQ: dlqHandler,
Inbox: inboxHandler,
+ DeviceToken: deviceTokenHandler,
+ WSToken: wsTokenHandler,
+ Webhook: webhookHandler,
})
// 14. HTTP server
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
index 450039b..2bc7721 100644
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -102,6 +102,7 @@ func main() {
templateRepo := repository.NewTemplateRepository(queries)
dlqRepo := repository.NewDeadLetterRepository(queries)
inappRepo := repository.NewInAppRepository(queries)
+ webhookRepo := repository.NewWebhookRepository(queries)
// 6. Services
validate := validator.New()
@@ -117,7 +118,16 @@ func main() {
}
}()
- // 8. Provider registry, processor, scheduler
+ webhookSvc := service.NewWebhookService(webhookRepo, producer, cfg.WebhookMaxAttempts, validate, logger)
+
+ // 8. Pre-create Kafka topics so consumers never join with empty assignments.
+ // CreateTopics is idempotent — safe to call on every startup.
+ if err := queue.EnsureTopics(context.Background(), cfg.KafkaBrokers, logger); err != nil {
+ logger.Error("failed to ensure kafka topics", "error", err)
+ os.Exit(1)
+ }
+
+ // 9. Provider registry, processor, scheduler
registry := buildRegistry(cfg, logger, inappRepo, redisClient)
processor := worker.NewProcessor(worker.ProcessorDeps{
@@ -138,6 +148,7 @@ func main() {
Logger: logger,
DLQPublisher: producer,
DLQEnabled: cfg.DLQEnabled,
+ Webhook: webhookSvc,
})
scheduler := service.NewScheduler(notifRepo, producer, schedulerInterval, schedulerBatch, cfg.WorkerRetryMaxAttempts, logger)
@@ -166,11 +177,12 @@ func main() {
slog.Int("inapp_concurrency", cfg.WorkerInAppConcurrency),
)
- // 12. Launch pools, DLQ consumers, and scheduler
+ // 12. Launch pools, DLQ consumers, scheduler, and webhook worker
var wg sync.WaitGroup
startPools(ctx, cfg, processor, logger, &wg)
startDLQConsumers(ctx, cfg, dlqRepo, metrics, logger, &wg)
startScheduler(ctx, scheduler, logger, &wg)
+ startWebhookWorker(ctx, cfg, webhookRepo, metrics, logger, &wg)
// 13. Block until signal, then drain in order
<-ctx.Done()
@@ -281,6 +293,30 @@ func startScheduler(ctx context.Context, sched *service.Scheduler, logger *slog.
}()
}
+// startWebhookWorker launches the outbound webhook delivery goroutine when enabled.
+func startWebhookWorker(ctx context.Context, cfg *config.Config, repo repository.WebhookRepository, metrics *observability.Metrics, logger *slog.Logger, wg *sync.WaitGroup) {
+ if !cfg.WebhookEnabled {
+ return
+ }
+ consumer := queue.NewConsumer(cfg.KafkaBrokers, queue.WebhookTopic, cfg.WebhookGroupID, logger)
+ w := worker.NewWebhookWorker(
+ consumer,
+ repo,
+ metrics,
+ time.Duration(cfg.WebhookRetryBaseMS)*time.Millisecond,
+ logger,
+ )
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ logger.Info("webhook worker started")
+ if err := w.Run(ctx); err != nil {
+ logger.Error("webhook worker error", slog.Any("error", err))
+ }
+ logger.Info("webhook worker stopped")
+ }()
+}
+
// buildRegistry registers real providers when credentials are configured,
// falling back to the mock for any channel without credentials.
func buildRegistry(cfg *config.Config, logger *slog.Logger, inappRepo repository.InAppRepository, rdb *redis.Client) *provider.Registry {
diff --git a/docs/api.yaml b/docs/api.yaml
new file mode 100644
index 0000000..baeb440
--- /dev/null
+++ b/docs/api.yaml
@@ -0,0 +1,626 @@
+openapi: 3.0.3
+info:
+ title: NotifyHub API
+ version: 0.1.0
+ description: Multi-channel notification API for tenant-scoped sends, templates, preferences, inbox, device tokens, webhooks, and internal operations.
+servers:
+ - url: http://localhost:8080
+security:
+ - ApiKeyAuth: []
+tags:
+ - name: Health
+ - name: Notifications
+ - name: Templates
+ - name: Preferences
+ - name: Inbox
+ - name: Device Tokens
+ - name: Webhooks
+ - name: Internal
+paths:
+ /health:
+ get:
+ tags: [Health]
+ security: []
+ summary: Deep health check
+ responses:
+ "200":
+ description: Dependencies are reachable.
+ /livez:
+ get:
+ tags: [Health]
+ security: []
+ summary: Liveness probe
+ responses:
+ "200":
+ description: Process is alive.
+ /readyz:
+ get:
+ tags: [Health]
+ security: []
+ summary: Readiness probe
+ responses:
+ "200":
+ description: Service is ready.
+ /api/v1/notifications:
+ post:
+ tags: [Notifications]
+ summary: Send a notification
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/SendRequest"
+ responses:
+ "202":
+ description: Notification accepted.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Notification"
+ get:
+ tags: [Notifications]
+ summary: List notifications
+ parameters:
+ - $ref: "#/components/parameters/Page"
+ - $ref: "#/components/parameters/PerPage"
+ - name: recipient_id
+ in: query
+ schema: { type: string }
+ - name: channel
+ in: query
+ schema: { $ref: "#/components/schemas/Channel" }
+ - name: status
+ in: query
+ schema: { $ref: "#/components/schemas/NotificationStatus" }
+ responses:
+ "200":
+ description: Paged notifications.
+ /api/v1/notifications/bulk:
+ post:
+ tags: [Notifications]
+ summary: Send notifications to up to 1000 recipients
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/BulkSendRequest"
+ responses:
+ "202":
+ description: Bulk send accepted with per-recipient results.
+ /api/v1/notifications/{id}:
+ get:
+ tags: [Notifications]
+ summary: Get a notification
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "200":
+ description: Notification.
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Notification"
+ delete:
+ tags: [Notifications]
+ summary: Cancel a pending notification
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "200":
+ description: Canceled notification.
+ /api/v1/templates:
+ post:
+ tags: [Templates]
+ summary: Create a template
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateTemplateRequest"
+ responses:
+ "201":
+ description: Created template.
+ get:
+ tags: [Templates]
+ summary: List templates
+ parameters:
+ - $ref: "#/components/parameters/Page"
+ - $ref: "#/components/parameters/PerPage"
+ - name: channel
+ in: query
+ schema: { $ref: "#/components/schemas/Channel" }
+ responses:
+ "200":
+ description: Paged templates.
+ /api/v1/templates/{id}:
+ get:
+ tags: [Templates]
+ summary: Get a template
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "200":
+ description: Template.
+ put:
+ tags: [Templates]
+ summary: Update a template
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/UpdateTemplateRequest"
+ responses:
+ "200":
+ description: Updated template.
+ delete:
+ tags: [Templates]
+ summary: Delete a template
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "200":
+ description: Deleted template.
+ /api/v1/templates/{id}/preview:
+ post:
+ tags: [Templates]
+ summary: Preview a rendered template
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PreviewTemplateRequest"
+ responses:
+ "200":
+ description: Rendered template preview.
+ /api/v1/users/{user_id}/preferences:
+ get:
+ tags: [Preferences]
+ summary: List preferences for a recipient
+ parameters:
+ - name: user_id
+ in: path
+ required: true
+ schema: { type: string }
+ responses:
+ "200":
+ description: Preferences.
+ /api/v1/users/{user_id}/preferences/{channel}:
+ put:
+ tags: [Preferences]
+ summary: Upsert a channel preference
+ parameters:
+ - name: user_id
+ in: path
+ required: true
+ schema: { type: string }
+ - name: channel
+ in: path
+ required: true
+ schema: { $ref: "#/components/schemas/Channel" }
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/UpsertPreferenceRequest"
+ responses:
+ "200":
+ description: Preference.
+ delete:
+ tags: [Preferences]
+ summary: Delete a channel preference
+ parameters:
+ - name: user_id
+ in: path
+ required: true
+ schema: { type: string }
+ - name: channel
+ in: path
+ required: true
+ schema: { $ref: "#/components/schemas/Channel" }
+ responses:
+ "204":
+ description: Deleted.
+ /api/v1/ws-token:
+ post:
+ tags: [Inbox]
+ summary: Issue a short-lived WebSocket JWT
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ required: [recipient_id]
+ properties:
+ recipient_id: { type: string }
+ responses:
+ "200":
+ description: WebSocket token.
+ /api/v1/inbox:
+ get:
+ tags: [Inbox]
+ summary: List in-app inbox messages
+ parameters:
+ - $ref: "#/components/parameters/RecipientHeader"
+ - name: unread
+ in: query
+ schema: { type: boolean }
+ - name: limit
+ in: query
+ schema: { type: integer, minimum: 1, maximum: 200 }
+ - name: after_id
+ in: query
+ schema: { type: string, format: uuid }
+ responses:
+ "200":
+ description: Inbox messages.
+ /api/v1/inbox/unread-count:
+ get:
+ tags: [Inbox]
+ summary: Count unread in-app messages
+ parameters:
+ - $ref: "#/components/parameters/RecipientHeader"
+ responses:
+ "200":
+ description: Unread count.
+ /api/v1/inbox/{id}/read:
+ post:
+ tags: [Inbox]
+ summary: Mark one message read
+ parameters:
+ - $ref: "#/components/parameters/RecipientHeader"
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "204":
+ description: Marked read.
+ /api/v1/inbox/read-all:
+ post:
+ tags: [Inbox]
+ summary: Mark all inbox messages read
+ parameters:
+ - $ref: "#/components/parameters/RecipientHeader"
+ responses:
+ "204":
+ description: Marked read.
+ /api/v1/inbox/stream:
+ get:
+ tags: [Inbox]
+ security: []
+ summary: WebSocket stream for in-app messages
+ parameters:
+ - name: token
+ in: query
+ required: true
+ schema: { type: string }
+ - name: since
+ in: query
+ schema: { type: string, format: uuid }
+ responses:
+ "101":
+ description: WebSocket upgrade.
+ /api/v1/device-tokens:
+ post:
+ tags: [Device Tokens]
+ summary: Register or refresh a device token
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/RegisterDeviceTokenRequest"
+ responses:
+ "201":
+ description: Device token.
+ /api/v1/device-tokens/{token}:
+ delete:
+ tags: [Device Tokens]
+ summary: Deregister a device token
+ parameters:
+ - name: token
+ in: path
+ required: true
+ schema: { type: string }
+ responses:
+ "204":
+ description: Deregistered.
+ /api/v1/webhooks:
+ post:
+ tags: [Webhooks]
+ summary: Create a webhook endpoint
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateWebhookEndpointRequest"
+ responses:
+ "201":
+ description: Webhook endpoint.
+ get:
+ tags: [Webhooks]
+ summary: List webhook endpoints
+ responses:
+ "200":
+ description: Webhook endpoints.
+ /api/v1/webhooks/{id}:
+ get:
+ tags: [Webhooks]
+ summary: Get a webhook endpoint
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "200":
+ description: Webhook endpoint.
+ put:
+ tags: [Webhooks]
+ summary: Update a webhook endpoint
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/UpdateWebhookEndpointRequest"
+ responses:
+ "200":
+ description: Updated endpoint.
+ delete:
+ tags: [Webhooks]
+ summary: Delete a webhook endpoint
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "204":
+ description: Deleted.
+ /internal/tenants:
+ post:
+ tags: [Internal]
+ security:
+ - AdminTokenAuth: []
+ summary: Create a tenant
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateTenantRequest"
+ responses:
+ "201":
+ description: Tenant.
+ /internal/tenants/{id}/api-keys:
+ post:
+ tags: [Internal]
+ security:
+ - AdminTokenAuth: []
+ summary: Create an API key for a tenant
+ parameters:
+ - $ref: "#/components/parameters/ID"
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateAPIKeyRequest"
+ responses:
+ "201":
+ description: API key, returned once.
+ /internal/dlq:
+ get:
+ tags: [Internal]
+ security:
+ - AdminTokenAuth: []
+ summary: List dead-letter messages
+ parameters:
+ - $ref: "#/components/parameters/TenantHeader"
+ - name: channel
+ in: query
+ schema: { type: string }
+ - name: unreplayed
+ in: query
+ schema: { type: boolean }
+ responses:
+ "200":
+ description: Dead-letter messages.
+ /internal/dlq/{id}:
+ get:
+ tags: [Internal]
+ security:
+ - AdminTokenAuth: []
+ summary: Get a dead-letter message
+ parameters:
+ - $ref: "#/components/parameters/TenantHeader"
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "200":
+ description: Dead-letter message.
+ delete:
+ tags: [Internal]
+ security:
+ - AdminTokenAuth: []
+ summary: Delete a dead-letter message
+ parameters:
+ - $ref: "#/components/parameters/TenantHeader"
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "204":
+ description: Deleted.
+ /internal/dlq/{id}/replay:
+ post:
+ tags: [Internal]
+ security:
+ - AdminTokenAuth: []
+ summary: Replay a dead-letter message
+ parameters:
+ - $ref: "#/components/parameters/TenantHeader"
+ - $ref: "#/components/parameters/ID"
+ responses:
+ "200":
+ description: Replayed.
+components:
+ securitySchemes:
+ ApiKeyAuth:
+ type: apiKey
+ in: header
+ name: X-API-Key
+ AdminTokenAuth:
+ type: apiKey
+ in: header
+ name: X-Admin-Token
+ parameters:
+ ID:
+ name: id
+ in: path
+ required: true
+ schema: { type: string, format: uuid }
+ Page:
+ name: page
+ in: query
+ schema: { type: integer, minimum: 1, default: 1 }
+ PerPage:
+ name: per_page
+ in: query
+ schema: { type: integer, minimum: 1, default: 20 }
+ RecipientHeader:
+ name: X-Recipient-ID
+ in: header
+ required: true
+ schema: { type: string }
+ TenantHeader:
+ name: X-Tenant-ID
+ in: header
+ required: true
+ schema: { type: string, format: uuid }
+ schemas:
+ Channel:
+ type: string
+ enum: [email, push, sms, inapp]
+ NotificationStatus:
+ type: string
+ enum: [pending, queued, processing, delivered, failed, dropped]
+ JSONMap:
+ type: object
+ additionalProperties: true
+ SendRequest:
+ type: object
+ required: [type, channel, recipient_id, recipient_address]
+ properties:
+ type: { type: string }
+ channel: { $ref: "#/components/schemas/Channel" }
+ recipient_id: { type: string }
+ recipient_address: { type: string }
+ template_id: { type: string, format: uuid }
+ payload: { $ref: "#/components/schemas/JSONMap" }
+ priority: { type: integer, minimum: 1, maximum: 10, default: 5 }
+ idempotency_key: { type: string }
+ scheduled_at: { type: string, format: date-time }
+ BulkSendRequest:
+ type: object
+ required: [type, channel, recipients]
+ properties:
+ type: { type: string }
+ channel: { $ref: "#/components/schemas/Channel" }
+ template_id: { type: string, format: uuid }
+ payload: { $ref: "#/components/schemas/JSONMap" }
+ priority: { type: integer, minimum: 1, maximum: 10, default: 5 }
+ recipients:
+ type: array
+ minItems: 1
+ maxItems: 1000
+ items:
+ type: object
+ required: [recipient_id, recipient_address]
+ properties:
+ recipient_id: { type: string }
+ recipient_address: { type: string }
+ payload: { $ref: "#/components/schemas/JSONMap" }
+ Notification:
+ allOf:
+ - $ref: "#/components/schemas/SendRequest"
+ - type: object
+ properties:
+ id: { type: string, format: uuid }
+ tenant_id: { type: string, format: uuid }
+ status: { $ref: "#/components/schemas/NotificationStatus" }
+ created_at: { type: string, format: date-time }
+ updated_at: { type: string, format: date-time }
+ CreateTemplateRequest:
+ type: object
+ required: [name, channel, body_template]
+ properties:
+ name: { type: string }
+ channel: { $ref: "#/components/schemas/Channel" }
+ subject_template: { type: string }
+ body_template: { type: string }
+ metadata: { $ref: "#/components/schemas/JSONMap" }
+ UpdateTemplateRequest:
+ type: object
+ properties:
+ subject_template: { type: string }
+ body_template: { type: string }
+ metadata: { $ref: "#/components/schemas/JSONMap" }
+ is_active: { type: boolean }
+ PreviewTemplateRequest:
+ type: object
+ required: [payload]
+ properties:
+ payload: { $ref: "#/components/schemas/JSONMap" }
+ UpsertPreferenceRequest:
+ type: object
+ properties:
+ enabled: { type: boolean }
+ quiet_hours_start: { type: string, example: "22:00" }
+ quiet_hours_end: { type: string, example: "08:00" }
+ frequency_cap: { type: integer, minimum: 1 }
+ frequency_window_minutes: { type: integer, minimum: 1 }
+ timezone: { type: string, example: Asia/Kolkata }
+ RegisterDeviceTokenRequest:
+ type: object
+ required: [user_id, token, platform]
+ properties:
+ user_id: { type: string }
+ token: { type: string }
+ platform:
+ type: string
+ enum: [ios, android, web]
+ CreateWebhookEndpointRequest:
+ type: object
+ required: [url, secret, events]
+ properties:
+ url: { type: string, format: uri }
+ secret: { type: string, minLength: 16 }
+ events:
+ type: array
+ items:
+ type: string
+ enum: [notification.delivered, notification.failed, notification.dropped, device_token.deactivated]
+ UpdateWebhookEndpointRequest:
+ type: object
+ required: [url, events]
+ properties:
+ url: { type: string, format: uri }
+ events:
+ type: array
+ items:
+ type: string
+ is_active: { type: boolean }
+ CreateTenantRequest:
+ type: object
+ required: [name]
+ properties:
+ name: { type: string }
+ CreateAPIKeyRequest:
+ type: object
+ required: [name]
+ properties:
+ name: { type: string }
diff --git a/docs/db-schema.md b/docs/db-schema.md
deleted file mode 100644
index cc61b95..0000000
--- a/docs/db-schema.md
+++ /dev/null
@@ -1,113 +0,0 @@
-# Database Schema
-
-```mermaid
-erDiagram
- api_clients {
- UUID id PK
- VARCHAR name
- VARCHAR api_key_hash UK
- BOOLEAN is_active
- TIMESTAMPTZ created_at
- TIMESTAMPTZ updated_at
- }
-
- templates {
- UUID id PK
- VARCHAR name UK
- VARCHAR channel
- TEXT subject_template "email only"
- TEXT body_template
- JSONB metadata
- INT version
- BOOLEAN is_active
- TIMESTAMPTZ created_at
- TIMESTAMPTZ updated_at
- }
-
- preferences {
- UUID id PK
- VARCHAR user_id
- VARCHAR channel
- BOOLEAN enabled
- TIME quiet_hours_start
- TIME quiet_hours_end
- INT frequency_cap
- INT frequency_window_minutes
- VARCHAR timezone
- TIMESTAMPTZ created_at
- TIMESTAMPTZ updated_at
- }
-
- notifications {
- UUID id PK
- VARCHAR idempotency_key UK
- VARCHAR type
- VARCHAR channel
- VARCHAR recipient_id
- VARCHAR recipient_address
- UUID template_id FK
- JSONB payload
- INT priority
- VARCHAR status "pending|queued|processing|delivered|failed|dropped"
- TIMESTAMPTZ scheduled_at "NULL = immediate"
- TIMESTAMPTZ created_at
- TIMESTAMPTZ updated_at
- }
-
- delivery_logs {
- UUID id PK
- UUID notification_id FK
- VARCHAR channel
- VARCHAR provider "ses|fcm|twilio"
- VARCHAR status "success|failed|bounced|rejected"
- VARCHAR provider_message_id
- TEXT error_message
- VARCHAR error_code
- INT retry_count
- TIMESTAMPTZ attempted_at
- TIMESTAMPTZ delivered_at
- }
-
- templates ||--o{ notifications : "referenced by"
- notifications ||--o{ delivery_logs : "has many"
- preferences }o--|| notifications : "checked against"
-```
-
-## Notification Lifecycle
-
-```mermaid
-stateDiagram-v2
- [*] --> pending : Created
- pending --> queued : Sent to Kafka
- pending --> dropped : Preference check failed
- queued --> processing : Worker picks up
- processing --> delivered : Provider success
- processing --> failed : Max retries exhausted
- processing --> processing : Retry on transient error
-```
-
-## Key Constraints
-
-| Table | Constraint | Purpose |
-| --------------- | -------------------------- | ----------------------------------- |
-| `api_clients` | `UNIQUE(api_key_hash)` | One key per client |
-| `templates` | `UNIQUE(name)` | Lookup by name |
-| `preferences` | `UNIQUE(user_id, channel)` | One preference per user per channel |
-| `notifications` | `UNIQUE(idempotency_key)` | Prevent duplicate sends |
-| `delivery_logs` | `FK(notification_id)` | Links attempts to notification |
-
-## Indexes
-
-| Index | Table | Purpose |
-| ------------------------------- | ------------- | ----------------------------------- |
-| `idx_templates_channel` | templates | Filter by channel |
-| `idx_templates_name` | templates | Lookup by name |
-| `idx_preferences_user` | preferences | Lookup by user_id |
-| `idx_notifications_status` | notifications | Filter by status |
-| `idx_notifications_recipient` | notifications | Filter by recipient |
-| `idx_notifications_scheduled` | notifications | Partial: pending + has scheduled_at |
-| `idx_notifications_idempotency` | notifications | Partial: non-null idempotency keys |
-| `idx_notifications_created` | notifications | Sort by creation time |
-| `idx_delivery_notification` | delivery_logs | Join to notification |
-| `idx_delivery_status` | delivery_logs | Filter by delivery status |
-| `idx_delivery_attempted` | delivery_logs | Sort by attempt time |
diff --git a/docs/deployment.md b/docs/deployment.md
deleted file mode 100644
index 9d3edd8..0000000
--- a/docs/deployment.md
+++ /dev/null
@@ -1,941 +0,0 @@
-# NotifyHub Deployment Guide
-
-This document covers two deployment strategies:
-
-1. **AWS Free Tier** — a cost-effective setup for development, demos, and low-traffic use
-2. **Production-Grade** — a scalable, HA architecture for real workloads
-
----
-
-## Table of Contents
-
-- [Prerequisites](#prerequisites)
-- [Part 1: AWS Free Tier Deployment](#part-1-aws-free-tier-deployment)
- - [Architecture Overview](#free-tier-architecture-overview)
- - [Step 1: AWS Account Setup](#step-1-aws-account-setup)
- - [Step 2: Networking (VPC)](#step-2-networking-vpc)
- - [Step 3: Database (RDS PostgreSQL)](#step-3-database-rds-postgresql)
- - [Step 4: Cache (ElastiCache Redis)](#step-4-cache-elasticache-redis)
- - [Step 5: Message Queue (MSK Serverless / Self-Hosted)](#step-5-message-queue)
- - [Step 6: Container Registry (ECR)](#step-6-container-registry-ecr)
- - [Step 7: Compute (ECS Fargate / EC2)](#step-7-compute)
- - [Step 8: Load Balancer & DNS](#step-8-load-balancer--dns)
- - [Step 9: CI/CD Pipeline](#step-9-cicd-pipeline)
- - [Step 10: Monitoring](#step-10-monitoring)
- - [Cost Breakdown](#free-tier-cost-breakdown)
-- [Part 2: Production Deployment](#part-2-production-deployment)
- - [Architecture Overview](#production-architecture-overview)
- - [Infrastructure as Code](#infrastructure-as-code)
- - [Compute Layer](#compute-layer)
- - [Data Layer](#data-layer)
- - [Networking & Security](#networking--security)
- - [Observability Stack](#observability-stack)
- - [CI/CD Pipeline](#production-cicd-pipeline)
- - [Disaster Recovery](#disaster-recovery)
- - [Cost Estimate](#production-cost-estimate)
-
----
-
-## Prerequisites
-
-- AWS account (12-month free tier eligibility for new accounts)
-- AWS CLI v2 installed and configured (`aws configure`)
-- Docker installed locally
-- Terraform (optional, for IaC)
-- Domain name (optional, for custom DNS)
-
----
-
-## Part 1: AWS Free Tier Deployment
-
-### Free Tier Architecture Overview
-
-```
- ┌─────────────┐
- │ Route 53 │ (optional)
- └──────┬──────┘
- │
- ┌──────▼──────┐
- │ ALB │ (free tier: not included,
- └──────┬──────┘ use EC2 direct or skip)
- │
- ┌────────────┼────────────┐
- │ │
- ┌──────▼──────┐ ┌──────▼──────┐
- │ EC2 (API) │ │ EC2 (Worker)│
- │ t2.micro │ │ t2.micro │
- └──────┬──────┘ └──────┬──────┘
- │ │
- ┌─────────┼─────────┬───────────────┤
- │ │ │ │
-┌───▼───┐ ┌──▼──┐ ┌────▼────┐ ┌───────▼───────┐
-│RDS PG │ │Redis│ │ Kafka │ │ SES (email) │
-│db.t3 │ │(EC2)│ │ (EC2) │ │ SNS (push) │
-│.micro │ │ │ │ │ └───────────────┘
-└───────┘ └─────┘ └─────────┘
-```
-
-**Strategy:** Run everything on a single `t2.micro` EC2 instance using Docker Compose for the simplest free-tier approach. For a slightly better setup, use RDS free tier for PostgreSQL and run everything else on EC2.
-
----
-
-### Step 1: AWS Account Setup
-
-1. Create an AWS account at https://aws.amazon.com/free
-2. Enable MFA on the root account
-3. Create an IAM user with programmatic access for deployments:
-
-```bash
-aws iam create-user --user-name notifyhub-deploy
-aws iam attach-user-policy --user-name notifyhub-deploy \
- --policy-arn arn:aws:iam::aws:policy/PowerUserAccess
-aws iam create-access-key --user-name notifyhub-deploy
-```
-
-4. Configure AWS CLI:
-
-```bash
-aws configure --profile notifyhub
-# Enter access key, secret key, region (us-east-1 recommended for most free tier services)
-```
-
----
-
-### Step 2: Networking (VPC)
-
-Use the default VPC for free-tier simplicity, or create a dedicated one:
-
-```bash
-# Use default VPC (simplest)
-VPC_ID=$(aws ec2 describe-vpcs --filters "Name=isDefault,Values=true" \
- --query "Vpcs[0].VpcId" --output text)
-
-# Create a security group for NotifyHub
-aws ec2 create-security-group \
- --group-name notifyhub-sg \
- --description "NotifyHub security group" \
- --vpc-id $VPC_ID
-
-# Allow inbound HTTP (API)
-aws ec2 authorize-security-group-ingress \
- --group-name notifyhub-sg \
- --protocol tcp --port 8080 --cidr 0.0.0.0/0
-
-# Allow SSH
-aws ec2 authorize-security-group-ingress \
- --group-name notifyhub-sg \
- --protocol tcp --port 22 --cidr /32
-```
-
----
-
-### Step 3: Database (RDS PostgreSQL)
-
-RDS offers 12 months of free tier: `db.t3.micro`, 20 GB storage, PostgreSQL 16.
-
-```bash
-aws rds create-db-instance \
- --db-instance-identifier notifyhub-db \
- --db-instance-class db.t3.micro \
- --engine postgres \
- --engine-version "16" \
- --master-username notifyhub \
- --master-user-password "" \
- --allocated-storage 20 \
- --storage-type gp2 \
- --no-multi-az \
- --publicly-accessible \
- --backup-retention-period 7 \
- --vpc-security-group-ids
-```
-
-After the instance is ready, run migrations:
-
-```bash
-# Get the endpoint
-RDS_ENDPOINT=$(aws rds describe-db-instances \
- --db-instance-identifier notifyhub-db \
- --query "DBInstances[0].Endpoint.Address" --output text)
-
-# Run migrations
-DATABASE_URL="postgres://notifyhub:@${RDS_ENDPOINT}:5432/notifyhub?sslmode=require"
-make migrate-up
-```
-
----
-
-### Step 4: Cache (ElastiCache Redis)
-
-ElastiCache is **not** in the free tier. For free-tier, run Redis on the EC2 instance via Docker.
-
-If budget allows (~$13/mo for `cache.t3.micro`):
-
-```bash
-aws elasticache create-cache-cluster \
- --cache-cluster-id notifyhub-redis \
- --cache-node-type cache.t3.micro \
- --engine redis \
- --engine-version "7.0" \
- --num-cache-nodes 1
-```
-
-**Free alternative:** Run Redis in Docker on the EC2 instance (covered in Step 7).
-
----
-
-### Step 5: Message Queue
-
-Amazon MSK (Managed Kafka) is **not** free tier eligible. Options:
-
-#### Option A: Self-hosted Kafka on EC2 (free tier)
-
-Run Kafka via Docker Compose on the same EC2 instance. This works for low traffic but is not recommended beyond demos.
-
-#### Option B: Amazon SQS as an alternative (~free tier eligible)
-
-SQS offers 1 million free requests/month. This would require code changes to replace the Kafka producer/consumer with SQS, so we'll stick with self-hosted Kafka for now.
-
----
-
-### Step 6: Container Registry (ECR)
-
-ECR free tier: 500 MB storage/month for 12 months.
-
-```bash
-# Create repositories
-aws ecr create-repository --repository-name notifyhub/api
-aws ecr create-repository --repository-name notifyhub/worker
-
-# Login to ECR
-aws ecr get-login-password --region us-east-1 | \
- docker login --username AWS --password-stdin .dkr.ecr.us-east-1.amazonaws.com
-
-# Build and push images
-docker build --target api -t notifyhub/api -f deployments/Dockerfile .
-docker tag notifyhub/api:latest .dkr.ecr.us-east-1.amazonaws.com/notifyhub/api:latest
-docker push .dkr.ecr.us-east-1.amazonaws.com/notifyhub/api:latest
-
-docker build --target worker -t notifyhub/worker -f deployments/Dockerfile .
-docker tag notifyhub/worker:latest .dkr.ecr.us-east-1.amazonaws.com/notifyhub/worker:latest
-docker push .dkr.ecr.us-east-1.amazonaws.com/notifyhub/worker:latest
-```
-
----
-
-### Step 7: Compute
-
-#### Option A: Single EC2 with Docker Compose (Simplest, True Free Tier)
-
-This runs everything on one `t2.micro` instance (1 vCPU, 1 GB RAM). Tight but workable for demos.
-
-```bash
-# Create a key pair
-aws ec2 create-key-pair --key-name notifyhub-key \
- --query "KeyMaterial" --output text > notifyhub-key.pem
-chmod 400 notifyhub-key.pem
-
-# Launch EC2 instance (Amazon Linux 2023, t2.micro)
-aws ec2 run-instances \
- --image-id ami-0c02fb55956c7d316 \
- --instance-type t2.micro \
- --key-name notifyhub-key \
- --security-groups notifyhub-sg \
- --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=notifyhub}]' \
- --user-data file://scripts/ec2-user-data.sh
-```
-
-Create `scripts/ec2-user-data.sh`:
-
-```bash
-#!/bin/bash
-set -euo pipefail
-
-# Install Docker
-yum update -y
-yum install -y docker git
-systemctl enable docker
-systemctl start docker
-usermod -aG docker ec2-user
-
-# Install Docker Compose
-DOCKER_COMPOSE_VERSION="v2.24.0"
-curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" \
- -o /usr/local/bin/docker-compose
-chmod +x /usr/local/bin/docker-compose
-
-# Clone and start
-cd /home/ec2-user
-git clone notifyhub
-cd notifyhub
-docker-compose -f deployments/docker-compose.yml up -d
-```
-
-**Important:** On `t2.micro` (1 GB RAM), you'll need to create a swap file and reduce Kafka memory:
-
-```bash
-# SSH into the instance
-ssh -i notifyhub-key.pem ec2-user@
-
-# Create 2GB swap
-sudo dd if=/dev/zero of=/swapfile bs=1M count=2048
-sudo chmod 600 /swapfile
-sudo mkswap /swapfile
-sudo swapon /swapfile
-echo '/swapfile swap swap defaults 0 0' | sudo tee -a /etc/fstab
-```
-
-Create a `deployments/docker-compose.freetier.yml` override to reduce memory:
-
-```yaml
-# Override for free-tier EC2 (1 GB RAM + 2 GB swap)
-services:
- kafka:
- environment:
- KAFKA_HEAP_OPTS: "-Xmx256m -Xms128m"
-
- postgres:
- command: postgres -c shared_buffers=64MB -c work_mem=4MB
-
- redis:
- command: redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru
-```
-
-Run with:
-
-```bash
-docker-compose -f deployments/docker-compose.yml \
- -f deployments/docker-compose.freetier.yml up -d
-```
-
-#### Option B: ECS Fargate (Not Free Tier, but Low Cost)
-
-If you're past free tier or want a more managed experience, ECS Fargate is a clean option. See the production section below for ECS patterns.
-
----
-
-### Step 8: Load Balancer & DNS
-
-For free tier, skip ALB (it costs ~$16/mo) and access the API directly via the EC2 public IP:
-
-```
-http://:8080
-```
-
-For a stable endpoint, allocate an Elastic IP (free when attached to a running instance):
-
-```bash
-ALLOCATION_ID=$(aws ec2 allocate-address --query "AllocationId" --output text)
-aws ec2 associate-address --allocation-id $ALLOCATION_ID --instance-id
-```
-
-**Optional:** Use Cloudflare (free tier) in front for DNS + free SSL:
-
-1. Register your domain with any registrar
-2. Add your domain to Cloudflare (free plan)
-3. Create an A record pointing to your Elastic IP
-4. Enable Cloudflare proxy for free SSL termination
-
----
-
-### Step 9: CI/CD Pipeline
-
-Use **GitHub Actions** (free for public repos, 2000 min/mo for private):
-
-Create `.github/workflows/deploy.yml`:
-
-```yaml
-name: Deploy to AWS
-
-on:
- push:
- branches: [main]
-
-env:
- AWS_REGION: us-east-1
- ECR_REGISTRY: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com
-
-jobs:
- test:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-go@v5
- with:
- go-version: "1.22"
- - run: make test
- - run: make lint
-
- build-and-deploy:
- needs: test
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Configure AWS credentials
- uses: aws-actions/configure-aws-credentials@v4
- with:
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- aws-region: ${{ env.AWS_REGION }}
-
- - name: Login to ECR
- uses: aws-actions/amazon-ecr-login@v2
-
- - name: Build and push API image
- run: |
- docker build --target api -t $ECR_REGISTRY/notifyhub/api:${{ github.sha }} \
- -f deployments/Dockerfile .
- docker push $ECR_REGISTRY/notifyhub/api:${{ github.sha }}
-
- - name: Build and push Worker image
- run: |
- docker build --target worker -t $ECR_REGISTRY/notifyhub/worker:${{ github.sha }} \
- -f deployments/Dockerfile .
- docker push $ECR_REGISTRY/notifyhub/worker:${{ github.sha }}
-
- - name: Deploy to EC2
- uses: appleboy/ssh-action@v1
- with:
- host: ${{ secrets.EC2_HOST }}
- username: ec2-user
- key: ${{ secrets.EC2_SSH_KEY }}
- script: |
- cd /home/ec2-user/notifyhub
- git pull origin main
- aws ecr get-login-password --region us-east-1 | \
- docker login --username AWS --password-stdin ${{ env.ECR_REGISTRY }}
- docker-compose -f deployments/docker-compose.yml \
- -f deployments/docker-compose.freetier.yml pull
- docker-compose -f deployments/docker-compose.yml \
- -f deployments/docker-compose.freetier.yml up -d
- docker image prune -f
-```
-
-GitHub Actions secrets to configure:
-- `AWS_ACCOUNT_ID`
-- `AWS_ACCESS_KEY_ID`
-- `AWS_SECRET_ACCESS_KEY`
-- `EC2_HOST` (Elastic IP)
-- `EC2_SSH_KEY` (contents of the .pem file)
-
----
-
-### Step 10: Monitoring
-
-#### Free options:
-
-1. **CloudWatch** (free tier: 10 custom metrics, 5 GB log ingestion, 3 dashboards)
-
-```bash
-# Install CloudWatch agent on EC2
-sudo yum install -y amazon-cloudwatch-agent
-```
-
-2. **Prometheus + Grafana** are already in the Docker Compose stack. Access:
- - Prometheus: `http://:9090`
- - Grafana: `http://:3000` (admin/admin)
-
-3. **Health check endpoint** — the API exposes a health endpoint. Set up a free uptime monitor via [UptimeRobot](https://uptimerobot.com) (50 monitors free).
-
----
-
-### Free Tier Cost Breakdown
-
-| Service | Free Tier Allowance | NotifyHub Usage | Monthly Cost |
-|---------|-------------------|-----------------|-------------|
-| EC2 t2.micro | 750 hrs/mo (12 months) | 1 instance, 24/7 | $0.00 |
-| RDS db.t3.micro | 750 hrs/mo (12 months) | 1 instance | $0.00 |
-| ECR | 500 MB (12 months) | ~100 MB (2 images) | $0.00 |
-| EBS (EC2 storage) | 30 GB gp2 | 20 GB | $0.00 |
-| RDS storage | 20 GB gp2 | 20 GB | $0.00 |
-| SES (email) | 3,000 msgs/mo (from EC2) | Low volume | $0.00 |
-| CloudWatch | 10 metrics, 5 GB logs | Basic monitoring | $0.00 |
-| Elastic IP | Free when attached | 1 IP | $0.00 |
-| Data Transfer | 100 GB/mo outbound | Low traffic | $0.00 |
-| **Total** | | | **$0.00** |
-
-> After 12 months, expect ~$15-25/mo for EC2 + RDS at the smallest instance sizes.
-
----
-
-## Part 2: Production Deployment
-
-This section describes a fully production-ready architecture. This is included for reference and future planning.
-
-### Production Architecture Overview
-
-```
- ┌──────────────┐
- │ Route 53 │
- │ (DNS + Health│
- │ Checks) │
- └──────┬───────┘
- │
- ┌──────▼───────┐
- │ CloudFront │
- │ (CDN) │
- └──────┬───────┘
- │
- ┌──────▼───────┐
- │ WAF │
- └──────┬───────┘
- │
- ┌───────────▼───────────┐
- │ ALB (public) │
- │ SSL termination │
- └───────────┬───────────┘
- │
- ┌──────────────┼──────────────┐
- │ │
- ┌──────▼──────┐ ┌──────▼──────┐
- │ ECS API │ │ ECS API │
- │ (Fargate) │ │ (Fargate) │
- │ AZ-a │ │ AZ-b │
- └──────┬──────┘ └──────┬──────┘
- │ │
- ┌────────────┼────────────┬────────────────┤
- │ │ │ │
- ▼ ▼ ▼ ▼
-┌───────┐ ┌────────┐ ┌──────────┐ ┌──────────┐
-│Aurora │ │Elasti- │ │ MSK │ │ECS Worker│
-│PG │ │Cache │ │ (Kafka) │ │(Fargate) │
-│Multi- │ │Redis │ │ 3-broker │ │ Auto- │
-│AZ │ │Cluster │ │ cluster │ │ scaling │
-└───────┘ └────────┘ └──────────┘ └──────────┘
-
- Private Subnets (no public IPs)
- ────────────────────────────────
- VPC with NAT Gateway for outbound
-```
-
----
-
-### Infrastructure as Code
-
-Use **Terraform** to manage all AWS resources. Suggested module layout:
-
-```
-terraform/
-├── environments/
-│ ├── staging/
-│ │ ├── main.tf
-│ │ ├── variables.tf
-│ │ └── terraform.tfvars
-│ └── production/
-│ ├── main.tf
-│ ├── variables.tf
-│ └── terraform.tfvars
-├── modules/
-│ ├── vpc/
-│ ├── ecs/
-│ ├── rds/
-│ ├── elasticache/
-│ ├── msk/
-│ ├── alb/
-│ ├── ecr/
-│ └── monitoring/
-└── backend.tf # S3 + DynamoDB state locking
-```
-
-State management:
-
-```hcl
-terraform {
- backend "s3" {
- bucket = "notifyhub-terraform-state"
- key = "production/terraform.tfstate"
- region = "us-east-1"
- dynamodb_table = "terraform-locks"
- encrypt = true
- }
-}
-```
-
----
-
-### Compute Layer
-
-#### ECS Fargate (recommended)
-
-```hcl
-# API service
-resource "aws_ecs_service" "api" {
- name = "notifyhub-api"
- cluster = aws_ecs_cluster.main.id
- task_definition = aws_ecs_task_definition.api.arn
- desired_count = 2 # minimum for HA
- launch_type = "FARGATE"
-
- network_configuration {
- subnets = var.private_subnets
- security_groups = [aws_security_group.api.id]
- assign_public_ip = false
- }
-
- load_balancer {
- target_group_arn = aws_lb_target_group.api.arn
- container_name = "api"
- container_port = 8080
- }
-}
-
-# API task definition
-resource "aws_ecs_task_definition" "api" {
- family = "notifyhub-api"
- requires_compatibilities = ["FARGATE"]
- network_mode = "awsvpc"
- cpu = 512 # 0.5 vCPU
- memory = 1024 # 1 GB
-
- container_definitions = jsonencode([{
- name = "api"
- image = "${aws_ecr_repository.api.repository_url}:latest"
- portMappings = [{
- containerPort = 8080
- protocol = "tcp"
- }]
- environment = [
- { name = "PORT", value = "8080" },
- { name = "ENVIRONMENT", value = "production" },
- { name = "LOG_LEVEL", value = "info" },
- ]
- secrets = [
- { name = "DATABASE_URL", valueFrom = aws_ssm_parameter.db_url.arn },
- { name = "REDIS_URL", valueFrom = aws_ssm_parameter.redis_url.arn },
- { name = "KAFKA_BROKERS", valueFrom = aws_ssm_parameter.kafka_brokers.arn },
- ]
- logConfiguration = {
- logDriver = "awslogs"
- options = {
- "awslogs-group" = "/ecs/notifyhub-api"
- "awslogs-region" = "us-east-1"
- "awslogs-stream-prefix" = "api"
- }
- }
- }])
-}
-
-# Worker auto-scaling based on Kafka consumer lag
-resource "aws_appautoscaling_target" "worker" {
- max_capacity = 10
- min_capacity = 2
- resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.worker.name}"
- scalable_dimension = "ecs:service:DesiredCount"
- service_namespace = "ecs"
-}
-
-resource "aws_appautoscaling_policy" "worker_cpu" {
- name = "worker-cpu-scaling"
- policy_type = "TargetTrackingScaling"
- resource_id = aws_appautoscaling_target.worker.resource_id
- scalable_dimension = aws_appautoscaling_target.worker.scalable_dimension
- service_namespace = aws_appautoscaling_target.worker.service_namespace
-
- target_tracking_scaling_policy_configuration {
- predefined_metric_specification {
- predefined_metric_type = "ECSServiceAverageCPUUtilization"
- }
- target_value = 70.0
- }
-}
-```
-
-#### Resource Sizing (Production)
-
-| Service | Instance/Size | Count | Purpose |
-|---------|-------------|-------|---------|
-| ECS API | 0.5 vCPU / 1 GB | 2-4 | HTTP request handling |
-| ECS Worker | 1 vCPU / 2 GB | 2-10 | Notification processing |
-| Aurora PG | db.r6g.large | 2 (writer + reader) | Primary data store |
-| ElastiCache | cache.r6g.large | 2 (primary + replica) | Rate limiting, caching |
-| MSK | kafka.m5.large | 3 | Event streaming |
-
----
-
-### Data Layer
-
-#### Aurora PostgreSQL (Multi-AZ)
-
-```hcl
-resource "aws_rds_cluster" "main" {
- cluster_identifier = "notifyhub"
- engine = "aurora-postgresql"
- engine_version = "16.1"
- database_name = "notifyhub"
- master_username = "notifyhub"
- master_password = var.db_password # Use AWS Secrets Manager in practice
- storage_encrypted = true
- deletion_protection = true
- backup_retention_period = 14
- preferred_backup_window = "03:00-04:00"
- vpc_security_group_ids = [aws_security_group.db.id]
- db_subnet_group_name = aws_db_subnet_group.main.name
-
- serverlessv2_scaling_configuration {
- min_capacity = 0.5
- max_capacity = 4.0
- }
-}
-
-resource "aws_rds_cluster_instance" "writer" {
- cluster_identifier = aws_rds_cluster.main.id
- instance_class = "db.serverless"
- engine = "aurora-postgresql"
-}
-
-resource "aws_rds_cluster_instance" "reader" {
- cluster_identifier = aws_rds_cluster.main.id
- instance_class = "db.serverless"
- engine = "aurora-postgresql"
-}
-```
-
-#### ElastiCache Redis (Cluster Mode)
-
-```hcl
-resource "aws_elasticache_replication_group" "main" {
- replication_group_id = "notifyhub-redis"
- description = "NotifyHub Redis cluster"
- node_type = "cache.r6g.large"
- num_cache_clusters = 2
- engine_version = "7.0"
- port = 6379
- at_rest_encryption_enabled = true
- transit_encryption_enabled = true
- automatic_failover_enabled = true
- subnet_group_name = aws_elasticache_subnet_group.main.name
- security_group_ids = [aws_security_group.redis.id]
-}
-```
-
-#### Amazon MSK (Managed Kafka)
-
-```hcl
-resource "aws_msk_cluster" "main" {
- cluster_name = "notifyhub"
- kafka_version = "3.6.0"
- number_of_broker_nodes = 3
-
- broker_node_group_info {
- instance_type = "kafka.m5.large"
- client_subnets = var.private_subnets
- security_groups = [aws_security_group.kafka.id]
-
- storage_info {
- ebs_storage_info {
- volume_size = 100
- }
- }
- }
-
- encryption_info {
- encryption_in_transit {
- client_broker = "TLS"
- in_cluster = true
- }
- }
-
- logging_info {
- broker_logs {
- cloudwatch_logs {
- enabled = true
- log_group = "/msk/notifyhub"
- }
- }
- }
-}
-```
-
----
-
-### Networking & Security
-
-#### VPC Layout
-
-```
-VPC: 10.0.0.0/16
-├── Public Subnets (ALB, NAT Gateway)
-│ ├── 10.0.1.0/24 (AZ-a)
-│ └── 10.0.2.0/24 (AZ-b)
-├── Private App Subnets (ECS tasks)
-│ ├── 10.0.10.0/24 (AZ-a)
-│ └── 10.0.11.0/24 (AZ-b)
-└── Private Data Subnets (RDS, ElastiCache, MSK)
- ├── 10.0.20.0/24 (AZ-a)
- └── 10.0.21.0/24 (AZ-b)
-```
-
-#### Security Groups
-
-| Resource | Inbound | Source |
-|----------|---------|--------|
-| ALB | 443 (HTTPS) | 0.0.0.0/0 |
-| API ECS | 8080 | ALB SG |
-| Worker ECS | — | — (outbound only) |
-| RDS | 5432 | App Subnet SG |
-| Redis | 6379 | App Subnet SG |
-| MSK | 9094 (TLS) | App Subnet SG |
-
-#### Secrets Management
-
-Store all sensitive configuration in AWS Systems Manager Parameter Store or Secrets Manager:
-
-```bash
-# Database URL
-aws ssm put-parameter --name "/notifyhub/prod/DATABASE_URL" \
- --type "SecureString" \
- --value "postgres://..."
-
-# API keys
-aws ssm put-parameter --name "/notifyhub/prod/SES_API_KEY" \
- --type "SecureString" \
- --value "..."
-```
-
----
-
-### Observability Stack
-
-#### Logging
-
-- **CloudWatch Logs** for all ECS tasks (auto-configured via `awslogs` driver)
-- Log retention: 30 days for production, 7 days for staging
-- Structured JSON logs (slog already outputs JSON)
-
-#### Metrics
-
-- **CloudWatch Container Insights** for ECS metrics (CPU, memory, network)
-- **Prometheus** (self-hosted or Amazon Managed Prometheus) for application metrics
-- **Grafana** (self-hosted or Amazon Managed Grafana) for dashboards
-
-#### Tracing
-
-- **AWS X-Ray** or **OpenTelemetry Collector** -> Jaeger/Tempo
-- Already instrumented via OpenTelemetry in the application
-
-#### Alerting
-
-```hcl
-# High error rate alert
-resource "aws_cloudwatch_metric_alarm" "api_5xx" {
- alarm_name = "notifyhub-api-5xx-high"
- comparison_operator = "GreaterThanThreshold"
- evaluation_periods = 2
- metric_name = "HTTPCode_Target_5XX_Count"
- namespace = "AWS/ApplicationELB"
- period = 300
- statistic = "Sum"
- threshold = 10
- alarm_actions = [aws_sns_topic.alerts.arn]
-}
-
-# Kafka consumer lag alert
-resource "aws_cloudwatch_metric_alarm" "consumer_lag" {
- alarm_name = "notifyhub-kafka-lag-high"
- comparison_operator = "GreaterThanThreshold"
- evaluation_periods = 3
- metric_name = "SumOffsetLag"
- namespace = "AWS/Kafka"
- period = 300
- statistic = "Maximum"
- threshold = 10000
- alarm_actions = [aws_sns_topic.alerts.arn]
-}
-```
-
----
-
-### Production CI/CD Pipeline
-
-```
-┌─────────┐ ┌──────┐ ┌──────────┐ ┌─────────┐ ┌────────────┐
-│ Push │───>│ Test │───>│ Build │───>│ Deploy │───>│ Deploy │
-│ to PR │ │ Lint │ │ Images │ │ Staging │ │ Production │
-└─────────┘ └──────┘ └──────────┘ └─────────┘ └────────────┘
- │ │
- Auto deploy Manual approval
- + canary rollout
-```
-
-Key practices:
-
-1. **Blue/Green deployments** via ECS deployment circuit breaker
-2. **Database migrations** run as a separate ECS task before deployment
-3. **Canary releases** — route 10% traffic to new version, monitor, then full rollout
-4. **Rollback** — ECS automatically rolls back if health checks fail
-
-```yaml
-# Production deploy job (GitHub Actions)
-deploy-production:
- needs: deploy-staging
- runs-on: ubuntu-latest
- environment:
- name: production # Requires manual approval in GitHub
- steps:
- - name: Run migrations
- run: |
- aws ecs run-task \
- --cluster notifyhub-prod \
- --task-definition notifyhub-migrate \
- --launch-type FARGATE \
- --network-configuration "..." \
- --overrides '{"containerOverrides":[{"name":"migrate","command":["up"]}]}'
-
- - name: Deploy API (rolling update)
- run: |
- aws ecs update-service \
- --cluster notifyhub-prod \
- --service notifyhub-api \
- --force-new-deployment \
- --deployment-configuration "maximumPercent=200,minimumHealthyPercent=100"
-
- - name: Deploy Worker
- run: |
- aws ecs update-service \
- --cluster notifyhub-prod \
- --service notifyhub-worker \
- --force-new-deployment
-```
-
----
-
-### Disaster Recovery
-
-| Strategy | RPO | RTO | Implementation |
-|----------|-----|-----|---------------|
-| **Database** | ~1 sec | < 1 min | Aurora Multi-AZ with auto-failover |
-| **Redis** | ~seconds | < 1 min | ElastiCache Multi-AZ with auto-failover |
-| **Kafka** | 0 (replicated) | < 5 min | MSK 3-broker cluster across AZs |
-| **Application** | N/A | < 5 min | ECS auto-restarts failed tasks |
-| **Cross-Region** | < 1 hour | < 30 min | Aurora Global Database + Route 53 failover |
-
-Backup strategy:
-- **RDS**: Automated daily snapshots, 14-day retention, copy to another region weekly
-- **Kafka**: Topic data retained for 7 days (configurable), critical topics replicated
-- **Application state**: Stateless — no backup needed, just redeploy
-
----
-
-### Production Cost Estimate
-
-| Service | Spec | Monthly Cost |
-|---------|------|-------------|
-| ECS Fargate (API, 2 tasks) | 0.5 vCPU / 1 GB each | ~$30 |
-| ECS Fargate (Worker, 2 tasks) | 1 vCPU / 2 GB each | ~$60 |
-| Aurora PostgreSQL Serverless v2 | 0.5-4 ACU, 50 GB | ~$80 |
-| ElastiCache Redis | cache.r6g.large x2 | ~$300 |
-| MSK (3 brokers) | kafka.m5.large, 100 GB | ~$650 |
-| ALB | 1 ALB + traffic | ~$25 |
-| NAT Gateway | 2 (one per AZ) | ~$65 |
-| CloudWatch | Logs + metrics | ~$30 |
-| ECR | Image storage | ~$5 |
-| SES | Email delivery | ~$10 |
-| Data Transfer | Outbound | ~$20 |
-| **Total** | | **~$1,275/mo** |
-
-Cost optimization options:
-- Use **Fargate Spot** for workers (up to 70% savings)
-- Use **Aurora Serverless v2** to scale to zero during low traffic
-- Use **Reserved Instances** for predictable workloads (up to 40% savings)
-- Replace MSK with **SQS + SNS** if strict ordering isn't needed (~$5/mo vs $650)
-- Use a single NAT Gateway (~$32/mo savings, less HA)
diff --git a/docs/production-grade.md b/docs/production-grade.md
deleted file mode 100644
index 296ce06..0000000
--- a/docs/production-grade.md
+++ /dev/null
@@ -1,294 +0,0 @@
-# Production-Grade Additions
-
-This document tracks production-engineering work beyond the core application plan.
-The goal: emulate what you'd actually do at a company shipping this to prod — all runnable locally or free-tier.
-
-**Legend**
-- ✅ Free / fully local — no cost
-- 🟡 Free tier exists — watch limits
-- 🔴 Paid — skip or stub
-
----
-
-## 1. Infrastructure as Code (Terraform)
-
-**What it shows:** You provision infra the same way every environment does it — no clicking in the console.
-
-| Resource | Tool | Cost | Notes |
-|---|---|---|---|
-| Local infra (Postgres, Redis, Kafka) | `docker-compose.yml` | ✅ Free | Already planned |
-| AWS VPC, subnets, security groups | Terraform | 🔴 Paid | Skip — use local |
-| EKS / ECS cluster | Terraform | 🔴 Paid | Use local k8s instead |
-| RDS PostgreSQL | Terraform | 🔴 Paid | Use local Postgres |
-| ElastiCache Redis | Terraform | 🔴 Paid | Use local Redis |
-| MSK (Managed Kafka) | Terraform | 🔴 Paid | Use local Kafka |
-| **Terraform module structure** | Terraform | ✅ Free | Write it, just don't `apply` to AWS |
-| **Remote state (local backend)** | Terraform | ✅ Free | Use local backend or free Terraform Cloud |
-| S3 + DynamoDB state backend | Terraform | 🟡 Free tier | Negligible cost |
-
-**Recommendation:** Write a full Terraform module tree (`modules/postgres`, `modules/redis`, `modules/kafka`, `modules/app`) targeting AWS, but use `terraform plan` only. The code is the artifact — it proves you know how to structure IaC.
-
-### Directory structure to create
-```
-deployments/
-└── terraform/
- ├── main.tf
- ├── variables.tf
- ├── outputs.tf
- ├── backend.tf
- └── modules/
- ├── networking/ # VPC, subnets, SGs
- ├── database/ # RDS / local Postgres
- ├── cache/ # ElastiCache / local Redis
- ├── messaging/ # MSK / local Kafka
- └── app/ # ECS/EKS task defs or k8s manifests
-```
-
----
-
-## 2. Kubernetes
-
-**What it shows:** You know how to deploy, scale, and harden a service in k8s — the standard deployment target at most companies.
-
-| Resource | Tool | Cost | Notes |
-|---|---|---|---|
-| Local cluster | `kind` or `minikube` | ✅ Free | Runs on your laptop |
-| API + Worker Deployments | k8s manifests / Helm | ✅ Free | Standard |
-| HorizontalPodAutoscaler (CPU/RPS) | k8s HPA | ✅ Free | Scales API pods |
-| **KEDA** (Kafka consumer lag scaling) | KEDA | ✅ Free | Scales workers by lag — very impressive |
-| PodDisruptionBudget | k8s PDB | ✅ Free | Guarantees availability during drains |
-| NetworkPolicy (pod traffic rules) | k8s NetworkPolicy | ✅ Free | Restrict inter-pod traffic |
-| ConfigMap + Secret | k8s | ✅ Free | Inject config without baking into image |
-| Liveness + readiness probes | k8s | ✅ Free | Wired to `/health` and `/ready` |
-| Resource requests + limits | k8s | ✅ Free | CPU/memory per container |
-| Helm chart | Helm | ✅ Free | Package everything as a chart |
-| Argo Rollouts (canary/blue-green) | Argo | ✅ Free | Install on local cluster |
-
-### Directory structure to create
-```
-deployments/
-└── k8s/
- ├── namespace.yaml
- ├── api/
- │ ├── deployment.yaml
- │ ├── service.yaml
- │ ├── hpa.yaml
- │ └── pdb.yaml
- ├── worker/
- │ ├── deployment.yaml
- │ ├── keda-scaledobject.yaml ← scales on Kafka consumer lag
- │ └── pdb.yaml
- ├── configmap.yaml
- └── networkpolicy.yaml
-
-deployments/
-└── helm/
- └── notifyhub/
- ├── Chart.yaml
- ├── values.yaml
- ├── values-dev.yaml
- ├── values-prod.yaml
- └── templates/
- ├── deployment-api.yaml
- ├── deployment-worker.yaml
- ├── service.yaml
- ├── hpa.yaml
- ├── keda-scaledobject.yaml
- └── _helpers.tpl
-```
-
-### KEDA ScaledObject (high-value showcase)
-```yaml
-apiVersion: keda.sh/v1alpha1
-kind: ScaledObject
-metadata:
- name: notifyhub-worker
-spec:
- scaleTargetRef:
- name: notifyhub-worker
- minReplicaCount: 1
- maxReplicaCount: 20
- triggers:
- - type: kafka
- metadata:
- bootstrapServers: kafka:29092
- consumerGroup: notifyhub-email-workers
- topic: notifyhub.notifications.email
- lagThreshold: "50" # 1 pod per 50 unprocessed messages
-```
-
-This is exactly what production Kafka consumer autoscaling looks like.
-
----
-
-## 3. Database
-
-| Addition | Cost | Notes |
-|---|---|---|
-| **PgBouncer** (connection pooling) | ✅ Free | Add as a sidecar or separate container in docker-compose |
-| Read replica (local) | ✅ Free | Second Postgres container in docker-compose, streaming replication |
-| Migration as init container | ✅ Free | Run `migrate up` as a k8s init container, not in app startup |
-| Automated backup script | ✅ Free | `pg_dump` to local volume, cron job |
-
-PgBouncer is a quick win — add it to `docker-compose.yml` and route the app through it. Shows you understand connection exhaustion under load.
-
----
-
-## 4. Observability (Local Stack)
-
-Everything below runs locally for free.
-
-| Addition | Tool | Cost | Notes |
-|---|---|---|---|
-| Metrics scraping | Prometheus | ✅ Free | Already in docker-compose |
-| Dashboards | Grafana | ✅ Free | Already in docker-compose |
-| **Grafana dashboard JSON** | Grafana | ✅ Free | Commit to `deployments/grafana/` |
-| **Alertmanager** | Alertmanager | ✅ Free | Add to docker-compose |
-| Alert rules | Prometheus | ✅ Free | `deployments/prometheus/alerts.yml` |
-| Distributed tracing | **Jaeger** (local) | ✅ Free | Add to docker-compose, wire OTel exporter |
-| Log aggregation | **Loki + Promtail** | ✅ Free | Add to docker-compose, view logs in Grafana |
-
-### Grafana dashboards to build
-- Notification throughput per channel (send rate, delivered, failed, dropped)
-- p50/p95/p99 delivery latency per channel + provider
-- Kafka consumer lag per group (linked to KEDA — shows why autoscaling fired)
-- Circuit breaker state per provider (0=closed, 1=half-open, 2=open)
-- Rate-limited notifications over time
-- DLQ depth (should always be near zero)
-
-### Alertmanager rules to write
-```yaml
-# deployments/prometheus/alerts.yml
-- alert: KafkaConsumerLagHigh
- expr: notifyhub_kafka_consumer_lag > 1000
- for: 2m
- annotations:
- summary: "Consumer {{ $labels.group }} is lagging behind"
-
-- alert: ProviderErrorRateHigh
- expr: rate(notifyhub_delivery_attempts_total{status="failed"}[5m]) > 0.1
- for: 1m
-
-- alert: CircuitBreakerOpen
- expr: notifyhub_circuit_breaker_state == 2
- for: 0s
- annotations:
- summary: "Circuit breaker open for provider {{ $labels.provider }}"
-
-- alert: DLQDepthSpike
- expr: notifyhub_kafka_consumer_lag{topic="notifyhub.dlq"} > 10
- for: 1m
-```
-
-### docker-compose additions
-```yaml
- jaeger:
- image: jaegertracing/all-in-one:latest
- ports:
- - "16686:16686" # Jaeger UI
- - "4317:4317" # OTLP gRPC
-
- alertmanager:
- image: prom/alertmanager:latest
- ports: ["9093:9093"]
- volumes: [./deployments/alertmanager.yml:/etc/alertmanager/alertmanager.yml]
-
- loki:
- image: grafana/loki:latest
- ports: ["3100:3100"]
-
- promtail:
- image: grafana/promtail:latest
- volumes:
- - /var/log:/var/log
- - ./deployments/promtail.yml:/etc/promtail/config.yml
-```
-
----
-
-## 5. Security
-
-| Addition | Cost | Notes |
-|---|---|---|
-| **gosec** (SAST) in CI | ✅ Free | `go install github.com/securego/gosec/v2/cmd/gosec@latest` |
-| **Trivy** container scanning in CI | ✅ Free | `trivy image notifyhub-api:latest` |
-| **TruffleHog** secrets scanning | ✅ Free | GitHub Action or local pre-commit hook |
-| API key rotation endpoint | ✅ Free | `POST /internal/tenants/:id/api-keys/rotate` |
-| Audit log table | ✅ Free | New migration: log every key issuance + rotation |
-| Rate limit per API key | ✅ Free | Already planned (`api/middleware/ratelimit.go`) |
-| TLS in local docker-compose | ✅ Free | Self-signed cert via `mkcert`, nginx sidecar |
-
----
-
-## 6. CI/CD Pipeline
-
-| Addition | Tool | Cost | Notes |
-|---|---|---|---|
-| Lint + test + build | GitHub Actions | ✅ Free | 2000 min/month on free tier |
-| Docker build + push to GHCR | GitHub Actions | ✅ Free | GitHub Container Registry is free |
-| Trivy image scan in CI | GitHub Actions | ✅ Free | |
-| gosec SAST in CI | GitHub Actions | ✅ Free | |
-| Integration tests (testcontainers) | GitHub Actions | ✅ Free | Containers spin up in the runner |
-| **Atlantis** (Terraform PR workflow) | Self-hosted | ✅ Free | Run locally or on a free VM |
-| Argo CD (GitOps) | Local k8s | ✅ Free | Deploy to local `kind` cluster from git |
-
-### Pipeline stages
-```
-PR opened:
- lint (golangci-lint)
- → gosec (SAST)
- → unit tests (go test -race)
- → integration tests (testcontainers)
- → build docker image
- → trivy scan image
- → terraform plan (comment on PR)
-
-Merge to main:
- → build + tag image → push to GHCR
- → helm upgrade (Argo CD syncs to local kind cluster)
- → smoke test (curl /health + send one canary notification)
-```
-
----
-
-## 7. Load Testing
-
-| Addition | Tool | Cost | Notes |
-|---|---|---|---|
-| k6 load test script | k6 | ✅ Free | Already planned in `scripts/loadtest/` |
-| k6 → InfluxDB → Grafana | All local | ✅ Free | Real-time load test dashboard |
-| Baseline + stress + soak scenarios | k6 | ✅ Free | Three separate scripts |
-
-### Test scenarios to write
-- **Baseline:** 10 RPS sustained for 5 min — establish p99 latency
-- **Stress:** Ramp from 10 → 500 RPS over 2 min — find the breaking point
-- **Soak:** 50 RPS for 30 min — catch memory leaks, goroutine leaks, connection pool exhaustion
-
----
-
-## 8. Operational Runbooks
-
-Simple markdown files in `docs/runbooks/`. Shows you've thought about what happens when things go wrong.
-
-| Runbook | What it covers |
-|---|---|
-| `dlq-overflow.md` | Why messages hit DLQ, how to inspect + replay them |
-| `circuit-breaker-open.md` | Which provider failed, how to verify it's back, how to reset |
-| `consumer-lag-spike.md` | How to check lag, scale workers manually, identify slow consumers |
-| `db-connection-exhaustion.md` | How to diagnose via pg_stat_activity, PgBouncer pool stats |
-| `rate-limit-incident.md` | How to temporarily lift a user's rate limit in Redis |
-
----
-
-## Priority Order (what to build first)
-
-1. **Kubernetes manifests + Helm chart** — highest visibility, most transferable skill
-2. **KEDA ScaledObject for workers** — production Kafka autoscaling, very impressive
-3. **Full observability stack** (Jaeger + Loki + Alertmanager) in docker-compose — shows you instrument everything
-4. **Grafana dashboards** — tangible artifact, easy to screenshot/demo
-5. **CI/CD pipeline** (GitHub Actions → GHCR → Argo CD → kind) — full GitOps loop
-6. **Terraform modules** (write, don't apply) — shows IaC discipline
-7. **PgBouncer** in docker-compose — quick win, real production concern
-8. **Load test scenarios** (k6 + Grafana) — run it, screenshot the dashboard under load
-9. **Runbooks** — soft skill that separates junior from senior engineers
-10. **Security tooling** (gosec + Trivy in CI) — table stakes at any serious company
diff --git a/go.mod b/go.mod
index 932d5f6..3702c24 100644
--- a/go.mod
+++ b/go.mod
@@ -8,8 +8,10 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.32.16
github.com/aws/aws-sdk-go-v2/service/sesv2 v1.60.3
github.com/aws/smithy-go v1.25.0
+ github.com/coder/websocket v1.8.14
github.com/go-chi/chi/v5 v5.2.5
github.com/go-playground/validator/v10 v10.30.1
+ github.com/golang-jwt/jwt/v5 v5.2.2
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.8.0
github.com/prometheus/client_golang v1.23.2
@@ -56,7 +58,6 @@ require (
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
- github.com/coder/websocket v1.8.14 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
@@ -68,7 +69,6 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
- github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/s2a-go v0.1.9 // indirect
diff --git a/internal/api/handler/device_token.go b/internal/api/handler/device_token.go
new file mode 100644
index 0000000..f93b13c
--- /dev/null
+++ b/internal/api/handler/device_token.go
@@ -0,0 +1,56 @@
+package handler
+
+import (
+ "net/http"
+
+ "github.com/go-chi/chi/v5"
+
+ "github.com/amitrajitdas31/notifyhub/internal/api/middleware"
+ "github.com/amitrajitdas31/notifyhub/internal/api/response"
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+ "github.com/amitrajitdas31/notifyhub/internal/service"
+)
+
+type DeviceTokenHandler struct {
+ svc service.DeviceTokenService
+}
+
+func NewDeviceTokenHandler(svc service.DeviceTokenService) *DeviceTokenHandler {
+ return &DeviceTokenHandler{svc: svc}
+}
+
+// POST /api/v1/device-tokens
+func (h *DeviceTokenHandler) Register(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+
+ var req domain.RegisterDeviceTokenRequest
+ if appErr := response.DecodeJSON(r, &req); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ dt, err := h.svc.Register(r.Context(), tenantID, req)
+ if appErr := toAppError(err); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ response.JSON(w, http.StatusCreated, dt, reqID)
+}
+
+// DELETE /api/v1/device-tokens/:token
+func (h *DeviceTokenHandler) Deregister(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+ token := chi.URLParam(r, "token")
+
+ if err := h.svc.Deregister(r.Context(), tenantID, token); err != nil {
+ if appErr := toAppError(err); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+ }
+
+ w.WriteHeader(http.StatusNoContent)
+}
diff --git a/internal/api/handler/inbox.go b/internal/api/handler/inbox.go
index 3d8ef14..0ff32eb 100644
--- a/internal/api/handler/inbox.go
+++ b/internal/api/handler/inbox.go
@@ -13,6 +13,7 @@ import (
"github.com/amitrajitdas31/notifyhub/internal/api/middleware"
"github.com/amitrajitdas31/notifyhub/internal/api/response"
+ "github.com/amitrajitdas31/notifyhub/internal/auth"
"github.com/amitrajitdas31/notifyhub/internal/domain"
"github.com/amitrajitdas31/notifyhub/internal/observability"
"github.com/amitrajitdas31/notifyhub/internal/realtime"
@@ -21,23 +22,25 @@ import (
// InboxHandler serves inbox REST endpoints and the WebSocket stream.
type InboxHandler struct {
- repo repository.InAppRepository
- hub *realtime.Hub
- metrics *observability.Metrics
- logger *slog.Logger
- heartbeat time.Duration
- readTimeout time.Duration
+ repo repository.InAppRepository
+ hub *realtime.Hub
+ wsToken *auth.WSTokenService
+ metrics *observability.Metrics
+ logger *slog.Logger
+ heartbeat time.Duration
+ readTimeout time.Duration
}
// InboxHandlerConfig holds tunable parameters for InboxHandler.
type InboxHandlerConfig struct {
- HeartbeatSeconds int
+ HeartbeatSeconds int
ReadTimeoutSeconds int
}
func NewInboxHandler(
repo repository.InAppRepository,
hub *realtime.Hub,
+ wsToken *auth.WSTokenService,
metrics *observability.Metrics,
logger *slog.Logger,
cfg InboxHandlerConfig,
@@ -53,6 +56,7 @@ func NewInboxHandler(
return &InboxHandler{
repo: repo,
hub: hub,
+ wsToken: wsToken,
metrics: metrics,
logger: logger,
heartbeat: hb,
@@ -93,9 +97,16 @@ func (h *InboxHandler) List(w http.ResponseWriter, r *http.Request) {
q.Limit = n
}
}
- if v := r.URL.Query().Get("offset"); v != "" {
- if n, err := strconv.Atoi(v); err == nil && n >= 0 {
- q.Offset = n
+ if v := r.URL.Query().Get("after_id"); v != "" {
+ if id, err := uuid.Parse(v); err == nil {
+ q.AfterID = &id
+ }
+ }
+ if q.AfterID == nil {
+ if v := r.URL.Query().Get("offset"); v != "" {
+ if n, err := strconv.Atoi(v); err == nil && n >= 0 {
+ q.Offset = n
+ }
}
}
@@ -176,17 +187,41 @@ func (h *InboxHandler) MarkAllRead(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
-// GET /api/v1/inbox/stream — WebSocket; sends JSON InAppMessage frames as they arrive.
+// GET /api/v1/inbox/stream?token=[&since=]
+//
+// WebSocket stream for real-time in-app messages. Identity comes from the
+// signed JWT; X-Recipient-ID is ignored.
+//
+// Reconnect protocol:
+// 1. Client stores the `id` of the last received message frame.
+// 2. On reconnect, client passes ?since=.
+// 3. Server subscribes to live pub/sub first (no-miss window), then fetches
+// the gap from DB and sends it as historical frames (type "history").
+// 4. Live frames follow (type "message"). Client deduplicates by `id`.
+// 5. Heartbeat pings (type "ping") keep the connection alive every ~25 s.
func (h *InboxHandler) Stream(w http.ResponseWriter, r *http.Request) {
- tenantID := middleware.ClientFromContext(r.Context()).TenantID
+ tokenStr := r.URL.Query().Get("token")
+ if tokenStr == "" {
+ http.Error(w, "token required", http.StatusUnauthorized)
+ return
+ }
- recipientID, appErr := recipientIDFromHeader(r)
- if appErr != nil {
- reqID := middleware.RequestIDFromContext(r.Context())
- response.JSONError(w, appErr, reqID)
+ claims, err := h.wsToken.Verify(tokenStr)
+ if err != nil {
+ http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
+ tenantID := claims.TenantID
+ recipientID := claims.RecipientID
+
+ var sinceID *uuid.UUID
+ if v := r.URL.Query().Get("since"); v != "" {
+ if id, err := uuid.Parse(v); err == nil {
+ sinceID = &id
+ }
+ }
+
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{
InsecureSkipVerify: true, // allow cross-origin from same cluster / dev
})
@@ -199,10 +234,32 @@ func (h *InboxHandler) Stream(w http.ResponseWriter, r *http.Request) {
h.metrics.InAppWSConnections.Inc()
defer h.metrics.InAppWSConnections.Dec()
+ // Subscribe before fetching historical so no messages fall through the gap.
ch, cancel := h.hub.Subscribe(tenantID.String(), recipientID)
defer cancel()
ctx := r.Context()
+
+ // Replay gap: send missed messages as historical frames, then go live.
+ // Client deduplicates by id in case a message arrives via both paths.
+ if sinceID != nil {
+ missed, err := h.repo.ListSince(ctx, tenantID, recipientID, *sinceID)
+ if err != nil {
+ h.logger.Warn("inapp ws: history fetch failed", slog.Any("error", err))
+ } else {
+ for _, msg := range missed {
+ frame := struct {
+ Type string `json:"type"`
+ Data domain.InAppMessage `json:"data"`
+ }{"history", msg}
+ b, _ := json.Marshal(frame)
+ if err := conn.Write(ctx, websocket.MessageText, b); err != nil {
+ return
+ }
+ }
+ }
+ }
+
ticker := time.NewTicker(h.heartbeat)
defer ticker.Stop()
@@ -232,12 +289,18 @@ func (h *InboxHandler) Stream(w http.ResponseWriter, r *http.Request) {
if err := conn.Write(ctx, websocket.MessageText, ping); err != nil {
return
}
- case msg, ok := <-ch:
+ case raw, ok := <-ch:
if !ok {
conn.Close(websocket.StatusNormalClosure, "")
return
}
- if err := conn.Write(ctx, websocket.MessageText, msg); err != nil {
+ // Wrap raw InAppMessage JSON with envelope so clients can distinguish
+ // live frames (type "message") from historical replays (type "history").
+ wrapped := make([]byte, 0, len(raw)+20)
+ wrapped = append(wrapped, `{"type":"message","data":`...)
+ wrapped = append(wrapped, raw...)
+ wrapped = append(wrapped, '}')
+ if err := conn.Write(ctx, websocket.MessageText, wrapped); err != nil {
h.metrics.InAppWSDropped.Inc()
return
}
diff --git a/internal/api/handler/webhook_admin.go b/internal/api/handler/webhook_admin.go
new file mode 100644
index 0000000..ae97277
--- /dev/null
+++ b/internal/api/handler/webhook_admin.go
@@ -0,0 +1,123 @@
+package handler
+
+import (
+ "net/http"
+
+ "github.com/go-chi/chi/v5"
+ "github.com/google/uuid"
+
+ "github.com/amitrajitdas31/notifyhub/internal/api/middleware"
+ "github.com/amitrajitdas31/notifyhub/internal/api/response"
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+ "github.com/amitrajitdas31/notifyhub/internal/service"
+)
+
+// WebhookHandler serves CRUD endpoints for webhook endpoint management.
+type WebhookHandler struct {
+ svc service.WebhookService
+}
+
+func NewWebhookHandler(svc service.WebhookService) *WebhookHandler {
+ return &WebhookHandler{svc: svc}
+}
+
+// POST /api/v1/webhooks
+func (h *WebhookHandler) Create(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+
+ var req domain.CreateWebhookEndpointRequest
+ if appErr := response.DecodeJSON(r, &req); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ ep, err := h.svc.Create(r.Context(), tenantID, req)
+ if appErr := toAppError(err); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ response.JSON(w, http.StatusCreated, ep, reqID)
+}
+
+// GET /api/v1/webhooks
+func (h *WebhookHandler) List(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+
+ eps, err := h.svc.List(r.Context(), tenantID)
+ if appErr := toAppError(err); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ response.JSON(w, http.StatusOK, eps, reqID)
+}
+
+// GET /api/v1/webhooks/{id}
+func (h *WebhookHandler) GetByID(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+
+ id, err := uuid.Parse(chi.URLParam(r, "id"))
+ if err != nil {
+ response.JSONError(w, domain.NewValidationError("invalid id", nil), reqID)
+ return
+ }
+
+ ep, err := h.svc.GetByID(r.Context(), tenantID, id)
+ if appErr := toAppError(err); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ response.JSON(w, http.StatusOK, ep, reqID)
+}
+
+// PUT /api/v1/webhooks/{id}
+func (h *WebhookHandler) Update(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+
+ id, err := uuid.Parse(chi.URLParam(r, "id"))
+ if err != nil {
+ response.JSONError(w, domain.NewValidationError("invalid id", nil), reqID)
+ return
+ }
+
+ var req domain.UpdateWebhookEndpointRequest
+ if appErr := response.DecodeJSON(r, &req); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ ep, err := h.svc.Update(r.Context(), tenantID, id, req)
+ if appErr := toAppError(err); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+
+ response.JSON(w, http.StatusOK, ep, reqID)
+}
+
+// DELETE /api/v1/webhooks/{id}
+func (h *WebhookHandler) Delete(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+
+ id, err := uuid.Parse(chi.URLParam(r, "id"))
+ if err != nil {
+ response.JSONError(w, domain.NewValidationError("invalid id", nil), reqID)
+ return
+ }
+
+ if err := h.svc.Delete(r.Context(), tenantID, id); err != nil {
+ if appErr := toAppError(err); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+ }
+
+ w.WriteHeader(http.StatusNoContent)
+}
diff --git a/internal/api/handler/ws_token.go b/internal/api/handler/ws_token.go
new file mode 100644
index 0000000..00223ef
--- /dev/null
+++ b/internal/api/handler/ws_token.go
@@ -0,0 +1,50 @@
+package handler
+
+import (
+ "net/http"
+
+ "github.com/amitrajitdas31/notifyhub/internal/api/middleware"
+ "github.com/amitrajitdas31/notifyhub/internal/api/response"
+ "github.com/amitrajitdas31/notifyhub/internal/auth"
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+)
+
+// WSTokenHandler issues short-lived JWTs for WebSocket authentication.
+type WSTokenHandler struct {
+ svc *auth.WSTokenService
+}
+
+func NewWSTokenHandler(svc *auth.WSTokenService) *WSTokenHandler {
+ return &WSTokenHandler{svc: svc}
+}
+
+// POST /api/v1/ws-token
+// Body: { "recipient_id": "..." }
+// Returns: { "token": "", "expires_in": 60 }
+func (h *WSTokenHandler) Issue(w http.ResponseWriter, r *http.Request) {
+ reqID := middleware.RequestIDFromContext(r.Context())
+ tenantID := middleware.ClientFromContext(r.Context()).TenantID
+
+ var body struct {
+ RecipientID string `json:"recipient_id"`
+ }
+ if appErr := response.DecodeJSON(r, &body); appErr != nil {
+ response.JSONError(w, appErr, reqID)
+ return
+ }
+ if body.RecipientID == "" {
+ response.JSONError(w, domain.NewValidationError("recipient_id required", nil), reqID)
+ return
+ }
+
+ token, err := h.svc.Issue(tenantID, body.RecipientID)
+ if err != nil {
+ response.JSONError(w, domain.NewInternalError("failed to issue token", err), reqID)
+ return
+ }
+
+ response.JSON(w, http.StatusOK, map[string]any{
+ "token": token,
+ "expires_in": 60,
+ }, reqID)
+}
diff --git a/internal/api/middleware/logging.go b/internal/api/middleware/logging.go
index 0caa38a..c3b3077 100644
--- a/internal/api/middleware/logging.go
+++ b/internal/api/middleware/logging.go
@@ -1,7 +1,10 @@
package middleware
import (
+ "bufio"
+ "fmt"
"log/slog"
+ "net"
"net/http"
"time"
)
@@ -17,6 +20,16 @@ func (sw *statusWriter) WriteHeader(status int) {
sw.ResponseWriter.WriteHeader(status)
}
+// Hijack forwards to the underlying ResponseWriter when it implements http.Hijacker.
+// Required so WebSocket upgrades work through this middleware.
+func (sw *statusWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+ h, ok := sw.ResponseWriter.(http.Hijacker)
+ if !ok {
+ return nil, nil, fmt.Errorf("underlying ResponseWriter does not implement http.Hijacker")
+ }
+ return h.Hijack()
+}
+
// Logging returns middleware that logs each request with method, path, status, and latency.
func Logging(logger *slog.Logger) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
diff --git a/internal/api/router.go b/internal/api/router.go
index 9a24554..1c366a6 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -29,8 +29,11 @@ type RouterDeps struct {
Template *handler.TemplateHandler
Preference *handler.PreferenceHandler
Tenant *handler.TenantHandler
- DLQ *handler.DLQHandler
- Inbox *handler.InboxHandler
+ DLQ *handler.DLQHandler
+ Inbox *handler.InboxHandler
+ DeviceToken *handler.DeviceTokenHandler
+ WSToken *handler.WSTokenHandler
+ Webhook *handler.WebhookHandler
}
func NewRouter(deps RouterDeps) http.Handler {
@@ -41,8 +44,13 @@ func NewRouter(deps RouterDeps) http.Handler {
r.Use(middleware.RequestID)
// otelhttp creates a server span per request and propagates W3C traceparent.
// It must run before Logging so the trace_id is available when we log.
+ // Filter skips the WebSocket stream route: otelhttp wraps ResponseWriter in a
+ // type that strips http.Hijacker, which coder/websocket requires for HTTP/1.1.
r.Use(otelhttp.NewMiddleware("notifyhub-api",
otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents),
+ otelhttp.WithFilter(func(req *http.Request) bool {
+ return req.URL.Path != "/api/v1/inbox/stream"
+ }),
))
r.Use(middleware.Logging(deps.Logger))
r.Use(middleware.Recovery(deps.Logger))
@@ -104,6 +112,19 @@ func NewRouter(deps RouterDeps) http.Handler {
r.Delete("/{channel}", deps.Preference.Delete)
})
+ // WebSocket JWT issuance
+ if deps.WSToken != nil {
+ r.Post("/ws-token", deps.WSToken.Issue)
+ }
+
+ // Device tokens (FCM push)
+ if deps.DeviceToken != nil {
+ r.Route("/device-tokens", func(r chi.Router) {
+ r.Post("/", deps.DeviceToken.Register)
+ r.Delete("/{token}", deps.DeviceToken.Deregister)
+ })
+ }
+
// In-app inbox
if deps.Inbox != nil {
r.Route("/inbox", func(r chi.Router) {
@@ -111,11 +132,27 @@ func NewRouter(deps RouterDeps) http.Handler {
r.Get("/unread-count", deps.Inbox.UnreadCount)
r.Post("/read-all", deps.Inbox.MarkAllRead)
r.Post("/{id}/read", deps.Inbox.MarkRead)
- r.Get("/stream", deps.Inbox.Stream)
+ })
+ }
+
+ // Outbound webhook endpoints
+ if deps.Webhook != nil {
+ r.Route("/webhooks", func(r chi.Router) {
+ r.Post("/", deps.Webhook.Create)
+ r.Get("/", deps.Webhook.List)
+ r.Get("/{id}", deps.Webhook.GetByID)
+ r.Put("/{id}", deps.Webhook.Update)
+ r.Delete("/{id}", deps.Webhook.Delete)
})
}
})
})
+ // WebSocket stream — JWT auth is handled by the handler itself; cannot use
+ // X-API-Key middleware because browsers cannot send custom headers on WS upgrade.
+ if deps.Inbox != nil {
+ r.Get("/api/v1/inbox/stream", deps.Inbox.Stream)
+ }
+
return r
}
diff --git a/internal/auth/jwt.go b/internal/auth/jwt.go
new file mode 100644
index 0000000..5912f08
--- /dev/null
+++ b/internal/auth/jwt.go
@@ -0,0 +1,71 @@
+package auth
+
+import (
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/golang-jwt/jwt/v5"
+ "github.com/google/uuid"
+)
+
+// WSClaims are the payload fields embedded in a WebSocket short-lived JWT.
+type WSClaims struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ RecipientID string `json:"recipient_id"`
+ jwt.RegisteredClaims
+}
+
+// WSTokenService issues and verifies WebSocket JWTs.
+type WSTokenService struct {
+ secret []byte
+ ttl time.Duration
+}
+
+func NewWSTokenService(secret string, ttlSeconds int) *WSTokenService {
+ return &WSTokenService{
+ secret: []byte(secret),
+ ttl: time.Duration(ttlSeconds) * time.Second,
+ }
+}
+
+// Issue signs a new JWT for the given tenant + recipient pair.
+func (s *WSTokenService) Issue(tenantID uuid.UUID, recipientID string) (string, error) {
+ now := time.Now()
+ claims := WSClaims{
+ TenantID: tenantID,
+ RecipientID: recipientID,
+ RegisteredClaims: jwt.RegisteredClaims{
+ IssuedAt: jwt.NewNumericDate(now),
+ ExpiresAt: jwt.NewNumericDate(now.Add(s.ttl)),
+ },
+ }
+ token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+ signed, err := token.SignedString(s.secret)
+ if err != nil {
+ return "", fmt.Errorf("ws jwt: sign: %w", err)
+ }
+ return signed, nil
+}
+
+// Verify parses and validates a WS JWT, returning its claims.
+func (s *WSTokenService) Verify(tokenStr string) (*WSClaims, error) {
+ token, err := jwt.ParseWithClaims(tokenStr, &WSClaims{}, func(t *jwt.Token) (any, error) {
+ if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
+ return nil, fmt.Errorf("ws jwt: unexpected signing method: %v", t.Header["alg"])
+ }
+ return s.secret, nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("ws jwt: %w", err)
+ }
+
+ claims, ok := token.Claims.(*WSClaims)
+ if !ok || !token.Valid {
+ return nil, errors.New("ws jwt: invalid claims")
+ }
+ if claims.RecipientID == "" {
+ return nil, errors.New("ws jwt: missing recipient_id")
+ }
+ return claims, nil
+}
diff --git a/internal/config/config.go b/internal/config/config.go
index 286d701..ec8ebd6 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -51,10 +51,14 @@ type Config struct {
RateLimitInAppPerHour int
// In-app WebSocket
- InAppWSHeartbeatSeconds int
- InAppWSBufferSize int
+ InAppWSHeartbeatSeconds int
+ InAppWSBufferSize int
InAppWSReadTimeoutSeconds int
+ // WebSocket JWT
+ WSJWTSecret string
+ WSJWTTTLSeconds int
+
// Admin
AdminToken string
@@ -63,6 +67,13 @@ type Config struct {
DLQConsumerEnabled bool
DLQGroupID string
+ // Webhooks
+ WebhookEnabled bool
+ WebhookGroupID string
+ WebhookConcurrency int
+ WebhookMaxAttempts int
+ WebhookRetryBaseMS int
+
// Observability — OpenTelemetry
// OTELEndpoint is the OTLP gRPC endpoint of the OTel Collector.
// Leave empty to disable tracing (a no-op provider will be used).
@@ -100,6 +111,8 @@ func Load() (*Config, error) {
DLQEnabled: getEnv("DLQ_ENABLED", "true") != "false",
DLQConsumerEnabled: getEnv("DLQ_CONSUMER_ENABLED", "true") != "false",
DLQGroupID: getEnv("DLQ_GROUP_ID", "notifyhub-dlq-consumer"),
+ WebhookEnabled: getEnv("WEBHOOK_ENABLED", "true") != "false",
+ WebhookGroupID: getEnv("WEBHOOK_GROUP_ID", "notifyhub-webhook-worker"),
OTELEndpoint: getEnv("OTEL_EXPORTER_OTLP_ENDPOINT", ""),
OTELInsecure: getEnv("OTEL_EXPORTER_OTLP_INSECURE", "true") != "false",
MetricsPort: getEnv("METRICS_PORT", "9091"),
@@ -170,10 +183,23 @@ func Load() (*Config, error) {
if cfg.InAppWSReadTimeoutSeconds, err = getEnvInt("INAPP_WS_READ_TIMEOUT_SECONDS", 60); err != nil {
return nil, fmt.Errorf("INAPP_WS_READ_TIMEOUT_SECONDS: %w", err)
}
+ cfg.WSJWTSecret = getEnv("WS_JWT_SECRET", "")
+ if cfg.WSJWTTTLSeconds, err = getEnvInt("WS_JWT_TTL_SECONDS", 60); err != nil {
+ return nil, fmt.Errorf("WS_JWT_TTL_SECONDS: %w", err)
+ }
if cfg.OTELSampleRatio, err = getEnvFloat("OTEL_TRACES_SAMPLER_ARG", 0.1); err != nil {
return nil, fmt.Errorf("OTEL_TRACES_SAMPLER_ARG: %w", err)
}
+ if cfg.WebhookConcurrency, err = getEnvInt("WEBHOOK_CONCURRENCY", 3); err != nil {
+ return nil, fmt.Errorf("WEBHOOK_CONCURRENCY: %w", err)
+ }
+ if cfg.WebhookMaxAttempts, err = getEnvInt("WEBHOOK_MAX_ATTEMPTS", 5); err != nil {
+ return nil, fmt.Errorf("WEBHOOK_MAX_ATTEMPTS: %w", err)
+ }
+ if cfg.WebhookRetryBaseMS, err = getEnvInt("WEBHOOK_RETRY_BASE_DELAY_MS", 1000); err != nil {
+ return nil, fmt.Errorf("WEBHOOK_RETRY_BASE_DELAY_MS: %w", err)
+ }
return cfg, nil
}
diff --git a/internal/db/device_token.sql.go b/internal/db/device_token.sql.go
new file mode 100644
index 0000000..a84eede
--- /dev/null
+++ b/internal/db/device_token.sql.go
@@ -0,0 +1,136 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+// sqlc v1.29.0
+// source: device_token.sql
+
+package db
+
+import (
+ "context"
+
+ "github.com/google/uuid"
+)
+
+const deactivateDeviceToken = `-- name: DeactivateDeviceToken :exec
+UPDATE device_tokens
+SET is_active = false, updated_at = NOW()
+WHERE tenant_id = $1 AND token = $2
+`
+
+type DeactivateDeviceTokenParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ Token string `json:"token"`
+}
+
+func (q *Queries) DeactivateDeviceToken(ctx context.Context, arg DeactivateDeviceTokenParams) error {
+ _, err := q.db.ExecContext(ctx, deactivateDeviceToken, arg.TenantID, arg.Token)
+ return err
+}
+
+const getDeviceToken = `-- name: GetDeviceToken :one
+SELECT id, tenant_id, user_id, token, platform, is_active, created_at, updated_at FROM device_tokens
+WHERE tenant_id = $1 AND token = $2
+`
+
+type GetDeviceTokenParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ Token string `json:"token"`
+}
+
+func (q *Queries) GetDeviceToken(ctx context.Context, arg GetDeviceTokenParams) (DeviceToken, error) {
+ row := q.db.QueryRowContext(ctx, getDeviceToken, arg.TenantID, arg.Token)
+ var i DeviceToken
+ err := row.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.UserID,
+ &i.Token,
+ &i.Platform,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ )
+ return i, err
+}
+
+const listActiveDeviceTokensByUser = `-- name: ListActiveDeviceTokensByUser :many
+SELECT id, tenant_id, user_id, token, platform, is_active, created_at, updated_at FROM device_tokens
+WHERE tenant_id = $1 AND user_id = $2 AND is_active = true
+ORDER BY created_at DESC
+`
+
+type ListActiveDeviceTokensByUserParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ UserID string `json:"user_id"`
+}
+
+func (q *Queries) ListActiveDeviceTokensByUser(ctx context.Context, arg ListActiveDeviceTokensByUserParams) ([]DeviceToken, error) {
+ rows, err := q.db.QueryContext(ctx, listActiveDeviceTokensByUser, arg.TenantID, arg.UserID)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ items := []DeviceToken{}
+ for rows.Next() {
+ var i DeviceToken
+ if err := rows.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.UserID,
+ &i.Token,
+ &i.Platform,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ ); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Close(); err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
+const upsertDeviceToken = `-- name: UpsertDeviceToken :one
+INSERT INTO device_tokens (tenant_id, user_id, token, platform)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (tenant_id, token) DO UPDATE
+ SET user_id = EXCLUDED.user_id,
+ platform = EXCLUDED.platform,
+ is_active = true,
+ updated_at = NOW()
+RETURNING id, tenant_id, user_id, token, platform, is_active, created_at, updated_at
+`
+
+type UpsertDeviceTokenParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ UserID string `json:"user_id"`
+ Token string `json:"token"`
+ Platform string `json:"platform"`
+}
+
+func (q *Queries) UpsertDeviceToken(ctx context.Context, arg UpsertDeviceTokenParams) (DeviceToken, error) {
+ row := q.db.QueryRowContext(ctx, upsertDeviceToken,
+ arg.TenantID,
+ arg.UserID,
+ arg.Token,
+ arg.Platform,
+ )
+ var i DeviceToken
+ err := row.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.UserID,
+ &i.Token,
+ &i.Platform,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ )
+ return i, err
+}
diff --git a/internal/db/inapp.sql.go b/internal/db/inapp.sql.go
index fc8a526..fd643e5 100644
--- a/internal/db/inapp.sql.go
+++ b/internal/db/inapp.sql.go
@@ -8,6 +8,7 @@ package db
import (
"context"
"encoding/json"
+ "time"
"github.com/google/uuid"
)
@@ -194,3 +195,116 @@ func (q *Queries) MarkInAppMessageRead(ctx context.Context, arg MarkInAppMessage
_, err := q.db.ExecContext(ctx, markInAppMessageRead, arg.ID, arg.TenantID, arg.RecipientID)
return err
}
+
+const listInboxAfterCursor = `-- name: ListInboxAfterCursor :many
+SELECT id, tenant_id, notification_id, recipient_id, title, body, payload, read_at, created_at FROM inapp_messages
+WHERE tenant_id = $1::uuid
+ AND recipient_id = $2::text
+ AND (NOT $3::boolean OR read_at IS NULL)
+ AND (created_at < $4 OR (created_at = $4 AND id < $5::uuid))
+ORDER BY created_at DESC, id DESC
+LIMIT $6::int
+`
+
+type ListInboxAfterCursorParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ RecipientID string `json:"recipient_id"`
+ UnreadOnly bool `json:"unread_only"`
+ CursorTime time.Time `json:"cursor_time"`
+ CursorID uuid.UUID `json:"cursor_id"`
+ Limit int32 `json:"limit_"`
+}
+
+func (q *Queries) ListInboxAfterCursor(ctx context.Context, arg ListInboxAfterCursorParams) ([]InappMessage, error) {
+ rows, err := q.db.QueryContext(ctx, listInboxAfterCursor,
+ arg.TenantID,
+ arg.RecipientID,
+ arg.UnreadOnly,
+ arg.CursorTime,
+ arg.CursorID,
+ arg.Limit,
+ )
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ items := []InappMessage{}
+ for rows.Next() {
+ var i InappMessage
+ if err := rows.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.NotificationID,
+ &i.RecipientID,
+ &i.Title,
+ &i.Body,
+ &i.Payload,
+ &i.ReadAt,
+ &i.CreatedAt,
+ ); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Close(); err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
+const listInboxSince = `-- name: ListInboxSince :many
+SELECT id, tenant_id, notification_id, recipient_id, title, body, payload, read_at, created_at FROM inapp_messages
+WHERE tenant_id = $1::uuid
+ AND recipient_id = $2::text
+ AND (created_at > $3 OR (created_at = $3 AND id > $4::uuid))
+ORDER BY created_at ASC, id ASC
+LIMIT 200
+`
+
+type ListInboxSinceParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ RecipientID string `json:"recipient_id"`
+ CursorTime time.Time `json:"cursor_time"`
+ CursorID uuid.UUID `json:"cursor_id"`
+}
+
+func (q *Queries) ListInboxSince(ctx context.Context, arg ListInboxSinceParams) ([]InappMessage, error) {
+ rows, err := q.db.QueryContext(ctx, listInboxSince,
+ arg.TenantID,
+ arg.RecipientID,
+ arg.CursorTime,
+ arg.CursorID,
+ )
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ items := []InappMessage{}
+ for rows.Next() {
+ var i InappMessage
+ if err := rows.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.NotificationID,
+ &i.RecipientID,
+ &i.Title,
+ &i.Body,
+ &i.Payload,
+ &i.ReadAt,
+ &i.CreatedAt,
+ ); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Close(); err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
diff --git a/internal/db/models.go b/internal/db/models.go
index c8b90e9..f1f466d 100644
--- a/internal/db/models.go
+++ b/internal/db/models.go
@@ -51,6 +51,17 @@ type DeliveryLog struct {
DeliveredAt sql.NullTime `json:"delivered_at"`
}
+type DeviceToken struct {
+ ID uuid.UUID `json:"id"`
+ TenantID uuid.UUID `json:"tenant_id"`
+ UserID string `json:"user_id"`
+ Token string `json:"token"`
+ Platform string `json:"platform"`
+ IsActive bool `json:"is_active"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
+
type InappMessage struct {
ID uuid.UUID `json:"id"`
TenantID uuid.UUID `json:"tenant_id"`
@@ -116,3 +127,29 @@ type Tenant struct {
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
+
+type WebhookDelivery struct {
+ ID uuid.UUID `json:"id"`
+ EndpointID uuid.UUID `json:"endpoint_id"`
+ NotificationID uuid.NullUUID `json:"notification_id"`
+ Event string `json:"event"`
+ Payload json.RawMessage `json:"payload"`
+ Status string `json:"status"`
+ Attempt int32 `json:"attempt"`
+ NextRetryAt sql.NullTime `json:"next_retry_at"`
+ LastError sql.NullString `json:"last_error"`
+ ResponseStatus sql.NullInt32 `json:"response_status"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
+
+type WebhookEndpoint struct {
+ ID uuid.UUID `json:"id"`
+ TenantID uuid.UUID `json:"tenant_id"`
+ Url string `json:"url"`
+ Secret string `json:"secret"`
+ Events json.RawMessage `json:"events"`
+ IsActive bool `json:"is_active"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
diff --git a/internal/db/querier.go b/internal/db/querier.go
index aba32ca..f7b42dc 100644
--- a/internal/db/querier.go
+++ b/internal/db/querier.go
@@ -17,11 +17,14 @@ type Querier interface {
CountTemplates(ctx context.Context, arg CountTemplatesParams) (int64, error)
CountUnread(ctx context.Context, arg CountUnreadParams) (int64, error)
CreateDeadLetter(ctx context.Context, arg CreateDeadLetterParams) (DeadLetterMessage, error)
+ DeactivateDeviceToken(ctx context.Context, arg DeactivateDeviceTokenParams) error
DeleteDeadLetter(ctx context.Context, arg DeleteDeadLetterParams) error
DeletePreference(ctx context.Context, arg DeletePreferenceParams) error
DeleteTemplate(ctx context.Context, arg DeleteTemplateParams) (Template, error)
+ DeleteWebhookEndpoint(ctx context.Context, arg DeleteWebhookEndpointParams) error
GetAPIClientByKeyHash(ctx context.Context, apiKeyHash string) (GetAPIClientByKeyHashRow, error)
GetDeadLetterByID(ctx context.Context, arg GetDeadLetterByIDParams) (DeadLetterMessage, error)
+ GetDeviceToken(ctx context.Context, arg GetDeviceTokenParams) (DeviceToken, error)
GetInAppMessage(ctx context.Context, arg GetInAppMessageParams) (InappMessage, error)
GetLatestDeliveryLog(ctx context.Context, notificationID uuid.UUID) (DeliveryLog, error)
GetNotificationByID(ctx context.Context, arg GetNotificationByIDParams) (Notification, error)
@@ -31,12 +34,17 @@ type Querier interface {
GetTemplateByName(ctx context.Context, arg GetTemplateByNameParams) (Template, error)
GetTenantByID(ctx context.Context, id uuid.UUID) (Tenant, error)
GetTenantByName(ctx context.Context, name string) (Tenant, error)
+ GetWebhookEndpoint(ctx context.Context, arg GetWebhookEndpointParams) (WebhookEndpoint, error)
InsertAPIClient(ctx context.Context, arg InsertAPIClientParams) (InsertAPIClientRow, error)
InsertDeliveryLog(ctx context.Context, arg InsertDeliveryLogParams) (DeliveryLog, error)
InsertInAppMessage(ctx context.Context, arg InsertInAppMessageParams) (InappMessage, error)
InsertNotification(ctx context.Context, arg InsertNotificationParams) (Notification, error)
InsertTemplate(ctx context.Context, arg InsertTemplateParams) (Template, error)
InsertTenant(ctx context.Context, name string) (Tenant, error)
+ InsertWebhookDelivery(ctx context.Context, arg InsertWebhookDeliveryParams) (WebhookDelivery, error)
+ InsertWebhookEndpoint(ctx context.Context, arg InsertWebhookEndpointParams) (WebhookEndpoint, error)
+ ListActiveDeviceTokensByUser(ctx context.Context, arg ListActiveDeviceTokensByUserParams) ([]DeviceToken, error)
+ ListActiveWebhookEndpointsForEvent(ctx context.Context, arg ListActiveWebhookEndpointsForEventParams) ([]WebhookEndpoint, error)
ListDeadLetters(ctx context.Context, arg ListDeadLettersParams) ([]DeadLetterMessage, error)
ListDeliveryLogsByNotification(ctx context.Context, notificationID uuid.UUID) ([]DeliveryLog, error)
ListDueScheduledNotifications(ctx context.Context, limit int32) ([]Notification, error)
@@ -44,11 +52,15 @@ type Querier interface {
ListNotifications(ctx context.Context, arg ListNotificationsParams) ([]Notification, error)
ListPreferencesByUser(ctx context.Context, arg ListPreferencesByUserParams) ([]Preference, error)
ListTemplates(ctx context.Context, arg ListTemplatesParams) ([]Template, error)
+ ListWebhookEndpoints(ctx context.Context, tenantID uuid.UUID) ([]WebhookEndpoint, error)
MarkAllInAppMessagesRead(ctx context.Context, arg MarkAllInAppMessagesReadParams) error
MarkDeadLetterReplayed(ctx context.Context, arg MarkDeadLetterReplayedParams) error
MarkInAppMessageRead(ctx context.Context, arg MarkInAppMessageReadParams) error
UpdateNotificationStatus(ctx context.Context, arg UpdateNotificationStatusParams) (Notification, error)
UpdateTemplate(ctx context.Context, arg UpdateTemplateParams) (Template, error)
+ UpdateWebhookDelivery(ctx context.Context, arg UpdateWebhookDeliveryParams) (WebhookDelivery, error)
+ UpdateWebhookEndpoint(ctx context.Context, arg UpdateWebhookEndpointParams) (WebhookEndpoint, error)
+ UpsertDeviceToken(ctx context.Context, arg UpsertDeviceTokenParams) (DeviceToken, error)
UpsertPreference(ctx context.Context, arg UpsertPreferenceParams) (Preference, error)
}
diff --git a/internal/db/webhook.sql.go b/internal/db/webhook.sql.go
new file mode 100644
index 0000000..a4e32e0
--- /dev/null
+++ b/internal/db/webhook.sql.go
@@ -0,0 +1,297 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+// sqlc v1.29.0
+// source: webhook.sql
+
+package db
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+
+ "github.com/google/uuid"
+)
+
+const deleteWebhookEndpoint = `-- name: DeleteWebhookEndpoint :exec
+DELETE FROM webhook_endpoints
+WHERE id = $1 AND tenant_id = $2
+`
+
+type DeleteWebhookEndpointParams struct {
+ ID uuid.UUID `json:"id"`
+ TenantID uuid.UUID `json:"tenant_id"`
+}
+
+func (q *Queries) DeleteWebhookEndpoint(ctx context.Context, arg DeleteWebhookEndpointParams) error {
+ _, err := q.db.ExecContext(ctx, deleteWebhookEndpoint, arg.ID, arg.TenantID)
+ return err
+}
+
+const getWebhookEndpoint = `-- name: GetWebhookEndpoint :one
+SELECT id, tenant_id, url, secret, events, is_active, created_at, updated_at FROM webhook_endpoints
+WHERE id = $1 AND tenant_id = $2
+`
+
+type GetWebhookEndpointParams struct {
+ ID uuid.UUID `json:"id"`
+ TenantID uuid.UUID `json:"tenant_id"`
+}
+
+func (q *Queries) GetWebhookEndpoint(ctx context.Context, arg GetWebhookEndpointParams) (WebhookEndpoint, error) {
+ row := q.db.QueryRowContext(ctx, getWebhookEndpoint, arg.ID, arg.TenantID)
+ var i WebhookEndpoint
+ err := row.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.Url,
+ &i.Secret,
+ &i.Events,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ )
+ return i, err
+}
+
+const insertWebhookDelivery = `-- name: InsertWebhookDelivery :one
+INSERT INTO webhook_deliveries (endpoint_id, notification_id, event, payload, status, attempt, next_retry_at)
+VALUES ($1, $2, $3, $4, $5, $6, $7)
+RETURNING id, endpoint_id, notification_id, event, payload, status, attempt, next_retry_at, last_error, response_status, created_at, updated_at
+`
+
+type InsertWebhookDeliveryParams struct {
+ EndpointID uuid.UUID `json:"endpoint_id"`
+ NotificationID uuid.NullUUID `json:"notification_id"`
+ Event string `json:"event"`
+ Payload json.RawMessage `json:"payload"`
+ Status string `json:"status"`
+ Attempt int32 `json:"attempt"`
+ NextRetryAt sql.NullTime `json:"next_retry_at"`
+}
+
+func (q *Queries) InsertWebhookDelivery(ctx context.Context, arg InsertWebhookDeliveryParams) (WebhookDelivery, error) {
+ row := q.db.QueryRowContext(ctx, insertWebhookDelivery,
+ arg.EndpointID,
+ arg.NotificationID,
+ arg.Event,
+ arg.Payload,
+ arg.Status,
+ arg.Attempt,
+ arg.NextRetryAt,
+ )
+ var i WebhookDelivery
+ err := row.Scan(
+ &i.ID,
+ &i.EndpointID,
+ &i.NotificationID,
+ &i.Event,
+ &i.Payload,
+ &i.Status,
+ &i.Attempt,
+ &i.NextRetryAt,
+ &i.LastError,
+ &i.ResponseStatus,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ )
+ return i, err
+}
+
+const insertWebhookEndpoint = `-- name: InsertWebhookEndpoint :one
+INSERT INTO webhook_endpoints (tenant_id, url, secret, events, is_active)
+VALUES ($1, $2, $3, $4, $5)
+RETURNING id, tenant_id, url, secret, events, is_active, created_at, updated_at
+`
+
+type InsertWebhookEndpointParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ Url string `json:"url"`
+ Secret string `json:"secret"`
+ Events json.RawMessage `json:"events"`
+ IsActive bool `json:"is_active"`
+}
+
+func (q *Queries) InsertWebhookEndpoint(ctx context.Context, arg InsertWebhookEndpointParams) (WebhookEndpoint, error) {
+ row := q.db.QueryRowContext(ctx, insertWebhookEndpoint,
+ arg.TenantID,
+ arg.Url,
+ arg.Secret,
+ arg.Events,
+ arg.IsActive,
+ )
+ var i WebhookEndpoint
+ err := row.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.Url,
+ &i.Secret,
+ &i.Events,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ )
+ return i, err
+}
+
+const listActiveWebhookEndpointsForEvent = `-- name: ListActiveWebhookEndpointsForEvent :many
+SELECT id, tenant_id, url, secret, events, is_active, created_at, updated_at FROM webhook_endpoints
+WHERE tenant_id = $1
+ AND is_active = TRUE
+ AND events @> jsonb_build_array($2::text)
+`
+
+type ListActiveWebhookEndpointsForEventParams struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ Column2 string `json:"column_2"`
+}
+
+func (q *Queries) ListActiveWebhookEndpointsForEvent(ctx context.Context, arg ListActiveWebhookEndpointsForEventParams) ([]WebhookEndpoint, error) {
+ rows, err := q.db.QueryContext(ctx, listActiveWebhookEndpointsForEvent, arg.TenantID, arg.Column2)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ items := []WebhookEndpoint{}
+ for rows.Next() {
+ var i WebhookEndpoint
+ if err := rows.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.Url,
+ &i.Secret,
+ &i.Events,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ ); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Close(); err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
+const listWebhookEndpoints = `-- name: ListWebhookEndpoints :many
+SELECT id, tenant_id, url, secret, events, is_active, created_at, updated_at FROM webhook_endpoints
+WHERE tenant_id = $1
+ORDER BY created_at DESC
+`
+
+func (q *Queries) ListWebhookEndpoints(ctx context.Context, tenantID uuid.UUID) ([]WebhookEndpoint, error) {
+ rows, err := q.db.QueryContext(ctx, listWebhookEndpoints, tenantID)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ items := []WebhookEndpoint{}
+ for rows.Next() {
+ var i WebhookEndpoint
+ if err := rows.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.Url,
+ &i.Secret,
+ &i.Events,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ ); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Close(); err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
+const updateWebhookDelivery = `-- name: UpdateWebhookDelivery :one
+UPDATE webhook_deliveries
+SET status = $2, attempt = $3, next_retry_at = $4, last_error = $5, response_status = $6, updated_at = now()
+WHERE id = $1
+RETURNING id, endpoint_id, notification_id, event, payload, status, attempt, next_retry_at, last_error, response_status, created_at, updated_at
+`
+
+type UpdateWebhookDeliveryParams struct {
+ ID uuid.UUID `json:"id"`
+ Status string `json:"status"`
+ Attempt int32 `json:"attempt"`
+ NextRetryAt sql.NullTime `json:"next_retry_at"`
+ LastError sql.NullString `json:"last_error"`
+ ResponseStatus sql.NullInt32 `json:"response_status"`
+}
+
+func (q *Queries) UpdateWebhookDelivery(ctx context.Context, arg UpdateWebhookDeliveryParams) (WebhookDelivery, error) {
+ row := q.db.QueryRowContext(ctx, updateWebhookDelivery,
+ arg.ID,
+ arg.Status,
+ arg.Attempt,
+ arg.NextRetryAt,
+ arg.LastError,
+ arg.ResponseStatus,
+ )
+ var i WebhookDelivery
+ err := row.Scan(
+ &i.ID,
+ &i.EndpointID,
+ &i.NotificationID,
+ &i.Event,
+ &i.Payload,
+ &i.Status,
+ &i.Attempt,
+ &i.NextRetryAt,
+ &i.LastError,
+ &i.ResponseStatus,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ )
+ return i, err
+}
+
+const updateWebhookEndpoint = `-- name: UpdateWebhookEndpoint :one
+UPDATE webhook_endpoints
+SET url = $3, events = $4, is_active = $5, updated_at = now()
+WHERE id = $1 AND tenant_id = $2
+RETURNING id, tenant_id, url, secret, events, is_active, created_at, updated_at
+`
+
+type UpdateWebhookEndpointParams struct {
+ ID uuid.UUID `json:"id"`
+ TenantID uuid.UUID `json:"tenant_id"`
+ Url string `json:"url"`
+ Events json.RawMessage `json:"events"`
+ IsActive bool `json:"is_active"`
+}
+
+func (q *Queries) UpdateWebhookEndpoint(ctx context.Context, arg UpdateWebhookEndpointParams) (WebhookEndpoint, error) {
+ row := q.db.QueryRowContext(ctx, updateWebhookEndpoint,
+ arg.ID,
+ arg.TenantID,
+ arg.Url,
+ arg.Events,
+ arg.IsActive,
+ )
+ var i WebhookEndpoint
+ err := row.Scan(
+ &i.ID,
+ &i.TenantID,
+ &i.Url,
+ &i.Secret,
+ &i.Events,
+ &i.IsActive,
+ &i.CreatedAt,
+ &i.UpdatedAt,
+ )
+ return i, err
+}
diff --git a/internal/domain/device_token.go b/internal/domain/device_token.go
new file mode 100644
index 0000000..be0d453
--- /dev/null
+++ b/internal/domain/device_token.go
@@ -0,0 +1,24 @@
+package domain
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+)
+
+type DeviceToken struct {
+ ID uuid.UUID `json:"id"`
+ TenantID uuid.UUID `json:"tenant_id"`
+ UserID string `json:"user_id"`
+ Token string `json:"token"`
+ Platform string `json:"platform"`
+ IsActive bool `json:"is_active"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
+
+type RegisterDeviceTokenRequest struct {
+ UserID string `json:"user_id" validate:"required"`
+ Token string `json:"token" validate:"required"`
+ Platform string `json:"platform" validate:"required,oneof=ios android web"`
+}
diff --git a/internal/domain/inapp.go b/internal/domain/inapp.go
index 101dd20..43f8f2f 100644
--- a/internal/domain/inapp.go
+++ b/internal/domain/inapp.go
@@ -24,4 +24,5 @@ type InboxQuery struct {
UnreadOnly bool
Limit int
Offset int
+ AfterID *uuid.UUID // keyset cursor; if set, Offset is ignored
}
diff --git a/internal/domain/webhook.go b/internal/domain/webhook.go
new file mode 100644
index 0000000..00c8503
--- /dev/null
+++ b/internal/domain/webhook.go
@@ -0,0 +1,72 @@
+package domain
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+)
+
+// Webhook event types fired after a notification reaches a terminal status.
+const (
+ WebhookEventDelivered = "notification.delivered"
+ WebhookEventFailed = "notification.failed"
+ WebhookEventDropped = "notification.dropped"
+ WebhookEventTokenDeactivated = "device_token.deactivated"
+)
+
+// AllWebhookEvents is the exhaustive list of valid event names for validation.
+var AllWebhookEvents = []string{
+ WebhookEventDelivered,
+ WebhookEventFailed,
+ WebhookEventDropped,
+ WebhookEventTokenDeactivated,
+}
+
+type WebhookDeliveryStatus string
+
+const (
+ WebhookDeliveryPending WebhookDeliveryStatus = "pending"
+ WebhookDeliveryDelivered WebhookDeliveryStatus = "delivered"
+ WebhookDeliveryFailed WebhookDeliveryStatus = "failed"
+)
+
+// WebhookEndpoint is a tenant-registered URL that receives HTTP POST callbacks
+// for subscribed notification lifecycle events.
+type WebhookEndpoint struct {
+ ID uuid.UUID `json:"id"`
+ TenantID uuid.UUID `json:"tenant_id"`
+ URL string `json:"url"`
+ Secret string `json:"-"`
+ Events []string `json:"events"`
+ IsActive bool `json:"is_active"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
+
+// WebhookDelivery tracks a single HTTP dispatch attempt to an endpoint.
+type WebhookDelivery struct {
+ ID uuid.UUID `json:"id"`
+ EndpointID uuid.UUID `json:"endpoint_id"`
+ NotificationID *uuid.UUID `json:"notification_id,omitempty"`
+ Event string `json:"event"`
+ Payload map[string]any `json:"payload"`
+ Status WebhookDeliveryStatus `json:"status"`
+ Attempt int `json:"attempt"`
+ NextRetryAt *time.Time `json:"next_retry_at,omitempty"`
+ LastError *string `json:"last_error,omitempty"`
+ ResponseStatus *int `json:"response_status,omitempty"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
+
+type CreateWebhookEndpointRequest struct {
+ URL string `json:"url" validate:"required,url"`
+ Secret string `json:"secret" validate:"required,min=16"`
+ Events []string `json:"events" validate:"required,min=1"`
+}
+
+type UpdateWebhookEndpointRequest struct {
+ URL string `json:"url" validate:"required,url"`
+ Events []string `json:"events" validate:"required,min=1"`
+ IsActive bool `json:"is_active"`
+}
diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go
index 38f89dd..1615371 100644
--- a/internal/observability/metrics.go
+++ b/internal/observability/metrics.go
@@ -76,6 +76,10 @@ type Metrics struct {
InAppWSConnections prometheus.Gauge
InAppWSDropped prometheus.Counter
InAppPubSubPublished prometheus.Counter
+
+ // Outbound webhooks.
+ WebhookDelivered *prometheus.CounterVec // labels: event
+ WebhookFailed *prometheus.CounterVec // labels: event
}
// NewMetrics builds a Metrics struct backed by a fresh registry.
@@ -272,6 +276,25 @@ func NewMetrics(version, commit, env string) *Metrics {
Name: "pubsub_published_total",
Help: "In-app messages published to Redis pub/sub.",
}),
+
+ WebhookDelivered: prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "notifyhub",
+ Subsystem: "webhook",
+ Name: "delivered_total",
+ Help: "Outbound webhook deliveries successfully acknowledged by the endpoint.",
+ },
+ []string{"event"},
+ ),
+ WebhookFailed: prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Namespace: "notifyhub",
+ Subsystem: "webhook",
+ Name: "failed_total",
+ Help: "Outbound webhook deliveries that exhausted all retry attempts.",
+ },
+ []string{"event"},
+ ),
}
reg.MustRegister(
@@ -286,6 +309,7 @@ func NewMetrics(version, commit, env string) *Metrics {
m.DLQPersisted, m.DLQReplayed, m.DLQDepth,
m.InAppPersisted, m.InAppWSConnections,
m.InAppWSDropped, m.InAppPubSubPublished,
+ m.WebhookDelivered, m.WebhookFailed,
)
m.BuildInfo.WithLabelValues(version, commit, env).Set(1)
diff --git a/internal/queue/admin.go b/internal/queue/admin.go
new file mode 100644
index 0000000..6802eb7
--- /dev/null
+++ b/internal/queue/admin.go
@@ -0,0 +1,88 @@
+package queue
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "net"
+ "time"
+
+ "github.com/segmentio/kafka-go"
+
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+)
+
+// allTopics returns every topic the worker reads from.
+// Called at startup to pre-create topics before consumers join,
+// preventing the empty-assignment deadlock that occurs when consumers
+// join a group before topics exist and kafka-go never re-triggers rebalance.
+func allTopics() []string {
+ channels := []domain.Channel{
+ domain.ChannelEmail,
+ domain.ChannelPush,
+ domain.ChannelSMS,
+ domain.ChannelInApp,
+ }
+ topics := make([]string, 0, len(channels)*2+1)
+ for _, ch := range channels {
+ topics = append(topics, TopicForChannel(ch), DLQTopicForChannel(ch))
+ }
+ topics = append(topics, WebhookTopic)
+ return topics
+}
+
+// EnsureTopics creates all worker topics on the broker if they do not already
+// exist. The operation is idempotent — calling it on every startup is safe.
+// Uses a 30-second timeout so a slow broker doesn't hang indefinitely.
+func EnsureTopics(ctx context.Context, brokers []string, logger *slog.Logger) error {
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+
+ // Resolve a broker address to connect to.
+ addr, err := net.ResolveTCPAddr("tcp", brokers[0])
+ if err != nil {
+ return fmt.Errorf("resolve broker address: %w", err)
+ }
+
+ conn, err := kafka.DialLeader(ctx, "tcp", addr.String(), "__consumer_offsets", 0)
+ if err != nil {
+ // Fallback: use plain Dial if DialLeader fails (broker may not have the
+ // internal topic yet on a completely fresh cluster).
+ conn, err = kafka.Dial("tcp", addr.String())
+ if err != nil {
+ return fmt.Errorf("connect to kafka broker: %w", err)
+ }
+ }
+ defer conn.Close()
+
+ controller, err := conn.Controller()
+ if err != nil {
+ return fmt.Errorf("get controller: %w", err)
+ }
+
+ controllerConn, err := kafka.Dial("tcp", fmt.Sprintf("%s:%d", controller.Host, controller.Port))
+ if err != nil {
+ return fmt.Errorf("connect to controller: %w", err)
+ }
+ defer controllerConn.Close()
+
+ topics := allTopics()
+ specs := make([]kafka.TopicConfig, len(topics))
+ for i, t := range topics {
+ specs[i] = kafka.TopicConfig{
+ Topic: t,
+ NumPartitions: 1,
+ ReplicationFactor: 1,
+ }
+ }
+
+ err = controllerConn.CreateTopics(specs...)
+ if err != nil {
+ // TopicAlreadyExists (error code 36) is not fatal — log and continue.
+ logger.Info("kafka topics already exist or partially created", slog.Any("error", err))
+ } else {
+ logger.Info("kafka topics ensured", slog.Int("count", len(topics)))
+ }
+
+ return nil
+}
diff --git a/internal/queue/consumer.go b/internal/queue/consumer.go
index f4b51d2..c7ff65f 100644
--- a/internal/queue/consumer.go
+++ b/internal/queue/consumer.go
@@ -27,7 +27,7 @@ func NewConsumer(brokers []string, topic, groupID string, logger *slog.Logger) *
MaxBytes: 10 << 20, // 10 MB cap per fetch response
MaxWait: 1 * time.Second, // max idle wait; keeps ctx cancellation responsive
CommitInterval: 0, // disable auto-commit; we commit manually
- StartOffset: kafka.LastOffset, // new groups start from latest, not beginning
+ StartOffset: kafka.FirstOffset, // new groups replay from earliest uncommitted offset
Logger: kafka.LoggerFunc(func(msg string, args ...interface{}) {
logger.Debug(fmt.Sprintf(msg, args...))
}),
diff --git a/internal/queue/message.go b/internal/queue/message.go
index fc5c4ac..5c1d607 100644
--- a/internal/queue/message.go
+++ b/internal/queue/message.go
@@ -47,3 +47,21 @@ type DLQMessage struct {
LastAttempt int `json:"last_attempt"`
DeadLetteredAt time.Time `json:"dead_lettered_at"`
}
+
+// WebhookTopic is the Kafka topic for outbound webhook dispatch.
+const WebhookTopic = "notifyhub.webhooks.outbound"
+
+// WebhookMessage is the Kafka envelope for one outbound webhook dispatch attempt.
+type WebhookMessage struct {
+ TenantID uuid.UUID `json:"tenant_id"`
+ EndpointID uuid.UUID `json:"endpoint_id"`
+ DeliveryID uuid.UUID `json:"delivery_id"`
+ NotificationID *uuid.UUID `json:"notification_id,omitempty"`
+ Event string `json:"event"`
+ Payload map[string]any `json:"payload"`
+ EndpointURL string `json:"endpoint_url"`
+ Secret string `json:"secret"`
+ Attempt int `json:"attempt"`
+ MaxAttempts int `json:"max_attempts"`
+ EnqueuedAt time.Time `json:"enqueued_at"`
+}
diff --git a/internal/repository/device_token.go b/internal/repository/device_token.go
new file mode 100644
index 0000000..c3dab6f
--- /dev/null
+++ b/internal/repository/device_token.go
@@ -0,0 +1,81 @@
+package repository
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+
+ "github.com/google/uuid"
+
+ "github.com/amitrajitdas31/notifyhub/internal/db"
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+)
+
+type DeviceTokenRepository interface {
+ Upsert(ctx context.Context, tenantID uuid.UUID, req domain.RegisterDeviceTokenRequest) (*domain.DeviceToken, error)
+ Deactivate(ctx context.Context, tenantID uuid.UUID, token string) error
+ ListActiveByUser(ctx context.Context, tenantID uuid.UUID, userID string) ([]domain.DeviceToken, error)
+}
+
+type postgresDeviceTokenRepo struct {
+ q *db.Queries
+}
+
+func NewDeviceTokenRepository(q *db.Queries) DeviceTokenRepository {
+ return &postgresDeviceTokenRepo{q: q}
+}
+
+func (r *postgresDeviceTokenRepo) Upsert(ctx context.Context, tenantID uuid.UUID, req domain.RegisterDeviceTokenRequest) (*domain.DeviceToken, error) {
+ row, err := r.q.UpsertDeviceToken(ctx, db.UpsertDeviceTokenParams{
+ TenantID: tenantID,
+ UserID: req.UserID,
+ Token: req.Token,
+ Platform: req.Platform,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return toDeviceToken(row), nil
+}
+
+func (r *postgresDeviceTokenRepo) Deactivate(ctx context.Context, tenantID uuid.UUID, token string) error {
+ err := r.q.DeactivateDeviceToken(ctx, db.DeactivateDeviceTokenParams{
+ TenantID: tenantID,
+ Token: token,
+ })
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return domain.NewNotFoundError("device token not found")
+ }
+ return err
+ }
+ return nil
+}
+
+func (r *postgresDeviceTokenRepo) ListActiveByUser(ctx context.Context, tenantID uuid.UUID, userID string) ([]domain.DeviceToken, error) {
+ rows, err := r.q.ListActiveDeviceTokensByUser(ctx, db.ListActiveDeviceTokensByUserParams{
+ TenantID: tenantID,
+ UserID: userID,
+ })
+ if err != nil {
+ return nil, err
+ }
+ tokens := make([]domain.DeviceToken, len(rows))
+ for i, row := range rows {
+ tokens[i] = *toDeviceToken(row)
+ }
+ return tokens, nil
+}
+
+func toDeviceToken(row db.DeviceToken) *domain.DeviceToken {
+ return &domain.DeviceToken{
+ ID: row.ID,
+ TenantID: row.TenantID,
+ UserID: row.UserID,
+ Token: row.Token,
+ Platform: row.Platform,
+ IsActive: row.IsActive,
+ CreatedAt: row.CreatedAt,
+ UpdatedAt: row.UpdatedAt,
+ }
+}
diff --git a/internal/repository/inapp.go b/internal/repository/inapp.go
index b8fb747..10bdf4c 100644
--- a/internal/repository/inapp.go
+++ b/internal/repository/inapp.go
@@ -14,6 +14,9 @@ type InAppRepository interface {
Insert(ctx context.Context, msg domain.InAppMessage) (*domain.InAppMessage, error)
GetByID(ctx context.Context, tenantID, id uuid.UUID) (*domain.InAppMessage, error)
List(ctx context.Context, q domain.InboxQuery) ([]domain.InAppMessage, error)
+ // ListSince returns all messages newer than sinceID, ordered oldest-first.
+ // Used by the WS stream handler to replay missed messages after reconnect.
+ ListSince(ctx context.Context, tenantID uuid.UUID, recipientID string, sinceID uuid.UUID) ([]domain.InAppMessage, error)
CountUnread(ctx context.Context, tenantID uuid.UUID, recipientID string) (int64, error)
MarkRead(ctx context.Context, tenantID, id uuid.UUID, recipientID string) error
MarkAllRead(ctx context.Context, tenantID uuid.UUID, recipientID string) error
@@ -60,6 +63,31 @@ func (r *postgresInAppRepo) List(ctx context.Context, q domain.InboxQuery) ([]do
if limit <= 0 {
limit = 50
}
+ if q.AfterID != nil {
+ cursor, err := r.q.GetInAppMessage(ctx, db.GetInAppMessageParams{
+ ID: *q.AfterID,
+ TenantID: q.TenantID,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("list inbox cursor lookup: %w", err)
+ }
+ rows, err := r.q.ListInboxAfterCursor(ctx, db.ListInboxAfterCursorParams{
+ TenantID: q.TenantID,
+ RecipientID: q.RecipientID,
+ UnreadOnly: q.UnreadOnly,
+ CursorTime: cursor.CreatedAt,
+ CursorID: cursor.ID,
+ Limit: int32(limit),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("list inbox after cursor: %w", err)
+ }
+ out := make([]domain.InAppMessage, len(rows))
+ for i, row := range rows {
+ out[i] = fromDBInApp(row)
+ }
+ return out, nil
+ }
rows, err := r.q.ListInbox(ctx, db.ListInboxParams{
TenantID: q.TenantID,
RecipientID: q.RecipientID,
@@ -77,6 +105,30 @@ func (r *postgresInAppRepo) List(ctx context.Context, q domain.InboxQuery) ([]do
return out, nil
}
+func (r *postgresInAppRepo) ListSince(ctx context.Context, tenantID uuid.UUID, recipientID string, sinceID uuid.UUID) ([]domain.InAppMessage, error) {
+ cursor, err := r.q.GetInAppMessage(ctx, db.GetInAppMessageParams{
+ ID: sinceID,
+ TenantID: tenantID,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("list inbox since cursor lookup: %w", err)
+ }
+ rows, err := r.q.ListInboxSince(ctx, db.ListInboxSinceParams{
+ TenantID: tenantID,
+ RecipientID: recipientID,
+ CursorTime: cursor.CreatedAt,
+ CursorID: cursor.ID,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("list inbox since: %w", err)
+ }
+ out := make([]domain.InAppMessage, len(rows))
+ for i, row := range rows {
+ out[i] = fromDBInApp(row)
+ }
+ return out, nil
+}
+
func (r *postgresInAppRepo) CountUnread(ctx context.Context, tenantID uuid.UUID, recipientID string) (int64, error) {
count, err := r.q.CountUnread(ctx, db.CountUnreadParams{
TenantID: tenantID,
diff --git a/internal/repository/webhook.go b/internal/repository/webhook.go
new file mode 100644
index 0000000..b4b7747
--- /dev/null
+++ b/internal/repository/webhook.go
@@ -0,0 +1,207 @@
+package repository
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+
+ "github.com/google/uuid"
+
+ "github.com/amitrajitdas31/notifyhub/internal/db"
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+)
+
+// WebhookRepository defines DB operations for webhook endpoints and deliveries.
+type WebhookRepository interface {
+ CreateEndpoint(ctx context.Context, ep domain.WebhookEndpoint) (*domain.WebhookEndpoint, error)
+ GetEndpoint(ctx context.Context, tenantID, id uuid.UUID) (*domain.WebhookEndpoint, error)
+ ListEndpoints(ctx context.Context, tenantID uuid.UUID) ([]domain.WebhookEndpoint, error)
+ UpdateEndpoint(ctx context.Context, ep domain.WebhookEndpoint) (*domain.WebhookEndpoint, error)
+ DeleteEndpoint(ctx context.Context, tenantID, id uuid.UUID) error
+ ListActiveForEvent(ctx context.Context, tenantID uuid.UUID, event string) ([]domain.WebhookEndpoint, error)
+ CreateDelivery(ctx context.Context, d domain.WebhookDelivery) (*domain.WebhookDelivery, error)
+ UpdateDelivery(ctx context.Context, d domain.WebhookDelivery) (*domain.WebhookDelivery, error)
+}
+
+type postgresWebhookRepo struct {
+ q *db.Queries
+}
+
+func NewWebhookRepository(q *db.Queries) WebhookRepository {
+ return &postgresWebhookRepo{q: q}
+}
+
+func (r *postgresWebhookRepo) CreateEndpoint(ctx context.Context, ep domain.WebhookEndpoint) (*domain.WebhookEndpoint, error) {
+ eventsJSON, err := json.Marshal(ep.Events)
+ if err != nil {
+ return nil, domain.NewInternalError("marshal events", err)
+ }
+ row, err := r.q.InsertWebhookEndpoint(ctx, db.InsertWebhookEndpointParams{
+ TenantID: ep.TenantID,
+ Url: ep.URL,
+ Secret: ep.Secret,
+ Events: eventsJSON,
+ IsActive: ep.IsActive,
+ })
+ if err != nil {
+ return nil, domain.NewInternalError("insert webhook endpoint", err)
+ }
+ out := dbWebhookEndpointToDomain(row)
+ return &out, nil
+}
+
+func (r *postgresWebhookRepo) GetEndpoint(ctx context.Context, tenantID, id uuid.UUID) (*domain.WebhookEndpoint, error) {
+ row, err := r.q.GetWebhookEndpoint(ctx, db.GetWebhookEndpointParams{
+ ID: id,
+ TenantID: tenantID,
+ })
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, domain.NewNotFoundError("webhook endpoint not found")
+ }
+ return nil, domain.NewInternalError("get webhook endpoint", err)
+ }
+ out := dbWebhookEndpointToDomain(row)
+ return &out, nil
+}
+
+func (r *postgresWebhookRepo) ListEndpoints(ctx context.Context, tenantID uuid.UUID) ([]domain.WebhookEndpoint, error) {
+ rows, err := r.q.ListWebhookEndpoints(ctx, tenantID)
+ if err != nil {
+ return nil, domain.NewInternalError("list webhook endpoints", err)
+ }
+ out := make([]domain.WebhookEndpoint, len(rows))
+ for i, row := range rows {
+ out[i] = dbWebhookEndpointToDomain(row)
+ }
+ return out, nil
+}
+
+func (r *postgresWebhookRepo) UpdateEndpoint(ctx context.Context, ep domain.WebhookEndpoint) (*domain.WebhookEndpoint, error) {
+ eventsJSON, err := json.Marshal(ep.Events)
+ if err != nil {
+ return nil, domain.NewInternalError("marshal events", err)
+ }
+ row, err := r.q.UpdateWebhookEndpoint(ctx, db.UpdateWebhookEndpointParams{
+ ID: ep.ID,
+ TenantID: ep.TenantID,
+ Url: ep.URL,
+ Events: eventsJSON,
+ IsActive: ep.IsActive,
+ })
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, domain.NewNotFoundError("webhook endpoint not found")
+ }
+ return nil, domain.NewInternalError("update webhook endpoint", err)
+ }
+ out := dbWebhookEndpointToDomain(row)
+ return &out, nil
+}
+
+func (r *postgresWebhookRepo) DeleteEndpoint(ctx context.Context, tenantID, id uuid.UUID) error {
+ if err := r.q.DeleteWebhookEndpoint(ctx, db.DeleteWebhookEndpointParams{
+ ID: id,
+ TenantID: tenantID,
+ }); err != nil {
+ return domain.NewInternalError("delete webhook endpoint", err)
+ }
+ return nil
+}
+
+func (r *postgresWebhookRepo) ListActiveForEvent(ctx context.Context, tenantID uuid.UUID, event string) ([]domain.WebhookEndpoint, error) {
+ rows, err := r.q.ListActiveWebhookEndpointsForEvent(ctx, db.ListActiveWebhookEndpointsForEventParams{
+ TenantID: tenantID,
+ Column2: event,
+ })
+ if err != nil {
+ return nil, domain.NewInternalError("list active webhook endpoints", err)
+ }
+ out := make([]domain.WebhookEndpoint, len(rows))
+ for i, row := range rows {
+ out[i] = dbWebhookEndpointToDomain(row)
+ }
+ return out, nil
+}
+
+func (r *postgresWebhookRepo) CreateDelivery(ctx context.Context, d domain.WebhookDelivery) (*domain.WebhookDelivery, error) {
+ payloadJSON, err := json.Marshal(d.Payload)
+ if err != nil {
+ return nil, domain.NewInternalError("marshal delivery payload", err)
+ }
+ row, err := r.q.InsertWebhookDelivery(ctx, db.InsertWebhookDeliveryParams{
+ EndpointID: d.EndpointID,
+ NotificationID: toNullUUID(d.NotificationID),
+ Event: d.Event,
+ Payload: payloadJSON,
+ Status: string(d.Status),
+ Attempt: int32(d.Attempt),
+ NextRetryAt: toNullTime(d.NextRetryAt),
+ })
+ if err != nil {
+ return nil, domain.NewInternalError("insert webhook delivery", err)
+ }
+ out := dbWebhookDeliveryToDomain(row)
+ return &out, nil
+}
+
+func (r *postgresWebhookRepo) UpdateDelivery(ctx context.Context, d domain.WebhookDelivery) (*domain.WebhookDelivery, error) {
+ row, err := r.q.UpdateWebhookDelivery(ctx, db.UpdateWebhookDeliveryParams{
+ ID: d.ID,
+ Status: string(d.Status),
+ Attempt: int32(d.Attempt),
+ NextRetryAt: toNullTime(d.NextRetryAt),
+ LastError: toNullString(d.LastError),
+ ResponseStatus: toNullInt32(d.ResponseStatus),
+ })
+ if err != nil {
+ return nil, domain.NewInternalError("update webhook delivery", err)
+ }
+ out := dbWebhookDeliveryToDomain(row)
+ return &out, nil
+}
+
+func dbWebhookEndpointToDomain(row db.WebhookEndpoint) domain.WebhookEndpoint {
+ var events []string
+ _ = json.Unmarshal(row.Events, &events)
+ if events == nil {
+ events = []string{}
+ }
+ return domain.WebhookEndpoint{
+ ID: row.ID,
+ TenantID: row.TenantID,
+ URL: row.Url,
+ Secret: row.Secret,
+ Events: events,
+ IsActive: row.IsActive,
+ CreatedAt: row.CreatedAt,
+ UpdatedAt: row.UpdatedAt,
+ }
+}
+
+func dbWebhookDeliveryToDomain(row db.WebhookDelivery) domain.WebhookDelivery {
+ var payload map[string]any
+ _ = json.Unmarshal(row.Payload, &payload)
+ d := domain.WebhookDelivery{
+ ID: row.ID,
+ EndpointID: row.EndpointID,
+ Event: row.Event,
+ Payload: payload,
+ Status: domain.WebhookDeliveryStatus(row.Status),
+ Attempt: int(row.Attempt),
+ CreatedAt: row.CreatedAt,
+ UpdatedAt: row.UpdatedAt,
+ LastError: fromNullString(row.LastError),
+ NextRetryAt: fromNullTime(row.NextRetryAt),
+ }
+ if row.NotificationID.Valid {
+ id := row.NotificationID.UUID
+ d.NotificationID = &id
+ }
+ if row.ResponseStatus.Valid {
+ v := int(row.ResponseStatus.Int32)
+ d.ResponseStatus = &v
+ }
+ return d
+}
diff --git a/internal/service/device_token.go b/internal/service/device_token.go
new file mode 100644
index 0000000..d9552c8
--- /dev/null
+++ b/internal/service/device_token.go
@@ -0,0 +1,48 @@
+package service
+
+import (
+ "context"
+
+ "github.com/go-playground/validator/v10"
+ "github.com/google/uuid"
+
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+ "github.com/amitrajitdas31/notifyhub/internal/repository"
+)
+
+type DeviceTokenService interface {
+ Register(ctx context.Context, tenantID uuid.UUID, req domain.RegisterDeviceTokenRequest) (*domain.DeviceToken, error)
+ Deregister(ctx context.Context, tenantID uuid.UUID, token string) error
+ ListActiveByUser(ctx context.Context, tenantID uuid.UUID, userID string) ([]domain.DeviceToken, error)
+}
+
+type deviceTokenService struct {
+ repo repository.DeviceTokenRepository
+ validate *validator.Validate
+}
+
+func NewDeviceTokenService(repo repository.DeviceTokenRepository, validate *validator.Validate) DeviceTokenService {
+ return &deviceTokenService{repo: repo, validate: validate}
+}
+
+func (s *deviceTokenService) Register(ctx context.Context, tenantID uuid.UUID, req domain.RegisterDeviceTokenRequest) (*domain.DeviceToken, error) {
+ if err := s.validate.Struct(req); err != nil {
+ return nil, toValidationError(err)
+ }
+ dt, err := s.repo.Upsert(ctx, tenantID, req)
+ if err != nil {
+ return nil, domain.NewInternalError("failed to register device token", err)
+ }
+ return dt, nil
+}
+
+func (s *deviceTokenService) Deregister(ctx context.Context, tenantID uuid.UUID, token string) error {
+ if token == "" {
+ return domain.NewValidationError("token required", nil)
+ }
+ return s.repo.Deactivate(ctx, tenantID, token)
+}
+
+func (s *deviceTokenService) ListActiveByUser(ctx context.Context, tenantID uuid.UUID, userID string) ([]domain.DeviceToken, error) {
+ return s.repo.ListActiveByUser(ctx, tenantID, userID)
+}
diff --git a/internal/service/webhook.go b/internal/service/webhook.go
new file mode 100644
index 0000000..9383013
--- /dev/null
+++ b/internal/service/webhook.go
@@ -0,0 +1,197 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/go-playground/validator/v10"
+ "github.com/google/uuid"
+
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+ "github.com/amitrajitdas31/notifyhub/internal/queue"
+ "github.com/amitrajitdas31/notifyhub/internal/repository"
+)
+
+// WebhookService manages webhook endpoint registration and fanout dispatch.
+type WebhookService interface {
+ Create(ctx context.Context, tenantID uuid.UUID, req domain.CreateWebhookEndpointRequest) (*domain.WebhookEndpoint, error)
+ List(ctx context.Context, tenantID uuid.UUID) ([]domain.WebhookEndpoint, error)
+ GetByID(ctx context.Context, tenantID, id uuid.UUID) (*domain.WebhookEndpoint, error)
+ Update(ctx context.Context, tenantID, id uuid.UUID, req domain.UpdateWebhookEndpointRequest) (*domain.WebhookEndpoint, error)
+ Delete(ctx context.Context, tenantID, id uuid.UUID) error
+ // Dispatch fans out a webhook event for a notification to all active subscribed endpoints.
+ // Called by the worker processor after each terminal notification outcome.
+ Dispatch(ctx context.Context, n *domain.Notification, event string) error
+}
+
+// webhookPublisher is the subset of queue.Producer used by WebhookService.
+type webhookPublisher interface {
+ Publish(ctx context.Context, topic, key string, payload []byte) error
+}
+
+type webhookService struct {
+ repo repository.WebhookRepository
+ publisher webhookPublisher
+ maxAttempts int
+ validate *validator.Validate
+ logger *slog.Logger
+}
+
+func NewWebhookService(
+ repo repository.WebhookRepository,
+ publisher webhookPublisher,
+ maxAttempts int,
+ validate *validator.Validate,
+ logger *slog.Logger,
+) WebhookService {
+ return &webhookService{
+ repo: repo,
+ publisher: publisher,
+ maxAttempts: maxAttempts,
+ validate: validate,
+ logger: logger,
+ }
+}
+
+func (s *webhookService) Create(ctx context.Context, tenantID uuid.UUID, req domain.CreateWebhookEndpointRequest) (*domain.WebhookEndpoint, error) {
+ if err := s.validate.Struct(req); err != nil {
+ return nil, domain.NewValidationError("invalid webhook endpoint", nil)
+ }
+ if err := validateEvents(req.Events); err != nil {
+ return nil, err
+ }
+ ep := domain.WebhookEndpoint{
+ TenantID: tenantID,
+ URL: req.URL,
+ Secret: req.Secret,
+ Events: req.Events,
+ IsActive: true,
+ }
+ return s.repo.CreateEndpoint(ctx, ep)
+}
+
+func (s *webhookService) List(ctx context.Context, tenantID uuid.UUID) ([]domain.WebhookEndpoint, error) {
+ return s.repo.ListEndpoints(ctx, tenantID)
+}
+
+func (s *webhookService) GetByID(ctx context.Context, tenantID, id uuid.UUID) (*domain.WebhookEndpoint, error) {
+ return s.repo.GetEndpoint(ctx, tenantID, id)
+}
+
+func (s *webhookService) Update(ctx context.Context, tenantID, id uuid.UUID, req domain.UpdateWebhookEndpointRequest) (*domain.WebhookEndpoint, error) {
+ if err := s.validate.Struct(req); err != nil {
+ return nil, domain.NewValidationError("invalid webhook endpoint", nil)
+ }
+ if err := validateEvents(req.Events); err != nil {
+ return nil, err
+ }
+ ep := domain.WebhookEndpoint{
+ ID: id,
+ TenantID: tenantID,
+ URL: req.URL,
+ Events: req.Events,
+ IsActive: req.IsActive,
+ }
+ return s.repo.UpdateEndpoint(ctx, ep)
+}
+
+func (s *webhookService) Delete(ctx context.Context, tenantID, id uuid.UUID) error {
+ return s.repo.DeleteEndpoint(ctx, tenantID, id)
+}
+
+// Dispatch looks up active endpoints subscribed to event, creates a delivery
+// record for each, and publishes a WebhookMessage to the outbound Kafka topic.
+// Errors per endpoint are logged and skipped — other endpoints still receive the event.
+func (s *webhookService) Dispatch(ctx context.Context, n *domain.Notification, event string) error {
+ endpoints, err := s.repo.ListActiveForEvent(ctx, n.TenantID, event)
+ if err != nil {
+ return fmt.Errorf("list active webhook endpoints: %w", err)
+ }
+ if len(endpoints) == 0 {
+ return nil
+ }
+
+ eventPayload := buildEventPayload(n, event)
+
+ for _, ep := range endpoints {
+ delivery, err := s.repo.CreateDelivery(ctx, domain.WebhookDelivery{
+ EndpointID: ep.ID,
+ NotificationID: &n.ID,
+ Event: event,
+ Payload: eventPayload,
+ Status: domain.WebhookDeliveryPending,
+ Attempt: 0,
+ })
+ if err != nil {
+ s.logger.ErrorContext(ctx, "webhook: failed to create delivery record",
+ slog.String("endpoint_id", ep.ID.String()),
+ slog.String("event", event),
+ slog.Any("error", err),
+ )
+ continue
+ }
+
+ msg := queue.WebhookMessage{
+ TenantID: n.TenantID,
+ EndpointID: ep.ID,
+ DeliveryID: delivery.ID,
+ NotificationID: &n.ID,
+ Event: event,
+ Payload: eventPayload,
+ EndpointURL: ep.URL,
+ Secret: ep.Secret,
+ Attempt: 1,
+ MaxAttempts: s.maxAttempts,
+ EnqueuedAt: time.Now().UTC(),
+ }
+
+ msgBytes, err := json.Marshal(msg)
+ if err != nil {
+ s.logger.ErrorContext(ctx, "webhook: failed to marshal message",
+ slog.String("delivery_id", delivery.ID.String()),
+ slog.Any("error", err),
+ )
+ continue
+ }
+
+ if err := s.publisher.Publish(ctx, queue.WebhookTopic, ep.ID.String(), msgBytes); err != nil {
+ s.logger.ErrorContext(ctx, "webhook: failed to publish message",
+ slog.String("delivery_id", delivery.ID.String()),
+ slog.Any("error", err),
+ )
+ }
+ }
+ return nil
+}
+
+// buildEventPayload constructs the JSON body sent to the webhook endpoint.
+func buildEventPayload(n *domain.Notification, event string) map[string]any {
+ return map[string]any{
+ "event": event,
+ "notification_id": n.ID.String(),
+ "tenant_id": n.TenantID.String(),
+ "channel": string(n.Channel),
+ "recipient_id": n.RecipientID,
+ "status": string(n.Status),
+ "timestamp": time.Now().UTC().Format(time.RFC3339),
+ }
+}
+
+// validateEvents rejects unknown event names.
+func validateEvents(events []string) *domain.AppError {
+ valid := make(map[string]bool, len(domain.AllWebhookEvents))
+ for _, e := range domain.AllWebhookEvents {
+ valid[e] = true
+ }
+ for _, e := range events {
+ if !valid[e] {
+ return domain.NewValidationError("unknown event: "+e, map[string]any{
+ "valid_events": domain.AllWebhookEvents,
+ })
+ }
+ }
+ return nil
+}
diff --git a/internal/worker/processor.go b/internal/worker/processor.go
index 060da59..e58c292 100644
--- a/internal/worker/processor.go
+++ b/internal/worker/processor.go
@@ -25,6 +25,12 @@ type DLQPublisher interface {
PublishDLQ(ctx context.Context, msg queue.DLQMessage) error
}
+// WebhookDispatcher fans out webhook events to registered tenant endpoints
+// after a notification reaches a terminal status. Nil disables dispatch.
+type WebhookDispatcher interface {
+ Dispatch(ctx context.Context, n *domain.Notification, event string) error
+}
+
// ProcessorDeps groups all dependencies required to build a Processor.
// Using a struct keeps the constructor signature within linter limits and
// makes the call site self-documenting.
@@ -39,8 +45,9 @@ type ProcessorDeps struct {
RateCaps map[domain.Channel]int // per-channel hourly cap from config
RetryDelay time.Duration // base delay; actual = RetryDelay * 2^(attempt-1)
Logger *slog.Logger
- DLQPublisher DLQPublisher // nil disables DLQ routing
+ DLQPublisher DLQPublisher // nil disables DLQ routing
DLQEnabled bool
+ Webhook WebhookDispatcher // nil disables webhook fanout
}
// Processor runs the full delivery pipeline for a single queue.Message.
@@ -168,6 +175,7 @@ func (p *Processor) deliver(ctx context.Context, msg queue.Message, n *domain.No
p.logDelivery(ctx, msg, n, domain.DeliveryFailed, &errMsg, nil, nil)
p.deadLetter(ctx, msg, domain.DLQReasonTemplateRender, errMsg)
p.updateStatus(ctx, n.ID, domain.StatusFailed)
+ p.dispatchWebhook(ctx, n, domain.WebhookEventFailed)
return observability.StatusFailed, nil
}
payload["_subject"] = subject
@@ -185,6 +193,7 @@ func (p *Processor) deliver(ctx context.Context, msg queue.Message, n *domain.No
p.logDelivery(ctx, msg, n, domain.DeliveryFailed, &errMsg, nil, nil)
p.deadLetter(ctx, msg, domain.DLQReasonNoProvider, errMsg)
p.updateStatus(ctx, n.ID, domain.StatusFailed)
+ p.dispatchWebhook(ctx, n, domain.WebhookEventFailed)
return observability.StatusFailed, nil
}
@@ -200,6 +209,7 @@ func (p *Processor) deliver(ctx context.Context, msg queue.Message, n *domain.No
p.logDelivery(ctx, msg, n, domain.DeliveryFailed, &errMsg, nil, nil)
p.deadLetter(ctx, msg, reason, errMsg)
p.updateStatus(ctx, n.ID, domain.StatusFailed)
+ p.dispatchWebhook(ctx, n, domain.WebhookEventFailed)
log.ErrorContext(ctx, "notification failed", slog.Any("error", sendErr))
return observability.StatusFailed, nil
}
@@ -207,6 +217,7 @@ func (p *Processor) deliver(ctx context.Context, msg queue.Message, n *domain.No
now := time.Now().UTC()
p.logDelivery(ctx, msg, n, domain.DeliverySuccess, nil, nil, &now)
p.updateStatus(ctx, n.ID, domain.StatusDelivered)
+ p.dispatchWebhook(ctx, n, domain.WebhookEventDelivered)
log.InfoContext(ctx, "notification delivered")
return observability.StatusDelivered, nil
}
@@ -300,6 +311,21 @@ func setSpanErr(span trace.Span, op string, err error) {
func (p *Processor) drop(ctx context.Context, msg queue.Message, n *domain.Notification, reason string) {
p.logDelivery(ctx, msg, n, domain.DeliveryFailed, &reason, nil, nil)
p.updateStatus(ctx, n.ID, domain.StatusDropped)
+ p.dispatchWebhook(ctx, n, domain.WebhookEventDropped)
+}
+
+// dispatchWebhook fans out a webhook event non-fatally; errors are logged only.
+func (p *Processor) dispatchWebhook(ctx context.Context, n *domain.Notification, event string) {
+ if p.deps.Webhook == nil {
+ return
+ }
+ if err := p.deps.Webhook.Dispatch(ctx, n, event); err != nil {
+ p.deps.Logger.ErrorContext(ctx, "webhook dispatch failed",
+ slog.String("notification_id", n.ID.String()),
+ slog.String("event", event),
+ slog.Any("error", err),
+ )
+ }
}
// logDelivery persists a delivery attempt record. Non-fatal: errors are logged only.
diff --git a/internal/worker/webhook_worker.go b/internal/worker/webhook_worker.go
new file mode 100644
index 0000000..59cf946
--- /dev/null
+++ b/internal/worker/webhook_worker.go
@@ -0,0 +1,200 @@
+package worker
+
+import (
+ "bytes"
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "math"
+ "net/http"
+ "time"
+
+ "github.com/amitrajitdas31/notifyhub/internal/domain"
+ "github.com/amitrajitdas31/notifyhub/internal/observability"
+ "github.com/amitrajitdas31/notifyhub/internal/queue"
+ "github.com/amitrajitdas31/notifyhub/internal/repository"
+)
+
+// WebhookWorker consumes from the outbound webhook Kafka topic and delivers
+// HTTP POST callbacks to registered tenant endpoints with HMAC signatures.
+// Inline retries with exponential backoff; delivery record updated after each attempt.
+type WebhookWorker struct {
+ consumer *queue.Consumer
+ repo repository.WebhookRepository
+ metrics *observability.Metrics
+ logger *slog.Logger
+ httpClient *http.Client
+ retryDelay time.Duration
+}
+
+// NewWebhookWorker wires up a WebhookWorker.
+func NewWebhookWorker(
+ consumer *queue.Consumer,
+ repo repository.WebhookRepository,
+ metrics *observability.Metrics,
+ retryDelay time.Duration,
+ logger *slog.Logger,
+) *WebhookWorker {
+ return &WebhookWorker{
+ consumer: consumer,
+ repo: repo,
+ metrics: metrics,
+ logger: logger,
+ retryDelay: retryDelay,
+ httpClient: &http.Client{Timeout: 10 * time.Second},
+ }
+}
+
+// Run blocks, consuming the webhook topic until ctx is cancelled.
+func (w *WebhookWorker) Run(ctx context.Context) error {
+ for {
+ raw, err := w.consumer.Fetch(ctx)
+ if err != nil {
+ if ctx.Err() != nil {
+ break
+ }
+ w.logger.ErrorContext(ctx, "webhook: fetch error", slog.Any("error", err))
+ continue
+ }
+
+ var msg queue.WebhookMessage
+ if err := json.Unmarshal(raw.Value, &msg); err != nil {
+ w.logger.ErrorContext(ctx, "webhook: decode failed, skipping",
+ slog.Any("error", err),
+ slog.Int("bytes", len(raw.Value)),
+ )
+ _ = w.consumer.Commit(ctx, raw)
+ continue
+ }
+
+ if err := w.process(ctx, msg); err != nil {
+ // Infrastructure error — don't commit; Kafka will redeliver.
+ w.logger.ErrorContext(ctx, "webhook: process error, not committing",
+ slog.String("delivery_id", msg.DeliveryID.String()),
+ slog.Any("error", err),
+ )
+ continue
+ }
+
+ if err := w.consumer.Commit(ctx, raw); err != nil {
+ w.logger.ErrorContext(ctx, "webhook: offset commit failed", slog.Any("error", err))
+ }
+ }
+
+ if err := w.consumer.Close(); err != nil {
+ w.logger.Error("webhook: consumer close error", slog.Any("error", err))
+ }
+ return nil
+}
+
+// process delivers the webhook with inline retries. Returns non-nil only on
+// infrastructure errors (DB write failure) that must block the offset commit.
+func (w *WebhookWorker) process(ctx context.Context, msg queue.WebhookMessage) error {
+ body, err := json.Marshal(msg.Payload)
+ if err != nil {
+ // Payload is already validated upstream; marshal failure is permanent.
+ w.logger.ErrorContext(ctx, "webhook: marshal payload failed", slog.Any("error", err))
+ return w.markFailed(ctx, msg, 0, "marshal payload: "+err.Error())
+ }
+
+ attempt := msg.Attempt
+ for {
+ statusCode, sendErr := w.send(ctx, msg.EndpointURL, msg.Secret, msg.Event, msg.DeliveryID.String(), body)
+
+ if sendErr == nil {
+ w.metrics.WebhookDelivered.WithLabelValues(msg.Event).Inc()
+ w.logger.InfoContext(ctx, "webhook: delivered",
+ slog.String("delivery_id", msg.DeliveryID.String()),
+ slog.String("url", msg.EndpointURL),
+ slog.Int("attempt", attempt),
+ )
+ return w.markDelivered(ctx, msg, attempt, statusCode)
+ }
+
+ w.logger.WarnContext(ctx, "webhook: send failed",
+ slog.String("delivery_id", msg.DeliveryID.String()),
+ slog.String("url", msg.EndpointURL),
+ slog.Int("attempt", attempt),
+ slog.Any("error", sendErr),
+ )
+
+ if attempt >= msg.MaxAttempts {
+ w.metrics.WebhookFailed.WithLabelValues(msg.Event).Inc()
+ return w.markFailed(ctx, msg, statusCode, sendErr.Error())
+ }
+
+ delay := w.retryDelay * time.Duration(math.Pow(2, float64(attempt-1)))
+ select {
+ case <-ctx.Done():
+ return fmt.Errorf("context cancelled during webhook retry: %w", ctx.Err())
+ case <-time.After(delay):
+ }
+ attempt++
+ }
+}
+
+// send POSTs the event payload to the endpoint URL with HMAC signature headers.
+// Returns the HTTP status code and any error.
+func (w *WebhookWorker) send(ctx context.Context, url, secret, event, deliveryID string, body []byte) (int, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
+ if err != nil {
+ return 0, fmt.Errorf("build request: %w", err)
+ }
+
+ sig := computeHMAC(body, secret)
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("X-NotifyHub-Signature", "sha256="+sig)
+ req.Header.Set("X-NotifyHub-Event", event)
+ req.Header.Set("X-NotifyHub-Delivery-ID", deliveryID)
+
+ resp, err := w.httpClient.Do(req)
+ if err != nil {
+ return 0, fmt.Errorf("http post: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return resp.StatusCode, fmt.Errorf("non-2xx response: %d", resp.StatusCode)
+ }
+ return resp.StatusCode, nil
+}
+
+func (w *WebhookWorker) markDelivered(ctx context.Context, msg queue.WebhookMessage, attempt, statusCode int) error {
+ _, err := w.repo.UpdateDelivery(ctx, domain.WebhookDelivery{
+ ID: msg.DeliveryID,
+ Status: domain.WebhookDeliveryDelivered,
+ Attempt: attempt,
+ ResponseStatus: &statusCode,
+ })
+ if err != nil {
+ return fmt.Errorf("mark delivery delivered: %w", err)
+ }
+ return nil
+}
+
+func (w *WebhookWorker) markFailed(ctx context.Context, msg queue.WebhookMessage, statusCode int, errMsg string) error {
+ d := domain.WebhookDelivery{
+ ID: msg.DeliveryID,
+ Status: domain.WebhookDeliveryFailed,
+ Attempt: msg.MaxAttempts,
+ LastError: &errMsg,
+ }
+ if statusCode != 0 {
+ d.ResponseStatus = &statusCode
+ }
+ if _, err := w.repo.UpdateDelivery(ctx, d); err != nil {
+ return fmt.Errorf("mark delivery failed: %w", err)
+ }
+ return nil
+}
+
+// computeHMAC returns the hex-encoded HMAC-SHA256 of body using secret.
+func computeHMAC(body []byte, secret string) string {
+ mac := hmac.New(sha256.New, []byte(secret))
+ mac.Write(body)
+ return hex.EncodeToString(mac.Sum(nil))
+}
diff --git a/migrations/000005_add_device_tokens.down.sql b/migrations/000005_add_device_tokens.down.sql
new file mode 100644
index 0000000..99a806d
--- /dev/null
+++ b/migrations/000005_add_device_tokens.down.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS device_tokens;
diff --git a/migrations/000005_add_device_tokens.up.sql b/migrations/000005_add_device_tokens.up.sql
new file mode 100644
index 0000000..c4b7632
--- /dev/null
+++ b/migrations/000005_add_device_tokens.up.sql
@@ -0,0 +1,13 @@
+CREATE TABLE device_tokens (
+ id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+ tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
+ user_id VARCHAR(255) NOT NULL,
+ token TEXT NOT NULL,
+ platform VARCHAR(20) NOT NULL CHECK (platform IN ('ios', 'android', 'web')),
+ is_active BOOLEAN NOT NULL DEFAULT true,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ UNIQUE (tenant_id, token)
+);
+
+CREATE INDEX idx_device_tokens_user ON device_tokens(tenant_id, user_id) WHERE is_active = true;
diff --git a/migrations/000006_webhooks.down.sql b/migrations/000006_webhooks.down.sql
new file mode 100644
index 0000000..dafd96b
--- /dev/null
+++ b/migrations/000006_webhooks.down.sql
@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS webhook_deliveries;
+DROP TABLE IF EXISTS webhook_endpoints;
diff --git a/migrations/000006_webhooks.up.sql b/migrations/000006_webhooks.up.sql
new file mode 100644
index 0000000..42e66f9
--- /dev/null
+++ b/migrations/000006_webhooks.up.sql
@@ -0,0 +1,31 @@
+CREATE TABLE webhook_endpoints (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
+ url TEXT NOT NULL,
+ secret TEXT NOT NULL,
+ events JSONB NOT NULL DEFAULT '[]',
+ is_active BOOLEAN NOT NULL DEFAULT TRUE,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX idx_webhook_endpoints_tenant ON webhook_endpoints(tenant_id);
+
+CREATE TABLE webhook_deliveries (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ endpoint_id UUID NOT NULL REFERENCES webhook_endpoints(id) ON DELETE CASCADE,
+ notification_id UUID REFERENCES notifications(id) ON DELETE SET NULL,
+ event TEXT NOT NULL,
+ payload JSONB NOT NULL,
+ status TEXT NOT NULL DEFAULT 'pending',
+ attempt INT NOT NULL DEFAULT 0,
+ next_retry_at TIMESTAMPTZ,
+ last_error TEXT,
+ response_status INT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX idx_webhook_deliveries_endpoint ON webhook_deliveries(endpoint_id);
+CREATE INDEX idx_webhook_deliveries_retry ON webhook_deliveries(next_retry_at)
+ WHERE status = 'pending' AND next_retry_at IS NOT NULL;
diff --git a/queries/device_token.sql b/queries/device_token.sql
new file mode 100644
index 0000000..ed8dd6e
--- /dev/null
+++ b/queries/device_token.sql
@@ -0,0 +1,23 @@
+-- name: UpsertDeviceToken :one
+INSERT INTO device_tokens (tenant_id, user_id, token, platform)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (tenant_id, token) DO UPDATE
+ SET user_id = EXCLUDED.user_id,
+ platform = EXCLUDED.platform,
+ is_active = true,
+ updated_at = NOW()
+RETURNING *;
+
+-- name: DeactivateDeviceToken :exec
+UPDATE device_tokens
+SET is_active = false, updated_at = NOW()
+WHERE tenant_id = $1 AND token = $2;
+
+-- name: ListActiveDeviceTokensByUser :many
+SELECT * FROM device_tokens
+WHERE tenant_id = $1 AND user_id = $2 AND is_active = true
+ORDER BY created_at DESC;
+
+-- name: GetDeviceToken :one
+SELECT * FROM device_tokens
+WHERE tenant_id = $1 AND token = $2;
diff --git a/queries/inapp.sql b/queries/inapp.sql
index 9145259..4322c39 100644
--- a/queries/inapp.sql
+++ b/queries/inapp.sql
@@ -39,3 +39,20 @@ WHERE id = $1 AND tenant_id = $2 AND recipient_id = $3 AND read_at IS NULL;
UPDATE inapp_messages
SET read_at = now()
WHERE tenant_id = $1 AND recipient_id = $2 AND read_at IS NULL;
+
+-- name: ListInboxAfterCursor :many
+SELECT * FROM inapp_messages
+WHERE tenant_id = $1::uuid
+ AND recipient_id = $2::text
+ AND (NOT $3::boolean OR read_at IS NULL)
+ AND (created_at < $4 OR (created_at = $4 AND id < $5::uuid))
+ORDER BY created_at DESC, id DESC
+LIMIT $6::int;
+
+-- name: ListInboxSince :many
+SELECT * FROM inapp_messages
+WHERE tenant_id = $1::uuid
+ AND recipient_id = $2::text
+ AND (created_at > $3 OR (created_at = $3 AND id > $4::uuid))
+ORDER BY created_at ASC, id ASC
+LIMIT 200;
diff --git a/queries/webhook.sql b/queries/webhook.sql
new file mode 100644
index 0000000..0975a8a
--- /dev/null
+++ b/queries/webhook.sql
@@ -0,0 +1,40 @@
+-- name: InsertWebhookEndpoint :one
+INSERT INTO webhook_endpoints (tenant_id, url, secret, events, is_active)
+VALUES ($1, $2, $3, $4, $5)
+RETURNING *;
+
+-- name: GetWebhookEndpoint :one
+SELECT * FROM webhook_endpoints
+WHERE id = $1 AND tenant_id = $2;
+
+-- name: ListWebhookEndpoints :many
+SELECT * FROM webhook_endpoints
+WHERE tenant_id = $1
+ORDER BY created_at DESC;
+
+-- name: UpdateWebhookEndpoint :one
+UPDATE webhook_endpoints
+SET url = $3, events = $4, is_active = $5, updated_at = now()
+WHERE id = $1 AND tenant_id = $2
+RETURNING *;
+
+-- name: DeleteWebhookEndpoint :exec
+DELETE FROM webhook_endpoints
+WHERE id = $1 AND tenant_id = $2;
+
+-- name: ListActiveWebhookEndpointsForEvent :many
+SELECT * FROM webhook_endpoints
+WHERE tenant_id = $1
+ AND is_active = TRUE
+ AND events @> jsonb_build_array($2::text);
+
+-- name: InsertWebhookDelivery :one
+INSERT INTO webhook_deliveries (endpoint_id, notification_id, event, payload, status, attempt, next_retry_at)
+VALUES ($1, $2, $3, $4, $5, $6, $7)
+RETURNING *;
+
+-- name: UpdateWebhookDelivery :one
+UPDATE webhook_deliveries
+SET status = $2, attempt = $3, next_retry_at = $4, last_error = $5, response_status = $6, updated_at = now()
+WHERE id = $1
+RETURNING *;
diff --git a/scripts/loadtest/inapp_realtime.js b/scripts/loadtest/inapp_realtime.js
new file mode 100644
index 0000000..21c32f1
--- /dev/null
+++ b/scripts/loadtest/inapp_realtime.js
@@ -0,0 +1,224 @@
+import http from 'k6/http';
+import ws from 'k6/ws';
+import { check, fail, sleep } from 'k6';
+import { Counter, Rate, Trend } from 'k6/metrics';
+
+const scenario = __ENV.K6_SCENARIO || 'smoke';
+const baseURL = (__ENV.NOTIFYHUB_BASE_URL || 'http://localhost:8080').replace(/\/$/, '');
+const wsURL = (__ENV.NOTIFYHUB_WS_URL || baseURL.replace(/^http/, 'ws')).replace(/\/$/, '');
+const apiKey = __ENV.NOTIFYHUB_API_KEY || 'loadtest-api-key';
+const recipientPrefix = __ENV.NOTIFYHUB_RECIPIENT_PREFIX || 'k6-inapp-user';
+const connectJitterMs = intEnv('NOTIFYHUB_CONNECT_JITTER_MS', 1000);
+const firstSendDelayMs = intEnv('NOTIFYHUB_FIRST_SEND_DELAY_MS', 1000);
+const sendIntervalMs = intEnv('NOTIFYHUB_SEND_INTERVAL_MS', 5000);
+const sendsPerClient = intEnv('NOTIFYHUB_SENDS_PER_CLIENT', 1);
+
+export const wsConnected = new Counter('notifyhub_ws_connected');
+export const wsMessages = new Counter('notifyhub_ws_messages');
+export const wsExpectedMessages = new Counter('notifyhub_ws_expected_messages');
+export const wsMissedMessages = new Counter('notifyhub_ws_missed_messages');
+export const wsConnectionErrors = new Rate('notifyhub_ws_connection_errors');
+export const sendErrors = new Rate('notifyhub_send_errors');
+export const deliveryLatency = new Trend('notifyhub_ws_delivery_latency', true);
+export const tokenLatency = new Trend('notifyhub_ws_token_latency', true);
+
+const scenarioOptions = {
+ smoke: {
+ executor: 'constant-vus',
+ vus: 5,
+ duration: '30s',
+ },
+ baseline: {
+ executor: 'ramping-vus',
+ stages: [
+ { duration: '30s', target: 100 },
+ { duration: '2m', target: 100 },
+ { duration: '30s', target: 0 },
+ ],
+ },
+ prod_1000: {
+ executor: 'ramping-vus',
+ gracefulRampDown: '30s',
+ stages: [
+ { duration: '2m', target: 1000 },
+ { duration: '5m', target: 1000 },
+ { duration: '1m', target: 0 },
+ ],
+ },
+};
+
+export const options = {
+ scenarios: {
+ inapp_realtime: scenarioOptions[scenario] || scenarioOptions.smoke,
+ },
+ thresholds: {
+ http_req_failed: ['rate<0.01'],
+ notifyhub_send_errors: ['rate<0.01'],
+ notifyhub_ws_connection_errors: ['rate<0.01'],
+ notifyhub_ws_delivery_latency: ['p(95)<2000', 'p(99)<5000'],
+ },
+};
+
+export function setup() {
+ const health = http.get(`${baseURL}/health`);
+ if (health.status !== 200) {
+ fail(`NotifyHub health check failed: status=${health.status} body=${health.body}`);
+ }
+}
+
+export default function () {
+ const recipientID = `${recipientPrefix}-${__VU}`;
+ const tokenStarted = Date.now();
+ const tokenRes = http.post(
+ `${baseURL}/api/v1/ws-token`,
+ JSON.stringify({ recipient_id: recipientID }),
+ {
+ headers: {
+ 'Content-Type': 'application/json',
+ 'X-API-Key': apiKey,
+ },
+ tags: { endpoint: 'ws_token', scenario },
+ },
+ );
+ tokenLatency.add(Date.now() - tokenStarted);
+
+ const tokenOK = check(tokenRes, {
+ 'ws token issued': (r) => r.status === 200 && !!responseData(r.body).token,
+ });
+ if (!tokenOK) {
+ wsConnectionErrors.add(true);
+ sleep(1);
+ return;
+ }
+
+ const token = responseData(tokenRes.body).token;
+ const url = `${wsURL}/api/v1/inbox/stream?token=${encodeURIComponent(token)}`;
+
+ const expected = {};
+ let sent = 0;
+ let received = 0;
+
+ const res = ws.connect(url, { tags: { endpoint: 'inbox_stream', scenario } }, (socket) => {
+ socket.on('open', () => {
+ wsConnected.add(1);
+
+ const jitter = Math.floor(Math.random() * connectJitterMs);
+ socket.setTimeout(() => {
+ sendOne(recipientID, expected, sent);
+ sent += 1;
+ wsExpectedMessages.add(1);
+
+ if (sendsPerClient > 1) {
+ const interval = socket.setInterval(() => {
+ if (sent >= sendsPerClient) {
+ socket.clearInterval(interval);
+ return;
+ }
+ sendOne(recipientID, expected, sent);
+ sent += 1;
+ wsExpectedMessages.add(1);
+ }, sendIntervalMs);
+ }
+ }, firstSendDelayMs + jitter);
+ });
+
+ socket.on('message', (raw) => {
+ const frame = safeJSON(raw);
+ if (!frame || (frame.type !== 'message' && frame.type !== 'history')) {
+ return;
+ }
+
+ const data = frame.data || {};
+ const payload = data.payload || {};
+ const id = payload.client_msg_id;
+ const sentAt = payload.sent_at_ms;
+ if (!id || !expected[id]) {
+ return;
+ }
+
+ received += 1;
+ wsMessages.add(1);
+ deliveryLatency.add(Date.now() - Number(sentAt));
+ delete expected[id];
+ });
+
+ socket.on('error', () => {
+ wsConnectionErrors.add(true);
+ });
+
+ socket.setTimeout(() => {
+ const missed = Object.keys(expected).length;
+ if (missed > 0) {
+ wsMissedMessages.add(missed);
+ }
+ check(received, {
+ 'received all expected ws messages': () => missed === 0,
+ });
+ socket.close();
+ }, testWindowMs());
+ });
+
+ check(res, {
+ 'ws connected with 101': (r) => r && r.status === 101,
+ });
+ wsConnectionErrors.add(!res || res.status !== 101);
+ sleep(1);
+}
+
+function sendOne(recipientID, expected, sequence) {
+ const now = Date.now();
+ const clientMsgID = `k6-${scenario}-${__VU}-${__ITER}-${sequence}-${now}`;
+ expected[clientMsgID] = true;
+
+ const res = http.post(
+ `${baseURL}/api/v1/notifications`,
+ JSON.stringify({
+ type: 'loadtest',
+ channel: 'inapp',
+ recipient_id: recipientID,
+ recipient_address: recipientID,
+ payload: {
+ title: 'Realtime load test',
+ body: `message ${sequence} for ${recipientID}`,
+ client_msg_id: clientMsgID,
+ sent_at_ms: now,
+ source: 'k6-inapp-realtime',
+ },
+ priority: 5,
+ }),
+ {
+ headers: {
+ 'Content-Type': 'application/json',
+ 'X-API-Key': apiKey,
+ },
+ tags: { endpoint: 'send_notification', channel: 'inapp', scenario },
+ },
+ );
+
+ const ok = check(res, {
+ 'notification accepted': (r) => r.status === 202,
+ });
+ sendErrors.add(!ok || res.status >= 500);
+}
+
+function safeJSON(raw) {
+ try {
+ return JSON.parse(raw);
+ } catch (_) {
+ return {};
+ }
+}
+
+function responseData(raw) {
+ const parsed = safeJSON(raw);
+ return parsed.data || parsed;
+}
+
+function intEnv(name, fallback) {
+ const value = Number.parseInt(__ENV[name] || '', 10);
+ return Number.isFinite(value) ? value : fallback;
+}
+
+function testWindowMs() {
+ return firstSendDelayMs + connectJitterMs + Math.max(5000, sendsPerClient * sendIntervalMs + 10000);
+}
diff --git a/scripts/loadtest/send_notification.js b/scripts/loadtest/send_notification.js
new file mode 100644
index 0000000..e45a85d
--- /dev/null
+++ b/scripts/loadtest/send_notification.js
@@ -0,0 +1,112 @@
+import http from 'k6/http';
+import { check, fail, sleep } from 'k6';
+import { Counter, Rate, Trend } from 'k6/metrics';
+
+const scenario = __ENV.K6_SCENARIO || 'baseline';
+const baseURL = (__ENV.NOTIFYHUB_BASE_URL || 'http://localhost:8080').replace(/\/$/, '');
+const apiKey = __ENV.NOTIFYHUB_API_KEY || 'loadtest-api-key';
+const channel = __ENV.NOTIFYHUB_CHANNEL || 'inapp';
+const recipientPrefix = __ENV.NOTIFYHUB_RECIPIENT_PREFIX || 'k6-user';
+const idempotency = (__ENV.NOTIFYHUB_IDEMPOTENCY || 'false') === 'true';
+
+export const sendErrors = new Rate('notifyhub_send_errors');
+export const accepted = new Counter('notifyhub_accepted');
+export const sendLatency = new Trend('notifyhub_send_latency', true);
+
+const scenarios = {
+ smoke: {
+ executor: 'constant-vus',
+ vus: 1,
+ duration: '15s',
+ },
+ baseline: {
+ executor: 'ramping-arrival-rate',
+ startRate: 5,
+ timeUnit: '1s',
+ preAllocatedVUs: 20,
+ maxVUs: 100,
+ stages: [
+ { duration: '30s', target: 10 },
+ { duration: '1m', target: 25 },
+ { duration: '30s', target: 0 },
+ ],
+ },
+ stress: {
+ executor: 'ramping-arrival-rate',
+ startRate: 10,
+ timeUnit: '1s',
+ preAllocatedVUs: 50,
+ maxVUs: 300,
+ stages: [
+ { duration: '1m', target: 50 },
+ { duration: '2m', target: 100 },
+ { duration: '1m', target: 200 },
+ { duration: '1m', target: 0 },
+ ],
+ },
+};
+
+export const options = {
+ scenarios: {
+ send_notifications: scenarios[scenario] || scenarios.baseline,
+ },
+ thresholds: {
+ http_req_failed: ['rate<0.01'],
+ http_req_duration: ['p(95)<1500', 'p(99)<3000'],
+ notifyhub_send_errors: ['rate<0.01'],
+ },
+};
+
+export function setup() {
+ const health = http.get(`${baseURL}/health`);
+ if (health.status !== 200) {
+ fail(`NotifyHub health check failed: status=${health.status} body=${health.body}`);
+ }
+}
+
+export default function () {
+ const n = `${__VU}-${__ITER}`;
+ const payload = {
+ type: 'loadtest',
+ channel,
+ recipient_id: `${recipientPrefix}-${n}`,
+ recipient_address: `${recipientPrefix}-${n}`,
+ payload: {
+ title: 'Load test',
+ body: `k6 ${scenario} notification ${n}`,
+ source: 'k6',
+ },
+ priority: 5,
+ };
+
+ if (idempotency) {
+ payload.idempotency_key = `k6-${scenario}-${n}`;
+ }
+
+ const started = Date.now();
+ const res = http.post(`${baseURL}/api/v1/notifications`, JSON.stringify(payload), {
+ headers: {
+ 'Content-Type': 'application/json',
+ 'X-API-Key': apiKey,
+ },
+ tags: {
+ endpoint: 'send_notification',
+ channel,
+ scenario,
+ },
+ });
+
+ sendLatency.add(Date.now() - started);
+
+ const ok = check(res, {
+ 'accepted or rate-limited': (r) => r.status === 202 || r.status === 429,
+ 'accepted': (r) => r.status === 202,
+ });
+
+ if (res.status === 202) {
+ accepted.add(1);
+ }
+ sendErrors.add(!ok || res.status >= 500);
+
+ sleep(0.1);
+}
diff --git a/worker b/worker
deleted file mode 100755
index 43371f8..0000000
Binary files a/worker and /dev/null differ