From eadade99a79a9593181849557a31b337d1ab40e8 Mon Sep 17 00:00:00 2001 From: Proxynth Date: Wed, 8 Jul 2026 10:56:04 +0200 Subject: [PATCH] feat: add processed webhook events projection --- ..._create_processed_webhook_events_table.php | 74 +++++++++++++++++++ docs/architecture.md | 10 ++- .../Application/UseCases/ReceiveWebhook.php | 11 +++ .../Ports/ProcessedWebhookRecorder.php | 15 ++++ .../EloquentWebhookDuplicateDetector.php | 7 +- .../EloquentProcessedWebhookRecorder.php | 27 +++++++ .../Models/ProcessedWebhookEvent.php | 24 ++++++ .../Providers/LarawebhookServiceProvider.php | 4 + .../FakeProcessedWebhookRecorder.php | 24 ++++++ .../ValidateWebhookMiddlewareTest.php | 59 +++++++++++++++ tests/Helpers/SignatureTestHelpers.php | 7 ++ tests/Testcase.php | 3 + .../UseCases/ReceiveWebhookTest.php | 59 +++++++++++++-- .../EloquentWebhookDuplicateDetectorTest.php | 17 ++--- .../EloquentProcessedWebhookRecorderTest.php | 34 +++++++++ 15 files changed, 355 insertions(+), 20 deletions(-) create mode 100644 database/migrations/003_create_processed_webhook_events_table.php create mode 100644 src/Processing/Application/Ports/ProcessedWebhookRecorder.php create mode 100644 src/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorder.php create mode 100644 src/Processing/Infrastructure/Persistence/Models/ProcessedWebhookEvent.php create mode 100644 tests/Fakes/Processing/FakeProcessedWebhookRecorder.php create mode 100644 tests/Unit/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorderTest.php diff --git a/database/migrations/003_create_processed_webhook_events_table.php b/database/migrations/003_create_processed_webhook_events_table.php new file mode 100644 index 0000000..a931c00 --- /dev/null +++ b/database/migrations/003_create_processed_webhook_events_table.php @@ -0,0 +1,74 @@ +id(); + $table->string('service'); + $table->string('idempotency_key'); + $table->string('external_id')->nullable(); + $table->string('event')->nullable(); + $table->timestamp('processed_at'); + $table->timestamps(); + + $table->unique( + ['service', 'idempotency_key'], + 'processed_webhook_events_service_idempotency_key_unique' + ); + + $table->index(['service', 'external_id']); + $table->index(['service', 'event']); + }); + + $this->backfillProcessedWebhookEvents(); + } + + public function down(): void + { + Schema::dropIfExists('processed_webhook_events'); + } + + /** + * Backfill processed webhook events from historical successful audit logs. + * + * Only successful logs with an idempotency_key are considered processed. + * webhook_logs remains an audit trail and is not used for deduplication after this migration. + */ + private function backfillProcessedWebhookEvents(): void + { + DB::table('webhook_logs') + ->select('service', 'idempotency_key', DB::raw('MIN(id) as id')) + ->where('status', 'success') + ->whereNotNull('idempotency_key') + ->groupBy('service', 'idempotency_key') + ->get() + ->each(function (object $row): void { + $log = DB::table('webhook_logs') + ->where('id', $row->id) + ->first(); + + if ($log === null) { + return; + } + + DB::table('processed_webhook_events')->insertOrIgnore([ + 'service' => $log->service, + 'idempotency_key' => $log->idempotency_key, + 'external_id' => $log->external_id, + 'event' => $log->event, + 'processed_at' => $log->created_at ?? now(), + 'created_at' => now(), + 'updated_at' => now(), + ]); + }); + } +}; diff --git a/docs/architecture.md b/docs/architecture.md index 84db321..8718e27 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,10 +53,16 @@ $log = Larawebhook::validateAndLog($request->getContent(), $signature, 'stripe', ## Identity and Deduplication - `external_id` stores the provider delivery or event identifier when the provider exposes one. -- `idempotency_key` stores the deduplication key used by the application. +- `idempotency_key` stores the deduplication key resolved by the application. - Providers without a stable external id use a payload-hash fallback for idempotency. -The database unique constraint applies to `service + idempotency_key`, not `service + external_id`. +Deduplication is backed by the `processed_webhook_events` projection. + +`webhook_logs` is an audit trail and read model source. It may contain several rows for the same webhook across receive, retry or replay attempts. + +`processed_webhook_events` owns the uniqueness constraint on `service + idempotency_key`. + +For compatibility, `webhook_logs` may still contain a legacy uniqueness constraint on `service + idempotency_key`, but it is no longer used as the source of truth for deduplication. ### Historical migration diff --git a/src/Ingestion/Application/UseCases/ReceiveWebhook.php b/src/Ingestion/Application/UseCases/ReceiveWebhook.php index 7050980..fb8a597 100644 --- a/src/Ingestion/Application/UseCases/ReceiveWebhook.php +++ b/src/Ingestion/Application/UseCases/ReceiveWebhook.php @@ -19,6 +19,7 @@ use Proxynth\Larawebhook\Ingestion\Domain\ValueObjects\Provider; use Proxynth\Larawebhook\Ingestion\Domain\ValueObjects\RawPayload; use Proxynth\Larawebhook\Processing\Application\Ports\IdempotencyResolver; +use Proxynth\Larawebhook\Processing\Application\Ports\ProcessedWebhookRecorder; use Proxynth\Larawebhook\Processing\Application\Ports\RetryConfigurationResolver; use Proxynth\Larawebhook\Processing\Application\Ports\WebhookDuplicateDetector; use Proxynth\Larawebhook\Processing\Domain\Entities\WebhookEvent; @@ -36,6 +37,7 @@ public function __construct( private WebhookPayloadParserResolver $payloadParserResolver, private WebhookSecretResolver $secretResolver, private RetryConfigurationResolver $retryConfigurationResolver, + private ProcessedWebhookRecorder $processedWebhookRecorder, private ValidateWebhook $validateWebhook, private RecordWebhookLog $recordWebhookLog, ) {} @@ -124,6 +126,15 @@ public function handle(ReceiveWebhookCommand $command): ReceiveWebhookResult $webhookEvent->markProcessed(); + if ($idempotencyKey !== null) { + $this->processedWebhookRecorder->recordProcessed( + service: $provider->value(), + idempotencyKey: $idempotencyKey->value(), + externalId: $externalId, + event: $validation->event, + ); + } + return ReceiveWebhookResult::success($log, [ $receivedEvent, $validatedEvent, diff --git a/src/Processing/Application/Ports/ProcessedWebhookRecorder.php b/src/Processing/Application/Ports/ProcessedWebhookRecorder.php new file mode 100644 index 0000000..1a7887d --- /dev/null +++ b/src/Processing/Application/Ports/ProcessedWebhookRecorder.php @@ -0,0 +1,15 @@ +where('service', $service) + ->where('idempotency_key', $idempotencyKey) + ->exists(); } } diff --git a/src/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorder.php b/src/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorder.php new file mode 100644 index 0000000..03100ac --- /dev/null +++ b/src/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorder.php @@ -0,0 +1,27 @@ +firstOrCreate([ + 'service' => $service, + 'idempotency_key' => $idempotencyKey, + ], [ + 'external_id' => $externalId, + 'event' => $event, + 'processed_at' => now(), + ]); + } +} diff --git a/src/Processing/Infrastructure/Persistence/Models/ProcessedWebhookEvent.php b/src/Processing/Infrastructure/Persistence/Models/ProcessedWebhookEvent.php new file mode 100644 index 0000000..873aa68 --- /dev/null +++ b/src/Processing/Infrastructure/Persistence/Models/ProcessedWebhookEvent.php @@ -0,0 +1,24 @@ + 'datetime', + ]; +} diff --git a/src/Shared/Infrastructure/Laravel/Providers/LarawebhookServiceProvider.php b/src/Shared/Infrastructure/Laravel/Providers/LarawebhookServiceProvider.php index 2290fd1..a7f2937 100644 --- a/src/Shared/Infrastructure/Laravel/Providers/LarawebhookServiceProvider.php +++ b/src/Shared/Infrastructure/Laravel/Providers/LarawebhookServiceProvider.php @@ -34,6 +34,7 @@ use Proxynth\Larawebhook\Ingestion\Infrastructure\Validation\ProviderSignatureValidator; use Proxynth\Larawebhook\Ingestion\Infrastructure\Validation\WebhookValidatorFactory; use Proxynth\Larawebhook\Processing\Application\Ports\IdempotencyResolver; +use Proxynth\Larawebhook\Processing\Application\Ports\ProcessedWebhookRecorder; use Proxynth\Larawebhook\Processing\Application\Ports\ReplayableWebhookRepository; use Proxynth\Larawebhook\Processing\Application\Ports\RetryConfigurationResolver; use Proxynth\Larawebhook\Processing\Application\Ports\RetryPolicyResolver; @@ -44,6 +45,7 @@ use Proxynth\Larawebhook\Processing\Infrastructure\Config\ConfigRetryPolicyResolver; use Proxynth\Larawebhook\Processing\Infrastructure\Deduplication\EloquentWebhookDuplicateDetector; use Proxynth\Larawebhook\Processing\Infrastructure\Idempotency\DefaultIdempotencyResolver; +use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\EloquentProcessedWebhookRecorder; use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\EloquentReplayableWebhookRepository; use Proxynth\Larawebhook\Shared\Application\EventBus; use Proxynth\Larawebhook\Shared\Application\Larawebhook; @@ -179,6 +181,8 @@ private function registerServices(): void $this->app->singleton(RetryPolicyResolver::class, ConfigRetryPolicyResolver::class); $this->app->singleton(RetryConfigurationResolver::class, ConfigRetryConfigurationResolver::class); + + $this->app->singleton(ProcessedWebhookRecorder::class, EloquentProcessedWebhookRecorder::class); } /** diff --git a/tests/Fakes/Processing/FakeProcessedWebhookRecorder.php b/tests/Fakes/Processing/FakeProcessedWebhookRecorder.php new file mode 100644 index 0000000..2c7ed42 --- /dev/null +++ b/tests/Fakes/Processing/FakeProcessedWebhookRecorder.php @@ -0,0 +1,24 @@ +records[] = [ + 'service' => $service, + 'idempotencyKey' => $idempotencyKey, + 'externalId' => $externalId, + 'event' => $event, + ]; + } +} diff --git a/tests/Feature/Middleware/ValidateWebhookMiddlewareTest.php b/tests/Feature/Middleware/ValidateWebhookMiddlewareTest.php index 39e70c3..9708699 100644 --- a/tests/Feature/Middleware/ValidateWebhookMiddlewareTest.php +++ b/tests/Feature/Middleware/ValidateWebhookMiddlewareTest.php @@ -9,7 +9,9 @@ use Proxynth\Larawebhook\Ingestion\Domain\Events\WebhookReceived; use Proxynth\Larawebhook\Ingestion\Domain\Events\WebhookRejected; use Proxynth\Larawebhook\Ingestion\Domain\Events\WebhookValidated; +use Proxynth\Larawebhook\Ingestion\Domain\ValueObjects\Signature; use Proxynth\Larawebhook\Processing\Infrastructure\Laravel\Jobs\RetryWebhookJob; +use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\Models\ProcessedWebhookEvent; use Proxynth\Larawebhook\Shared\Application\EventBus; use Proxynth\Larawebhook\Tests\Fakes\Shared\FakeEventBus; @@ -1045,3 +1047,60 @@ ->and($log->external_id)->toBe('delivery_123') ->and($log->idempotency_key)->toBe('delivery_123'); }); + +it('does not process the same webhook twice', function () { + $payload = json_encode([ + 'action' => 'opened', + 'pull_request' => [ + 'id' => 123, + ], + ], JSON_THROW_ON_ERROR); + + $signature = githubSignature($payload, 'test_github_secret'); + + $response = $this->call( + 'POST', + 'test-github-webhook', + [], + [], + [], + [ + 'HTTP_X_HUB_SIGNATURE_256' => $signature->value(), + 'HTTP_X_GITHUB_EVENT' => 'ping', + 'HTTP_X_GITHUB_DELIVERY' => 'delivery_123', + 'CONTENT_TYPE' => 'application/json', + ], + $payload + ); + + expect(WebhookLog::query()->count())->toBe(1) + ->and(ProcessedWebhookEvent::query()->count())->toBe(1); + + $processed = ProcessedWebhookEvent::query()->first(); + + expect($processed->service)->toBe('github') + ->and($processed->idempotency_key)->toBe('delivery_123') + ->and($processed->external_id)->toBe('delivery_123') + ->and($processed->event)->not->toBeNull(); + + $this->call( + 'POST', + 'test-github-webhook', + [], + [], + [], + [ + 'HTTP_X_HUB_SIGNATURE_256' => $signature->value(), + 'HTTP_X_GITHUB_EVENT' => 'ping', + 'HTTP_X_GITHUB_DELIVERY' => 'delivery_123', + 'CONTENT_TYPE' => 'application/json', + ], + $payload + )->assertOk() + ->assertJson([ + 'status' => 'already_processed', + ]); + + expect(WebhookLog::query()->count())->toBe(1) + ->and(ProcessedWebhookEvent::query()->count())->toBe(1); +}); diff --git a/tests/Helpers/SignatureTestHelpers.php b/tests/Helpers/SignatureTestHelpers.php index bde970b..6de51c9 100644 --- a/tests/Helpers/SignatureTestHelpers.php +++ b/tests/Helpers/SignatureTestHelpers.php @@ -17,3 +17,10 @@ function slackIncomingSignature(string $value, int|string $timestamp): Signature timestamp: (string) $timestamp, ); } + +function githubSignature(string $payload, string $secret = 'github_secret'): Signature +{ + return incomingSignature( + 'sha256='.hash_hmac('sha256', $payload, $secret) + ); +} diff --git a/tests/Testcase.php b/tests/Testcase.php index 4aa2fcd..3b62651 100644 --- a/tests/Testcase.php +++ b/tests/Testcase.php @@ -39,5 +39,8 @@ public function getEnvironmentSetUp($app) $migration2 = include __DIR__.'/../database/migrations/002_add_idempotency_key_to_webhook_logs_table.php'; $migration2->up(); + + $migration3 = include __DIR__.'/../database/migrations/003_create_processed_webhook_events_table.php'; + $migration3->up(); } } diff --git a/tests/Unit/Ingestion/Application/UseCases/ReceiveWebhookTest.php b/tests/Unit/Ingestion/Application/UseCases/ReceiveWebhookTest.php index 987e5be..0894292 100644 --- a/tests/Unit/Ingestion/Application/UseCases/ReceiveWebhookTest.php +++ b/tests/Unit/Ingestion/Application/UseCases/ReceiveWebhookTest.php @@ -14,10 +14,12 @@ use Proxynth\Larawebhook\Ingestion\Domain\Events\WebhookValidated; use Proxynth\Larawebhook\Ingestion\Domain\ValueObjects\Signature; use Proxynth\Larawebhook\Processing\Application\Data\RetryConfiguration; +use Proxynth\Larawebhook\Processing\Application\Ports\ProcessedWebhookRecorder; use Proxynth\Larawebhook\Processing\Application\Ports\RetryConfigurationResolver; use Proxynth\Larawebhook\Processing\Application\Ports\WebhookDuplicateDetector; use Proxynth\Larawebhook\Shared\Domain\Enums\WebhookService; use Proxynth\Larawebhook\Tests\Fakes\Ingestion\FakeWebhookSecretResolver; +use Proxynth\Larawebhook\Tests\Fakes\Processing\FakeProcessedWebhookRecorder; use Proxynth\Larawebhook\Tests\Fakes\Processing\FakeRetryConfigurationResolver; use Proxynth\Larawebhook\Tests\Fakes\Processing\FakeWebhookDuplicateDetector; @@ -33,13 +35,6 @@ function githubPayload(string $deliveryId = 'delivery_123'): string ], JSON_THROW_ON_ERROR); } -function githubSignature(string $payload, string $secret = 'github_secret'): Signature -{ - return incomingSignature( - 'sha256='.hash_hmac('sha256', $payload, $secret) - ); -} - beforeEach(function () { Queue::fake(); @@ -317,3 +312,53 @@ function githubSignature(string $payload, string $secret = 'github_secret'): Sig expect($result->status)->toBe('secret_not_configured'); }); + +it('records processed webhook event after succesful receive', function () { + $processedRecorder = new FakeProcessedWebhookRecorder; + + app()->instance(ProcessedWebhookRecorder::class, $processedRecorder); + + app()->instance(WebhookSecretResolver::class, new FakeWebhookSecretResolver([ + 'github' => 'github_secret', + ])); + + $payload = json_encode([ + 'type' => 'custom.event', + 'data' => [ + 'foo' => 'bar', + ], + ], JSON_THROW_ON_ERROR); + + $signature = githubSignature($payload); + + $result = app(ReceiveWebhook::class)->handle(new ReceiveWebhookCommand( + service: WebhookService::Github, + payload: $payload, + signature: $signature, + externalIdHeaderValue: 'delivery_123', + )); + + expect($result->isSuccess())->toBeTrue() + ->and($processedRecorder->records)->toHaveCount(1) + ->and($processedRecorder->records[0])->toMatchArray([ + 'service' => 'github', + 'idempotencyKey' => 'delivery_123', + 'externalId' => 'delivery_123', + ]); +}); + +it('does not record processed webhook event when validation fails', function () { + $processedRecorder = new FakeProcessedWebhookRecorder; + + app()->instance(ProcessedWebhookRecorder::class, $processedRecorder); + + $result = app(ReceiveWebhook::class)->handle(new ReceiveWebhookCommand( + service: WebhookService::Github, + payload: githubPayload(), + signature: Signature::fromString('sha256=invalid'), + externalIdHeaderValue: 'delivery_123', + )); + + expect($result->isSuccess())->toBeFalse() + ->and($processedRecorder->records)->toBeEmpty(); +}); diff --git a/tests/Unit/Processing/Infrastructure/Deduplication/EloquentWebhookDuplicateDetectorTest.php b/tests/Unit/Processing/Infrastructure/Deduplication/EloquentWebhookDuplicateDetectorTest.php index d6de9cb..ea1914f 100644 --- a/tests/Unit/Processing/Infrastructure/Deduplication/EloquentWebhookDuplicateDetectorTest.php +++ b/tests/Unit/Processing/Infrastructure/Deduplication/EloquentWebhookDuplicateDetectorTest.php @@ -1,29 +1,28 @@ create([ +it('detects already processed webhook using processed events table', function () { + ProcessedWebhookEvent::query()->create([ 'service' => 'github', 'idempotency_key' => 'delivery_123', 'external_id' => 'delivery_123', + 'event' => 'push', + 'processed_at' => now(), ]); $detector = app(EloquentWebhookDuplicateDetector::class); - expect($detector->alreadyProcessed('github', 'delivery_123'))->toBeTrue() - ->and($detector->alreadyProcessed('github', 'missing'))->toBeFalse() - ->and($detector->alreadyProcessed('stripe', 'delivery_123'))->toBeFalse(); + expect($detector->alreadyProcessed('github', 'delivery_123'))->toBeTrue(); }); -it('does not use external id as duplicate key when idempotency key is missing', function () { +it('does not use webhook logs for duplicate detection anymore', function () { WebhookLog::factory()->create([ 'service' => 'github', 'external_id' => 'delivery_123', - 'idempotency_key' => null, + 'idempotency_key' => 'delivery_123', 'status' => 'success', ]); diff --git a/tests/Unit/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorderTest.php b/tests/Unit/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorderTest.php new file mode 100644 index 0000000..0c1136d --- /dev/null +++ b/tests/Unit/Processing/Infrastructure/Persistence/EloquentProcessedWebhookRecorderTest.php @@ -0,0 +1,34 @@ +recordProcessed( + service: 'github', + idempotencyKey: 'delivery_123', + externalId: 'delivery_123', + event: 'push', + ); + + expect(ProcessedWebhookEvent::query()->count())->toBe(1); + + $processed = ProcessedWebhookEvent::query()->first(); + + expect($processed->service)->toBe('github') + ->and($processed->idempotency_key)->toBe('delivery_123') + ->and($processed->external_id)->toBe('delivery_123') + ->and($processed->event)->toBe('push') + ->and($processed->processed_at)->not->toBeNull(); +}); + +it('does not create duplicate processed event for same service and idempotency key', function () { + $recorder = app(EloquentProcessedWebhookRecorder::class); + + $recorder->recordProcessed('github', 'delivery_123', 'delivery_123', 'push'); + $recorder->recordProcessed('github', 'delivery_123', 'delivery_123', 'push'); + + expect(ProcessedWebhookEvent::query()->count())->toBe(1); +});