Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions database/migrations/003_create_processed_webhook_events_table.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
<?php

declare(strict_types=1);

use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;

return new class extends Migration
{
public function up(): void
{
Schema::create('processed_webhook_events', function (Blueprint $table): void {
$table->id();
$table->string('service');
$table->string('idempotency_key');
$table->string('external_id')->nullable();
$table->string('event')->nullable();
$table->timestamp('processed_at');
$table->timestamps();

$table->unique(
['service', 'idempotency_key'],
'processed_webhook_events_service_idempotency_key_unique'
);

$table->index(['service', 'external_id']);
$table->index(['service', 'event']);
});

$this->backfillProcessedWebhookEvents();
}

public function down(): void
{
Schema::dropIfExists('processed_webhook_events');
}

/**
* Backfill processed webhook events from historical successful audit logs.
*
* Only successful logs with an idempotency_key are considered processed.
* webhook_logs remains an audit trail and is not used for deduplication after this migration.
*/
private function backfillProcessedWebhookEvents(): void
{
DB::table('webhook_logs')
->select('service', 'idempotency_key', DB::raw('MIN(id) as id'))
->where('status', 'success')
->whereNotNull('idempotency_key')
->groupBy('service', 'idempotency_key')
->get()
->each(function (object $row): void {
$log = DB::table('webhook_logs')
->where('id', $row->id)
->first();

if ($log === null) {
return;
}

DB::table('processed_webhook_events')->insertOrIgnore([
'service' => $log->service,
'idempotency_key' => $log->idempotency_key,
'external_id' => $log->external_id,
'event' => $log->event,
'processed_at' => $log->created_at ?? now(),
'created_at' => now(),
'updated_at' => now(),
]);
});
}
};
10 changes: 8 additions & 2 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,16 @@ $log = Larawebhook::validateAndLog($request->getContent(), $signature, 'stripe',
## Identity and Deduplication

- `external_id` stores the provider delivery or event identifier when the provider exposes one.
- `idempotency_key` stores the deduplication key used by the application.
- `idempotency_key` stores the deduplication key resolved by the application.
- Providers without a stable external id use a payload-hash fallback for idempotency.

The database unique constraint applies to `service + idempotency_key`, not `service + external_id`.
Deduplication is backed by the `processed_webhook_events` projection.

`webhook_logs` is an audit trail and read model source. It may contain several rows for the same webhook across receive, retry or replay attempts.

`processed_webhook_events` owns the uniqueness constraint on `service + idempotency_key`.

For compatibility, `webhook_logs` may still contain a legacy uniqueness constraint on `service + idempotency_key`, but it is no longer used as the source of truth for deduplication.

### Historical migration

Expand Down
11 changes: 11 additions & 0 deletions src/Ingestion/Application/UseCases/ReceiveWebhook.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
use Proxynth\Larawebhook\Ingestion\Domain\ValueObjects\Provider;
use Proxynth\Larawebhook\Ingestion\Domain\ValueObjects\RawPayload;
use Proxynth\Larawebhook\Processing\Application\Ports\IdempotencyResolver;
use Proxynth\Larawebhook\Processing\Application\Ports\ProcessedWebhookRecorder;
use Proxynth\Larawebhook\Processing\Application\Ports\RetryConfigurationResolver;
use Proxynth\Larawebhook\Processing\Application\Ports\WebhookDuplicateDetector;
use Proxynth\Larawebhook\Processing\Domain\Entities\WebhookEvent;
Expand All @@ -36,6 +37,7 @@ public function __construct(
private WebhookPayloadParserResolver $payloadParserResolver,
private WebhookSecretResolver $secretResolver,
private RetryConfigurationResolver $retryConfigurationResolver,
private ProcessedWebhookRecorder $processedWebhookRecorder,
private ValidateWebhook $validateWebhook,
private RecordWebhookLog $recordWebhookLog,
) {}
Expand Down Expand Up @@ -124,6 +126,15 @@ public function handle(ReceiveWebhookCommand $command): ReceiveWebhookResult

$webhookEvent->markProcessed();

if ($idempotencyKey !== null) {
$this->processedWebhookRecorder->recordProcessed(
service: $provider->value(),
idempotencyKey: $idempotencyKey->value(),
externalId: $externalId,
event: $validation->event,
);
}

return ReceiveWebhookResult::success($log, [
$receivedEvent,
$validatedEvent,
Expand Down
15 changes: 15 additions & 0 deletions src/Processing/Application/Ports/ProcessedWebhookRecorder.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<?php

declare(strict_types=1);

namespace Proxynth\Larawebhook\Processing\Application\Ports;

interface ProcessedWebhookRecorder
{
public function recordProcessed(
string $service,
string $idempotencyKey,
?string $externalId,
?string $event,
): void;
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@

namespace Proxynth\Larawebhook\Processing\Infrastructure\Deduplication;

use Proxynth\Larawebhook\Audit\Infrastructure\Laravel\Persistence\Models\WebhookLog;
use Proxynth\Larawebhook\Processing\Application\Ports\WebhookDuplicateDetector;
use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\Models\ProcessedWebhookEvent;

final readonly class EloquentWebhookDuplicateDetector implements WebhookDuplicateDetector
{
public function alreadyProcessed(string $service, string $idempotencyKey): bool
{
return WebhookLog::existsForIdempotencyKey($service, $idempotencyKey);
return ProcessedWebhookEvent::query()
->where('service', $service)
->where('idempotency_key', $idempotencyKey)
->exists();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<?php

declare(strict_types=1);

namespace Proxynth\Larawebhook\Processing\Infrastructure\Persistence;

use Proxynth\Larawebhook\Processing\Application\Ports\ProcessedWebhookRecorder;
use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\Models\ProcessedWebhookEvent;

class EloquentProcessedWebhookRecorder implements ProcessedWebhookRecorder
{
public function recordProcessed(
string $service,
string $idempotencyKey,
?string $externalId,
?string $event,
): void {
ProcessedWebhookEvent::query()->firstOrCreate([
'service' => $service,
'idempotency_key' => $idempotencyKey,
], [
'external_id' => $externalId,
'event' => $event,
'processed_at' => now(),
]);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<?php

declare(strict_types=1);

namespace Proxynth\Larawebhook\Processing\Infrastructure\Persistence\Models;

use Illuminate\Database\Eloquent\Model;

final class ProcessedWebhookEvent extends Model
{
protected $table = 'processed_webhook_events';

protected $fillable = [
'service',
'idempotency_key',
'external_id',
'event',
'processed_at',
];

protected $casts = [
'processed_at' => 'datetime',
];
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
use Proxynth\Larawebhook\Ingestion\Infrastructure\Validation\ProviderSignatureValidator;
use Proxynth\Larawebhook\Ingestion\Infrastructure\Validation\WebhookValidatorFactory;
use Proxynth\Larawebhook\Processing\Application\Ports\IdempotencyResolver;
use Proxynth\Larawebhook\Processing\Application\Ports\ProcessedWebhookRecorder;
use Proxynth\Larawebhook\Processing\Application\Ports\ReplayableWebhookRepository;
use Proxynth\Larawebhook\Processing\Application\Ports\RetryConfigurationResolver;
use Proxynth\Larawebhook\Processing\Application\Ports\RetryPolicyResolver;
Expand All @@ -44,6 +45,7 @@
use Proxynth\Larawebhook\Processing\Infrastructure\Config\ConfigRetryPolicyResolver;
use Proxynth\Larawebhook\Processing\Infrastructure\Deduplication\EloquentWebhookDuplicateDetector;
use Proxynth\Larawebhook\Processing\Infrastructure\Idempotency\DefaultIdempotencyResolver;
use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\EloquentProcessedWebhookRecorder;
use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\EloquentReplayableWebhookRepository;
use Proxynth\Larawebhook\Shared\Application\EventBus;
use Proxynth\Larawebhook\Shared\Application\Larawebhook;
Expand Down Expand Up @@ -179,6 +181,8 @@ private function registerServices(): void
$this->app->singleton(RetryPolicyResolver::class, ConfigRetryPolicyResolver::class);

$this->app->singleton(RetryConfigurationResolver::class, ConfigRetryConfigurationResolver::class);

$this->app->singleton(ProcessedWebhookRecorder::class, EloquentProcessedWebhookRecorder::class);
}

/**
Expand Down
24 changes: 24 additions & 0 deletions tests/Fakes/Processing/FakeProcessedWebhookRecorder.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<?php

namespace Proxynth\Larawebhook\Tests\Fakes\Processing;

use Proxynth\Larawebhook\Processing\Application\Ports\ProcessedWebhookRecorder;

class FakeProcessedWebhookRecorder implements ProcessedWebhookRecorder
{
public array $records = [];

public function recordProcessed(
string $service,
string $idempotencyKey,
?string $externalId,
?string $event,
): void {
$this->records[] = [
'service' => $service,
'idempotencyKey' => $idempotencyKey,
'externalId' => $externalId,
'event' => $event,
];
}
}
59 changes: 59 additions & 0 deletions tests/Feature/Middleware/ValidateWebhookMiddlewareTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
use Proxynth\Larawebhook\Ingestion\Domain\Events\WebhookReceived;
use Proxynth\Larawebhook\Ingestion\Domain\Events\WebhookRejected;
use Proxynth\Larawebhook\Ingestion\Domain\Events\WebhookValidated;
use Proxynth\Larawebhook\Ingestion\Domain\ValueObjects\Signature;
use Proxynth\Larawebhook\Processing\Infrastructure\Laravel\Jobs\RetryWebhookJob;
use Proxynth\Larawebhook\Processing\Infrastructure\Persistence\Models\ProcessedWebhookEvent;
use Proxynth\Larawebhook\Shared\Application\EventBus;
use Proxynth\Larawebhook\Tests\Fakes\Shared\FakeEventBus;

Expand Down Expand Up @@ -1045,3 +1047,60 @@
->and($log->external_id)->toBe('delivery_123')
->and($log->idempotency_key)->toBe('delivery_123');
});

it('does not process the same webhook twice', function () {
$payload = json_encode([
'action' => 'opened',
'pull_request' => [
'id' => 123,
],
], JSON_THROW_ON_ERROR);

$signature = githubSignature($payload, 'test_github_secret');

$response = $this->call(
'POST',
'test-github-webhook',
[],
[],
[],
[
'HTTP_X_HUB_SIGNATURE_256' => $signature->value(),
'HTTP_X_GITHUB_EVENT' => 'ping',
'HTTP_X_GITHUB_DELIVERY' => 'delivery_123',
'CONTENT_TYPE' => 'application/json',
],
$payload
);

expect(WebhookLog::query()->count())->toBe(1)
->and(ProcessedWebhookEvent::query()->count())->toBe(1);

$processed = ProcessedWebhookEvent::query()->first();

expect($processed->service)->toBe('github')
->and($processed->idempotency_key)->toBe('delivery_123')
->and($processed->external_id)->toBe('delivery_123')
->and($processed->event)->not->toBeNull();

$this->call(
'POST',
'test-github-webhook',
[],
[],
[],
[
'HTTP_X_HUB_SIGNATURE_256' => $signature->value(),
'HTTP_X_GITHUB_EVENT' => 'ping',
'HTTP_X_GITHUB_DELIVERY' => 'delivery_123',
'CONTENT_TYPE' => 'application/json',
],
$payload
)->assertOk()
->assertJson([
'status' => 'already_processed',
]);

expect(WebhookLog::query()->count())->toBe(1)
->and(ProcessedWebhookEvent::query()->count())->toBe(1);
});
7 changes: 7 additions & 0 deletions tests/Helpers/SignatureTestHelpers.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,10 @@ function slackIncomingSignature(string $value, int|string $timestamp): Signature
timestamp: (string) $timestamp,
);
}

function githubSignature(string $payload, string $secret = 'github_secret'): Signature
{
return incomingSignature(
'sha256='.hash_hmac('sha256', $payload, $secret)
);
}
3 changes: 3 additions & 0 deletions tests/Testcase.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,8 @@ public function getEnvironmentSetUp($app)

$migration2 = include __DIR__.'/../database/migrations/002_add_idempotency_key_to_webhook_logs_table.php';
$migration2->up();

$migration3 = include __DIR__.'/../database/migrations/003_create_processed_webhook_events_table.php';
$migration3->up();
}
}
Loading