SaaS Webhook System: Delivery, Retry Logic, Signature Verification, and Subscriber Management

Webhooks are the standard integration primitive in SaaS — Stripe, GitHub, Twilio, Shopify all use them. When you build your own webhook system, reliability is the hard part: HTTP calls fail, customer endpoints go down, and you can't lose events. You need delivery guarantees, retry with exponential backoff, signature verification so receivers can trust the payload, and a delivery log for debugging.

This post covers the full implementation: database schema, event publishing, the delivery worker with retry backoff, HMAC-SHA256 signature verification, subscriber management UI endpoints, and dead letter handling.

System Architecture

Your app emits event
        │
        ▼
webhook_events table (append-only log)
        │
        ▼
Delivery worker (polls pending events)
        │
        ▼
For each subscriber:
  webhook_deliveries record created
        │
        ▼
HTTP POST to endpoint (signed with HMAC-SHA256)
        ├── 2xx → mark delivered
        ├── 4xx → mark failed (don't retry — client error)
        └── 5xx / timeout → retry with exponential backoff
              │
              └── After max retries → dead letter (alert user)

---

1. Database Schema

-- Webhook subscribers: one per integration endpoint
CREATE TABLE webhook_subscribers (
  id              UUID        PRIMARY KEY DEFAULT gen_random_uuid(),
  tenant_id       UUID        NOT NULL REFERENCES accounts(id) ON DELETE CASCADE,
  name            TEXT        NOT NULL,
  endpoint_url    TEXT        NOT NULL,
  secret          TEXT        NOT NULL,  -- HMAC signing key (stored encrypted)
  
  -- Event type filter (NULL = receive all events)
  event_types     TEXT[],    -- e.g. ['order.created', 'payment.succeeded']
  
  is_active       BOOLEAN     NOT NULL DEFAULT true,
  created_at      TIMESTAMPTZ NOT NULL DEFAULT NOW(),
  
  INDEX idx_subscribers_tenant (tenant_id, is_active)
);

-- Canonical event log (source of truth)
CREATE TABLE webhook_events (
  id              UUID        PRIMARY KEY DEFAULT gen_random_uuid(),
  tenant_id       UUID        NOT NULL REFERENCES accounts(id),
  event_type      TEXT        NOT NULL,   -- 'order.created', 'payment.failed', etc.
  payload         JSONB       NOT NULL,
  idempotency_key TEXT        UNIQUE,     -- Prevent duplicate events on retry
  created_at      TIMESTAMPTZ NOT NULL DEFAULT NOW(),
  
  INDEX idx_events_tenant_type (tenant_id, event_type, created_at DESC)
);

-- Delivery attempts: one per subscriber per event
CREATE TABLE webhook_deliveries (
  id              UUID        PRIMARY KEY DEFAULT gen_random_uuid(),
  event_id        UUID        NOT NULL REFERENCES webhook_events(id),
  subscriber_id   UUID        NOT NULL REFERENCES webhook_subscribers(id) ON DELETE CASCADE,
  
  status          TEXT        NOT NULL DEFAULT 'pending'
                              CHECK (status IN ('pending', 'delivered', 'failed', 'dead')),
  
  -- Delivery metadata
  attempt_count   INTEGER     NOT NULL DEFAULT 0,
  next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
  last_attempted_at TIMESTAMPTZ,
  
  -- Response from subscriber endpoint
  response_status  INTEGER,
  response_body    TEXT,
  response_time_ms INTEGER,
  
  -- Error details
  error_message   TEXT,
  
  delivered_at    TIMESTAMPTZ,
  dead_at         TIMESTAMPTZ,
  
  UNIQUE (event_id, subscriber_id),
  INDEX idx_deliveries_pending (status, next_attempt_at) WHERE status = 'pending'
);

---

2. Event Publishing

// src/services/webhooks/publisher.ts
import { db } from '../../lib/db';
import { createHmac } from 'crypto';

export interface WebhookEvent {
  tenantId: string;
  eventType: string;
  payload: Record<string, unknown>;
  idempotencyKey?: string;
}

export async function publishWebhookEvent(event: WebhookEvent): Promise<string> {
  // Store event + create delivery records in a single transaction
  const result = await db.$transaction(async (tx) => {
    // Insert canonical event (idempotent)
    const webhookEvent = await tx.webhookEvent.upsert({
      where: { idempotencyKey: event.idempotencyKey ?? `${event.tenantId}-${Date.now()}` },
      create: {
        tenantId: event.tenantId,
        eventType: event.eventType,
        payload: event.payload,
        idempotencyKey: event.idempotencyKey,
      },
      update: {}, // Already exists — no-op
    });

    // Find all active subscribers that want this event type
    const subscribers = await tx.webhookSubscriber.findMany({
      where: {
        tenantId: event.tenantId,
        isActive: true,
        OR: [
          { eventTypes: { isEmpty: true } },        // Receive all events
          { eventTypes: { has: event.eventType } },  // Explicitly subscribed
        ],
      },
      select: { id: true },
    });

    // Create pending delivery for each subscriber
    if (subscribers.length > 0) {
      await tx.webhookDelivery.createMany({
        data: subscribers.map((sub) => ({
          eventId: webhookEvent.id,
          subscriberId: sub.id,
          nextAttemptAt: new Date(),
        })),
        skipDuplicates: true,
      });
    }

    return webhookEvent.id;
  });

  return result;
}

---

3. Delivery Worker

// src/workers/webhook-delivery.worker.ts
import { db } from '../lib/db';
import { createHmac } from 'crypto';
import { decrypt } from '../lib/crypto';

const MAX_ATTEMPTS = 7;
const RETRY_DELAYS_MS = [
  0,          // Attempt 1: immediate
  30_000,     // Attempt 2: 30 seconds
  300_000,    // Attempt 3: 5 minutes
  1_800_000,  // Attempt 4: 30 minutes
  7_200_000,  // Attempt 5: 2 hours
  21_600_000, // Attempt 6: 6 hours
  86_400_000, // Attempt 7: 24 hours
];

export async function processWebhookDeliveries(): Promise<void> {
  // Claim a batch of pending deliveries (skip locked = no double-processing)
  const deliveries = await db.$queryRaw<Array<{
    id: string;
    event_id: string;
    subscriber_id: string;
    attempt_count: number;
    event_type: string;
    payload: object;
    endpoint_url: string;
    secret: string;
  }>>`
    SELECT
      d.id, d.event_id, d.subscriber_id, d.attempt_count,
      e.event_type, e.payload,
      s.endpoint_url, s.secret
    FROM webhook_deliveries d
    JOIN webhook_events e ON d.event_id = e.id
    JOIN webhook_subscribers s ON d.subscriber_id = s.id
    WHERE d.status = 'pending'
      AND d.next_attempt_at <= NOW()
      AND s.is_active = true
    ORDER BY d.next_attempt_at ASC
    LIMIT 50
    FOR UPDATE OF d SKIP LOCKED
  `;

  await Promise.allSettled(
    deliveries.map((delivery) => processDelivery(delivery))
  );
}

async function processDelivery(delivery: {
  id: string;
  event_id: string;
  subscriber_id: string;
  attempt_count: number;
  event_type: string;
  payload: object;
  endpoint_url: string;
  secret: string;
}): Promise<void> {
  const attemptNumber = delivery.attempt_count + 1;
  const startMs = Date.now();

  // Build signed payload
  const body = JSON.stringify({
    id: delivery.event_id,
    type: delivery.event_type,
    data: delivery.payload,
    attempt: attemptNumber,
    timestamp: new Date().toISOString(),
  });

  const secret = await decrypt(delivery.secret);
  const signature = signPayload(body, secret);

  let responseStatus: number | null = null;
  let responseBody: string | null = null;
  let errorMessage: string | null = null;

  try {
    const response = await fetch(delivery.endpoint_url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'X-Webhook-Signature': `sha256=${signature}`,
        'X-Webhook-Event': delivery.event_type,
        'X-Webhook-Delivery': delivery.id,
        'User-Agent': 'Viprasol-Webhooks/1.0',
      },
      body,
      signal: AbortSignal.timeout(30_000), // 30 second timeout
    });

    responseStatus = response.status;
    responseBody = (await response.text()).slice(0, 1000); // Cap stored response
  } catch (err: any) {
    errorMessage = err.message ?? 'Network error';
  }

  const responseTimeMs = Date.now() - startMs;
  const isSuccess = responseStatus !== null && responseStatus >= 200 && responseStatus < 300;
  const isClientError = responseStatus !== null && responseStatus >= 400 && responseStatus < 500;
  const isDeadLetter = !isSuccess && attemptNumber >= MAX_ATTEMPTS;

  if (isSuccess) {
    await db.webhookDelivery.update({
      where: { id: delivery.id },
      data: {
        status: 'delivered',
        attemptCount: attemptNumber,
        lastAttemptedAt: new Date(),
        responseStatus,
        responseBody,
        responseTimeMs,
        deliveredAt: new Date(),
      },
    });
    return;
  }

  if (isDeadLetter || isClientError) {
    await db.webhookDelivery.update({
      where: { id: delivery.id },
      data: {
        status: isDeadLetter ? 'dead' : 'failed',
        attemptCount: attemptNumber,
        lastAttemptedAt: new Date(),
        responseStatus,
        responseBody,
        responseTimeMs,
        errorMessage,
        deadAt: isDeadLetter ? new Date() : undefined,
      },
    });

    if (isDeadLetter) {
      await notifyDeadLetter(delivery);
    }
    return;
  }

  // Transient failure: schedule retry with exponential backoff
  const nextDelay = RETRY_DELAYS_MS[attemptNumber] ?? RETRY_DELAYS_MS[RETRY_DELAYS_MS.length - 1];
  await db.webhookDelivery.update({
    where: { id: delivery.id },
    data: {
      status: 'pending',
      attemptCount: attemptNumber,
      lastAttemptedAt: new Date(),
      nextAttemptAt: new Date(Date.now() + nextDelay),
      responseStatus,
      responseBody,
      responseTimeMs,
      errorMessage,
    },
  });
}

function signPayload(body: string, secret: string): string {
  return createHmac('sha256', secret).update(body).digest('hex');
}

async function notifyDeadLetter(delivery: { id: string; event_id: string; subscriber_id: string }) {
  // Notify tenant: webhook endpoint is failing
  // Could send email, Slack notification, or create an in-app alert
  console.error(`Dead letter: delivery ${delivery.id} for event ${delivery.event_id}`);
}

---

4. Signature Verification (Receiver Side)

// How your customers verify webhook authenticity
// src/examples/webhook-receiver.ts

import { createHmac, timingSafeEqual } from 'crypto';
import express from 'express';

const app = express();
app.use(express.raw({ type: 'application/json' })); // Raw body for signature check!

app.post('/webhooks/viprasol', (req, res) => {
  const signature = req.headers['x-webhook-signature'] as string;
  const rawBody = req.body as Buffer;

  if (!verifyWebhookSignature(rawBody, signature, process.env.WEBHOOK_SECRET!)) {
    return res.status(401).json({ error: 'Invalid signature' });
  }

  const event = JSON.parse(rawBody.toString());

  // Process event (respond quickly, process async if needed)
  processEvent(event).catch(console.error);

  // Return 2xx immediately — processing happens async
  res.status(200).json({ received: true });
});

function verifyWebhookSignature(
  body: Buffer,
  signatureHeader: string,
  secret: string
): boolean {
  if (!signatureHeader?.startsWith('sha256=')) return false;

  const receivedSig = Buffer.from(signatureHeader.slice(7), 'hex');
  const expectedSig = createHmac('sha256', secret).update(body).digest();

  // timingSafeEqual prevents timing attacks
  if (receivedSig.length !== expectedSig.length) return false;
  return timingSafeEqual(receivedSig, expectedSig);
}

---

5. Subscriber Management API

// src/app/api/webhooks/subscribers/route.ts
import { NextRequest, NextResponse } from 'next/server';
import { z } from 'zod';
import { db } from '../../../../lib/db';
import { encrypt } from '../../../../lib/crypto';
import { nanoid } from 'nanoid';

const CreateSubscriberSchema = z.object({
  name: z.string().min(1).max(100),
  endpointUrl: z.string().url(),
  eventTypes: z.array(z.string()).optional(),
});

export async function POST(req: NextRequest) {
  const session = await getServerSession();
  if (!session) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });

  const body = CreateSubscriberSchema.safeParse(await req.json());
  if (!body.success) {
    return NextResponse.json({ error: body.error.flatten() }, { status: 400 });
  }

  // Generate signing secret — shown once, then stored encrypted
  const secret = `whsec_${nanoid(32)}`;
  const encryptedSecret = await encrypt(secret);

  const subscriber = await db.webhookSubscriber.create({
    data: {
      tenantId: session.user.tenantId,
      name: body.data.name,
      endpointUrl: body.data.endpointUrl,
      secret: encryptedSecret,
      eventTypes: body.data.eventTypes ?? [],
    },
    select: { id: true, name: true, endpointUrl: true, eventTypes: true, createdAt: true },
  });

  // Return secret in creation response only — never again
  return NextResponse.json({ ...subscriber, secret }, { status: 201 });
}

---

Cost Reference

---

See Also

• Stripe Webhook Handling: Signature Verification and Idempotency
• SaaS Referral System: Tracking, Reward Logic, and Analytics
• API Gateway Authentication: JWT, API Keys, and Rate Limiting
• SaaS Audit Logging: Immutable Trails and SOC2 Compliance
• Event-Driven Architecture: EventBridge, SNS, and SQS Patterns

---

Working With Viprasol

Building a SaaS product that needs to notify customer systems reliably via webhooks? We implement end-to-end webhook infrastructure — event publishing, delivery with exponential backoff, HMAC signing, dead letter alerts, and a subscriber management portal — that's operationally transparent and self-healing.

Talk to our team → | See our web development services →