Idempotency Keys and Signed Webhooks in Node.js: Retries Without Duplicates

Short answer: async APIs fail in two places. The caller sends a request twice, or the sender delivers an event twice. Fix the first with an idempotency key, fix the second with a delivery id, and protect the webhook with a signature check. This post builds all three in one Node.js file with no packages, and runs six scenarios so you can see what each one does.

I build Libraryminds, whose API uses these patterns, and I describe what its docs say near the end. The demo itself is generic. Its header names are my own.

The three problems

1. A caller sends POST /jobs, the network times out, and the caller sends it again. Now there may be two jobs, and two bills.

2. A provider sends your server an event, your answer is slow or lost, and the provider sends it again. Now you may handle the same event twice.

3. Anyone can send a POST to a public URL. You need a way to know the request came from the provider.

The receiver

// ---------- Receiver: verify, dedupe, save, then answer fast ----------
const seen = new Map(); // delivery id -> event. In production use a database.
let failFirst = 0;      // simulate an outage: answer 500 for the next N requests

function sign(rawBody, secret) {
  return "sha256=" + crypto.createHmac("sha256", secret).update(rawBody).digest("hex");
}

function validSignature(rawBody, received) {
  const expected = sign(rawBody, SECRET);
  return received.length === expected.length &&
    crypto.timingSafeEqual(Buffer.from(received), Buffer.from(expected));
}

const receiver = http.createServer((req, res) => {
  const chunks = [];
  req.on("data", (c) => chunks.push(c));
  req.on("end", () => {
    const rawBody = Buffer.concat(chunks); // hash the raw bytes, not parsed JSON
    const id = req.headers["x-delivery-id"] || "";
    const sig = req.headers["x-signature"] || "";

    if (!validSignature(rawBody, sig)) {
      console.log(`  receiver: ${id} bad signature -> 401`);
      return res.writeHead(401).end();
    }
    if (failFirst > 0) {
      failFirst--;
      console.log(`  receiver: ${id} simulated outage -> 500`);
      return res.writeHead(500).end();
    }
    if (seen.has(id)) {
      console.log(`  receiver: ${id} already handled -> 200, not processed again`);
      return res.writeHead(200).end();
    }
    seen.set(id, JSON.parse(rawBody.toString())); // save first
    console.log(`  receiver: ${id} saved -> 200`);
    res.writeHead(200).end();                      // then answer; do slow work later
  });
});

Four choices matter here:

1. It hashes the raw bytes of the body. If you parse the JSON and serialize it again, the bytes can change, and every real request would fail.

2. It compares signatures with timingSafeEqual, after a length check. The length check is needed because timingSafeEqual throws when the lengths differ.

3. It saves the event first and answers 200 after. If your server crashes after the answer, you still have the event, and you can do the slow work later from what you saved.

4. For a delivery id it has already seen, it still answers 200. That stops the sender from retrying, and the event is not processed twice.

The sender

// ---------- Sender: retry with backoff, same delivery id every time ----------
async function deliver(url, event, id, { attempts = 5, baseMs = 20, tamper = false } = {}) {
  const body = JSON.stringify(event);
  const signature = sign(body, SECRET);
  const sentBody = tamper ? body.replace("job_1", "job_2") : body;
  for (let attempt = 1; attempt <= attempts; attempt++) {
    console.log(`  sender: ${id} attempt ${attempt}`);
    try {
      const r = await fetch(url, {
        method: "POST",
        headers: { "content-type": "application/json", "x-delivery-id": id, "x-signature": signature },
        body: sentBody,
      });
      if (r.ok) return { ok: true, attempt };
      if (r.status === 401) return { ok: false, attempt, reason: "401, will not retry" };
    } catch {}
    await new Promise((r) => setTimeout(r, baseMs * 2 ** (attempt - 1)));
  }
  return { ok: false, attempt: attempts, reason: "gave up" };
}

The sender retries on a 500 or a network error, and waits twice as long each time. It sends the same delivery id on every attempt, which is what lets the receiver spot a repeat. It does not retry on a 401. That is my choice for this demo, because a wrong signature will not fix itself.

Idempotency keys on the API side

// ---------- Idempotency keys on the API side ----------
const keys = new Map(); // key -> { bodyHash, response }
let nextJob = 1;
const api = http.createServer((req, res) => {
  const chunks = [];
  req.on("data", (c) => chunks.push(c));
  req.on("end", () => {
    const body = Buffer.concat(chunks).toString();
    const key = req.headers["idempotency-key"];
    const hash = crypto.createHash("sha256").update(body).digest("hex");
    if (key && keys.has(key)) {
      const saved = keys.get(key);
      if (saved.bodyHash !== hash) return res.writeHead(422).end(JSON.stringify({ error: "idempotency_key_reused" }));
      return res.writeHead(202).end(saved.response);
    }
    const response = JSON.stringify({ id: `job_${nextJob++}` });
    if (key) keys.set(key, { bodyHash: hash, response });
    res.writeHead(202).end(response);
  });
});

async function post(url, body, key) {
  const r = await fetch(url, {
    method: "POST",
    headers: { "content-type": "application/json", ...(key ? { "idempotency-key": key } : {}) },
    body: JSON.stringify(body),
  });
  return `${r.status} ${await r.text()}`;
}

The rule has three cases. The same key with the same body returns the first response. The same key with a different body is an error, 422 here. No key means a new job every time.

Six scenarios, real output

The output below is from a real run on Node.js 22. Global fetch needs Node 18 or newer.

1. A normal delivery
  sender: dlv_1 attempt 1
  receiver: dlv_1 saved -> 200
  result: { ok: true, attempt: 1 }

2. The same delivery id sent again (a retry after a lost answer)
  sender: dlv_1 attempt 1
  receiver: dlv_1 already handled -> 200, not processed again
  result: { ok: true, attempt: 1 }

3. The body changed on the way (wrong signature)
  sender: dlv_2 attempt 1
  receiver: dlv_2 bad signature -> 401
  result: { ok: false, attempt: 1, reason: '401, will not retry' }

4. The receiver is down for the first 2 attempts
  sender: dlv_3 attempt 1
  receiver: dlv_3 simulated outage -> 500
  sender: dlv_3 attempt 2
  receiver: dlv_3 simulated outage -> 500
  sender: dlv_3 attempt 3
  receiver: dlv_3 saved -> 200
  result: { ok: true, attempt: 3 }

5. The receiver is down for every attempt
  sender: dlv_4 attempt 1
  receiver: dlv_4 simulated outage -> 500
  sender: dlv_4 attempt 2
  receiver: dlv_4 simulated outage -> 500
  sender: dlv_4 attempt 3
  receiver: dlv_4 simulated outage -> 500
  result: { ok: false, attempt: 3, reason: 'gave up' }

6. Idempotency key on the API
  first call           : 202 {"id":"job_1"}
  same key, same body  : 202 {"id":"job_1"}
  same key, other body : 422 {"error":"idempotency_key_reused"}
  no key               : 202 {"id":"job_2"}

Saved deliveries: dlv_1, dlv_3

What it shows:

1. A normal delivery is saved and answered with 200 on the first attempt.

2. The same delivery id sent again gets a 200, and the receiver does not process it again. The saved list at the end has dlv_1 only once.

3. A body that changed on the way fails the signature check with 401, and the sender stops.

4. With the receiver down for two attempts, the sender succeeds on the third, with the same delivery id every time. The event is saved once.

5. With the receiver down for every attempt, the sender gives up after three tries. In a real system this is the case you need to store and alert on.

6. With the same idempotency key, the second call returns job_1 again. A different body with that key returns 422. A call without a key makes job_2.

What this demo does not do

1. It keeps delivery ids in memory. In production, use a database with a unique constraint on the id, so two deliveries that arrive at the same moment cannot both pass.

2. It has no queue. It shows "save, then answer", but the processing step is left out.

3. It has no replay protection. Someone who captures a signed request can send it again. Keeping ids for long enough helps, and some providers also put a timestamp inside the signed data. Check yours.

4. It has no dead-letter storage and no alert for scenario 5.

5. Its wait time has no jitter, so many senders retrying together would retry in step.

How one real API does it

The Libraryminds developer docs describe these rules:

1. An Idempotency-Key header of up to 255 characters on job creation. The same key with the same body returns the original response. The same key with a different body returns 422.

2. Webhooks signed with HMAC-SHA256, in a header named X-Libraryminds-Signature, with a value that starts with sha256=.

3. Up to five delivery attempts with backoff, and a stable delivery ID across retries so that you can ignore repeats.

4. After three consecutive dead-lettered deliveries, the webhook is disabled until you enable it again.

5. Webhooks need the Plus plan or higher.

These are claims from the developer docs, not something this demo tests. The header names in the demo above, x-signature and x-delivery-id, are mine.

What to take from this

1. Give every retryable POST an idempotency key, and decide what a reused key with a new body should do.

2. Verify the signature on the raw body before you trust anything in it.

3. Save first, answer fast, process later.

4. Answer 200 for an id you have already handled.

5. Decide what happens when every retry fails, before it happens.

The full script

Save this as webhook_demo.mjs and run node webhook_demo.mjs.

import http from "node:http";
import crypto from "node:crypto";

const SECRET = "whsec_demo_secret";

// ---------- Receiver: verify, dedupe, save, then answer fast ----------
const seen = new Map(); // delivery id -> event. In production use a database.
let failFirst = 0;      // simulate an outage: answer 500 for the next N requests

function sign(rawBody, secret) {
  return "sha256=" + crypto.createHmac("sha256", secret).update(rawBody).digest("hex");
}

function validSignature(rawBody, received) {
  const expected = sign(rawBody, SECRET);
  return received.length === expected.length &&
    crypto.timingSafeEqual(Buffer.from(received), Buffer.from(expected));
}

const receiver = http.createServer((req, res) => {
  const chunks = [];
  req.on("data", (c) => chunks.push(c));
  req.on("end", () => {
    const rawBody = Buffer.concat(chunks); // hash the raw bytes, not parsed JSON
    const id = req.headers["x-delivery-id"] || "";
    const sig = req.headers["x-signature"] || "";

    if (!validSignature(rawBody, sig)) {
      console.log(`  receiver: ${id} bad signature -> 401`);
      return res.writeHead(401).end();
    }
    if (failFirst > 0) {
      failFirst--;
      console.log(`  receiver: ${id} simulated outage -> 500`);
      return res.writeHead(500).end();
    }
    if (seen.has(id)) {
      console.log(`  receiver: ${id} already handled -> 200, not processed again`);
      return res.writeHead(200).end();
    }
    seen.set(id, JSON.parse(rawBody.toString())); // save first
    console.log(`  receiver: ${id} saved -> 200`);
    res.writeHead(200).end();                      // then answer; do slow work later
  });
});

// ---------- Sender: retry with backoff, same delivery id every time ----------
async function deliver(url, event, id, { attempts = 5, baseMs = 20, tamper = false } = {}) {
  const body = JSON.stringify(event);
  const signature = sign(body, SECRET);
  const sentBody = tamper ? body.replace("job_1", "job_2") : body;
  for (let attempt = 1; attempt <= attempts; attempt++) {
    console.log(`  sender: ${id} attempt ${attempt}`);
    try {
      const r = await fetch(url, {
        method: "POST",
        headers: { "content-type": "application/json", "x-delivery-id": id, "x-signature": signature },
        body: sentBody,
      });
      if (r.ok) return { ok: true, attempt };
      if (r.status === 401) return { ok: false, attempt, reason: "401, will not retry" };
    } catch {}
    await new Promise((r) => setTimeout(r, baseMs * 2 ** (attempt - 1)));
  }
  return { ok: false, attempt: attempts, reason: "gave up" };
}

// ---------- Idempotency keys on the API side ----------
const keys = new Map(); // key -> { bodyHash, response }
let nextJob = 1;
const api = http.createServer((req, res) => {
  const chunks = [];
  req.on("data", (c) => chunks.push(c));
  req.on("end", () => {
    const body = Buffer.concat(chunks).toString();
    const key = req.headers["idempotency-key"];
    const hash = crypto.createHash("sha256").update(body).digest("hex");
    if (key && keys.has(key)) {
      const saved = keys.get(key);
      if (saved.bodyHash !== hash) return res.writeHead(422).end(JSON.stringify({ error: "idempotency_key_reused" }));
      return res.writeHead(202).end(saved.response);
    }
    const response = JSON.stringify({ id: `job_${nextJob++}` });
    if (key) keys.set(key, { bodyHash: hash, response });
    res.writeHead(202).end(response);
  });
});

async function post(url, body, key) {
  const r = await fetch(url, {
    method: "POST",
    headers: { "content-type": "application/json", ...(key ? { "idempotency-key": key } : {}) },
    body: JSON.stringify(body),
  });
  return `${r.status} ${await r.text()}`;
}

// ---------- Run the scenarios ----------
const listen = (s) => new Promise((ok) => s.listen(0, "127.0.0.1", () => ok(s.address().port)));
const hookUrl = `http://127.0.0.1:${await listen(receiver)}/hook`;
const apiUrl = `http://127.0.0.1:${await listen(api)}/jobs`;
const event = { type: "job.completed", jobId: "job_1" };

console.log("\n1. A normal delivery");
console.log("  result:", await deliver(hookUrl, event, "dlv_1"));

console.log("\n2. The same delivery id sent again (a retry after a lost answer)");
console.log("  result:", await deliver(hookUrl, event, "dlv_1"));

console.log("\n3. The body changed on the way (wrong signature)");
console.log("  result:", await deliver(hookUrl, event, "dlv_2", { tamper: true }));

console.log("\n4. The receiver is down for the first 2 attempts");
failFirst = 2;
console.log("  result:", await deliver(hookUrl, { type: "job.completed", jobId: "job_3" }, "dlv_3"));

console.log("\n5. The receiver is down for every attempt");
failFirst = 99;
console.log("  result:", await deliver(hookUrl, { type: "job.completed", jobId: "job_4" }, "dlv_4", { attempts: 3 }));
failFirst = 0;

console.log("\n6. Idempotency key on the API");
console.log("  first call           :", await post(apiUrl, { url: "https://example.com/a" }, "key-1"));
console.log("  same key, same body  :", await post(apiUrl, { url: "https://example.com/a" }, "key-1"));
console.log("  same key, other body :", await post(apiUrl, { url: "https://example.com/b" }, "key-1"));
console.log("  no key               :", await post(apiUrl, { url: "https://example.com/a" }));

console.log("\nSaved deliveries:", [...seen.keys()].join(", "));
receiver.close();
api.close();

If your provider does something different, tell me in the comments and I will add it to the list.