Notebook · code/module-05/03-poison-pills-and-dead-letter-topics.ipynb

Download 03-poison-pills-and-dead-letter-topics.ipynb

Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.

Poison pills and dead-letter topics — practical

600 meter readings, 30 of them forwarded unchanged from legacy 3.9 chargers. A naive worker crashes on the first legacy one, again and again. A careful worker parks the bad records in a dead-letter topic and rides out a ledger outage. Then a worker that parks every error throws away good readings.

Lab services: kafka-1, kafka-2, kafka-3, schema-registry (docker compose --profile schemas up -d --wait) · Script: module-05/telemetry_worker.py · Lesson: modules/05-data-contracts/03-poison-pills-and-dead-letter-topics.md

# Setup: find the lab's shared helpers and check the lab is running
import sys, pathlib
lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir())
sys.path.insert(0, str(lab_root))

import sqlite3, time
from collections import Counter
from common import settings, lab
from common.scenario import Brightlane, parse_iso

lab.require_kafka()
lab.require_http("Schema Registry", settings.SCHEMA_REGISTRY_URL + "/subjects",
                 start_hint="docker compose --profile schemas up -d --wait")
lab.print_versions()

Step 1: The readings, with the gateway's legacy bug

The gateway serializes readings with the Avro schema, except on its legacy path: readings from 3.9 chargers are forwarded as the charger sent them.

from confluent_kafka import Producer, Consumer, TopicPartition
from confluent_kafka.admin import AdminClient
from confluent_kafka.schema_registry import SchemaRegistryClient
from confluent_kafka.schema_registry.avro import AvroSerializer
from confluent_kafka.serialization import SerializationContext, MessageField

TOPIC, DLQ = "m05l03-telemetry", "m05l03-telemetry.dlq"
admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
lab.recreate_topic(admin, TOPIC, partitions=1)
lab.recreate_topic(admin, DLQ, partitions=1)
registry = SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL})
serialize = AvroSerializer(registry, (lab_root / "schemas" / "meter_reading.avsc").read_text(),
                           lambda r, ctx: {**r, "ts": parse_iso(r["ts"])})
context = SerializationContext(TOPIC, MessageField.VALUE)

telemetry = Brightlane().telemetry_with_legacy(600)
producer = Producer(settings.PRODUCER_DEFAULTS)
for key, reading, legacy_bytes in telemetry:
    producer.produce(TOPIC, key=key, value=legacy_bytes or serialize(reading, context))
producer.flush(10)
legacy = [b for _, _, b in telemetry if b]
print(f"{len(telemetry)} readings, {len(legacy)} of them legacy payloads, for example:")
print(legacy[0])
print(legacy[2])

Step 2: A naive worker

The naive worker lets any error stop it. The notebook restarts it three times, like a container platform would. run_worker() starts the script and waits until it has exited or has nothing left to read.

DATA = lab_root / ".lab-data"
DATA.mkdir(exist_ok=True)

def run_worker(name, mode, *extra):
    group, ledger = f"m05l03-{name}", DATA / f"m05l03-{name}.db"
    ledger.unlink(missing_ok=True)
    args = ["--mode", mode, "--topic", TOPIC, "--group", group, "--ledger", f".lab-data/{ledger.name}", *extra]
    worker = lab.start_script("module-05/telemetry_worker.py", *args, name=f"m05l03-{name}")
    lab.wait_until(lambda: not worker.running or worker.lines("idle:"), timeout=120, interval=1,
                   what="the worker to finish")
    lab.stop_script(worker)
    return worker

admin.delete_consumer_groups(["m05l03-naive"])
for attempt in range(1, 4):
    worker = run_worker("naive", "naive")
    print(f"run {attempt}:", worker.lines("CRASH")[-1].split(" ", 1)[1])
print("lag:", lab.group_lag(admin, "m05l03-naive", TOPIC, 1))

Every run dies on the same record, offset 19, and the lag never shrinks: the record can't be processed, and the worker never gets past it. That's a poison pill.

Step 3: A careful worker with a dead-letter topic

The careful worker parks records it can't decode or validate in m05l03-telemetry.dlq, and retries when the ledger is unavailable. This run also has a 2-second ledger outage after 300 readings.

careful = run_worker("careful", "careful", "--outage-at", "300", "--outage-s", "2")
with sqlite3.connect(DATA / "m05l03-careful.db") as db:
    in_ledger = db.execute("SELECT count(*) FROM readings").fetchone()[0]
print("readings in the ledger:", in_ledger, "  parked:", len(careful.lines("PARKED")))
for line in careful.lines("RETRY"):
    print(line)

Step 4: What's in the dead-letter topic?

Each parked record keeps its original bytes, and its headers say where it came from and why it was parked.

def read_dlq():
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m05l03-inspect",
                         "enable.auto.commit": False})
    end = consumer.get_watermark_offsets(TopicPartition(DLQ, 0), timeout=10)[1]
    consumer.assign([TopicPartition(DLQ, 0, 0)])
    records = [consumer.poll(10) for _ in range(end)]
    consumer.close()
    return records

parked = read_dlq()
first = parked[0]
for key, value in first.headers():
    print(f"{key:<22} {value.decode()}")
print("value:", first.value())
print(Counter(dict(r.headers())["dlq.error.message"].decode() for r in parked))

Break it: park every error

A shortcut many workers take: except Exception: send it to the dead-letter topic. Predict first: what happens to good readings during the same 2-second ledger outage?

lab.recreate_topic(admin, DLQ, partitions=1)
sloppy = run_worker("park-all", "park-all", "--outage-at", "300", "--outage-s", "2")
with sqlite3.connect(DATA / "m05l03-park-all.db") as db:
    in_ledger = db.execute("SELECT count(*) FROM readings").fetchone()[0]
reasons = Counter(dict(r.headers())["dlq.error.class"].decode() for r in read_dlq())
print("readings in the ledger:", in_ledger, "  parked:", len(sloppy.lines("PARKED")))
print("parked, by error:", dict(reasons))

The worker parked good readings as if they were bad data. They are missing from the ledger now, and somebody has to find and replay them.

Clean up

Delete the topics, the groups and the ledgers. If anything gets stuck, docker compose down -v in the code/ folder resets the whole lab.

lab.delete_topics(admin, [TOPIC, DLQ])
registry.delete_subject(f"{TOPIC}-value")
registry.delete_subject(f"{TOPIC}-value", permanent=True)
admin.delete_consumer_groups(["m05l03-naive", "m05l03-careful", "m05l03-park-all", "m05l03-inspect"])
for f in DATA.glob("m05l03-*.db*"):
    f.unlink()
print("cleaned up")