Download 01-exactly-once-billing.ipynb
Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.
Exactly-once billing with transactions — practical
The billing service reads session events and writes one charge per finished session. It crashes halfway, twice: once writing charges and offsets separately, once in transactions. Then two copies of the service run with the same transactional id.
Lab services: kafka-1, kafka-2, kafka-3 · Script: module-07/billing_service.py
· Lesson: modules/07-stream-processing/01-exactly-once-billing.md
# Setup: find the lab's shared helpers and check the lab is running import sys, pathlib lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir()) sys.path.insert(0, str(lab_root)) import time from common import settings, lab from common.scenario import Brightlane, to_json lab.require_kafka() lab.print_versions()
Step 1: 400 finished sessions
800 events: a start and a stop for each session. A correct run writes exactly 400 charges.
from confluent_kafka import Consumer, Producer, TopicPartition from confluent_kafka.admin import AdminClient SESSIONS, CHARGES = "m07l01-sessions", "m07l01-charges" admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP}) lab.recreate_topic(admin, SESSIONS, partitions=3) producer = Producer(settings.PRODUCER_DEFAULTS) for charger_id, event in Brightlane().stream(sessions=400, readings=False): producer.produce(SESSIONS, key=charger_id, value=to_json(event)) print("not yet stored:", producer.flush(30))
Step 2: A crash, with offsets committed separately
crash_test() starts the service with --crash-after 150: it dies right after its 150th charge
reached the broker, before committing anything. Then it starts the service again and waits until
the group has caught up. audit() reads the charges topic like an auditor would, with the given
isolation level, and counts charges and different sessions.
def crash_test(mode, group): lab.recreate_topic(admin, CHARGES, partitions=3) admin.delete_consumer_groups([group]) args = ["--mode", mode, "--sessions", SESSIONS, "--charges", CHARGES, "--group", group] first = lab.start_script("module-07/billing_service.py", *args, "--crash-after", "150", name=f"{group}-1") lab.wait_until(lambda: not first.running, timeout=60, what="the crash") print(first.lines("CRASH")[-1]) second = lab.start_script("module-07/billing_service.py", *args, name=f"{group}-2") lab.wait_until(lambda: sum(lag for *_, lag in lab.group_lag(admin, group, SESSIONS, 3).values()) == 0, timeout=120, interval=1, what="the group to catch up") lab.stop_script(second) def audit(isolation): consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l01-audit", "isolation.level": isolation, "enable.auto.commit": False}) consumer.assign([TopicPartition(CHARGES, p, 0) for p in range(3)]) sessions = [] while (msg := consumer.poll(3)) is not None: if not msg.error(): sessions.append(msg.key()) consumer.close() return f"{len(sessions)} charges for {len(set(sessions))} sessions" crash_test("plain", "m07l01-plain") print("audit:", audit("read_committed"))
Some sessions were charged twice: the charges of the batch in progress reached the topic, but the offsets weren't committed, so the restarted service handled that batch again.
Step 3: The same crash, with transactions
In txn mode, each batch's charges and the consumer's offsets go into one transaction.
crash_test("txn", "m07l01-txn") print("audit (read_committed): ", audit("read_committed")) print("audit (read_uncommitted):", audit("read_uncommitted"))
A read_committed reader sees exactly 400 charges. A read_uncommitted reader also sees the
charges of the transaction that the crash left open, and that was aborted.
Break it: two copies with one transactional id
Someone starts a second copy of the service with the same configuration, so with the same
transactional.id. Predict first: what happens to the first copy, and to the charges?
lab.recreate_topic(admin, CHARGES, partitions=3) admin.delete_consumer_groups(["m07l01-twins"]) args = ["--mode", "txn", "--sessions", SESSIONS, "--charges", CHARGES, "--group", "m07l01-twins", "--work-ms", "20"] a = lab.start_script("module-07/billing_service.py", *args, "--name", "A", name="m07l01-A") time.sleep(5) b = lab.start_script("module-07/billing_service.py", *args, "--name", "B", name="m07l01-B") lab.wait_until(lambda: sum(lag for *_, lag in lab.group_lag(admin, "m07l01-twins", SESSIONS, 3).values()) == 0, timeout=120, interval=1, what="the group to catch up") for s in (a, b): lab.stop_script(s) print("\n".join(line for line in a.lines() if not line.startswith("%"))) print(b.lines("stopped")[-1]) print("audit (read_committed):", audit("read_committed"))
Clean up
Delete the topics and groups. If anything gets stuck, docker compose down -v in the code/
folder resets the whole lab.
lab.delete_topics(admin, [SESSIONS, CHARGES]) admin.delete_consumer_groups(["m07l01-plain", "m07l01-txn", "m07l01-twins", "m07l01-audit"]) print("cleaned up")