Notebook · code/module-07/01-exactly-once-billing.ipynb

Download 01-exactly-once-billing.ipynb

Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.

Exactly-once billing with transactions — practical

The billing service reads session events and writes one charge per finished session. It crashes halfway, twice: once writing charges and offsets separately, once in transactions. Then two copies of the service run with the same transactional id.

Lab services: kafka-1, kafka-2, kafka-3 · Script: module-07/billing_service.py · Lesson: modules/07-stream-processing/01-exactly-once-billing.md

# Setup: find the lab's shared helpers and check the lab is running
import sys, pathlib
lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir())
sys.path.insert(0, str(lab_root))

import time
from common import settings, lab
from common.scenario import Brightlane, to_json

lab.require_kafka()
lab.print_versions()

Step 1: 400 finished sessions

800 events: a start and a stop for each session. A correct run writes exactly 400 charges.

from confluent_kafka import Consumer, Producer, TopicPartition
from confluent_kafka.admin import AdminClient

SESSIONS, CHARGES = "m07l01-sessions", "m07l01-charges"
admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
lab.recreate_topic(admin, SESSIONS, partitions=3)
producer = Producer(settings.PRODUCER_DEFAULTS)
for charger_id, event in Brightlane().stream(sessions=400, readings=False):
    producer.produce(SESSIONS, key=charger_id, value=to_json(event))
print("not yet stored:", producer.flush(30))

Step 2: A crash, with offsets committed separately

crash_test() starts the service with --crash-after 150: it dies right after its 150th charge reached the broker, before committing anything. Then it starts the service again and waits until the group has caught up. audit() reads the charges topic like an auditor would, with the given isolation level, and counts charges and different sessions.

def crash_test(mode, group):
    lab.recreate_topic(admin, CHARGES, partitions=3)
    admin.delete_consumer_groups([group])
    args = ["--mode", mode, "--sessions", SESSIONS, "--charges", CHARGES, "--group", group]
    first = lab.start_script("module-07/billing_service.py", *args, "--crash-after", "150", name=f"{group}-1")
    lab.wait_until(lambda: not first.running, timeout=60, what="the crash")
    print(first.lines("CRASH")[-1])
    second = lab.start_script("module-07/billing_service.py", *args, name=f"{group}-2")
    lab.wait_until(lambda: sum(lag for *_, lag in lab.group_lag(admin, group, SESSIONS, 3).values()) == 0,
                   timeout=120, interval=1, what="the group to catch up")
    lab.stop_script(second)

def audit(isolation):
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l01-audit",
                         "isolation.level": isolation, "enable.auto.commit": False})
    consumer.assign([TopicPartition(CHARGES, p, 0) for p in range(3)])
    sessions = []
    while (msg := consumer.poll(3)) is not None:
        if not msg.error():
            sessions.append(msg.key())
    consumer.close()
    return f"{len(sessions)} charges for {len(set(sessions))} sessions"

crash_test("plain", "m07l01-plain")
print("audit:", audit("read_committed"))

Some sessions were charged twice: the charges of the batch in progress reached the topic, but the offsets weren't committed, so the restarted service handled that batch again.

Step 3: The same crash, with transactions

In txn mode, each batch's charges and the consumer's offsets go into one transaction.

crash_test("txn", "m07l01-txn")
print("audit (read_committed):  ", audit("read_committed"))
print("audit (read_uncommitted):", audit("read_uncommitted"))

A read_committed reader sees exactly 400 charges. A read_uncommitted reader also sees the charges of the transaction that the crash left open, and that was aborted.

Break it: two copies with one transactional id

Someone starts a second copy of the service with the same configuration, so with the same transactional.id. Predict first: what happens to the first copy, and to the charges?

lab.recreate_topic(admin, CHARGES, partitions=3)
admin.delete_consumer_groups(["m07l01-twins"])
args = ["--mode", "txn", "--sessions", SESSIONS, "--charges", CHARGES, "--group", "m07l01-twins", "--work-ms", "20"]
a = lab.start_script("module-07/billing_service.py", *args, "--name", "A", name="m07l01-A")
time.sleep(5)
b = lab.start_script("module-07/billing_service.py", *args, "--name", "B", name="m07l01-B")
lab.wait_until(lambda: sum(lag for *_, lag in lab.group_lag(admin, "m07l01-twins", SESSIONS, 3).values()) == 0,
               timeout=120, interval=1, what="the group to catch up")
for s in (a, b):
    lab.stop_script(s)
print("\n".join(line for line in a.lines() if not line.startswith("%")))
print(b.lines("stopped")[-1])
print("audit (read_committed):", audit("read_committed"))

Clean up

Delete the topics and groups. If anything gets stuck, docker compose down -v in the code/ folder resets the whole lab.

lab.delete_topics(admin, [SESSIONS, CHARGES])
admin.delete_consumer_groups(["m07l01-plain", "m07l01-txn", "m07l01-twins", "m07l01-audit"])
print("cleaned up")