Notebook · code/module-07/02-state-stream-to-table.ipynb

Download 02-state-stream-to-table.ipynb

Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.

State: turning a stream into a table — practical

The pricing service prices each finished session with its site's current tariff. It keeps the tariffs as a table built from the CDC topic, and its own state (open sessions, driver totals) in memory, backed by changelog topics. You change a price while it runs, restart it cleanly, crash it and restore it, and then run it without a changelog.

Lab services: kafka-1..3, schema-registry, postgres, connect (docker compose --profile connect up -d --wait) · Script: module-07/pricing_service.py · Lesson: modules/07-stream-processing/02-state-stream-to-table.md

# Setup: find the lab's shared helpers and check the lab is running
import sys, pathlib
lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir())
sys.path.insert(0, str(lab_root))

import json, time
from collections import defaultdict
from common import settings, lab
from common.scenario import Brightlane, to_json

lab.require_kafka()
for name, url in [("Schema Registry", settings.SCHEMA_REGISTRY_URL + "/subjects"), ("Kafka Connect", settings.CONNECT_URL)]:
    lab.require_http(name, url, start_hint="docker compose --profile connect up -d --wait")
lab.print_versions()

Step 1: The tariffs table, from CDC

The same flattening connector as in lesson 6.1 keeps the latest tariff per id in a compacted topic.

from confluent_kafka import Consumer, Producer, TopicPartition
from confluent_kafka.admin import AdminClient

TARIFFS, SESSIONS, APP = "m07l02_pricing.tariffs", "m07l02-sessions", "m07l02-pricing"
admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
lab.remove_connectors("m07l02-")
lab.reset_pricing_db()
lab.delete_topics(admin, [t for t in admin.list_topics(timeout=10).topics if t.startswith("m07l02")])
for f in (lab_root / ".lab-data").glob("m07l02-*.checkpoint.json"):
    f.unlink()
admin.delete_consumer_groups([APP, "m07l02-naive"])
lab.recreate_topic(admin, TARIFFS, partitions=1, config={"cleanup.policy": "compact"})
lab.recreate_topic(admin, SESSIONS, partitions=3)
lab.connect_api("PUT", "/connectors/m07l02-tariffs/config", {
    "connector.class": "io.debezium.connector.postgresql.PostgresConnector", "plugin.name": "pgoutput",
    "database.hostname": "postgres", "database.port": "5432", "database.dbname": "pricing",
    "database.user": "brightlane", "database.password": "brightlane-lab",
    "topic.prefix": "m07l02_src", "table.include.list": "public.tariffs",
    "slot.name": "m07l02_tariffs", "publication.name": "m07l02_tariffs", "publication.autocreate.mode": "filtered",
    "time.precision.mode": "connect",
    "transforms": "unwrap,route", "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
    "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter",
    "transforms.route.regex": r"m07l02_src\.public\.tariffs", "transforms.route.replacement": TARIFFS})
print("tariffs connector:", lab.wait_connector("m07l02-tariffs")["connector"]["state"])
peek = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l02-peek"})
lab.wait_until(lambda: peek.get_watermark_offsets(TopicPartition(TARIFFS, 0), timeout=10)[1] >= 36,
               timeout=60, what="the 36 tariffs")
peek.close()
print("tariffs in the table topic: 36")

Step 2: Start the service on the first sessions

400 sessions give 800 events, keyed by charger. The first 400 events go in now; some sessions have started but not finished yet.

events = Brightlane().stream(sessions=400, readings=False)
producer = Producer(settings.PRODUCER_DEFAULTS)

def write(first, last):
    for charger_id, event in events[first:last]:
        producer.produce(SESSIONS, key=charger_id, value=to_json(event))
    producer.flush(10)

def run(*extra, app=APP, name=None):
    return lab.start_script("module-07/pricing_service.py", "--app", app, "--sessions", SESSIONS,
                            "--tariffs", TARIFFS, *extra, name=name or app)

def caught_up(app=APP):
    return sum(lag for *_, lag in lab.group_lag(admin, app, SESSIONS, 3).values()) == 0

write(0, 400)
service = run()
lab.wait_until(caught_up, timeout=120, interval=1, what="the service to catch up")
for line in service.lines():
    print(line)

Step 3: A price change while it runs

The pricing team raises the price at site-N03 in PostgreSQL. Then the next 150 events arrive.

lab.pg("UPDATE tariffs SET price_per_kwh = 0.6500 WHERE tariff_id = 'tar-site-N03-1'")
time.sleep(4)
write(400, 550)
lab.wait_until(caught_up, timeout=120, interval=1, what="the service to catch up")
print(service.lines("tariff changed")[-1])

Step 4: A clean restart, then a crash

A clean stop writes a checkpoint: the state, and how far the changelogs went. The next start loads it and reads only what the changelogs got since. A crash leaves no checkpoint.

lab.stop_script(service)
service = run(name=f"{APP}-2")
lab.wait_until(lambda: service.lines("restored"), timeout=60, what="the restore")
print(service.lines("restored")[-1])
lab.stop_script(service)

service = run("--crash-after", "20", name=f"{APP}-3")
write(550, 700)
lab.wait_until(lambda: not service.running, timeout=60, what="the crash")
print(service.lines("CRASH")[-1])

service = run(name=f"{APP}-4")
lab.wait_until(lambda: service.lines("restored"), timeout=60, what="the restore")
print(service.lines("restored")[-1])
write(700, len(events))
lab.wait_until(caught_up, timeout=120, interval=1, what="the service to catch up")
time.sleep(2)
lab.stop_script(service)
print(service.lines("stopped")[-1])

Step 5: Is the state right?

Read the committed charges and compare them with the service's driver totals (from the checkpoint its clean stop just wrote).

def read_charges(topic):
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l02-audit",
                         "isolation.level": "read_committed", "enable.auto.commit": False})
    consumer.assign([TopicPartition(topic, p, 0) for p in range(3)])
    charges = []
    while (msg := consumer.poll(3)) is not None:
        if not msg.error():
            charges.append(json.loads(msg.value()))
    consumer.close()
    return charges

def check(app):
    charges = read_charges(f"{app}-charges")
    print(f"charges: {len(charges)} for {len({c['session_id'] for c in charges})} sessions (400 finished)")
    checkpoint = json.loads((lab_root / ".lab-data" / f"{app}.checkpoint.json").read_text())["state"]
    summed = defaultdict(float)
    for c in charges:
        summed[c["driver_id"]] += c["amount_eur"]
    wrong = [d for d, t in checkpoint["driver-totals"].items() if abs(t["amount_eur"] - summed[d]) > 0.011]
    print(f"driver totals: {len(checkpoint['driver-totals'])}, different from their charges: {len(wrong)}, "
          f"open sessions left: {len(checkpoint['open-sessions'])}")
    return charges

charges = check(APP)
print("prices charged at site-N03:", sorted({c["price_per_kwh"] for c in charges if c["site_id"] == "site-N03"}))

Break it: no changelog

The same service with --no-changelog keeps its state in memory only, like a quick script would. It processes the same events, crashes after 150 sessions, and starts again. Predict first: what happens to the sessions that started before the crash and finish after it?

naive = run("--no-changelog", "--crash-after", "150", app="m07l02-naive")
lab.wait_until(lambda: not naive.running, timeout=60, what="the crash")
naive = run("--no-changelog", app="m07l02-naive", name="m07l02-naive-2")
lab.wait_until(lambda: caught_up("m07l02-naive"), timeout=120, interval=1, what="the naive service to catch up")
lab.stop_script(naive)
print(naive.lines("restored")[-1])
print(naive.lines("stopped")[-1])
print(f"charges: {len(read_charges('m07l02-naive-charges'))} (400 sessions finished)")

Clean up

Delete the connector, its slot, the topics, groups, checkpoint files and schemas, and reset the database. If anything gets stuck, docker compose down -v in the code/ folder resets the lab.

from confluent_kafka.schema_registry import SchemaRegistryClient

lab.remove_connectors("m07l02-")
lab.delete_topics(admin, [t for t in admin.list_topics(timeout=10).topics if t.startswith("m07l02")])
admin.delete_consumer_groups([APP, "m07l02-naive", "m07l02-audit", f"{APP}-tariffs", "m07l02-naive-tariffs"])
registry = SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL})
for subject in [s for s in registry.get_subjects() if s.startswith("m07l02")]:
    registry.delete_subject(subject)
    registry.delete_subject(subject, permanent=True)
for f in (lab_root / ".lab-data").glob("m07l02-*.checkpoint.json"):
    f.unlink()
lab.reset_pricing_db()
print("cleaned up")