Download 02-state-stream-to-table.ipynb
Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.
State: turning a stream into a table — practical
The pricing service prices each finished session with its site's current tariff. It keeps the tariffs as a table built from the CDC topic, and its own state (open sessions, driver totals) in memory, backed by changelog topics. You change a price while it runs, restart it cleanly, crash it and restore it, and then run it without a changelog.
Lab services: kafka-1..3, schema-registry, postgres, connect (docker compose --profile connect up -d --wait)
· Script: module-07/pricing_service.py · Lesson: modules/07-stream-processing/02-state-stream-to-table.md
# Setup: find the lab's shared helpers and check the lab is running import sys, pathlib lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir()) sys.path.insert(0, str(lab_root)) import json, time from collections import defaultdict from common import settings, lab from common.scenario import Brightlane, to_json lab.require_kafka() for name, url in [("Schema Registry", settings.SCHEMA_REGISTRY_URL + "/subjects"), ("Kafka Connect", settings.CONNECT_URL)]: lab.require_http(name, url, start_hint="docker compose --profile connect up -d --wait") lab.print_versions()
Step 1: The tariffs table, from CDC
The same flattening connector as in lesson 6.1 keeps the latest tariff per id in a compacted topic.
from confluent_kafka import Consumer, Producer, TopicPartition from confluent_kafka.admin import AdminClient TARIFFS, SESSIONS, APP = "m07l02_pricing.tariffs", "m07l02-sessions", "m07l02-pricing" admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP}) lab.remove_connectors("m07l02-") lab.reset_pricing_db() lab.delete_topics(admin, [t for t in admin.list_topics(timeout=10).topics if t.startswith("m07l02")]) for f in (lab_root / ".lab-data").glob("m07l02-*.checkpoint.json"): f.unlink() admin.delete_consumer_groups([APP, "m07l02-naive"]) lab.recreate_topic(admin, TARIFFS, partitions=1, config={"cleanup.policy": "compact"}) lab.recreate_topic(admin, SESSIONS, partitions=3) lab.connect_api("PUT", "/connectors/m07l02-tariffs/config", { "connector.class": "io.debezium.connector.postgresql.PostgresConnector", "plugin.name": "pgoutput", "database.hostname": "postgres", "database.port": "5432", "database.dbname": "pricing", "database.user": "brightlane", "database.password": "brightlane-lab", "topic.prefix": "m07l02_src", "table.include.list": "public.tariffs", "slot.name": "m07l02_tariffs", "publication.name": "m07l02_tariffs", "publication.autocreate.mode": "filtered", "time.precision.mode": "connect", "transforms": "unwrap,route", "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState", "transforms.route.type": "org.apache.kafka.connect.transforms.RegexRouter", "transforms.route.regex": r"m07l02_src\.public\.tariffs", "transforms.route.replacement": TARIFFS}) print("tariffs connector:", lab.wait_connector("m07l02-tariffs")["connector"]["state"]) peek = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l02-peek"}) lab.wait_until(lambda: peek.get_watermark_offsets(TopicPartition(TARIFFS, 0), timeout=10)[1] >= 36, timeout=60, what="the 36 tariffs") peek.close() print("tariffs in the table topic: 36")
Step 2: Start the service on the first sessions
400 sessions give 800 events, keyed by charger. The first 400 events go in now; some sessions have started but not finished yet.
events = Brightlane().stream(sessions=400, readings=False) producer = Producer(settings.PRODUCER_DEFAULTS) def write(first, last): for charger_id, event in events[first:last]: producer.produce(SESSIONS, key=charger_id, value=to_json(event)) producer.flush(10) def run(*extra, app=APP, name=None): return lab.start_script("module-07/pricing_service.py", "--app", app, "--sessions", SESSIONS, "--tariffs", TARIFFS, *extra, name=name or app) def caught_up(app=APP): return sum(lag for *_, lag in lab.group_lag(admin, app, SESSIONS, 3).values()) == 0 write(0, 400) service = run() lab.wait_until(caught_up, timeout=120, interval=1, what="the service to catch up") for line in service.lines(): print(line)
Step 3: A price change while it runs
The pricing team raises the price at site-N03 in PostgreSQL. Then the next 150 events arrive.
lab.pg("UPDATE tariffs SET price_per_kwh = 0.6500 WHERE tariff_id = 'tar-site-N03-1'") time.sleep(4) write(400, 550) lab.wait_until(caught_up, timeout=120, interval=1, what="the service to catch up") print(service.lines("tariff changed")[-1])
Step 4: A clean restart, then a crash
A clean stop writes a checkpoint: the state, and how far the changelogs went. The next start loads it and reads only what the changelogs got since. A crash leaves no checkpoint.
lab.stop_script(service) service = run(name=f"{APP}-2") lab.wait_until(lambda: service.lines("restored"), timeout=60, what="the restore") print(service.lines("restored")[-1]) lab.stop_script(service) service = run("--crash-after", "20", name=f"{APP}-3") write(550, 700) lab.wait_until(lambda: not service.running, timeout=60, what="the crash") print(service.lines("CRASH")[-1]) service = run(name=f"{APP}-4") lab.wait_until(lambda: service.lines("restored"), timeout=60, what="the restore") print(service.lines("restored")[-1]) write(700, len(events)) lab.wait_until(caught_up, timeout=120, interval=1, what="the service to catch up") time.sleep(2) lab.stop_script(service) print(service.lines("stopped")[-1])
Step 5: Is the state right?
Read the committed charges and compare them with the service's driver totals (from the checkpoint its clean stop just wrote).
def read_charges(topic): consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l02-audit", "isolation.level": "read_committed", "enable.auto.commit": False}) consumer.assign([TopicPartition(topic, p, 0) for p in range(3)]) charges = [] while (msg := consumer.poll(3)) is not None: if not msg.error(): charges.append(json.loads(msg.value())) consumer.close() return charges def check(app): charges = read_charges(f"{app}-charges") print(f"charges: {len(charges)} for {len({c['session_id'] for c in charges})} sessions (400 finished)") checkpoint = json.loads((lab_root / ".lab-data" / f"{app}.checkpoint.json").read_text())["state"] summed = defaultdict(float) for c in charges: summed[c["driver_id"]] += c["amount_eur"] wrong = [d for d, t in checkpoint["driver-totals"].items() if abs(t["amount_eur"] - summed[d]) > 0.011] print(f"driver totals: {len(checkpoint['driver-totals'])}, different from their charges: {len(wrong)}, " f"open sessions left: {len(checkpoint['open-sessions'])}") return charges charges = check(APP) print("prices charged at site-N03:", sorted({c["price_per_kwh"] for c in charges if c["site_id"] == "site-N03"}))
Break it: no changelog
The same service with --no-changelog keeps its state in memory only, like a quick script would.
It processes the same events, crashes after 150 sessions, and starts again. Predict first: what
happens to the sessions that started before the crash and finish after it?
naive = run("--no-changelog", "--crash-after", "150", app="m07l02-naive") lab.wait_until(lambda: not naive.running, timeout=60, what="the crash") naive = run("--no-changelog", app="m07l02-naive", name="m07l02-naive-2") lab.wait_until(lambda: caught_up("m07l02-naive"), timeout=120, interval=1, what="the naive service to catch up") lab.stop_script(naive) print(naive.lines("restored")[-1]) print(naive.lines("stopped")[-1]) print(f"charges: {len(read_charges('m07l02-naive-charges'))} (400 sessions finished)")
Clean up
Delete the connector, its slot, the topics, groups, checkpoint files and schemas, and reset the
database. If anything gets stuck, docker compose down -v in the code/ folder resets the lab.
from confluent_kafka.schema_registry import SchemaRegistryClient lab.remove_connectors("m07l02-") lab.delete_topics(admin, [t for t in admin.list_topics(timeout=10).topics if t.startswith("m07l02")]) admin.delete_consumer_groups([APP, "m07l02-naive", "m07l02-audit", f"{APP}-tariffs", "m07l02-naive-tariffs"]) registry = SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL}) for subject in [s for s in registry.get_subjects() if s.startswith("m07l02")]: registry.delete_subject(subject) registry.delete_subject(subject, permanent=True) for f in (lab_root / ".lab-data").glob("m07l02-*.checkpoint.json"): f.unlink() lab.reset_pricing_db() print("cleaned up")