"""Pricing service: prices every finished session with its site's current tariff, and keeps a
running total per driver.
Inputs: session events (JSON, keyed by charger id), and the tariffs table: a compacted topic with
the latest tariff per id (Avro, written by the CDC connector of lesson 6.1).
State: open sessions (session id -> driver and site) and driver totals, kept in memory. Every
change is also written to a compacted changelog topic per store, in the same transaction
as the charge and the consumer offsets.
Restart: after a clean stop the service loads its checkpoint file and reads only the changelog
records written since; after a crash it rebuilds its state from the changelogs.
--no-changelog keeps state in memory only, like a naive service.
The notebook of lesson 7.2 runs this script. From the code/ folder:
python module-07/pricing_service.py --app m07l02-pricing --sessions m07l02-sessions --tariffs m07l02_pricing.tariffs
"""
import argparse
import json
import os
import signal
import sys
import time
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer, Producer, TopicPartition # noqa: E402
from confluent_kafka.admin import AdminClient, NewTopic # noqa: E402
from confluent_kafka.schema_registry import SchemaRegistryClient # noqa: E402
from confluent_kafka.schema_registry.avro import AvroDeserializer # noqa: E402
from confluent_kafka.serialization import SerializationContext, MessageField # noqa: E402
from common import settings # noqa: E402
from common.scenario import to_json # noqa: E402
STORES = ("open-sessions", "driver-totals")
def now():
return datetime.now().strftime("%H:%M:%S.%f")[:-3]
def read_to_end(topic, on_record, isolation="read_committed", start=None):
"""Read every partition of `topic` up to its current end; call on_record(msg) for each record.
`start` maps partition -> first offset (default: the beginning). Returns the end offsets."""
consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "restore",
"enable.auto.commit": False, "isolation.level": isolation})
partitions = consumer.list_topics(topic, timeout=10).topics[topic].partitions
ends = {p: consumer.get_watermark_offsets(TopicPartition(topic, p), timeout=10)[1] for p in partitions}
starts = {p: (start or {}).get(p, 0) for p in partitions}
consumer.assign([TopicPartition(topic, p, starts[p]) for p in partitions])
waiting = [TopicPartition(topic, p) for p in partitions if ends[p] > starts[p]]
while waiting:
msg = consumer.poll(0.5)
if msg is not None and not msg.error():
on_record(msg)
# Done when the position has passed the end: the last offsets may be transaction markers.
waiting = [tp for tp in consumer.position(waiting) if tp.offset < ends[tp.partition]]
consumer.close()
return ends
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--app", default="m07l02-pricing", help="names the group, transactional id and changelogs")
ap.add_argument("--sessions", default="m07l02-sessions")
ap.add_argument("--tariffs", default="m07l02_pricing.tariffs")
ap.add_argument("--charges", help="output topic (default: <app>-charges)")
ap.add_argument("--no-changelog", action="store_true", help="keep state in memory only")
ap.add_argument("--crash-after", type=int, default=0, help="die after pricing this many sessions")
args = ap.parse_args()
charges_topic = args.charges or f"{args.app}-charges"
changelog = {store: f"{args.app}-{store}-changelog" for store in STORES}
checkpoint = settings.LAB_ROOT / ".lab-data" / f"{args.app}.checkpoint.json"
stop = False
def on_signal(*_):
nonlocal stop
stop = True
signal.signal(signal.SIGINT, on_signal)
signal.signal(signal.SIGTERM, on_signal)
# 1. The tariffs table: every tariff, by site, from the compacted topic.
deserialize = AvroDeserializer(SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL}))
tariffs = {}
def apply_tariff(msg):
key = deserialize(msg.key(), SerializationContext(msg.topic(), MessageField.KEY))
row = deserialize(msg.value(), SerializationContext(msg.topic(), MessageField.VALUE)) if msg.value() else None
for site, tariff in list(tariffs.items()):
if tariff["tariff_id"] == key["tariff_id"]:
del tariffs[site] # deleted, or about to be replaced
if row:
tariffs[row["site_id"]] = row
return row
admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP}) # create the service's own topics
missing = [NewTopic(t, 3, 3, config={"cleanup.policy": "compact"}) for t in changelog.values()]
missing.append(NewTopic(charges_topic, 3, 3))
existing = admin.list_topics(timeout=10).topics
missing = [t for t in missing if t.topic not in existing]
for future in (admin.create_topics(missing).values() if missing else []):
future.result()
read_to_end(args.tariffs, apply_tariff)
table = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": f"{args.app}-tariffs",
"enable.auto.commit": False})
parts = table.list_topics(args.tariffs, timeout=10).topics[args.tariffs].partitions
table.assign([TopicPartition(args.tariffs, p, table.get_watermark_offsets(TopicPartition(args.tariffs, p), timeout=10)[1])
for p in parts]) # from here on: only changes
print(f"{now()} tariffs table loaded: {len(tariffs)} sites", flush=True)
# 2. The state: from the checkpoint and the changelogs, or from the changelogs only.
state = {store: {} for store in STORES}
started = time.monotonic()
if args.no_changelog:
how = "no changelog: starting with empty state"
else:
positions = {}
if checkpoint.exists():
saved = json.loads(checkpoint.read_text())
state, positions = saved["state"], {s: {int(p): o for p, o in saved["positions"][s].items()} for s in STORES}
checkpoint.unlink() # a crash from now on must not reuse it
how = "checkpoint + changelog records written since"
else:
how = "changelogs only, no checkpoint"
read = 0
for store in STORES:
def apply(msg, store=store):
nonlocal read
read += 1
key = msg.key().decode()
if msg.value() is None:
state[store].pop(key, None)
else:
state[store][key] = json.loads(msg.value())
read_to_end(changelog[store], apply, start=positions.get(store))
how += f", {read} changelog records read"
print(f"{now()} state restored in {time.monotonic() - started:.2f} s ({how}): "
f"{len(state['open-sessions'])} open sessions, {len(state['driver-totals'])} driver totals", flush=True)
# 3. Process sessions: charges, state changes and offsets in one transaction.
producer = Producer({**settings.PRODUCER_DEFAULTS, "transactional.id": f"{args.app}-1"})
producer.init_transactions(30)
consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.app,
"auto.offset.reset": "earliest", "enable.auto.commit": False,
"isolation.level": "read_committed", "session.timeout.ms": 6000})
consumer.subscribe([args.sessions])
def put(store, key, value):
if value is None:
state[store].pop(key, None)
else:
state[store][key] = value
if not args.no_changelog:
producer.produce(changelog[store], key=key, value=None if value is None else to_json(value))
priced = unknown = 0
idle_since = time.monotonic()
while not stop:
while (change := table.poll(0)) is not None: # keep the tariffs table up to date
if not change.error():
row = apply_tariff(change)
if row:
print(f"{now()} tariff changed: {row['site_id']} now {row['price_per_kwh']} EUR/kWh", flush=True)
msgs = consumer.consume(num_messages=50, timeout=1.0)
if not msgs:
if time.monotonic() - idle_since > 5:
print(f"{now()} idle: priced={priced} unknown={unknown}", flush=True)
idle_since = time.monotonic()
continue
idle_since = time.monotonic()
producer.begin_transaction()
for msg in msgs:
event = json.loads(msg.value())
if event["type"] == "session_started":
put("open-sessions", event["session_id"], {"driver_id": event["driver_id"], "site_id": event["site_id"]})
continue
session = state["open-sessions"].get(event["session_id"])
if session is None:
unknown += 1 # its start was never seen: lost state
continue
tariff = tariffs[session["site_id"]]
amount = round(event["energy_kwh"] * float(tariff["price_per_kwh"]), 2)
total = state["driver-totals"].get(session["driver_id"], {"sessions": 0, "amount_eur": 0.0})
put("driver-totals", session["driver_id"], {"sessions": total["sessions"] + 1,
"amount_eur": round(total["amount_eur"] + amount, 2)})
put("open-sessions", event["session_id"], None)
producer.produce(charges_topic, key=event["session_id"], value=to_json({
"session_id": event["session_id"], "driver_id": session["driver_id"], "site_id": session["site_id"],
"tariff_id": tariff["tariff_id"], "price_per_kwh": float(tariff["price_per_kwh"]),
"energy_kwh": event["energy_kwh"], "amount_eur": amount}))
priced += 1
if priced == args.crash_after:
print(f"{now()} CRASH after {priced} priced sessions", flush=True)
os._exit(1)
producer.send_offsets_to_transaction(consumer.position(consumer.assignment()),
consumer.consumer_group_metadata())
producer.commit_transaction()
consumer.close()
table.close()
if not args.no_changelog: # a clean stop: save a checkpoint
positions = {}
for store in STORES:
probe = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "checkpoint"})
topic_parts = probe.list_topics(changelog[store], timeout=10).topics[changelog[store]].partitions
positions[store] = {p: probe.get_watermark_offsets(TopicPartition(changelog[store], p), timeout=10)[1]
for p in topic_parts}
probe.close()
checkpoint.parent.mkdir(exist_ok=True)
checkpoint.write_text(json.dumps({"state": state, "positions": positions}))
print(f"{now()} stopped: priced={priced} unknown={unknown}", flush=True)
if __name__ == "__main__":
main()