pricing_service.py

Script · code/module-07/pricing_service.py

Download pricing_service.py

"""Pricing service: prices every finished session with its site's current tariff, and keeps a
running total per driver.

Inputs:  session events (JSON, keyed by charger id), and the tariffs table: a compacted topic with
         the latest tariff per id (Avro, written by the CDC connector of lesson 6.1).
State:   open sessions (session id -> driver and site) and driver totals, kept in memory. Every
         change is also written to a compacted changelog topic per store, in the same transaction
         as the charge and the consumer offsets.
Restart: after a clean stop the service loads its checkpoint file and reads only the changelog
         records written since; after a crash it rebuilds its state from the changelogs.
         --no-changelog keeps state in memory only, like a naive service.

The notebook of lesson 7.2 runs this script. From the code/ folder:

    python module-07/pricing_service.py --app m07l02-pricing --sessions m07l02-sessions --tariffs m07l02_pricing.tariffs
"""
import argparse
import json
import os
import signal
import sys
import time
from datetime import datetime
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer, Producer, TopicPartition  # noqa: E402
from confluent_kafka.admin import AdminClient, NewTopic  # noqa: E402
from confluent_kafka.schema_registry import SchemaRegistryClient  # noqa: E402
from confluent_kafka.schema_registry.avro import AvroDeserializer  # noqa: E402
from confluent_kafka.serialization import SerializationContext, MessageField  # noqa: E402
from common import settings  # noqa: E402
from common.scenario import to_json  # noqa: E402

STORES = ("open-sessions", "driver-totals")


def now():
    return datetime.now().strftime("%H:%M:%S.%f")[:-3]


def read_to_end(topic, on_record, isolation="read_committed", start=None):
    """Read every partition of `topic` up to its current end; call on_record(msg) for each record.
    `start` maps partition -> first offset (default: the beginning). Returns the end offsets."""
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "restore",
                         "enable.auto.commit": False, "isolation.level": isolation})
    partitions = consumer.list_topics(topic, timeout=10).topics[topic].partitions
    ends = {p: consumer.get_watermark_offsets(TopicPartition(topic, p), timeout=10)[1] for p in partitions}
    starts = {p: (start or {}).get(p, 0) for p in partitions}
    consumer.assign([TopicPartition(topic, p, starts[p]) for p in partitions])
    waiting = [TopicPartition(topic, p) for p in partitions if ends[p] > starts[p]]
    while waiting:
        msg = consumer.poll(0.5)
        if msg is not None and not msg.error():
            on_record(msg)
        # Done when the position has passed the end: the last offsets may be transaction markers.
        waiting = [tp for tp in consumer.position(waiting) if tp.offset < ends[tp.partition]]
    consumer.close()
    return ends


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--app", default="m07l02-pricing", help="names the group, transactional id and changelogs")
    ap.add_argument("--sessions", default="m07l02-sessions")
    ap.add_argument("--tariffs", default="m07l02_pricing.tariffs")
    ap.add_argument("--charges", help="output topic (default: <app>-charges)")
    ap.add_argument("--no-changelog", action="store_true", help="keep state in memory only")
    ap.add_argument("--crash-after", type=int, default=0, help="die after pricing this many sessions")
    args = ap.parse_args()
    charges_topic = args.charges or f"{args.app}-charges"
    changelog = {store: f"{args.app}-{store}-changelog" for store in STORES}
    checkpoint = settings.LAB_ROOT / ".lab-data" / f"{args.app}.checkpoint.json"

    stop = False

    def on_signal(*_):
        nonlocal stop
        stop = True

    signal.signal(signal.SIGINT, on_signal)
    signal.signal(signal.SIGTERM, on_signal)

    # 1. The tariffs table: every tariff, by site, from the compacted topic.
    deserialize = AvroDeserializer(SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL}))
    tariffs = {}

    def apply_tariff(msg):
        key = deserialize(msg.key(), SerializationContext(msg.topic(), MessageField.KEY))
        row = deserialize(msg.value(), SerializationContext(msg.topic(), MessageField.VALUE)) if msg.value() else None
        for site, tariff in list(tariffs.items()):
            if tariff["tariff_id"] == key["tariff_id"]:
                del tariffs[site]                    # deleted, or about to be replaced
        if row:
            tariffs[row["site_id"]] = row
        return row

    admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})   # create the service's own topics
    missing = [NewTopic(t, 3, 3, config={"cleanup.policy": "compact"}) for t in changelog.values()]
    missing.append(NewTopic(charges_topic, 3, 3))
    existing = admin.list_topics(timeout=10).topics
    missing = [t for t in missing if t.topic not in existing]
    for future in (admin.create_topics(missing).values() if missing else []):
        future.result()

    read_to_end(args.tariffs, apply_tariff)
    table = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": f"{args.app}-tariffs",
                      "enable.auto.commit": False})
    parts = table.list_topics(args.tariffs, timeout=10).topics[args.tariffs].partitions
    table.assign([TopicPartition(args.tariffs, p, table.get_watermark_offsets(TopicPartition(args.tariffs, p), timeout=10)[1])
                  for p in parts])                   # from here on: only changes
    print(f"{now()} tariffs table loaded: {len(tariffs)} sites", flush=True)

    # 2. The state: from the checkpoint and the changelogs, or from the changelogs only.
    state = {store: {} for store in STORES}
    started = time.monotonic()
    if args.no_changelog:
        how = "no changelog: starting with empty state"
    else:
        positions = {}
        if checkpoint.exists():
            saved = json.loads(checkpoint.read_text())
            state, positions = saved["state"], {s: {int(p): o for p, o in saved["positions"][s].items()} for s in STORES}
            checkpoint.unlink()                      # a crash from now on must not reuse it
            how = "checkpoint + changelog records written since"
        else:
            how = "changelogs only, no checkpoint"
        read = 0
        for store in STORES:
            def apply(msg, store=store):
                nonlocal read
                read += 1
                key = msg.key().decode()
                if msg.value() is None:
                    state[store].pop(key, None)
                else:
                    state[store][key] = json.loads(msg.value())
            read_to_end(changelog[store], apply, start=positions.get(store))
        how += f", {read} changelog records read"
    print(f"{now()} state restored in {time.monotonic() - started:.2f} s ({how}): "
          f"{len(state['open-sessions'])} open sessions, {len(state['driver-totals'])} driver totals", flush=True)

    # 3. Process sessions: charges, state changes and offsets in one transaction.
    producer = Producer({**settings.PRODUCER_DEFAULTS, "transactional.id": f"{args.app}-1"})
    producer.init_transactions(30)
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.app,
                         "auto.offset.reset": "earliest", "enable.auto.commit": False,
                         "isolation.level": "read_committed", "session.timeout.ms": 6000})
    consumer.subscribe([args.sessions])

    def put(store, key, value):
        if value is None:
            state[store].pop(key, None)
        else:
            state[store][key] = value
        if not args.no_changelog:
            producer.produce(changelog[store], key=key, value=None if value is None else to_json(value))

    priced = unknown = 0
    idle_since = time.monotonic()
    while not stop:
        while (change := table.poll(0)) is not None:     # keep the tariffs table up to date
            if not change.error():
                row = apply_tariff(change)
                if row:
                    print(f"{now()} tariff changed: {row['site_id']} now {row['price_per_kwh']} EUR/kWh", flush=True)
        msgs = consumer.consume(num_messages=50, timeout=1.0)
        if not msgs:
            if time.monotonic() - idle_since > 5:
                print(f"{now()} idle: priced={priced} unknown={unknown}", flush=True)
                idle_since = time.monotonic()
            continue
        idle_since = time.monotonic()
        producer.begin_transaction()
        for msg in msgs:
            event = json.loads(msg.value())
            if event["type"] == "session_started":
                put("open-sessions", event["session_id"], {"driver_id": event["driver_id"], "site_id": event["site_id"]})
                continue
            session = state["open-sessions"].get(event["session_id"])
            if session is None:
                unknown += 1                         # its start was never seen: lost state
                continue
            tariff = tariffs[session["site_id"]]
            amount = round(event["energy_kwh"] * float(tariff["price_per_kwh"]), 2)
            total = state["driver-totals"].get(session["driver_id"], {"sessions": 0, "amount_eur": 0.0})
            put("driver-totals", session["driver_id"], {"sessions": total["sessions"] + 1,
                                                        "amount_eur": round(total["amount_eur"] + amount, 2)})
            put("open-sessions", event["session_id"], None)
            producer.produce(charges_topic, key=event["session_id"], value=to_json({
                "session_id": event["session_id"], "driver_id": session["driver_id"], "site_id": session["site_id"],
                "tariff_id": tariff["tariff_id"], "price_per_kwh": float(tariff["price_per_kwh"]),
                "energy_kwh": event["energy_kwh"], "amount_eur": amount}))
            priced += 1
            if priced == args.crash_after:
                print(f"{now()} CRASH after {priced} priced sessions", flush=True)
                os._exit(1)
        producer.send_offsets_to_transaction(consumer.position(consumer.assignment()),
                                             consumer.consumer_group_metadata())
        producer.commit_transaction()

    consumer.close()
    table.close()
    if not args.no_changelog:                        # a clean stop: save a checkpoint
        positions = {}
        for store in STORES:
            probe = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "checkpoint"})
            topic_parts = probe.list_topics(changelog[store], timeout=10).topics[changelog[store]].partitions
            positions[store] = {p: probe.get_watermark_offsets(TopicPartition(changelog[store], p), timeout=10)[1]
                                for p in topic_parts}
            probe.close()
        checkpoint.parent.mkdir(exist_ok=True)
        checkpoint.write_text(json.dumps({"state": state, "positions": positions}))
    print(f"{now()} stopped: priced={priced} unknown={unknown}", flush=True)


if __name__ == "__main__":
    main()