"""Billing worker: one member of a consumer group that charges finished charging sessions.
It reads session events. For each session_stopped it computes the charge and, with --ledger, writes
it to a SQLite ledger. --work-ms simulates how long handling one event takes. Run several copies
with different --name values to share the work. From the code/ folder:
python module-03/billing_worker.py --name A --topic m03l01-sessions --work-ms 60
Stop it with Ctrl+C. The options for offset commits (--commit), the group protocol (--protocol),
static membership (--static) and a slow handler (--stall-at) are explained in lessons 3.2 and 3.3.
"""
import argparse
import json
import signal
import sqlite3
import sys
import time
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer, KafkaException # noqa: E402
from common import settings # noqa: E402
PRICE_PER_KWH = 0.49 # EUR; lesson 7.2 takes prices from the tariffs table instead
def now():
return datetime.now().strftime("%H:%M:%S.%f")[:-3]
def open_ledger(path, idempotent):
db = sqlite3.connect(path, timeout=30, isolation_level=None)
db.execute("PRAGMA journal_mode=WAL")
key = "PRIMARY KEY" if idempotent else ""
db.execute(f"CREATE TABLE IF NOT EXISTS charges (session_id TEXT {key}, charger_id TEXT, "
"energy_kwh REAL, amount_eur REAL, worker TEXT, charged_at TEXT)")
return db
def consumer_config(args):
config = {"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.group,
"client.id": args.name, "auto.offset.reset": "earliest"}
if args.protocol == "consumer":
config["group.protocol"] = "consumer" # KIP-848: the broker assigns partitions
elif args.assignor:
config["partition.assignment.strategy"] = args.assignor
if args.static:
config["group.instance.id"] = args.name # same id after a restart: no rebalance
if args.max_poll_ms:
config["max.poll.interval.ms"] = args.max_poll_ms
if args.session_timeout_ms: # classic protocol only
config["session.timeout.ms"] = args.session_timeout_ms
if args.commit == "auto": # the client's defaults
pass
elif args.commit == "store": # auto-commit only what we've processed
config["enable.auto.offset.store"] = False
else: # "after" or "before": commit ourselves
config["enable.auto.commit"] = False
return config
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--name", required=True, help="worker name, also used as client.id")
ap.add_argument("--topic", default="m03l01-sessions")
ap.add_argument("--group", default="billing")
ap.add_argument("--work-ms", type=float, default=0, help="time to handle one event (default 0)")
ap.add_argument("--batch", type=int, default=1, help="events fetched and handled per batch (default 1)")
ap.add_argument("--commit", choices=["auto", "store", "after", "before"], default="auto",
help="how offsets are committed (lesson 3.2; default: the client's auto-commit)")
ap.add_argument("--ledger", help="SQLite file to write charges to")
ap.add_argument("--idempotent", action="store_true", help="ledger ignores a session already charged")
ap.add_argument("--protocol", choices=["classic", "consumer"], default="classic")
ap.add_argument("--assignor", help="classic protocol only, e.g. cooperative-sticky")
ap.add_argument("--static", action="store_true", help="static membership: group.instance.id = name")
ap.add_argument("--max-poll-ms", type=int, help="max.poll.interval.ms")
ap.add_argument("--session-timeout-ms", type=int, help="session.timeout.ms (classic protocol only)")
ap.add_argument("--stall-at", type=int, default=0, help="after this many events, stall once")
ap.add_argument("--stall-ms", type=int, default=0, help="how long the stall lasts")
args = ap.parse_args()
stop = False
def on_signal(*_):
nonlocal stop
stop = True
signal.signal(signal.SIGINT, on_signal)
signal.signal(signal.SIGTERM, on_signal)
def on_assign(consumer, partitions):
print(f"{now()} {args.name} ASSIGNED {sorted(p.partition for p in partitions)}", flush=True)
def on_revoke(consumer, partitions):
print(f"{now()} {args.name} REVOKED {sorted(p.partition for p in partitions)}", flush=True)
def on_lost(consumer, partitions):
print(f"{now()} {args.name} LOST {sorted(p.partition for p in partitions)}", flush=True)
ledger = open_ledger(args.ledger, args.idempotent) if args.ledger else None
consumer = Consumer(consumer_config(args))
consumer.subscribe([args.topic], on_assign=on_assign, on_revoke=on_revoke, on_lost=on_lost)
print(f"{now()} {args.name} joined group {args.group} on {args.topic} "
f"(protocol={args.protocol}, commit={args.commit})", flush=True)
processed = charged = 0
last_report = time.monotonic()
while not stop:
msgs = consumer.consume(num_messages=args.batch, timeout=0.5)
if time.monotonic() - last_report >= 2:
last_report = time.monotonic()
print(f"{now()} {args.name} processed={processed} charged={charged}", flush=True)
if not msgs:
continue
try:
if args.commit == "before": # commit first, then process
consumer.commit(asynchronous=False)
for msg in msgs:
if msg.error():
print(f"{now()} {args.name} ERROR {msg.error()}", flush=True)
continue
event = json.loads(msg.value())
if args.work_ms:
time.sleep(args.work_ms / 1000)
if event["type"] == "session_stopped":
amount = round(event["energy_kwh"] * PRICE_PER_KWH, 2)
if ledger:
verb = "INSERT OR IGNORE" if args.idempotent else "INSERT"
ledger.execute(f"{verb} INTO charges VALUES (?, ?, ?, ?, ?, ?)",
(event["session_id"], event["charger_id"], event["energy_kwh"],
amount, args.name, now()))
charged += 1
processed += 1
if args.commit == "store": # mark as done; auto-commit sends it later
consumer.store_offsets(message=msg)
if args.commit == "after": # process first, then commit
consumer.commit(asynchronous=False)
except KafkaException as e:
print(f"{now()} {args.name} ERROR {e}", flush=True)
if args.stall_at and processed >= args.stall_at and args.stall_ms:
print(f"{now()} {args.name} STALLING for {args.stall_ms} ms", flush=True)
time.sleep(args.stall_ms / 1000)
args.stall_ms = 0 # stall only once
consumer.close() # commits stored offsets (auto/store modes) and leaves the group
print(f"{now()} {args.name} stopped: processed={processed} charged={charged}", flush=True)
if __name__ == "__main__":
main()