"""DR billing: charges finished sessions and writes each charge to a ledger file.
Lesson 9.1 runs it first against the primary cluster, then, after the primary "burns down",
against the disaster-recovery cluster with the same consumer group. The ledger, one JSON line per
charge, is how the lesson counts sessions charged twice or not at all. From the code/ folder:
python module-09/dr_billing.py --cluster primary --topics '^m09l01-sessions$' --group m09l01-billing
python module-09/dr_billing.py --cluster dr --topics '^(primary\\.)?m09l01-sessions$' --group m09l01-billing
A topic pattern that starts with ^ is a regular expression: on the DR cluster billing reads the
copy of the primary's topic (primary.m09l01-sessions) and the topic the gateway writes to there.
"""
import argparse
import json
import signal
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer # noqa: E402
from common import settings # noqa: E402
CLUSTERS = {"primary": settings.KAFKA_BOOTSTRAP, "dr": settings.DR_BOOTSTRAP}
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--cluster", choices=CLUSTERS, default="primary")
ap.add_argument("--topics", default="^m09l01-sessions$", help="topic name or ^regex")
ap.add_argument("--group", default="m09l01-billing")
ap.add_argument("--ledger", default=".lab-data/m09l01-ledger.jsonl", help="file the charges go to")
ap.add_argument("--reset", default="earliest", help="auto.offset.reset for a group without offsets")
args = ap.parse_args()
stop = False
def on_signal(*_):
nonlocal stop
stop = True
signal.signal(signal.SIGINT, on_signal)
signal.signal(signal.SIGTERM, on_signal)
ledger = Path(settings.LAB_ROOT / args.ledger)
ledger.parent.mkdir(parents=True, exist_ok=True)
consumer = Consumer({"bootstrap.servers": CLUSTERS[args.cluster], "group.id": args.group,
"auto.offset.reset": args.reset,
"enable.auto.offset.store": False}) # offsets of processed records only
consumer.subscribe(args.topics.split(","),
on_assign=lambda c, parts: print(f"{time.strftime('%H:%M:%S')} assigned: " + ", ".join(
sorted({p.topic for p in parts})), flush=True))
print(f"{time.strftime('%H:%M:%S')} dr billing on {args.cluster} ({CLUSTERS[args.cluster]}) "
f"as {args.group}: {args.topics}", flush=True)
charged, last = 0, time.monotonic()
with ledger.open("a") as out:
while not stop:
msg = consumer.poll(1.0)
if msg is not None and not msg.error():
event = json.loads(msg.value())
if event["type"] == "session_stopped":
out.write(json.dumps({"session_id": event["session_id"], "cluster": args.cluster,
"topic": msg.topic(), "offset": msg.offset()}) + "\n")
out.flush()
charged += 1
consumer.store_offsets(message=msg) # committed every 5 s (auto-commit)
if time.monotonic() - last >= 5:
last = time.monotonic()
print(f"{time.strftime('%H:%M:%S')} charged={charged}", flush=True)
consumer.close()
print(f"{time.strftime('%H:%M:%S')} dr billing stopped: charged={charged}", flush=True)
if __name__ == "__main__":
main()