"""Billing service: reads session events and writes one charge per finished session.
--mode plain write the charges, then commit the consumer's offsets: at-least-once
--mode txn write the charges and the consumer's offsets in one Kafka transaction: exactly once
--crash-after N makes the service die (like kill -9) right after its N-th charge reached the
broker, before anything is committed. The notebook of lesson 7.1 runs this script. From the
code/ folder:
python module-07/billing_service.py --mode txn --sessions m07l01-sessions --charges m07l01-charges
"""
import argparse
import json
import os
import signal
import sys
import time
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import OFFSET_BEGINNING, Consumer, KafkaError, KafkaException, Producer, TopicPartition # noqa: E402
from common import settings # noqa: E402
from common.scenario import to_json # noqa: E402
PRICE_PER_KWH = 0.49
def now():
return datetime.now().strftime("%H:%M:%S.%f")[:-3]
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--mode", choices=["plain", "txn"], default="txn")
ap.add_argument("--sessions", default="m07l01-sessions", help="topic with session events")
ap.add_argument("--charges", default="m07l01-charges", help="topic the charges go to")
ap.add_argument("--group", default="m07l01-billing")
ap.add_argument("--txn-id", default="billing-1", help="transactional.id (txn mode)")
ap.add_argument("--name", default="billing")
ap.add_argument("--batch", type=int, default=50, help="events per transaction (default 50)")
ap.add_argument("--work-ms", type=float, default=0, help="time to handle one event")
ap.add_argument("--crash-after", type=int, default=0, help="die after this many charges")
args = ap.parse_args()
txn = args.mode == "txn"
stop = False
def on_signal(*_):
nonlocal stop
stop = True
signal.signal(signal.SIGINT, on_signal)
signal.signal(signal.SIGTERM, on_signal)
consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.group,
"auto.offset.reset": "earliest", "enable.auto.commit": False,
"isolation.level": "read_committed", "session.timeout.ms": 6000})
if txn:
producer = Producer({**settings.PRODUCER_DEFAULTS, "transactional.id": args.txn_id})
producer.init_transactions(30) # fences any older instance with the same id
else:
producer = Producer(settings.PRODUCER_DEFAULTS)
consumer.subscribe([args.sessions])
print(f"{now()} {args.name} started: mode={args.mode} transactional.id={args.txn_id if txn else '-'}", flush=True)
def rewind():
"""Go back to the last committed offsets, so an aborted batch is read again."""
for tp in consumer.committed(consumer.assignment(), timeout=10):
consumer.seek(tp if tp.offset >= 0 else TopicPartition(tp.topic, tp.partition, OFFSET_BEGINNING))
charged = 0
idle_since = time.monotonic()
while not stop:
msgs = consumer.consume(num_messages=args.batch, timeout=1.0)
if not msgs:
if time.monotonic() - idle_since > 5:
print(f"{now()} {args.name} idle: charged={charged}", flush=True)
idle_since = time.monotonic()
continue
idle_since = time.monotonic()
try:
if txn:
producer.begin_transaction()
for msg in msgs:
event = json.loads(msg.value())
time.sleep(args.work_ms / 1000)
if event["type"] != "session_stopped":
continue
charge = {"session_id": event["session_id"], "charger_id": event["charger_id"],
"energy_kwh": event["energy_kwh"], "price_per_kwh": PRICE_PER_KWH,
"amount_eur": round(event["energy_kwh"] * PRICE_PER_KWH, 2), "billed_by": args.name}
producer.produce(args.charges, key=event["session_id"], value=to_json(charge))
charged += 1
if charged == args.crash_after:
producer.flush(10) # the charges are on the broker ...
print(f"{now()} {args.name} CRASH after {charged} charges", flush=True)
os._exit(1) # ... and the service dies before committing
if txn:
producer.send_offsets_to_transaction(consumer.position(consumer.assignment()),
consumer.consumer_group_metadata())
producer.commit_transaction() # charges and offsets: both or neither
else:
producer.flush(10)
consumer.commit(asynchronous=False)
except KafkaException as e:
error = e.args[0]
if error.fatal() or error.code() == KafkaError._FATAL: # e.g. fenced by a newer instance
try:
producer.poll(0) # raises the underlying fatal error
except KafkaException as fatal:
error = fatal.args[0]
print(f"{now()} {args.name} FATAL {error.name()}: {error.str()}", flush=True)
break
if error.txn_requires_abort():
print(f"{now()} {args.name} ABORT {error.name()}: {error.str()}", flush=True)
producer.abort_transaction()
rewind()
continue
raise
consumer.close()
print(f"{now()} {args.name} stopped: charged={charged}", flush=True)
if __name__ == "__main__":
main()