billing_service.py

Script · code/module-07/billing_service.py

Download billing_service.py

"""Billing service: reads session events and writes one charge per finished session.

--mode plain   write the charges, then commit the consumer's offsets: at-least-once
--mode txn     write the charges and the consumer's offsets in one Kafka transaction: exactly once

--crash-after N makes the service die (like kill -9) right after its N-th charge reached the
broker, before anything is committed. The notebook of lesson 7.1 runs this script. From the
code/ folder:

    python module-07/billing_service.py --mode txn --sessions m07l01-sessions --charges m07l01-charges
"""
import argparse
import json
import os
import signal
import sys
import time
from datetime import datetime
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import OFFSET_BEGINNING, Consumer, KafkaError, KafkaException, Producer, TopicPartition  # noqa: E402
from common import settings  # noqa: E402
from common.scenario import to_json  # noqa: E402

PRICE_PER_KWH = 0.49


def now():
    return datetime.now().strftime("%H:%M:%S.%f")[:-3]


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--mode", choices=["plain", "txn"], default="txn")
    ap.add_argument("--sessions", default="m07l01-sessions", help="topic with session events")
    ap.add_argument("--charges", default="m07l01-charges", help="topic the charges go to")
    ap.add_argument("--group", default="m07l01-billing")
    ap.add_argument("--txn-id", default="billing-1", help="transactional.id (txn mode)")
    ap.add_argument("--name", default="billing")
    ap.add_argument("--batch", type=int, default=50, help="events per transaction (default 50)")
    ap.add_argument("--work-ms", type=float, default=0, help="time to handle one event")
    ap.add_argument("--crash-after", type=int, default=0, help="die after this many charges")
    args = ap.parse_args()
    txn = args.mode == "txn"

    stop = False

    def on_signal(*_):
        nonlocal stop
        stop = True

    signal.signal(signal.SIGINT, on_signal)
    signal.signal(signal.SIGTERM, on_signal)

    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.group,
                         "auto.offset.reset": "earliest", "enable.auto.commit": False,
                         "isolation.level": "read_committed", "session.timeout.ms": 6000})
    if txn:
        producer = Producer({**settings.PRODUCER_DEFAULTS, "transactional.id": args.txn_id})
        producer.init_transactions(30)               # fences any older instance with the same id
    else:
        producer = Producer(settings.PRODUCER_DEFAULTS)
    consumer.subscribe([args.sessions])
    print(f"{now()} {args.name} started: mode={args.mode} transactional.id={args.txn_id if txn else '-'}", flush=True)

    def rewind():
        """Go back to the last committed offsets, so an aborted batch is read again."""
        for tp in consumer.committed(consumer.assignment(), timeout=10):
            consumer.seek(tp if tp.offset >= 0 else TopicPartition(tp.topic, tp.partition, OFFSET_BEGINNING))

    charged = 0
    idle_since = time.monotonic()
    while not stop:
        msgs = consumer.consume(num_messages=args.batch, timeout=1.0)
        if not msgs:
            if time.monotonic() - idle_since > 5:
                print(f"{now()} {args.name} idle: charged={charged}", flush=True)
                idle_since = time.monotonic()
            continue
        idle_since = time.monotonic()
        try:
            if txn:
                producer.begin_transaction()
            for msg in msgs:
                event = json.loads(msg.value())
                time.sleep(args.work_ms / 1000)
                if event["type"] != "session_stopped":
                    continue
                charge = {"session_id": event["session_id"], "charger_id": event["charger_id"],
                          "energy_kwh": event["energy_kwh"], "price_per_kwh": PRICE_PER_KWH,
                          "amount_eur": round(event["energy_kwh"] * PRICE_PER_KWH, 2), "billed_by": args.name}
                producer.produce(args.charges, key=event["session_id"], value=to_json(charge))
                charged += 1
                if charged == args.crash_after:
                    producer.flush(10)                   # the charges are on the broker ...
                    print(f"{now()} {args.name} CRASH after {charged} charges", flush=True)
                    os._exit(1)                          # ... and the service dies before committing
            if txn:
                producer.send_offsets_to_transaction(consumer.position(consumer.assignment()),
                                                     consumer.consumer_group_metadata())
                producer.commit_transaction()            # charges and offsets: both or neither
            else:
                producer.flush(10)
                consumer.commit(asynchronous=False)
        except KafkaException as e:
            error = e.args[0]
            if error.fatal() or error.code() == KafkaError._FATAL:   # e.g. fenced by a newer instance
                try:
                    producer.poll(0)                     # raises the underlying fatal error
                except KafkaException as fatal:
                    error = fatal.args[0]
                print(f"{now()} {args.name} FATAL {error.name()}: {error.str()}", flush=True)
                break
            if error.txn_requires_abort():
                print(f"{now()} {args.name} ABORT {error.name()}: {error.str()}", flush=True)
                producer.abort_transaction()
                rewind()
                continue
            raise

    consumer.close()
    print(f"{now()} {args.name} stopped: charged={charged}", flush=True)


if __name__ == "__main__":
    main()