metered_worker.py

Script · code/module-08/metered_worker.py

Download metered_worker.py

"""Metered billing worker: charges finished sessions and exports its own metrics for Prometheus.

The Python client has no JMX. Instead, with statistics.interval.ms set, librdkafka hands the
application a JSON document of statistics every few seconds; this worker turns the parts that
matter into Prometheus metrics on http://127.0.0.1:8001/metrics, next to its own business
metrics. The notebook of lesson 8.3 runs it. From the code/ folder:

    python module-08/metered_worker.py --topic m08l03-sessions --group m08l03-billing
"""
import argparse
import json
import signal
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer  # noqa: E402
from prometheus_client import Counter, Gauge, start_http_server  # noqa: E402
from common import settings  # noqa: E402

CHARGED = Counter("billing_sessions_charged", "Finished sessions charged")
LAG = Gauge("billing_consumer_lag", "Records between the last one processed and the end of the partition", ["partition"])
EVENT_AGE = Gauge("billing_last_event_age_seconds", "Age of the last processed event when it was processed")
REBALANCES = Counter("billing_rebalances", "Partition assignments received")


def on_stats(raw):
    """Turn librdkafka's statistics JSON into Prometheus metrics."""
    stats = json.loads(raw)
    for topic in stats.get("topics", {}).values():
        for partition, p in topic["partitions"].items():
            if partition != "-1" and p.get("consumer_lag_stored", -1) >= 0:   # from the last processed record
                LAG.labels(partition).set(p["consumer_lag_stored"])


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--topic", default="m08l03-sessions")
    ap.add_argument("--group", default="m08l03-billing")
    ap.add_argument("--work-ms", type=float, default=10, help="time to handle one event (default 10)")
    ap.add_argument("--port", type=int, default=8001, help="metrics port (default 8001)")
    args = ap.parse_args()

    stop = False

    def on_signal(*_):
        nonlocal stop
        stop = True

    signal.signal(signal.SIGINT, on_signal)
    signal.signal(signal.SIGTERM, on_signal)

    start_http_server(args.port)
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.group,
                         "auto.offset.reset": "earliest", "enable.auto.offset.store": False,
                         "statistics.interval.ms": 5000, "stats_cb": on_stats})
    consumer.subscribe([args.topic], on_assign=lambda c, parts: REBALANCES.inc())
    print(f"metered worker: {args.topic} as {args.group}, metrics on :{args.port}/metrics", flush=True)

    while not stop:
        msg = consumer.poll(1.0)
        if msg is None or msg.error():
            continue
        event = json.loads(msg.value())
        time.sleep(args.work_ms / 1000)
        if event["type"] == "session_stopped":
            CHARGED.inc()
        EVENT_AGE.set(time.time() - msg.timestamp()[1] / 1000)
        consumer.store_offsets(message=msg)

    consumer.close()
    print("metered worker stopped", flush=True)


if __name__ == "__main__":
    main()