"""Metered billing worker: charges finished sessions and exports its own metrics for Prometheus.
The Python client has no JMX. Instead, with statistics.interval.ms set, librdkafka hands the
application a JSON document of statistics every few seconds; this worker turns the parts that
matter into Prometheus metrics on http://127.0.0.1:8001/metrics, next to its own business
metrics. The notebook of lesson 8.3 runs it. From the code/ folder:
python module-08/metered_worker.py --topic m08l03-sessions --group m08l03-billing
"""
import argparse
import json
import signal
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer # noqa: E402
from prometheus_client import Counter, Gauge, start_http_server # noqa: E402
from common import settings # noqa: E402
CHARGED = Counter("billing_sessions_charged", "Finished sessions charged")
LAG = Gauge("billing_consumer_lag", "Records between the last one processed and the end of the partition", ["partition"])
EVENT_AGE = Gauge("billing_last_event_age_seconds", "Age of the last processed event when it was processed")
REBALANCES = Counter("billing_rebalances", "Partition assignments received")
def on_stats(raw):
"""Turn librdkafka's statistics JSON into Prometheus metrics."""
stats = json.loads(raw)
for topic in stats.get("topics", {}).values():
for partition, p in topic["partitions"].items():
if partition != "-1" and p.get("consumer_lag_stored", -1) >= 0: # from the last processed record
LAG.labels(partition).set(p["consumer_lag_stored"])
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--topic", default="m08l03-sessions")
ap.add_argument("--group", default="m08l03-billing")
ap.add_argument("--work-ms", type=float, default=10, help="time to handle one event (default 10)")
ap.add_argument("--port", type=int, default=8001, help="metrics port (default 8001)")
args = ap.parse_args()
stop = False
def on_signal(*_):
nonlocal stop
stop = True
signal.signal(signal.SIGINT, on_signal)
signal.signal(signal.SIGTERM, on_signal)
start_http_server(args.port)
consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.group,
"auto.offset.reset": "earliest", "enable.auto.offset.store": False,
"statistics.interval.ms": 5000, "stats_cb": on_stats})
consumer.subscribe([args.topic], on_assign=lambda c, parts: REBALANCES.inc())
print(f"metered worker: {args.topic} as {args.group}, metrics on :{args.port}/metrics", flush=True)
while not stop:
msg = consumer.poll(1.0)
if msg is None or msg.error():
continue
event = json.loads(msg.value())
time.sleep(args.work_ms / 1000)
if event["type"] == "session_stopped":
CHARGED.inc()
EVENT_AGE.set(time.time() - msg.timestamp()[1] / 1000)
consumer.store_offsets(message=msg)
consumer.close()
print("metered worker stopped", flush=True)
if __name__ == "__main__":
main()