steady_writer.py

Script · code/module-04/steady_writer.py

Download steady_writer.py

"""Steady writer: writes meter readings at a fixed rate and reports what happened every second.

Each reading carries a sequence number ("seq"), so a reader can check later that every delivered
reading is in the topic exactly once. The writer uses the course's safe producer settings (acks=all,
idempotence on). Run it from the code/ folder and stop it with Ctrl+C:

    python module-04/steady_writer.py --topic m04l01-telemetry --rate 50
"""
import argparse
import signal
import sys
import time
from collections import Counter
from datetime import datetime
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Producer  # noqa: E402
from common import settings  # noqa: E402
from common.scenario import Brightlane, to_json  # noqa: E402


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--topic", default="m04l01-telemetry", help="topic to write to (must exist)")
    ap.add_argument("--rate", type=float, default=50, help="readings per second (default 50)")
    ap.add_argument("--duration", type=float, default=0, help="stop after this many seconds (0 = until Ctrl+C)")
    ap.add_argument("--timeout-ms", type=int, default=120000,
                    help="message.timeout.ms: give up on a reading after this long (default 120000)")
    args = ap.parse_args()

    stop = False

    def on_signal(*_):
        nonlocal stop
        stop = True

    signal.signal(signal.SIGINT, on_signal)
    signal.signal(signal.SIGTERM, on_signal)

    producer = Producer({**settings.PRODUCER_DEFAULTS, "message.timeout.ms": args.timeout_ms})
    readings = [(key, e) for key, e in Brightlane().stream(sessions=200) if "type" not in e]   # readings only
    print(f"{datetime.now():%H:%M:%S} steady writer: {args.rate:g} readings/s -> {args.topic}", flush=True)

    second = {"delivered": 0, "slowest_ms": 0.0, "errors": Counter()}
    totals = Counter()

    def report(err, msg, sent_at):
        if err:
            second["errors"][err.name()] += 1
            totals["failed"] += 1
        else:
            second["delivered"] += 1
            second["slowest_ms"] = max(second["slowest_ms"], (time.monotonic() - sent_at) * 1000)
            totals["delivered"] += 1

    start = last = time.monotonic()
    seq = sent_this_second = 0
    while not stop and (not args.duration or time.monotonic() - start < args.duration):
        time.sleep(max(0.0, start + seq / args.rate - time.monotonic()))   # keep the steady rate
        key, reading = readings[seq % len(readings)]
        sent_at = time.monotonic()
        producer.produce(args.topic, key=key, value=to_json({**reading, "seq": seq}),
                         on_delivery=lambda err, msg, t=sent_at: report(err, msg, t))
        producer.poll(0)
        seq += 1
        sent_this_second += 1
        if time.monotonic() - last >= 1:
            last = time.monotonic()
            errors = " ".join(f"{k}={v}" for k, v in second["errors"].items())
            print(f"{datetime.now():%H:%M:%S} sent={sent_this_second} delivered={second['delivered']} "
                  f"waiting={len(producer)} slowest={second['slowest_ms']:.0f}ms {errors}".rstrip(), flush=True)
            second = {"delivered": 0, "slowest_ms": 0.0, "errors": Counter()}
            sent_this_second = 0

    left = producer.flush(30)
    print(f"{datetime.now():%H:%M:%S} stopped: sent={seq} delivered={totals['delivered']} "
          f"failed={totals['failed']} not answered={left}", flush=True)


if __name__ == "__main__":
    main()