telemetry_worker.py

Script · code/module-05/telemetry_worker.py

Download telemetry_worker.py

"""Telemetry worker: reads Avro meter readings and adds each one to the site-load ledger (SQLite).

--mode naive      any error stops the worker (and its restart meets the same record again)
--mode park-all   any error sends the record to the dead-letter topic
--mode careful    bad data goes to the dead-letter topic; a ledger that is unavailable is retried

--outage-at N --outage-s S simulates the ledger being unavailable ("database is locked") for S
seconds once N readings have been added. The notebook of lesson 5.3 runs this script. From the
code/ folder:

    python module-05/telemetry_worker.py --mode careful --topic m05l03-telemetry --ledger .lab-data/m05l03.db
"""
import argparse
import signal
import sqlite3
import sys
import time
from datetime import datetime
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer, Producer  # noqa: E402
from confluent_kafka.schema_registry import SchemaRegistryClient  # noqa: E402
from confluent_kafka.schema_registry.avro import AvroDeserializer  # noqa: E402
from confluent_kafka.serialization import SerializationContext, MessageField, SerializationError  # noqa: E402
from common import settings  # noqa: E402


def now():
    return datetime.now().strftime("%H:%M:%S.%f")[:-3]


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--mode", choices=["naive", "park-all", "careful"], default="careful")
    ap.add_argument("--topic", default="m05l03-telemetry")
    ap.add_argument("--group", default="m05l03-site-load")
    ap.add_argument("--dlq", help="dead-letter topic (default: <topic>.dlq)")
    ap.add_argument("--ledger", required=True, help="SQLite file the readings are added to")
    ap.add_argument("--work-ms", type=float, default=20, help="time to handle one reading (default 20)")
    ap.add_argument("--outage-at", type=int, default=0, help="ledger unavailable after this many readings")
    ap.add_argument("--outage-s", type=float, default=0, help="for this many seconds")
    args = ap.parse_args()
    dlq = args.dlq or f"{args.topic}.dlq"

    stop = False

    def on_signal(*_):
        nonlocal stop
        stop = True

    signal.signal(signal.SIGINT, on_signal)
    signal.signal(signal.SIGTERM, on_signal)

    db = sqlite3.connect(args.ledger, timeout=30, isolation_level=None)
    db.execute("CREATE TABLE IF NOT EXISTS readings (charger_id TEXT, session_id TEXT, ts TEXT, power_kw REAL)")
    outage_until = None

    def add_to_ledger(reading):
        nonlocal outage_until
        if args.outage_at and added == args.outage_at and outage_until is None:
            outage_until = time.monotonic() + args.outage_s              # the outage starts now
        if outage_until and time.monotonic() < outage_until:
            raise sqlite3.OperationalError("database is locked")
        db.execute("INSERT INTO readings VALUES (?, ?, ?, ?)", (reading["charger_id"], reading["session_id"],
                                                                reading["ts"].isoformat(), reading["power_kw"]))

    def validate(reading):
        """Rules the schema can't express."""
        if not 0 <= reading["power_kw"] <= 400:
            raise ValueError(f"power_kw out of range: {reading['power_kw']}")

    registry = SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL})
    deserialize = AvroDeserializer(registry)
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.group,
                         "auto.offset.reset": "earliest", "enable.auto.commit": False,
                         "session.timeout.ms": 6000})
    consumer.subscribe([args.topic])
    producer = Producer(settings.PRODUCER_DEFAULTS)

    def park(msg, error):
        """Send the original bytes to the dead-letter topic, with the reason in headers."""
        headers = [("dlq.error.class", type(error).__name__), ("dlq.error.message", str(error)[:500]),
                   ("dlq.source.topic", msg.topic()), ("dlq.source.partition", str(msg.partition())),
                   ("dlq.source.offset", str(msg.offset())), ("dlq.parked.at", now())]
        producer.produce(dlq, key=msg.key(), value=msg.value(), headers=headers)
        producer.flush(10)                                   # stored before we move past the record
        print(f"{now()} PARKED p{msg.partition()}@{msg.offset()}: {type(error).__name__}: {error}", flush=True)

    print(f"{now()} worker started: mode={args.mode} topic={args.topic}", flush=True)
    added = parked = 0
    idle_since = time.monotonic()
    while not stop:
        msg = consumer.poll(1.0)
        if msg is None:
            if time.monotonic() - idle_since > 5:
                print(f"{now()} idle: added={added} parked={parked}", flush=True)
                idle_since = time.monotonic()
            continue
        idle_since = time.monotonic()
        if msg.error():
            print(f"{now()} ERROR {msg.error()}", flush=True)
            continue
        context = SerializationContext(msg.topic(), MessageField.VALUE)
        time.sleep(args.work_ms / 1000)

        if args.mode == "naive":
            try:
                reading = deserialize(msg.value(), context)
                validate(reading)
                add_to_ledger(reading)
            except Exception as e:
                print(f"{now()} CRASH on p{msg.partition()}@{msg.offset()}: {type(e).__name__}: {e}", flush=True)
                raise                                        # the process dies, like an unhandled error

        elif args.mode == "park-all":
            try:
                reading = deserialize(msg.value(), context)
                validate(reading)
                add_to_ledger(reading)
                added += 1
            except Exception as e:                           # every error is treated as bad data
                park(msg, e)
                parked += 1

        else:                                                # careful
            try:
                reading = deserialize(msg.value(), context)
                validate(reading)
            except (SerializationError, ValueError) as e:    # bad data: retrying won't help, so park it
                park(msg, e)
                parked += 1
                consumer.commit(message=msg, asynchronous=False)
                continue
            attempt = 0
            while True:                                      # the ledger may be down: wait and retry
                try:
                    add_to_ledger(reading)
                    break
                except sqlite3.OperationalError as e:
                    attempt += 1
                    delay = min(0.25 * 2 ** attempt, 4)
                    print(f"{now()} RETRY {attempt} in {delay:.1f}s: {e}", flush=True)
                    time.sleep(delay)
            added += 1

        if args.mode == "naive":
            added += 1
        consumer.commit(message=msg, asynchronous=False)    # one commit per record keeps the demo simple

    consumer.close()
    print(f"{now()} worker stopped: added={added} parked={parked}", flush=True)


if __name__ == "__main__":
    main()