"""Telemetry worker: reads Avro meter readings and adds each one to the site-load ledger (SQLite).
--mode naive any error stops the worker (and its restart meets the same record again)
--mode park-all any error sends the record to the dead-letter topic
--mode careful bad data goes to the dead-letter topic; a ledger that is unavailable is retried
--outage-at N --outage-s S simulates the ledger being unavailable ("database is locked") for S
seconds once N readings have been added. The notebook of lesson 5.3 runs this script. From the
code/ folder:
python module-05/telemetry_worker.py --mode careful --topic m05l03-telemetry --ledger .lab-data/m05l03.db
"""
import argparse
import signal
import sqlite3
import sys
import time
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer, Producer # noqa: E402
from confluent_kafka.schema_registry import SchemaRegistryClient # noqa: E402
from confluent_kafka.schema_registry.avro import AvroDeserializer # noqa: E402
from confluent_kafka.serialization import SerializationContext, MessageField, SerializationError # noqa: E402
from common import settings # noqa: E402
def now():
return datetime.now().strftime("%H:%M:%S.%f")[:-3]
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--mode", choices=["naive", "park-all", "careful"], default="careful")
ap.add_argument("--topic", default="m05l03-telemetry")
ap.add_argument("--group", default="m05l03-site-load")
ap.add_argument("--dlq", help="dead-letter topic (default: <topic>.dlq)")
ap.add_argument("--ledger", required=True, help="SQLite file the readings are added to")
ap.add_argument("--work-ms", type=float, default=20, help="time to handle one reading (default 20)")
ap.add_argument("--outage-at", type=int, default=0, help="ledger unavailable after this many readings")
ap.add_argument("--outage-s", type=float, default=0, help="for this many seconds")
args = ap.parse_args()
dlq = args.dlq or f"{args.topic}.dlq"
stop = False
def on_signal(*_):
nonlocal stop
stop = True
signal.signal(signal.SIGINT, on_signal)
signal.signal(signal.SIGTERM, on_signal)
db = sqlite3.connect(args.ledger, timeout=30, isolation_level=None)
db.execute("CREATE TABLE IF NOT EXISTS readings (charger_id TEXT, session_id TEXT, ts TEXT, power_kw REAL)")
outage_until = None
def add_to_ledger(reading):
nonlocal outage_until
if args.outage_at and added == args.outage_at and outage_until is None:
outage_until = time.monotonic() + args.outage_s # the outage starts now
if outage_until and time.monotonic() < outage_until:
raise sqlite3.OperationalError("database is locked")
db.execute("INSERT INTO readings VALUES (?, ?, ?, ?)", (reading["charger_id"], reading["session_id"],
reading["ts"].isoformat(), reading["power_kw"]))
def validate(reading):
"""Rules the schema can't express."""
if not 0 <= reading["power_kw"] <= 400:
raise ValueError(f"power_kw out of range: {reading['power_kw']}")
registry = SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL})
deserialize = AvroDeserializer(registry)
consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": args.group,
"auto.offset.reset": "earliest", "enable.auto.commit": False,
"session.timeout.ms": 6000})
consumer.subscribe([args.topic])
producer = Producer(settings.PRODUCER_DEFAULTS)
def park(msg, error):
"""Send the original bytes to the dead-letter topic, with the reason in headers."""
headers = [("dlq.error.class", type(error).__name__), ("dlq.error.message", str(error)[:500]),
("dlq.source.topic", msg.topic()), ("dlq.source.partition", str(msg.partition())),
("dlq.source.offset", str(msg.offset())), ("dlq.parked.at", now())]
producer.produce(dlq, key=msg.key(), value=msg.value(), headers=headers)
producer.flush(10) # stored before we move past the record
print(f"{now()} PARKED p{msg.partition()}@{msg.offset()}: {type(error).__name__}: {error}", flush=True)
print(f"{now()} worker started: mode={args.mode} topic={args.topic}", flush=True)
added = parked = 0
idle_since = time.monotonic()
while not stop:
msg = consumer.poll(1.0)
if msg is None:
if time.monotonic() - idle_since > 5:
print(f"{now()} idle: added={added} parked={parked}", flush=True)
idle_since = time.monotonic()
continue
idle_since = time.monotonic()
if msg.error():
print(f"{now()} ERROR {msg.error()}", flush=True)
continue
context = SerializationContext(msg.topic(), MessageField.VALUE)
time.sleep(args.work_ms / 1000)
if args.mode == "naive":
try:
reading = deserialize(msg.value(), context)
validate(reading)
add_to_ledger(reading)
except Exception as e:
print(f"{now()} CRASH on p{msg.partition()}@{msg.offset()}: {type(e).__name__}: {e}", flush=True)
raise # the process dies, like an unhandled error
elif args.mode == "park-all":
try:
reading = deserialize(msg.value(), context)
validate(reading)
add_to_ledger(reading)
added += 1
except Exception as e: # every error is treated as bad data
park(msg, e)
parked += 1
else: # careful
try:
reading = deserialize(msg.value(), context)
validate(reading)
except (SerializationError, ValueError) as e: # bad data: retrying won't help, so park it
park(msg, e)
parked += 1
consumer.commit(message=msg, asynchronous=False)
continue
attempt = 0
while True: # the ledger may be down: wait and retry
try:
add_to_ledger(reading)
break
except sqlite3.OperationalError as e:
attempt += 1
delay = min(0.25 * 2 ** attempt, 4)
print(f"{now()} RETRY {attempt} in {delay:.1f}s: {e}", flush=True)
time.sleep(delay)
added += 1
if args.mode == "naive":
added += 1
consumer.commit(message=msg, asynchronous=False) # one commit per record keeps the demo simple
consumer.close()
print(f"{now()} worker stopped: added={added} parked={parked}", flush=True)
if __name__ == "__main__":
main()