grid_report.py

Script · code/module-07/grid_report.py

Download grid_report.py

"""Grid report: energy per site in 5-minute windows, from meter readings.

--time arrival   put each reading in the window of the time it reached Kafka (processing time)
--time event     put each reading in the window of the time the charger measured it (event time);
                 a window closes when the stream time (the latest event time seen) passes its end
                 plus --grace seconds. Readings for a closed window are late: counted, not added.
--max-ahead S    (event time) ignore event times more than S seconds after the arrival time

It reads the topic from the beginning to its current end, writes every closed window to the
output topic, and prints a summary. The notebook of lesson 7.3 runs it. From the code/ folder:

    python module-07/grid_report.py --topic m07l03-telemetry --time event --grace 120
"""
import argparse
import json
import sys
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer, Producer, TopicPartition  # noqa: E402
from confluent_kafka.admin import AdminClient, NewTopic  # noqa: E402
from common import settings  # noqa: E402
from common.scenario import iso, parse_iso, to_json  # noqa: E402

READING_S = 10                                       # one reading covers 10 seconds of charging


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--topic", default="m07l03-telemetry")
    ap.add_argument("--output", help="topic for the closed windows (default: <topic>-site-load-5m)")
    ap.add_argument("--time", choices=["arrival", "event"], default="event")
    ap.add_argument("--window", type=int, default=300, help="window length in seconds (default 300)")
    ap.add_argument("--grace", type=int, default=120, help="grace period in seconds (default 120)")
    ap.add_argument("--max-ahead", type=int, default=0, help="ignore event times this far after arrival (0 = off)")
    ap.add_argument("--show-site", default="site-N03", help="site to print window by window")
    args = ap.parse_args()
    output = args.output or f"{args.topic}-site-load-5m"
    width = timedelta(seconds=args.window)

    def window_of(t):
        return datetime.fromtimestamp(t.timestamp() // args.window * args.window, tz=timezone.utc)

    open_windows = defaultdict(lambda: {"energy_kwh": 0.0, "readings": 0})   # (site, start) -> totals
    closed = {}                                       # (site, start) -> result
    late = defaultdict(int)                           # (site, start) -> readings that came too late
    ignored = 0
    stream_time = None
    admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
    if output not in admin.list_topics(timeout=10).topics:           # create the output topic once
        admin.create_topics([NewTopic(output, 1, 3)])[output].result()
    producer = Producer(settings.PRODUCER_DEFAULTS)

    def close_until(t):
        """Close every open window whose end (plus grace) is at or before t."""
        for site, start in sorted(k for k in open_windows if k[1] + width + timedelta(seconds=args.grace) <= t):
            totals = open_windows.pop((site, start))
            result = {"site_id": site, "window_start": iso(start), "window_end": iso(start + width),
                      "energy_kwh": round(totals["energy_kwh"], 3), "readings": totals["readings"]}
            closed[(site, start)] = result
            producer.produce(output, key=site, value=to_json(result))

    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l03-report",
                         "enable.auto.commit": False})
    end = consumer.get_watermark_offsets(TopicPartition(args.topic, 0), timeout=10)[1]
    consumer.assign([TopicPartition(args.topic, 0, 0)])
    for _ in range(end):
        msg = consumer.poll(10)
        reading = json.loads(msg.value())
        arrival = datetime.fromtimestamp(msg.timestamp()[1] / 1000, tz=timezone.utc)
        measured = parse_iso(reading["ts"])
        site = reading["charger_id"].replace("BL-", "site-")[:8]
        if args.time == "arrival":
            t = arrival
        else:
            if args.max_ahead and measured > arrival + timedelta(seconds=args.max_ahead):
                ignored += 1                          # a clock far in the future: don't trust it
                continue
            t = measured
        stream_time = t if stream_time is None or t > stream_time else stream_time
        start = window_of(t)
        if (site, start) in closed:
            late[(site, start)] += 1                  # its window has already been reported
        else:
            totals = open_windows[(site, start)]
            totals["energy_kwh"] += reading["power_kw"] * READING_S / 3600
            totals["readings"] += 1
        close_until(stream_time)
    consumer.close()
    close_until(datetime.max.replace(tzinfo=timezone.utc))   # end of input: close the rest
    producer.flush(30)

    print(f"time={args.time} grace={args.grace}s max-ahead={args.max_ahead or 'off'}")
    print(f"readings: {end}, windows reported: {len(closed)}, late readings: {sum(late.values())}, "
          f"ignored (clock ahead): {ignored}")
    starts = sorted({start for _, start in closed})
    for start in starts:                              # the chosen site, 17:55 to 18:25
        if "17:55" <= iso(start)[11:16] <= "18:20":
            result = closed.get((args.show_site, start), {"energy_kwh": 0.0, "readings": 0})
            print(f"{args.show_site} {iso(start)[11:16]}-{iso(start + width)[11:16]}  "
                  f"{result['energy_kwh']:6.2f} kWh  {result['readings']:4d} readings  "
                  f"late: {late[(args.show_site, start)]}")


if __name__ == "__main__":
    main()