"""Grid report: energy per site in 5-minute windows, from meter readings.
--time arrival put each reading in the window of the time it reached Kafka (processing time)
--time event put each reading in the window of the time the charger measured it (event time);
a window closes when the stream time (the latest event time seen) passes its end
plus --grace seconds. Readings for a closed window are late: counted, not added.
--max-ahead S (event time) ignore event times more than S seconds after the arrival time
It reads the topic from the beginning to its current end, writes every closed window to the
output topic, and prints a summary. The notebook of lesson 7.3 runs it. From the code/ folder:
python module-07/grid_report.py --topic m07l03-telemetry --time event --grace 120
"""
import argparse
import json
import sys
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from confluent_kafka import Consumer, Producer, TopicPartition # noqa: E402
from confluent_kafka.admin import AdminClient, NewTopic # noqa: E402
from common import settings # noqa: E402
from common.scenario import iso, parse_iso, to_json # noqa: E402
READING_S = 10 # one reading covers 10 seconds of charging
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--topic", default="m07l03-telemetry")
ap.add_argument("--output", help="topic for the closed windows (default: <topic>-site-load-5m)")
ap.add_argument("--time", choices=["arrival", "event"], default="event")
ap.add_argument("--window", type=int, default=300, help="window length in seconds (default 300)")
ap.add_argument("--grace", type=int, default=120, help="grace period in seconds (default 120)")
ap.add_argument("--max-ahead", type=int, default=0, help="ignore event times this far after arrival (0 = off)")
ap.add_argument("--show-site", default="site-N03", help="site to print window by window")
args = ap.parse_args()
output = args.output or f"{args.topic}-site-load-5m"
width = timedelta(seconds=args.window)
def window_of(t):
return datetime.fromtimestamp(t.timestamp() // args.window * args.window, tz=timezone.utc)
open_windows = defaultdict(lambda: {"energy_kwh": 0.0, "readings": 0}) # (site, start) -> totals
closed = {} # (site, start) -> result
late = defaultdict(int) # (site, start) -> readings that came too late
ignored = 0
stream_time = None
admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
if output not in admin.list_topics(timeout=10).topics: # create the output topic once
admin.create_topics([NewTopic(output, 1, 3)])[output].result()
producer = Producer(settings.PRODUCER_DEFAULTS)
def close_until(t):
"""Close every open window whose end (plus grace) is at or before t."""
for site, start in sorted(k for k in open_windows if k[1] + width + timedelta(seconds=args.grace) <= t):
totals = open_windows.pop((site, start))
result = {"site_id": site, "window_start": iso(start), "window_end": iso(start + width),
"energy_kwh": round(totals["energy_kwh"], 3), "readings": totals["readings"]}
closed[(site, start)] = result
producer.produce(output, key=site, value=to_json(result))
consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m07l03-report",
"enable.auto.commit": False})
end = consumer.get_watermark_offsets(TopicPartition(args.topic, 0), timeout=10)[1]
consumer.assign([TopicPartition(args.topic, 0, 0)])
for _ in range(end):
msg = consumer.poll(10)
reading = json.loads(msg.value())
arrival = datetime.fromtimestamp(msg.timestamp()[1] / 1000, tz=timezone.utc)
measured = parse_iso(reading["ts"])
site = reading["charger_id"].replace("BL-", "site-")[:8]
if args.time == "arrival":
t = arrival
else:
if args.max_ahead and measured > arrival + timedelta(seconds=args.max_ahead):
ignored += 1 # a clock far in the future: don't trust it
continue
t = measured
stream_time = t if stream_time is None or t > stream_time else stream_time
start = window_of(t)
if (site, start) in closed:
late[(site, start)] += 1 # its window has already been reported
else:
totals = open_windows[(site, start)]
totals["energy_kwh"] += reading["power_kw"] * READING_S / 3600
totals["readings"] += 1
close_until(stream_time)
consumer.close()
close_until(datetime.max.replace(tzinfo=timezone.utc)) # end of input: close the rest
producer.flush(30)
print(f"time={args.time} grace={args.grace}s max-ahead={args.max_ahead or 'off'}")
print(f"readings: {end}, windows reported: {len(closed)}, late readings: {sum(late.values())}, "
f"ignored (clock ahead): {ignored}")
starts = sorted({start for _, start in closed})
for start in starts: # the chosen site, 17:55 to 18:25
if "17:55" <= iso(start)[11:16] <= "18:20":
result = closed.get((args.show_site, start), {"energy_kwh": 0.0, "readings": 0})
print(f"{args.show_site} {iso(start)[11:16]}-{iso(start + width)[11:16]} "
f"{result['energy_kwh']:6.2f} kWh {result['readings']:4d} readings "
f"late: {late[(args.show_site, start)]}")
if __name__ == "__main__":
main()