Notebook · code/module-01/03-records-offsets-and-replay.ipynb

Download 03-records-offsets-and-replay.ipynb

Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.

Records, offsets and replay — practical

You look inside a single record, write an evening of charger events, and then answer a driver's dispute by reading one charger's events from a chosen point in time, twice.

Lab services: kafka-1, kafka-2, kafka-3 · Lesson: modules/01-first-events/03-records-offsets-and-replay.md

# Setup: find the lab's shared helpers and check the lab is running
import sys, pathlib
lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir())
sys.path.insert(0, str(lab_root))

from common import settings, lab
from datetime import timedelta
from common.scenario import Brightlane, BASE_TIME, iso, to_json, from_json, parse_iso

lab.require_kafka()
lab.print_versions()

Step 1: Write one record and look at all its parts

A record has a key, a value, optional headers and a timestamp. Kafka adds the partition and the offset when it stores the record.

from confluent_kafka import Producer, Consumer, TopicPartition
from confluent_kafka.admin import AdminClient, NewTopic

ANATOMY, TOPIC = "m01l03-anatomy", "m01l03-telemetry"
admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
lab.delete_topics(admin, [ANATOMY, TOPIC])
for name in (ANATOMY, TOPIC):
    admin.create_topics([NewTopic(name, num_partitions=3, replication_factor=3)])[name].result()

producer = Producer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
producer.produce(ANATOMY, key="BL-N03-07",
                 value=to_json({"charger_id": "BL-N03-07", "power_kw": 0.0, "ts": iso(BASE_TIME)}),
                 headers={"firmware": "4.2.1", "trace-id": "a91f3c"})
print("not yet stored:", producer.flush(10))

Read it back with a consumer that is assigned to partitions directly: no group membership, no commits, a plain reader. It reads up to each partition's current end and stops.

def read_all(tps, max_wait=30.0):
    """Read the given partitions from their start offsets up to their current end."""
    import time
    c = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m01l03-reader",
                  "enable.auto.commit": False})
    ends = {tp.partition: c.get_watermark_offsets(tp, timeout=10)[1] for tp in tps}
    todo = {tp.partition for tp in tps if ends[tp.partition] > tp.offset}
    c.assign(tps)
    out, deadline = [], time.monotonic() + max_wait
    while todo and time.monotonic() < deadline:
        for msg in c.consume(num_messages=1000, timeout=0.5):
            if msg.error():
                continue
            out.append(msg)
            if msg.offset() >= ends[msg.partition()] - 1:
                todo.discard(msg.partition())
    c.close()
    return out

msg = read_all([TopicPartition(ANATOMY, p, 0) for p in range(3)])[0]   # offset 0 = from the start
ts_type, ts_ms = msg.timestamp()
print("topic     ", msg.topic(), "| partition", msg.partition(), "| offset", msg.offset())
print("key       ", msg.key())
print("value     ", msg.value())
print("headers   ", msg.headers())
print("timestamp ", ts_ms, "(type 1 = CreateTime, set by the producer)" if ts_type == 1 else ts_type)

You should see the partition and offset Kafka chose, the key and value as bytes, the two headers, and a timestamp in milliseconds since 1970 (now, because we didn't set one).

Step 2: Write an evening of meter readings

The gateway relays every reading with the charger's own clock as the record timestamp. That's 40 sessions and their 10-second readings: several thousand records.

bl = Brightlane()
events = bl.stream(sessions=40)
for charger_id, event in events:
    ts_ms = int(parse_iso(event["ts"]).timestamp() * 1000)
    producer.produce(TOPIC, key=charger_id, value=to_json(event), timestamp=ts_ms)
    producer.poll(0)                                  # serve delivery events while producing
print(f"wrote {len(events)} records, not yet stored: {producer.flush(30)}")

You should see wrote 9123 records, not yet stored: 0.

Step 3: Find the disputed charger's partition

Records with the same key always go to the same partition, so one partition holds all of BL-N03-07's events. The partition's offsets run from 0 to its end.

everything = [TopicPartition(TOPIC, p, 0) for p in range(3)]
records = read_all(everything)
mine = [m for m in records if m.key() == b"BL-N03-07"]
partition = mine[0].partition()
low, high = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m01l03-reader"}) \
    .get_watermark_offsets(TopicPartition(TOPIC, partition), timeout=10)
print(f"BL-N03-07 -> partition {partition}: {len(mine)} records")
print(f"partition {partition} holds offsets {low} to {high - 1} ({high - low} records in total)")

You should see all of BL-N03-07's records in one partition, mixed with other chargers' records. The exact partition and counts are in the lesson.

Step 4: Jump to 17:40 by time

Support only needs events from 17:40. offsets_for_times asks the broker for the first offset whose timestamp is at or after a given time.

c = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m01l03-reader"})
at_1740 = int((BASE_TIME + timedelta(minutes=10)).timestamp() * 1000)   # BASE_TIME is 17:30 yesterday
found = c.offsets_for_times([TopicPartition(TOPIC, partition, at_1740)], timeout=10)[0]
c.close()
print(f"first offset at or after 17:40 in partition {partition}: {found.offset}")

since_1740 = [m for m in read_all([TopicPartition(TOPIC, partition, found.offset)]) if m.key() == b"BL-N03-07"]
for m in since_1740[:3]:
    e = from_json(m.value())
    print(m.offset(), e["ts"], e.get("type", "reading"), e.get("energy_kwh"))
print(f"... {len(since_1740)} records for BL-N03-07 since 17:40")

You should see the first three BL-N03-07 records after 17:40 and the total. The session started at 17:38:25, so the reading at 17:40:05 already shows about 2 kWh.

Step 5: Replay: read the same range again

Reading is not destructive. Ask for the same offset again and you get the same records, with the same offsets, in the same order.

again = [m for m in read_all([TopicPartition(TOPIC, partition, found.offset)]) if m.key() == b"BL-N03-07"]
same = [(m.offset(), m.value()) for m in again] == [(m.offset(), m.value()) for m in since_1740]
print(f"second read: {len(again)} records, identical to the first: {same}")

You should see identical to the first: True.

Change it: a reading that arrives late

Predict first: a 17:35 reading from BL-N03-07 reaches the gateway only now (the charger's mobile link was down). It is appended at the end of the partition. Where does it sit, and does a time search for 17:30 find it?

late = {"charger_id": "BL-N03-07", "session_id": "ses-test", "ts": iso(BASE_TIME + timedelta(minutes=5)),
        "power_kw": 7.0, "energy_kwh": 0.019, "firmware": "4.2.1"}
producer.produce(TOPIC, key="BL-N03-07", value=to_json(late),
                 timestamp=int(parse_iso(late["ts"]).timestamp() * 1000))
producer.flush(10)
tail = read_all([TopicPartition(TOPIC, partition, high)])
print("late record stored at offset", tail[0].offset(), "with timestamp", from_json(tail[0].value())["ts"])

c = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m01l03-reader"})
at_1730 = int(BASE_TIME.timestamp() * 1000)
print("time search for 17:30 starts at offset",
      c.offsets_for_times([TopicPartition(TOPIC, partition, at_1730)], timeout=10)[0].offset)
c.close()

The late record gets the next offset at the end of the log, even though its timestamp is earlier than its neighbours'. The time search still starts near the beginning: it returns the first offset with a timestamp at or after 17:30, and reading forward from there reaches the late record too. Offsets give order of arrival; timestamps can go backwards.

Clean up

Delete the topic. If anything gets stuck, docker compose down -v in the code/ folder resets the whole lab.

lab.delete_topics(admin, [ANATOMY, TOPIC])
print("cleaned up")