Notebook · code/module-05/02-evolving-a-schema.ipynb

Download 02-evolving-a-schema.ipynb

Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.

Evolving a schema without breaking readers — practical

The grid team wants a new field in meter readings, while billing still runs its old code. You add the field in a compatible way, read old and new readings with old and new readers, have a breaking change refused by the registry, and see what happens when the check is switched off.

Lab services: kafka-1, kafka-2, kafka-3, schema-registry (docker compose --profile schemas up -d --wait) · Lesson: modules/05-data-contracts/02-evolving-a-schema.md

# Setup: find the lab's shared helpers and check the lab is running
import sys, pathlib
lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir())
sys.path.insert(0, str(lab_root))

import json
from common import settings, lab
from common.scenario import Brightlane, parse_iso

lab.require_kafka()
lab.require_http("Schema Registry", settings.SCHEMA_REGISTRY_URL + "/subjects",
                 start_hint="docker compose --profile schemas up -d --wait")
lab.print_versions()

Step 1: Version 1, and 50 readings written with it

from confluent_kafka import Producer, Consumer, TopicPartition
from confluent_kafka.admin import AdminClient
from confluent_kafka.schema_registry import SchemaRegistryClient, Schema
from confluent_kafka.schema_registry.avro import AvroSerializer, AvroDeserializer
from confluent_kafka.serialization import SerializationContext, MessageField

TOPIC = "m05l02-telemetry"
SUBJECT = f"{TOPIC}-value"
admin = AdminClient({"bootstrap.servers": settings.KAFKA_BOOTSTRAP})
lab.recreate_topic(admin, TOPIC, partitions=1)
registry = SchemaRegistryClient({"url": settings.SCHEMA_REGISTRY_URL})
if SUBJECT in registry.get_subjects():
    registry.delete_subject(SUBJECT)
    registry.delete_subject(SUBJECT, permanent=True)
try:
    registry.delete_config(SUBJECT)          # forget a compatibility rule left by an earlier run
except Exception:
    pass

V1 = (lab_root / "schemas" / "meter_reading.avsc").read_text()
context = SerializationContext(TOPIC, MessageField.VALUE)
readings = [e for _, e in Brightlane().stream(sessions=10) if "type" not in e][:150]
producer = Producer(settings.PRODUCER_DEFAULTS)

def write(schema_str, batch, **extra):
    """Write readings with a given writer schema, registering it if needed."""
    serialize = AvroSerializer(registry, schema_str, lambda r, ctx: {**r, "ts": parse_iso(r["ts"]), **extra})
    for reading in batch:
        producer.produce(TOPIC, key=reading["charger_id"], value=serialize(reading, context))
    producer.flush(10)

write(V1, readings[:50])
print("versions:", registry.get_versions(SUBJECT), " compatibility:", registry.get_compatibility())

The registry's default rule is BACKWARD: a new version must be able to read data written with the previous one.

Step 2: Version 2 adds an optional field

The grid team wants phase_count: how many phases the charger uses. Old readings don't have it, so the field may be empty (null) and its default is null.

def changed(schema_str, edit):
    schema = json.loads(schema_str)
    edit(schema["fields"])
    return json.dumps(schema)

V2 = changed(V1, lambda fields: fields.append({"name": "phase_count", "type": ["null", "int"], "default": None}))
print("compatible with version 1:", registry.test_compatibility(SUBJECT, Schema(V2, "AVRO")))
write(V2, readings[50:100], phase_count=3)
print("versions:", registry.get_versions(SUBJECT))

Step 3: Old and new readers

Billing still reads with version 1, the grid report already with version 2. Giving the deserializer a schema makes it the reader schema: Avro translates each record from the schema it was written with into that one.

def read_all(reader_schema=None):
    deserialize = AvroDeserializer(registry, reader_schema)
    consumer = Consumer({"bootstrap.servers": settings.KAFKA_BOOTSTRAP, "group.id": "m05l02-reader",
                         "enable.auto.commit": False})
    end = consumer.get_watermark_offsets(TopicPartition(TOPIC, 0), timeout=10)[1]
    consumer.assign([TopicPartition(TOPIC, 0, 0)])
    out = []
    while len(out) < end:
        msg = consumer.poll(10)
        try:
            out.append(deserialize(msg.value(), context))
        except Exception as e:
            out.append(f"offset {msg.offset()}: {type(e).__name__}: {e}")
    consumer.close()
    return out

billing, grid = read_all(V1), read_all(V2)
print("billing (v1) reads", len(billing), "readings; fields of the newest:", sorted(billing[-1]))
print("grid (v2) on an old reading: phase_count =", grid[0]["phase_count"])
print("grid (v2) on a new reading:  phase_count =", grid[-1]["phase_count"])

Step 4: Version 3 renames a field

The gateway team wants energy_kwh called energy_wh (in watt-hours). Predict first: will the registry accept it?

V3 = changed(V2, lambda fields: [f.update(name="energy_wh") for f in fields if f["name"] == "energy_kwh"])
print("compatible with version 2:", registry.test_compatibility(SUBJECT, Schema(V3, "AVRO")))
try:
    registry.register_schema(SUBJECT, Schema(V3, "AVRO"))
except Exception as e:
    print(f"register: HTTP {e.http_status_code}: {str(e)[:230]}")

Refused with HTTP 409: a version 3 reader couldn't read the existing readings, because they have no energy_wh and the new field has no default. Nothing reached the topic.

Break it: switch the check off

Someone sets this subject's compatibility to NONE "just for this release", registers version 3 and writes 50 readings. Predict first: what does the grid report, still on version 2, see?

registry.set_compatibility(SUBJECT, "NONE")
write(V3, readings[100:150], phase_count=3, energy_wh=500.0)
grid = read_all(V2)
print("readable:", sum(isinstance(r, dict) for r in grid), " failed:", sum(isinstance(r, str) for r in grid))
print(next(r for r in grid if isinstance(r, str)))

Version 2 readers fail on every version 3 reading. Those readings are in the topic now; turning the check back on doesn't remove them.

Clean up

Delete the topic and the subject, so the check is back to the default next time. If anything gets stuck, docker compose down -v in the code/ folder resets the whole lab.

lab.delete_topics(admin, [TOPIC])
registry.delete_config(SUBJECT)
registry.delete_subject(SUBJECT)
registry.delete_subject(SUBJECT, permanent=True)
admin.delete_consumer_groups(["m05l02-reader"])
print("cleaned up")