Download 04-day-2-operations.ipynb
Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.
Day-2 operations: change without downtime — practical
Brightlane's topics live in module-08/topics.yaml. You apply the file to the cluster, catch a
change someone made by hand, add partitions for the growth plan and see which chargers move.
Then you restart all three brokers while a writer runs, read the cluster's feature versions,
and move replicas off kafka-1 with a throttle.
Lab services: kafka-1..3 (docker compose up -d --wait)
· Lesson: modules/08-security-and-operations/04-day-2-operations.md
# Setup: find the lab's shared helpers import sys, pathlib lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir()) sys.path.insert(0, str(lab_root)) import json, logging, os, re, time import yaml from confluent_kafka import Producer from confluent_kafka.admin import (AlterConfigOpType, ConfigEntry, ConfigResource, ConfigSource, NewPartitions, NewTopic, ResourceType) from common import lab, settings from common.scenario import Brightlane, to_json quiet = logging.getLogger("quiet") # brokers restart in this lesson: mute the clients' reconnect log quiet.addHandler(logging.NullHandler()) quiet.propagate = False lab.require_kafka() admin = lab.admin_client(logger=quiet) desired = yaml.safe_load((lab_root / "module-08" / "topics.yaml").read_text())["topics"] ARCHIVE = "charging.telemetry-archive" lab.delete_topics(admin, [*desired, ARCHIVE]) # start from a cluster without them print(list(desired))
Step 1: Plan, then apply
plan() compares the file with the cluster and lists the changes, like a dry run. Nothing
changes until apply() runs.
def plan(desired): """The changes that make the cluster match `desired`, as (change, topic, setting, detail) rows.""" current = admin.list_topics(timeout=10).topics existing = [name for name in desired if name in current] configs = {} if existing: futures = admin.describe_configs([ConfigResource(ResourceType.TOPIC, name) for name in existing]) configs = {resource.name: future.result() for resource, future in futures.items()} changes = [] for name, spec in desired.items(): if name not in current: changes.append(("create", name, "", f"{spec['partitions']} partitions, RF {spec['replication_factor']}")) continue have = len(current[name].partitions) if spec["partitions"] > have: changes.append(("add partitions", name, "", f"{have} -> {spec['partitions']}")) elif spec["partitions"] < have: changes.append(("can't apply", name, "", f"{have} -> {spec['partitions']}: partitions can't be removed")) if spec["replication_factor"] != len(current[name].partitions[0].replicas): changes.append(("can't apply", name, "", "replication factor: needs a reassignment (step 5)")) wanted = {key: str(value) for key, value in spec.get("config", {}).items()} for key, value in wanted.items(): if configs[name][key].value != value: changes.append(("set", name, key, f"{configs[name][key].value} -> {value}")) for key, entry in configs[name].items(): # set on the topic, but not in the file if ConfigSource(entry.source) == ConfigSource.DYNAMIC_TOPIC_CONFIG and key not in wanted: changes.append(("remove", name, key, f"{entry.value} -> cluster default")) return changes changes = plan(desired) lab.table(changes, ["change", "topic", "setting", "detail"])
apply() makes each change with the AdminClient: create topics, add partitions, set or remove
a topic's settings.
def apply(desired, changes): for change, name, key, detail in changes: spec = desired[name] if change == "create": topic = NewTopic(name, spec["partitions"], spec["replication_factor"], config={k: str(v) for k, v in spec.get("config", {}).items()}) admin.create_topics([topic])[name].result() elif change == "add partitions": admin.create_partitions([NewPartitions(name, spec["partitions"])])[name].result() elif change in ("set", "remove"): operation = AlterConfigOpType.SET if change == "set" else AlterConfigOpType.DELETE entry = ConfigEntry(key, str(spec["config"].get(key, "")), incremental_operation=operation) resource = ConfigResource(ResourceType.TOPIC, name, incremental_configs=[entry]) admin.incremental_alter_configs([resource])[resource].result() else: print(f"skipped {name}: {detail}") continue print(f"applied {change} {name} {key}") apply(desired, changes) print("plan now:", plan(desired))
Step 2: A change by hand, and the growth plan
During a disk alert last night, the on-call engineer shortened charging.sessions' retention to
one hour and raised its maximum record size, by hand. Today the growth plan needs telemetry to go
from 6 to 12 partitions; in real life that's an edit to topics.yaml in a pull request.
lab.kafka_cli("kafka-configs.sh", "--bootstrap-server", "localhost:19092", "--alter", "--entity-type", "topics", "--entity-name", "charging.sessions", "--add-config", "retention.ms=3600000,max.message.bytes=10485760") desired["charging.telemetry"]["partitions"] = 12 changes = plan(desired) lab.table(changes, ["change", "topic", "setting", "detail"])
Before applying, write one reading per charger and note the partition each lands in. After the change, do it again.
chargers = [c["charger_id"] for c in Brightlane().chargers] def partition_per_charger(): where = {} producer = Producer({**settings.PRODUCER_DEFAULTS, "logger": quiet}) for charger in chargers: producer.produce("charging.telemetry", key=charger, value=to_json({"charger_id": charger, "power_kw": 11.0}), on_delivery=lambda err, msg: where.__setitem__(msg.key().decode(), msg.partition())) producer.flush(15) return where before = partition_per_charger() apply(desired, changes) time.sleep(2) after = partition_per_charger() moved = [c for c in chargers if before[c] != after[c]] print(f"chargers in a different partition now: {len(moved)} of {len(chargers)}") print("for example:", ", ".join(f"{c} {before[c]} -> {after[c]}" for c in moved[:4]))
Step 3: A rolling restart with a writer running
A patch upgrade means restarting every broker. The steady writer from lesson 4.1 writes 50
readings a second to charging.telemetry while the brokers restart one by one. Before each
restart the notebook waits until every replica is back in sync.
def under_replicated(): """Partitions with fewer in-sync replicas than replicas, from the cluster's metadata.""" return sum(len(p.isrs) < len(p.replicas) for t in admin.list_topics(timeout=10).topics.values() for p in t.partitions.values()) def writer_report(writer): lines = writer.lines() slowest = max(int(m.group(1)) for line in lines if (m := re.search(r"slowest=(\d+)ms", line))) print(lines[-1].split(" ", 1)[1]) print(f"slowest delivery: {slowest} ms; seconds with nothing delivered: " f"{sum(' delivered=0 ' in line for line in lines[1:-1])}") writer = lab.start_script("module-04/steady_writer.py", "--topic", "charging.telemetry", "--rate", "50", name="m08l04-writer") time.sleep(5) for node in settings.KAFKA_NODES: lab.wait_until(lambda: under_replicated() == 0, timeout=120, interval=1, what="all replicas in sync") started = time.monotonic() lab.compose("restart", node, echo=False) restarted = time.monotonic() - started lab.wait_until(lambda: under_replicated() == 0, timeout=120, interval=1, what="all replicas in sync") print(f"{node}: restarted in {restarted:.0f} s, all replicas in sync after {time.monotonic() - started:.0f} s") time.sleep(3) lab.stop_script(writer) writer_report(writer)
Step 4: Feature versions
New broker software can come with new on-disk and protocol formats. They stay off until you raise the cluster's feature versions, after every broker runs the new software.
print(lab.kafka_cli("kafka-features.sh", "--bootstrap-server", "localhost:19092", "describe", echo=False))
Step 5: Move replicas off kafka-1, with a throttle
kafka-1's disk reports errors and will be replaced. The archive topic has two replicas per partition and 60 MB of data. Move its replicas to kafka-2 and kafka-3, copying at most 2 MB/s, so the copy doesn't crowd out billing's traffic.
admin.create_topics([NewTopic(ARCHIVE, num_partitions=3, replication_factor=-1, replica_assignment=[[1, 2], [2, 3], [3, 1]])])[ARCHIVE].result() time.sleep(2) producer = Producer({**settings.PRODUCER_DEFAULTS, "linger.ms": 50, "logger": quiet}) for i in range(6000): # 6,000 records of 10 KB: 60 MB, 20 MB per partition producer.produce(ARCHIVE, key=f"day-{i}", value=os.urandom(10_000), partition=i % 3) producer.poll(0) producer.flush(60) def replicas(): return {p.id: p.replicas for p in admin.list_topics(ARCHIVE, timeout=10).topics[ARCHIVE].partitions.values()} print("replicas now:", replicas())
move = {"version": 1, "partitions": [{"topic": ARCHIVE, "partition": 0, "replicas": [2, 3]}, {"topic": ARCHIVE, "partition": 1, "replicas": [2, 3]}, {"topic": ARCHIVE, "partition": 2, "replicas": [3, 2]}]} lab.put_file("kafka-1", "/tmp/move.json", json.dumps(move).encode()) reassign = ["kafka-reassign-partitions.sh", "--bootstrap-server", "localhost:19092", "--reassignment-json-file", "/tmp/move.json"] out = lab.kafka_cli(*reassign, "--execute", "--throttle", "2000000", echo=False) print(out.splitlines()[-1]) started = time.monotonic() while time.monotonic() - started < 120: time.sleep(3) out = lab.kafka_cli(*reassign, "--verify", echo=False) busy = "still in progress" in out print(f"+{time.monotonic() - started:3.0f} s replicas: {replicas()} {'copying' if busy else 'done'}") if not busy: break print("\n".join(line for line in out.splitlines() if "hrottle" in line))
Break it: two brokers at once
The same restart, but kafka-1 and kafka-2 restart together, as when someone restarts "all the brokers on that host" or doesn't wait for the cluster to catch up.
writer = lab.start_script("module-04/steady_writer.py", "--topic", "charging.telemetry", "--rate", "50", name="m08l04-writer-2") time.sleep(5) started = time.monotonic() lab.compose("restart", "kafka-1", "kafka-2", echo=False) lab.wait_until(lambda: under_replicated() == 0, timeout=180, interval=1, what="all replicas in sync") print(f"both back and in sync after {time.monotonic() - started:.0f} s") time.sleep(3) lab.stop_script(writer) writer_report(writer)
Clean up
lab.delete_topics(admin, [*desired, ARCHIVE]) print("cleaned up")