Notebook · code/module-08/04-day-2-operations.ipynb

Download 04-day-2-operations.ipynb

Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.

Day-2 operations: change without downtime — practical

Brightlane's topics live in module-08/topics.yaml. You apply the file to the cluster, catch a change someone made by hand, add partitions for the growth plan and see which chargers move. Then you restart all three brokers while a writer runs, read the cluster's feature versions, and move replicas off kafka-1 with a throttle.

Lab services: kafka-1..3 (docker compose up -d --wait) · Lesson: modules/08-security-and-operations/04-day-2-operations.md

# Setup: find the lab's shared helpers
import sys, pathlib
lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir())
sys.path.insert(0, str(lab_root))

import json, logging, os, re, time
import yaml
from confluent_kafka import Producer
from confluent_kafka.admin import (AlterConfigOpType, ConfigEntry, ConfigResource, ConfigSource,
                                   NewPartitions, NewTopic, ResourceType)
from common import lab, settings
from common.scenario import Brightlane, to_json

quiet = logging.getLogger("quiet")            # brokers restart in this lesson: mute the clients' reconnect log
quiet.addHandler(logging.NullHandler())
quiet.propagate = False

lab.require_kafka()
admin = lab.admin_client(logger=quiet)
desired = yaml.safe_load((lab_root / "module-08" / "topics.yaml").read_text())["topics"]
ARCHIVE = "charging.telemetry-archive"
lab.delete_topics(admin, [*desired, ARCHIVE])          # start from a cluster without them
print(list(desired))

Step 1: Plan, then apply

plan() compares the file with the cluster and lists the changes, like a dry run. Nothing changes until apply() runs.

def plan(desired):
    """The changes that make the cluster match `desired`, as (change, topic, setting, detail) rows."""
    current = admin.list_topics(timeout=10).topics
    existing = [name for name in desired if name in current]
    configs = {}
    if existing:
        futures = admin.describe_configs([ConfigResource(ResourceType.TOPIC, name) for name in existing])
        configs = {resource.name: future.result() for resource, future in futures.items()}
    changes = []
    for name, spec in desired.items():
        if name not in current:
            changes.append(("create", name, "", f"{spec['partitions']} partitions, RF {spec['replication_factor']}"))
            continue
        have = len(current[name].partitions)
        if spec["partitions"] > have:
            changes.append(("add partitions", name, "", f"{have} -> {spec['partitions']}"))
        elif spec["partitions"] < have:
            changes.append(("can't apply", name, "", f"{have} -> {spec['partitions']}: partitions can't be removed"))
        if spec["replication_factor"] != len(current[name].partitions[0].replicas):
            changes.append(("can't apply", name, "", "replication factor: needs a reassignment (step 5)"))
        wanted = {key: str(value) for key, value in spec.get("config", {}).items()}
        for key, value in wanted.items():
            if configs[name][key].value != value:
                changes.append(("set", name, key, f"{configs[name][key].value} -> {value}"))
        for key, entry in configs[name].items():                 # set on the topic, but not in the file
            if ConfigSource(entry.source) == ConfigSource.DYNAMIC_TOPIC_CONFIG and key not in wanted:
                changes.append(("remove", name, key, f"{entry.value} -> cluster default"))
    return changes

changes = plan(desired)
lab.table(changes, ["change", "topic", "setting", "detail"])

apply() makes each change with the AdminClient: create topics, add partitions, set or remove a topic's settings.

def apply(desired, changes):
    for change, name, key, detail in changes:
        spec = desired[name]
        if change == "create":
            topic = NewTopic(name, spec["partitions"], spec["replication_factor"],
                             config={k: str(v) for k, v in spec.get("config", {}).items()})
            admin.create_topics([topic])[name].result()
        elif change == "add partitions":
            admin.create_partitions([NewPartitions(name, spec["partitions"])])[name].result()
        elif change in ("set", "remove"):
            operation = AlterConfigOpType.SET if change == "set" else AlterConfigOpType.DELETE
            entry = ConfigEntry(key, str(spec["config"].get(key, "")), incremental_operation=operation)
            resource = ConfigResource(ResourceType.TOPIC, name, incremental_configs=[entry])
            admin.incremental_alter_configs([resource])[resource].result()
        else:
            print(f"skipped  {name}: {detail}")
            continue
        print(f"applied  {change} {name} {key}")

apply(desired, changes)
print("plan now:", plan(desired))

Step 2: A change by hand, and the growth plan

During a disk alert last night, the on-call engineer shortened charging.sessions' retention to one hour and raised its maximum record size, by hand. Today the growth plan needs telemetry to go from 6 to 12 partitions; in real life that's an edit to topics.yaml in a pull request.

lab.kafka_cli("kafka-configs.sh", "--bootstrap-server", "localhost:19092", "--alter", "--entity-type", "topics",
              "--entity-name", "charging.sessions", "--add-config", "retention.ms=3600000,max.message.bytes=10485760")
desired["charging.telemetry"]["partitions"] = 12
changes = plan(desired)
lab.table(changes, ["change", "topic", "setting", "detail"])

Before applying, write one reading per charger and note the partition each lands in. After the change, do it again.

chargers = [c["charger_id"] for c in Brightlane().chargers]

def partition_per_charger():
    where = {}
    producer = Producer({**settings.PRODUCER_DEFAULTS, "logger": quiet})
    for charger in chargers:
        producer.produce("charging.telemetry", key=charger, value=to_json({"charger_id": charger, "power_kw": 11.0}),
                         on_delivery=lambda err, msg: where.__setitem__(msg.key().decode(), msg.partition()))
    producer.flush(15)
    return where

before = partition_per_charger()
apply(desired, changes)
time.sleep(2)
after = partition_per_charger()
moved = [c for c in chargers if before[c] != after[c]]
print(f"chargers in a different partition now: {len(moved)} of {len(chargers)}")
print("for example:", ", ".join(f"{c} {before[c]} -> {after[c]}" for c in moved[:4]))

Step 3: A rolling restart with a writer running

A patch upgrade means restarting every broker. The steady writer from lesson 4.1 writes 50 readings a second to charging.telemetry while the brokers restart one by one. Before each restart the notebook waits until every replica is back in sync.

def under_replicated():
    """Partitions with fewer in-sync replicas than replicas, from the cluster's metadata."""
    return sum(len(p.isrs) < len(p.replicas)
               for t in admin.list_topics(timeout=10).topics.values() for p in t.partitions.values())

def writer_report(writer):
    lines = writer.lines()
    slowest = max(int(m.group(1)) for line in lines if (m := re.search(r"slowest=(\d+)ms", line)))
    print(lines[-1].split(" ", 1)[1])
    print(f"slowest delivery: {slowest} ms; seconds with nothing delivered: "
          f"{sum(' delivered=0 ' in line for line in lines[1:-1])}")

writer = lab.start_script("module-04/steady_writer.py", "--topic", "charging.telemetry", "--rate", "50",
                          name="m08l04-writer")
time.sleep(5)
for node in settings.KAFKA_NODES:
    lab.wait_until(lambda: under_replicated() == 0, timeout=120, interval=1, what="all replicas in sync")
    started = time.monotonic()
    lab.compose("restart", node, echo=False)
    restarted = time.monotonic() - started
    lab.wait_until(lambda: under_replicated() == 0, timeout=120, interval=1, what="all replicas in sync")
    print(f"{node}: restarted in {restarted:.0f} s, all replicas in sync after {time.monotonic() - started:.0f} s")
time.sleep(3)
lab.stop_script(writer)
writer_report(writer)

Step 4: Feature versions

New broker software can come with new on-disk and protocol formats. They stay off until you raise the cluster's feature versions, after every broker runs the new software.

print(lab.kafka_cli("kafka-features.sh", "--bootstrap-server", "localhost:19092", "describe", echo=False))

Step 5: Move replicas off kafka-1, with a throttle

kafka-1's disk reports errors and will be replaced. The archive topic has two replicas per partition and 60 MB of data. Move its replicas to kafka-2 and kafka-3, copying at most 2 MB/s, so the copy doesn't crowd out billing's traffic.

admin.create_topics([NewTopic(ARCHIVE, num_partitions=3, replication_factor=-1,
                              replica_assignment=[[1, 2], [2, 3], [3, 1]])])[ARCHIVE].result()
time.sleep(2)
producer = Producer({**settings.PRODUCER_DEFAULTS, "linger.ms": 50, "logger": quiet})
for i in range(6000):                                   # 6,000 records of 10 KB: 60 MB, 20 MB per partition
    producer.produce(ARCHIVE, key=f"day-{i}", value=os.urandom(10_000), partition=i % 3)
    producer.poll(0)
producer.flush(60)

def replicas():
    return {p.id: p.replicas for p in admin.list_topics(ARCHIVE, timeout=10).topics[ARCHIVE].partitions.values()}

print("replicas now:", replicas())
move = {"version": 1, "partitions": [{"topic": ARCHIVE, "partition": 0, "replicas": [2, 3]},
                                     {"topic": ARCHIVE, "partition": 1, "replicas": [2, 3]},
                                     {"topic": ARCHIVE, "partition": 2, "replicas": [3, 2]}]}
lab.put_file("kafka-1", "/tmp/move.json", json.dumps(move).encode())
reassign = ["kafka-reassign-partitions.sh", "--bootstrap-server", "localhost:19092",
            "--reassignment-json-file", "/tmp/move.json"]
out = lab.kafka_cli(*reassign, "--execute", "--throttle", "2000000", echo=False)
print(out.splitlines()[-1])
started = time.monotonic()
while time.monotonic() - started < 120:
    time.sleep(3)
    out = lab.kafka_cli(*reassign, "--verify", echo=False)
    busy = "still in progress" in out
    print(f"+{time.monotonic() - started:3.0f} s  replicas: {replicas()}  {'copying' if busy else 'done'}")
    if not busy:
        break
print("\n".join(line for line in out.splitlines() if "hrottle" in line))

Break it: two brokers at once

The same restart, but kafka-1 and kafka-2 restart together, as when someone restarts "all the brokers on that host" or doesn't wait for the cluster to catch up.

writer = lab.start_script("module-04/steady_writer.py", "--topic", "charging.telemetry", "--rate", "50",
                          name="m08l04-writer-2")
time.sleep(5)
started = time.monotonic()
lab.compose("restart", "kafka-1", "kafka-2", echo=False)
lab.wait_until(lambda: under_replicated() == 0, timeout=180, interval=1, what="all replicas in sync")
print(f"both back and in sync after {time.monotonic() - started:.0f} s")
time.sleep(3)
lab.stop_script(writer)
writer_report(writer)

Clean up

lab.delete_topics(admin, [*desired, ARCHIVE])
print("cleaned up")