Notebook · code/module-09/03-production-readiness-review.ipynb

Download 03-production-readiness-review.ipynb

Open it in JupyterLab or VS Code from the lab folder to run it. This page is a read-only view.

The production-readiness review — practical

Brightlane's next release is ready: its topics in module-09/release/topics.yaml, its services' client settings in clients.yaml. readiness_check.py reviews the release and the lab against the rules of this course. You run it, fix each finding the way the course taught, and run it again until it's green.

Lab services: kafka-1..3, kafka-secure, prometheus, kminion, grafana, kafka-dr, mirror-maker (the notebook starts them; together they need about 5 GB of the Docker VM's memory) · Lesson: modules/09-production-ready/03-production-readiness-review.md

# Setup: find the lab's shared helpers
import sys, pathlib
lab_root = next(p for p in [pathlib.Path.cwd(), *pathlib.Path.cwd().parents] if (p / "common").is_dir())
sys.path.insert(0, str(lab_root))

import shutil, subprocess, yaml
from confluent_kafka.admin import (AclBinding, AclBindingFilter, AclOperation, AclPermissionType, AlterConfigOpType,
                                   ConfigEntry, ConfigResource, NewTopic, ResourcePatternType, ResourceType)
from common import dr, lab, secure

lab.compose("--profile", "secure", "--profile", "monitoring", "--profile", "dr", "up", "-d", "--wait")
secure_admin = secure.ensure_ready()
RELEASE = lab_root / ".lab-data" / "release"                    # a working copy of the release files
shutil.rmtree(RELEASE, ignore_errors=True)
shutil.copytree(lab_root / "module-09" / "release", RELEASE)
print("release files copied to", RELEASE.relative_to(lab_root))

The setup puts the release's topics on the primary cluster, exactly as the file says, and the secure broker's ACLs as they are in production today.

topics = yaml.safe_load((RELEASE / "topics.yaml").read_text())["topics"]
dr.reset_mirroring("charging.", "billing.")                     # no topics, copies or groups from earlier
admin = lab.admin_client()
for name, future in admin.create_topics([NewTopic(name, spec["partitions"], spec["replication_factor"],
                                                  config={k: str(v) for k, v in spec["config"].items()})
                                         for name, spec in topics.items()]).items():
    future.result()
lab.wait_until(lambda: all("primary." + t in dr.admin("dr").list_topics(timeout=10).topics for t in topics),
               timeout=90, interval=3, what="MirrorMaker 2 to copy the topics")

def allow(user, resource_type, name, operation, pattern=ResourcePatternType.LITERAL):
    return AclBinding(resource_type, name, pattern, f"User:{user}", "*", operation, AclPermissionType.ALLOW)

everything = AclBindingFilter(ResourceType.ANY, None, ResourcePatternType.ANY, None, None, AclOperation.ANY,
                              AclPermissionType.ANY)
for future in secure_admin.delete_acls([everything]).values():
    future.result()
production_acls = [
    allow("svc-billing", ResourceType.TOPIC, "charging.sessions", AclOperation.READ),
    allow("svc-billing", ResourceType.GROUP, "billing", AclOperation.READ),
    allow("svc-billing", ResourceType.TOPIC, "billing.session-charges", AclOperation.WRITE),
    allow("svc-billing", ResourceType.TRANSACTIONAL_ID, "billing-", AclOperation.WRITE, ResourcePatternType.PREFIXED),
    allow("svc-gateway", ResourceType.TOPIC, "charging.", AclOperation.WRITE, ResourcePatternType.PREFIXED),
    allow("svc-grid", ResourceType.TOPIC, "*", AclOperation.ALL),     # "temporary", since the grid project
]
for future in secure_admin.create_acls(production_acls).values():
    future.result()
lab.remove_alert_rules("readiness")
print(f"{len(topics)} topics, {len(production_acls)} ACLs")

Step 1: The first review

def review():
    result = subprocess.run([sys.executable, "module-09/readiness_check.py", "--release", ".lab-data/release"],
                            cwd=lab_root, capture_output=True, text=True)
    print(result.stdout.strip() or result.stderr[-2000:])
    print(f"exit code {result.returncode}:", "a CI pipeline would stop the release" if result.returncode
          else "the release may go ahead")

review()

Step 2: Fix the topics

Change the file first, then apply the same change to the cluster, as lesson 8.4's apply() does. billing.session-charges needs two in-sync replicas like every other topic; the charger map goes back to the cluster's default, no unclean elections.

release_topics = yaml.safe_load((RELEASE / "topics.yaml").read_text())
release_topics["topics"]["billing.session-charges"]["config"]["min.insync.replicas"] = 2
del release_topics["topics"]["charging.charger-status"]["config"]["unclean.leader.election.enable"]
(RELEASE / "topics.yaml").write_text(yaml.safe_dump(release_topics, sort_keys=False))

changes = [ConfigResource(ResourceType.TOPIC, "billing.session-charges", incremental_configs=[
               ConfigEntry("min.insync.replicas", "2", incremental_operation=AlterConfigOpType.SET)]),
           ConfigResource(ResourceType.TOPIC, "charging.charger-status", incremental_configs=[
               ConfigEntry("unclean.leader.election.enable", None, incremental_operation=AlterConfigOpType.DELETE)])]
for resource, future in admin.incremental_alter_configs(changes).items():
    future.result()
    print("changed", resource.name)

Step 3: Fix the client settings

The gateway writes with acks=all and idempotence (lesson 2.2). Billing reads read_committed next to its transactional producer (lesson 7.1). Grid reporting stores an offset only after the record is processed (lesson 3.2).

clients = yaml.safe_load((RELEASE / "clients.yaml").read_text())
clients["services"]["gateway"]["producer"].update({"acks": "all", "enable.idempotence": True})
clients["services"]["billing"]["consumer"]["isolation.level"] = "read_committed"
clients["services"]["grid"]["consumer"]["enable.auto.offset.store"] = False
(RELEASE / "clients.yaml").write_text(yaml.safe_dump(clients, sort_keys=False))
print(yaml.safe_dump(clients["services"]["grid"], sort_keys=False))

Step 4: Fix the ACLs

Replace grid reporting's wildcard with what its job needs (exercise 8.1).

wildcard = AclBindingFilter(ResourceType.TOPIC, "*", ResourcePatternType.LITERAL, "User:svc-grid", None,
                            AclOperation.ALL, AclPermissionType.ALLOW)
for future in secure_admin.delete_acls([wildcard]).values():
    print("removed", len(future.result()), "ACL")
grid_acls = [allow("svc-grid", ResourceType.TOPIC, "charging.telemetry", AclOperation.READ),
             allow("svc-grid", ResourceType.GROUP, "grid-reporting", AclOperation.READ),
             allow("svc-grid", ResourceType.TOPIC, "grid.site-load-5m", AclOperation.WRITE)]
for future in secure_admin.create_acls(grid_acls).values():
    future.result()
print("added", len(grid_acls), "ACLs")

Step 5: Add the missing alert

Partitions under min ISR refuse acks=all writes: the gateway's telemetry stops. That must page.

rule = {"alert": "KafkaUnderMinIsrPartitions",
        "expr": "sum(kafka_server_replicamanager_underminisrpartitioncount) > 0",
        "for": "30s", "labels": {"severity": "page"},
        "annotations": {"summary": "{{ $value }} partitions have fewer in-sync replicas than min.insync.replicas: "
                                   "acks=all writes to them fail",
                        "runbook_url": "https://runbooks.brightlane.example/kafka/under-min-isr"}}
print(lab.add_alert_rules("readiness", yaml.safe_dump({"groups": [{"name": "readiness", "rules": [rule]}]})))

Step 6: Review again

review()

Break it: a bad night

The check isn't only for releases. Run every night against production, it notices what changed since: here, a broker that didn't come back after maintenance.

lab.compose("stop", "kafka-3")
review()
lab.compose("up", "-d", "--wait", "kafka-3")

Clean up

Remove the release's topics and copies, the ACLs and the extra alert rule. This is the course's last practical: to stop everything, docker compose --profile secure --profile monitoring --profile dr stop; to delete the lab with all its data, docker compose --profile secure --profile monitoring --profile dr --profile connect --profile schemas down -v.

dr.reset_mirroring("charging.", "billing.")
for future in secure_admin.delete_acls([everything]).values():
    future.result()
lab.remove_alert_rules("readiness")
shutil.rmtree(RELEASE, ignore_errors=True)
print("cleaned up")