"""Production-readiness check for Brightlane's Kafka setup (lesson 9.3, the capstone).
It reads a release's topic and client files, looks at the lab's clusters, and prints one line per
check: PASS, FAIL, or SKIP when the part of the lab it needs isn't running. Each FAIL names the
lesson that explains the fix. It exits with 1 if anything failed, so a CI pipeline can stop the
release. It only reads; it never changes the cluster. From the code/ folder:
python module-09/readiness_check.py --release module-09/release
"""
import argparse
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import requests # noqa: E402
import yaml # noqa: E402
from confluent_kafka import KafkaException # noqa: E402
from confluent_kafka.admin import (AclBindingFilter, AclOperation, AclPermissionType, AdminClient, # noqa: E402
ConfigResource, ResourcePatternType, ResourceType)
from common import lab, settings # noqa: E402
results = []
def check(area, name, ok, detail=""):
"""Record one check. ok is True, False or None (skipped)."""
results.append((area, name, {True: "PASS", False: "FAIL", None: "SKIP"}[ok], detail))
def reachable(client):
try:
client.list_topics(timeout=5)
return True
except KafkaException:
return False
def configs(admin, restype, name):
resource = ConfigResource(restype, name)
return {k: v.value for k, v in admin.describe_configs([resource])[resource].result().items()}
# ------------------------------------------------------------------ the primary cluster
def check_cluster(admin):
nodes = admin.describe_cluster(request_timeout=10).result().nodes
check("cluster", "3 or more brokers", len(nodes) >= 3, f"{len(nodes)} brokers")
c = configs(admin, ResourceType.BROKER, str(nodes[0].id))
check("cluster", "no topics created by accident", c["auto.create.topics.enable"] == "false",
"" if c["auto.create.topics.enable"] == "false" else "auto.create.topics.enable=true (lesson 8.1)")
ok = int(c["default.replication.factor"]) >= 3 and int(c["min.insync.replicas"]) >= 2
check("cluster", "new topics: 3 replicas, 2 in sync", ok,
f"default.replication.factor={c['default.replication.factor']}, min.insync.replicas={c['min.insync.replicas']}")
ok = (int(c["offsets.topic.replication.factor"]) >= 3 and int(c["transaction.state.log.replication.factor"]) >= 3)
check("cluster", "internal topics: 3 replicas", ok, "" if ok else "offsets or transaction log under 3 (lesson 4.1)")
check("cluster", "no unclean leader elections", c["unclean.leader.election.enable"] == "false")
def check_topics(admin, wanted):
existing = admin.list_topics(timeout=10).topics
for name, spec in wanted.items():
if name not in existing:
check("topics", name, False, "missing (lesson 8.4: apply topics.yaml)")
continue
partitions = existing[name].partitions
c = configs(admin, ResourceType.TOPIC, name)
problems = []
if len(partitions) != spec["partitions"]:
problems.append(f"{len(partitions)} partitions, file says {spec['partitions']} (lesson 8.4)")
if len(partitions[0].replicas) < 3:
problems.append(f"{len(partitions[0].replicas)} replicas (lesson 4.1)")
if int(c["min.insync.replicas"]) < 2:
problems.append(f"min.insync.replicas={c['min.insync.replicas']}: one broker's disk is the only copy (lesson 2.2)")
if c["unclean.leader.election.enable"] == "true":
problems.append("unclean.leader.election.enable=true: a stale replica may become leader (lesson 4.1)")
if c["cleanup.policy"] == "delete" and "retention.ms" not in spec.get("config", {}):
problems.append("no retention.ms chosen for it (lesson 4.2)")
summary = f"{len(partitions)} partitions, {len(partitions[0].replicas)} replicas, min ISR {c['min.insync.replicas']}"
check("topics", name, not problems, "; ".join(problems) or summary)
# ------------------------------------------------------------------ the services' client settings
def check_clients(release):
defaults = release.get("defaults", {})
for service, spec in release["services"].items():
producer = {**defaults, **spec.get("producer", {})} if "producer" in spec else None
consumer = {**defaults, **spec.get("consumer", {})} if "consumer" in spec else None
problems = []
for client in filter(None, (producer, consumer)):
if client.get("security.protocol") != "SASL_SSL":
problems.append("not SASL_SSL (lesson 8.1)")
break
if producer:
if str(producer.get("acks", "all")) not in ("all", "-1"):
problems.append(f"producer acks={producer['acks']}: a leader failure can lose writes (lesson 2.2)")
if str(producer.get("enable.idempotence", "false")).lower() != "true":
problems.append("producer without idempotence: retries can duplicate (lesson 2.2)")
if consumer:
if "auto.offset.reset" not in consumer:
problems.append("auto.offset.reset not chosen (lesson 3.2)")
auto_commit = str(consumer.get("enable.auto.commit", "true")).lower() == "true"
auto_store = str(consumer.get("enable.auto.offset.store", "true")).lower() == "true"
if auto_commit and auto_store:
problems.append("auto-commit stores offsets before processing: a crash skips records (lesson 3.2)")
if producer and "transactional.id" in producer and consumer.get("isolation.level") == "read_uncommitted":
problems.append("reads read_uncommitted next to a transactional producer (lesson 7.1)")
check("clients", service, not problems, "; ".join(problems) or "settings as in modules 2, 3 and 8")
# ------------------------------------------------------------------ security (the secure broker)
def check_security(release):
from common import secure
if not secure.CA_FILE.exists():
for name in ("deny by default", "no wildcard ACLs", "one login per service"):
check("security", name, None, "secure broker not set up (lesson 8.1)")
return
admin = AdminClient({**secure.client_config(*secure.ADMIN), "logger": lab._quiet})
if not reachable(admin):
for name in ("deny by default", "no wildcard ACLs", "one login per service"):
check("security", name, None, "start it: docker compose --profile secure up -d")
return
c = configs(admin, ResourceType.BROKER, "1")
check("security", "deny by default", bool(c.get("authorizer.class.name")),
c.get("authorizer.class.name", "").rsplit(".", 1)[-1] or "no authorizer (lesson 8.2)")
match_all = AclBindingFilter(ResourceType.ANY, None, ResourcePatternType.ANY, None, None,
AclOperation.ANY, AclPermissionType.ANY)
broad = [f"{b.principal} {b.operation.name} on {b.restype.name} {b.name}"
for b in admin.describe_acls(match_all).result()
if b.permission_type == AclPermissionType.ALLOW and (b.operation == AclOperation.ALL or b.name == "*")]
check("security", "no wildcard ACLs", not broad, "; ".join(broad) + " (lesson 8.2)" if broad else "")
users = admin.describe_user_scram_credentials()
try:
known = set(users.result().keys())
except AttributeError: # a dict of futures, one per user
known = {u for u, f in users.items() if not f.exception()}
missing = [s["principal"] for s in release["services"].values() if s["principal"] not in known]
check("security", "one login per service", not missing, "missing: " + ", ".join(missing) if missing else
", ".join(sorted(s["principal"] for s in release["services"].values())))
# ------------------------------------------------------------------ alerts (Prometheus)
NEEDED_ALERTS = {"broker down": 'up{job="kafka"}', "offline partitions": "offlinepartitionscount",
"partitions under min ISR": "underminisrpartitioncount",
"active controller count": "activecontrollercount", "consumer lag": "consumer_group_topic_lag"}
def check_alerts():
try:
groups = requests.get(settings.PROMETHEUS_URL + "/api/v1/rules", timeout=5).json()["data"]["groups"]
except requests.RequestException:
for name in NEEDED_ALERTS:
check("alerts", name, None, "start it: docker compose --profile monitoring up -d")
return
rules = [r for g in groups for r in g["rules"] if r["type"] == "alerting"]
for name, metric in NEEDED_ALERTS.items():
found = [r["name"] for r in rules if metric in r["query"]]
check("alerts", name, bool(found), found[0] if found else "no alert rule (lesson 8.3)")
# ------------------------------------------------------------------ disaster recovery (MirrorMaker 2)
def check_dr(wanted):
from common import dr
admin = dr.admin("dr")
if not reachable(admin):
check("dr", "MirrorMaker 2 is copying", None, "start it: docker compose --profile dr up -d")
check("dr", "every topic has a copy", None, "")
return
records = dr.read_records("dr", "primary.heartbeats", timeout=10)
age = time.time() - max((m.timestamp()[1] for m in records), default=0) / 1000
check("dr", "MirrorMaker 2 is copying", age < 30, f"last heartbeat {age:.0f} s ago" if age < 3600 else "no heartbeats")
copies = admin.list_topics(timeout=10).topics
missing = [t for t in wanted if "primary." + t not in copies]
check("dr", "every topic has a copy", not missing, "missing: " + ", ".join(missing) if missing else
f"{len(wanted)} topics copied to the dr cluster")
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--release", default="module-09/release", help="folder with topics.yaml and clients.yaml")
args = ap.parse_args()
folder = settings.LAB_ROOT / args.release
topics = yaml.safe_load((folder / "topics.yaml").read_text())["topics"]
clients = yaml.safe_load((folder / "clients.yaml").read_text())
admin = lab.admin_client()
if reachable(admin):
check_cluster(admin)
check_topics(admin, topics)
else:
check("cluster", "primary cluster", None, "start it: docker compose up -d --wait")
check_clients(clients)
check_security(clients)
check_alerts()
check_dr(topics)
lab.table(results, ["area", "check", "result", "detail"])
count = {r: sum(1 for x in results if x[2] == r) for r in ("PASS", "FAIL", "SKIP")}
print(f"\nreadiness: {count['PASS']} passed, {count['FAIL']} failed, {count['SKIP']} skipped")
sys.exit(1 if count["FAIL"] else 0)
if __name__ == "__main__":
main()