#!/usr/bin/env python3 """pia-gateway: PIA WireGuard egress gateway controller for minisforum. Installed by ansible/roles/pia-gateway. One program, several subcommands: check read-only status classification (journal + status/metrics files) recover one bounded recovery pass (timer-driven, every 60s) renew proactive same-key re-registration when the last verified registration is older than the configured age (daily timer) register explicit registration transaction (used by Ansible and operators) rewrite-config re-render pia-wg.conf from the last verified registration (no PIA API call), activate and verify status / pause / resume / retry-now / state-reset / notify Invariants this program must never break (see the role README): * The host's main default route and the global FORWARD policy are never touched. Only table `pia`, the `PIA-VLAN50` chain, and (in an emergency) one extra FORWARD rule scoped to the VLAN 50 source. * Every failure path leaves VLAN 50 closed. No path opens the gateway except a verified WireGuard activation. * No PIA credential, token, or private key is ever written to stdout, stderr, the journal, argv, a notification, or the metrics file. Only the Python standard library is used so the same file runs on the gateway (python3.13) and under the repo's local unittest suite. """ from __future__ import annotations import argparse import contextlib import dataclasses import datetime as _dt import fcntl import http.client import io import json import os import random import re import secrets import socket import ssl import subprocess import sys import tempfile import time import urllib.parse from typing import Any, Iterable, Optional STATE_SCHEMA_VERSION = 1 EMERGENCY_COMMENT = "pia-gateway-emergency" SERVERLIST_URL = "https://serverlist.piaservers.net/vpninfo/servers/v6" TOKEN_URL = "https://www.privateinternetaccess.com/api/client/v2/token" ADDKEY_PORT = 1337 CLASS_HEALTHY = "healthy" CLASS_BOOT_GRACE = "boot_grace" CLASS_PAUSED = "paused" CLASS_TUNNEL_STALE = "tunnel_stale" CLASS_TUNNEL_ABSENT = "tunnel_absent" CLASS_REGISTRATION_FAILURE = "registration_or_endpoint_failure" CLASS_SAFETY_DRIFT = "safety_drift" CLASS_HOST_CONNECTIVITY = "host_connectivity_failure" CLASS_UNKNOWN = "unknown" FAIL_AUTH = "auth_failure" FAIL_HOST_CONNECTIVITY = CLASS_HOST_CONNECTIVITY FAIL_REGISTRATION = CLASS_REGISTRATION_FAILURE FAIL_SAFETY = CLASS_SAFETY_DRIFT EXIT_OK = 0 EXIT_UNHEALTHY = 1 EXIT_STATE_ERROR = 2 EXIT_SAFETY = 3 EXIT_LOCKED = 4 EXIT_USAGE = 5 class GatewayError(Exception): """Base class; message must never contain secrets.""" class StateCorruptError(GatewayError): pass class SafetyError(GatewayError): pass class LockBusyError(GatewayError): pass class ApiError(GatewayError): def __init__(self, kind: str, detail: str): super().__init__(f"{kind}: {detail}") self.kind = kind self.detail = detail # -------------------------------------------------------------------------- # Configuration and credentials # -------------------------------------------------------------------------- @dataclasses.dataclass class Config: interface: str = "pia-wg" config_dir: str = "/etc/wireguard" state_dir: str = "/var/lib/pia-gateway" credentials_file: str = "/etc/pia-gateway/credentials.env" ca_file: str = "/etc/wireguard/pia-ca.crt" lock_file: str = "/run/lock/pia-gateway.lock" route_table: str = "pia" rule_priority: int = 100 vlan50_subnet: str = "10.10.50.0/24" iptables_chain: str = "PIA-VLAN50" lan_interface: str = "enp1s0" mss_clamp_enabled: bool = True region_candidates: list = dataclasses.field(default_factory=lambda: ["hk", "taiwan", "japan"]) handshake_max_age_sec: int = 180 failure_threshold: int = 3 startup_grace_sec: int = 120 restart_wait_sec: int = 90 activation_wait_sec: int = 90 recovery_max_duration_sec: int = 600 backoff_schedule_sec: list = dataclasses.field(default_factory=lambda: [300, 900, 1800, 3600]) backoff_jitter_fraction: float = 0.1 max_candidates_per_cycle: int = 6 max_rotation_candidates: int = 3 renewal_after_days: int = 21 http_connect_timeout_sec: int = 10 http_total_timeout_sec: int = 30 clock_skew_allowance_sec: int = 60 backup_keep: int = 5 lock_wait_sec: int = 600 alert_bridge_url: Optional[str] = None alert_source: str = "pia-gateway" metrics_file: str = "/var/lib/pia-gateway/metrics.prom" status_file: str = "/var/lib/pia-gateway/status.json" killswitch_unit: str = "pia-killswitch.service" @classmethod def load(cls, path: str) -> "Config": with open(path, "r", encoding="utf-8") as f: raw = json.load(f) known = {f.name for f in dataclasses.fields(cls)} unknown = set(raw) - known if unknown: raise GatewayError(f"config: unknown keys {sorted(unknown)}") cfg = cls(**raw) if not cfg.region_candidates: raise GatewayError("config: region_candidates must be non-empty") if cfg.failure_threshold < 1: raise GatewayError("config: failure_threshold must be >= 1") return cfg @property def wg_conf(self) -> str: return os.path.join(self.config_dir, f"{self.interface}.conf") @property def wg_key(self) -> str: return os.path.join(self.config_dir, f"{self.interface}.key") @property def state_file(self) -> str: return os.path.join(self.state_dir, "state.json") @property def wg_unit(self) -> str: return f"wg-quick@{self.interface}.service" @dataclasses.dataclass class Credentials: pia_user: str pia_password: str alert_bridge_api_key: Optional[str] = None def __repr__(self) -> str: # never leak via repr/str return "Credentials()" __str__ = __repr__ def load_credentials(path: str, require_pia: bool = True) -> Credentials: values: dict[str, str] = {} try: st = os.stat(path) except FileNotFoundError: if require_pia: raise GatewayError(f"credentials file {path} is missing") return Credentials("", "", None) if st.st_mode & 0o077: raise GatewayError(f"credentials file {path} must be mode 0600 (group/other bits set)") with open(path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line or line.startswith("#") or "=" not in line: continue k, v = line.split("=", 1) values[k.strip()] = v.strip() user = values.get("PIA_USER", "") password = values.get("PIA_PASSWORD", "") if require_pia and (not user or not password): raise GatewayError(f"credentials file {path} lacks PIA_USER/PIA_PASSWORD") return Credentials(user, password, values.get("ALERT_BRIDGE_API_KEY") or None) # -------------------------------------------------------------------------- # System boundary (commands, clock, files) — replaced by fakes in tests # -------------------------------------------------------------------------- @dataclasses.dataclass class CmdResult: rc: int out: str err: str class System: def run(self, argv: list[str], stdin: Optional[str] = None, timeout: int = 60) -> CmdResult: try: p = subprocess.run( argv, input=stdin, capture_output=True, text=True, timeout=timeout, check=False, ) except FileNotFoundError: return CmdResult(127, "", f"{argv[0]}: not found") except subprocess.TimeoutExpired: return CmdResult(124, "", f"{argv[0]}: timeout after {timeout}s") return CmdResult(p.returncode, p.stdout, p.stderr) def now(self) -> int: return int(time.time()) def monotonic(self) -> float: return time.monotonic() def sleep(self, sec: float) -> None: time.sleep(sec) def boot_id(self) -> str: try: with open("/proc/sys/kernel/random/boot_id", "r", encoding="utf-8") as f: return f.read().strip() except OSError: return "unknown" def read_text(self, path: str) -> str: with open(path, "r", encoding="utf-8") as f: return f.read() def exists(self, path: str) -> bool: return os.path.exists(path) def listdir(self, path: str) -> list[str]: try: return sorted(os.listdir(path)) except FileNotFoundError: return [] def unlink(self, path: str) -> None: with contextlib.suppress(FileNotFoundError): os.unlink(path) def rename(self, src: str, dst: str) -> None: os.replace(src, dst) def copy_file(self, src: str, dst: str, mode: int = 0o600) -> None: data = self.read_text(src) self.write_atomic(dst, data, mode) def mtime(self, path: str) -> float: return os.stat(path).st_mtime def makedirs(self, path: str, mode: int = 0o700) -> None: os.makedirs(path, mode=mode, exist_ok=True) os.chmod(path, mode) def write_atomic(self, path: str, data: str, mode: int = 0o600) -> None: directory = os.path.dirname(path) or "." fd, tmp = tempfile.mkstemp(dir=directory, prefix=f".{os.path.basename(path)}.", suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as f: f.write(data) f.flush() os.fsync(f.fileno()) os.chmod(tmp, mode) os.replace(tmp, path) dfd = os.open(directory, os.O_RDONLY) try: os.fsync(dfd) finally: os.close(dfd) except BaseException: with contextlib.suppress(FileNotFoundError): os.unlink(tmp) raise def random_jitter(self) -> float: return random.random() # -------------------------------------------------------------------------- # Logging (journal-friendly, secret-free by construction) # -------------------------------------------------------------------------- class Log: def __init__(self, stream: io.TextIOBase = None, json_mode: bool = False): self.stream = stream or sys.stdout self.json_mode = json_mode self.lines: list[str] = [] def emit(self, level: str, msg: str, **fields: Any) -> None: if self.json_mode: rec = {"level": level, "msg": msg} rec.update(fields) line = json.dumps(rec, sort_keys=True) else: extra = " ".join(f"{k}={_fmt(v)}" for k, v in fields.items()) line = f"{level} {msg}" + (f" {extra}" if extra else "") self.lines.append(line) print(line, file=self.stream, flush=True) def info(self, msg: str, **f: Any) -> None: self.emit("INFO", msg, **f) def warn(self, msg: str, **f: Any) -> None: self.emit("WARN", msg, **f) def error(self, msg: str, **f: Any) -> None: self.emit("ERROR", msg, **f) def _fmt(v: Any) -> str: if isinstance(v, (list, dict)): return json.dumps(v, separators=(",", ":")) return str(v) # -------------------------------------------------------------------------- # Persistent state # -------------------------------------------------------------------------- def default_state() -> dict: return { "schema_version": STATE_SCHEMA_VERSION, "boot_id": None, "boot_grace_until": 0, "restart_grace_until": 0, "consecutive_failures": 0, "consecutive_unknown": 0, "last_check": None, "last_healthy_epoch": None, "recovery": { "in_progress": False, "backoff_index": 0, "next_retry_epoch": None, "last_failure_class": None, "last_failure_epoch": None, "last_failure_detail": None, "cycles": 0, }, "last_recovery": None, "last_verified_registration": None, "active_registration": None, "renewal": {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None}, "paused": None, "emergency_close_active": False, "pending_notifications": [], } class StateStore: def __init__(self, cfg: Config, system: System): self.cfg = cfg self.sys = system def load(self) -> dict: path = self.cfg.state_file if not self.sys.exists(path): return default_state() try: raw = json.loads(self.sys.read_text(path)) except (ValueError, OSError) as e: raise StateCorruptError(f"state file {path} is unreadable or not JSON ({e.__class__.__name__})") if not isinstance(raw, dict) or raw.get("schema_version") != STATE_SCHEMA_VERSION: raise StateCorruptError( f"state file {path} has schema_version={raw.get('schema_version') if isinstance(raw, dict) else '?'}, " f"expected {STATE_SCHEMA_VERSION}; run `pia-gateway state-reset --confirm` after review" ) merged = default_state() for k, v in raw.items(): if isinstance(merged.get(k), dict) and isinstance(v, dict): merged[k].update(v) else: merged[k] = v return merged def save(self, state: dict) -> None: self.sys.makedirs(self.cfg.state_dir, 0o700) self.sys.write_atomic(self.cfg.state_file, json.dumps(state, indent=1, sort_keys=True) + "\n", 0o600) # -------------------------------------------------------------------------- # Lock # -------------------------------------------------------------------------- class Lock: """flock-based; released automatically by the kernel if the process dies.""" def __init__(self, path: str, system: System): self.path = path self.sys = system self.fd: Optional[int] = None def acquire(self, wait_sec: float = 0) -> None: directory = os.path.dirname(self.path) if directory and not os.path.isdir(directory): os.makedirs(directory, mode=0o755, exist_ok=True) fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600) start = self.sys.monotonic() while True: try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) self.fd = fd return except BlockingIOError: if self.sys.monotonic() - start >= wait_sec: os.close(fd) raise LockBusyError(f"lock {self.path} is held by another pia-gateway action") self.sys.sleep(1) def release(self) -> None: if self.fd is not None: with contextlib.suppress(OSError): fcntl.flock(self.fd, fcntl.LOCK_UN) os.close(self.fd) self.fd = None # -------------------------------------------------------------------------- # Status classification # -------------------------------------------------------------------------- @dataclasses.dataclass class Status: timestamp: int tunnel_class: str cls: str reasons: list[str] safety_ok: bool tunnel_ok: bool interface_up: bool peer_present: bool handshake_epoch: Optional[int] handshake_age_sec: Optional[int] peer_endpoint: Optional[str] expected_peer: Optional[str] route_state: str # open | closed | missing | unknown configured_peer_line: Optional[str] checks: dict def to_dict(self) -> dict: d = dataclasses.asdict(self) d["class"] = d.pop("cls") return d def parse_expected_peer(conf_text: str) -> Optional[str]: in_peer = False for line in conf_text.splitlines(): s = line.strip() if s.startswith("["): in_peer = s.lower() == "[peer]" continue if in_peer and s.startswith("PublicKey"): _, _, v = s.partition("=") return v.strip() return None def parse_configured_peer_line(conf_text: str) -> Optional[str]: for line in conf_text.splitlines(): if line.startswith("# PIA WireGuard server actually used"): return line[2:] return None def _split_rule_tokens(rule: str) -> list[str]: return rule.split() def _is_emergency_rule(rule: str) -> bool: return f"--comment {EMERGENCY_COMMENT}" in rule or f'--comment "{EMERGENCY_COMMENT}"' in rule class Classifier: def __init__(self, cfg: Config, system: System): self.cfg = cfg self.sys = system def _rc(self, argv: list[str]) -> CmdResult: return self.sys.run(argv, timeout=20) def classify(self, state: dict) -> Status: cfg = self.cfg now = self.sys.now() reasons: list[str] = [] safety_ok = True checks: dict[str, Any] = {} unknown = False # -- interface link = self._rc(["ip", "-o", "link", "show", "dev", cfg.interface]) interface_up = False if link.rc == 0 and link.out.strip(): m = re.search(r"<([^>]*)>", link.out) flags = m.group(1).split(",") if m else [] interface_up = "UP" in flags checks["interface"] = "up" if interface_up else "down" if not interface_up: reasons.append("interface is administratively down") else: checks["interface"] = "absent" reasons.append("interface absent") # -- expected peer from config expected_peer = None configured_line = None if self.sys.exists(cfg.wg_conf): try: conf_text = self.sys.read_text(cfg.wg_conf) expected_peer = parse_expected_peer(conf_text) configured_line = parse_configured_peer_line(conf_text) except OSError: reasons.append("config unreadable") else: reasons.append("config missing") checks["expected_peer_present_in_config"] = expected_peer is not None # -- handshake handshake_epoch: Optional[int] = None handshake_age: Optional[int] = None peer_present = False endpoint = None handshake_ok = False if interface_up: hs = self._rc(["wg", "show", cfg.interface, "latest-handshakes"]) ep = self._rc(["wg", "show", cfg.interface, "endpoints"]) if hs.rc != 0: reasons.append("wg show failed") else: peers = {} for line in hs.out.splitlines(): parts = line.split("\t") if len(parts) == 2 and parts[1].strip().lstrip("-").isdigit(): peers[parts[0].strip()] = int(parts[1].strip()) elif line.strip(): reasons.append("malformed wg output") if expected_peer is None: reasons.append("no expected peer to compare against") elif expected_peer not in peers: reasons.append("expected peer not present on interface") else: peer_present = True handshake_epoch = peers[expected_peer] if ep.rc == 0: for line in ep.out.splitlines(): parts = line.split("\t") if len(parts) == 2 and parts[0].strip() == expected_peer: endpoint = parts[1].strip() if endpoint in (None, "", "(none)"): reasons.append("peer has no endpoint") if handshake_epoch <= 0: reasons.append("no handshake recorded") elif handshake_epoch > now + cfg.clock_skew_allowance_sec: reasons.append(f"handshake timestamp {handshake_epoch - now}s in the future (clock jump?)") else: handshake_age = max(0, now - handshake_epoch) if handshake_age > cfg.handshake_max_age_sec: reasons.append(f"handshake {handshake_age}s old exceeds {cfg.handshake_max_age_sec}s") else: handshake_ok = True checks["handshake_age_sec"] = handshake_age # -- policy rule rule = self._rc(["ip", "rule", "show"]) rule_ok = False if rule.rc == 0: pat = re.compile(rf"^{cfg.rule_priority}:\s+from {re.escape(cfg.vlan50_subnet)} lookup {re.escape(cfg.route_table)}\s*$", re.M) rule_ok = bool(pat.search(rule.out)) else: unknown = True checks["policy_rule"] = rule_ok if not rule_ok: safety_ok = False reasons.append("policy rule missing") # -- table default route = self._rc(["ip", "route", "show", "table", cfg.route_table]) route_state = "unknown" if route.rc == 0: open_default = any( l.startswith("default") and f"dev {cfg.interface}" in l for l in route.out.splitlines() ) unreachable_default = any(l.strip().startswith("unreachable default") for l in route.out.splitlines()) if open_default: route_state = "open" elif unreachable_default: route_state = "closed" else: route_state = "missing" else: unknown = True checks["route_table_default"] = route_state if route_state in ("missing", "unknown"): safety_ok = False reasons.append(f"table {cfg.route_table} default route {route_state}") # -- FORWARD ordering fwd = self._rc(["iptables", "-S", "FORWARD"]) jump_first = False emergency_present = False if fwd.rc == 0: rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")] effective = [] for r in rules: if _is_emergency_rule(r): emergency_present = True continue effective.append(r) expected_jump = f"-A FORWARD -s {cfg.vlan50_subnet} -j {cfg.iptables_chain}" jump_first = bool(effective) and effective[0].strip() == expected_jump else: unknown = True checks["forward_jump_first"] = jump_first checks["emergency_rule_present"] = emergency_present if not jump_first: safety_ok = False reasons.append("FORWARD jump to kill-switch chain is not first") # -- chain contents chain = self._rc(["iptables", "-S", cfg.iptables_chain]) chain_ok = False accepts_ok = False if chain.rc == 0: rules = [l.strip() for l in chain.out.splitlines() if l.startswith(f"-A {cfg.iptables_chain}")] drop_idx = next((i for i, r in enumerate(rules) if r == f"-A {cfg.iptables_chain} -j DROP"), None) if drop_idx is not None and drop_idx == len(rules) - 1: chain_ok = True else: reasons.append("kill-switch chain lacks a terminal unconditional DROP") before_drop = rules[:drop_idx] if drop_idx is not None else rules out_accept = f"-s {cfg.vlan50_subnet} -o {cfg.interface} -j ACCEPT" in_accept_prefix = f"-d {cfg.vlan50_subnet} -i {cfg.interface}" has_out = any(r.endswith(out_accept) for r in before_drop) has_in = any(in_accept_prefix in r and r.endswith("-j ACCEPT") for r in before_drop) for r in before_drop: if r.endswith("-j ACCEPT") and f"-o {cfg.interface}" not in r and f"-i {cfg.interface}" not in r: chain_ok = False reasons.append("kill-switch chain has an ACCEPT not scoped to the tunnel interface") break accepts_ok = has_out and has_in else: unknown = True reasons.append("kill-switch chain missing") checks["chain_terminal_drop"] = chain_ok checks["chain_tunnel_accepts"] = accepts_ok if not chain_ok: safety_ok = False # -- host default route main = self._rc(["ip", "-4", "route", "show", "default"]) main_ok = False if main.rc == 0: main_ok = f"dev {cfg.lan_interface}" in main.out and f"dev {cfg.interface}" not in main.out else: unknown = True checks["host_default_route_ok"] = main_ok if not main_ok: safety_ok = False reasons.append("host default route is not via the LAN interface") tunnel_ok = interface_up and peer_present and handshake_ok and route_state == "open" and accepts_ok if interface_up and peer_present and handshake_ok and route_state != "open": reasons.append("handshake fresh but table route is not open") if interface_up and peer_present and handshake_ok and route_state == "open" and not accepts_ok: reasons.append("tunnel ACCEPT rules missing from kill-switch chain") # -- classify if not safety_ok: tunnel_class = CLASS_SAFETY_DRIFT elif unknown: tunnel_class = CLASS_UNKNOWN elif tunnel_ok: tunnel_class = CLASS_HEALTHY elif not interface_up or not peer_present or (handshake_ok and not tunnel_ok): tunnel_class = CLASS_TUNNEL_ABSENT else: tunnel_class = CLASS_TUNNEL_STALE cls = tunnel_class rec = state.get("recovery", {}) if tunnel_class in (CLASS_TUNNEL_STALE, CLASS_TUNNEL_ABSENT): grace = max(state.get("boot_grace_until") or 0, state.get("restart_grace_until") or 0) if now < grace: cls = CLASS_BOOT_GRACE elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_HOST_CONNECTIVITY,): cls = CLASS_HOST_CONNECTIVITY elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_REGISTRATION, FAIL_AUTH): cls = CLASS_REGISTRATION_FAILURE if state.get("paused") and cls not in (CLASS_HEALTHY, CLASS_SAFETY_DRIFT): cls = CLASS_PAUSED return Status( timestamp=now, tunnel_class=tunnel_class, cls=cls, reasons=reasons, safety_ok=safety_ok, tunnel_ok=tunnel_ok, interface_up=interface_up, peer_present=peer_present, handshake_epoch=handshake_epoch, handshake_age_sec=handshake_age, peer_endpoint=endpoint, expected_peer=expected_peer, route_state=route_state, configured_peer_line=configured_line, checks=checks, ) # -------------------------------------------------------------------------- # PIA API client (stdlib only) # -------------------------------------------------------------------------- class _PinnedHTTPSConnection(http.client.HTTPSConnection): """Connect to a fixed IP while verifying TLS against the hostname (SNI).""" def __init__(self, host: str, ip: str, port: int, context: ssl.SSLContext, connect_timeout: float, total_timeout: float): super().__init__(host, port, timeout=connect_timeout, context=context) self._pinned_ip = ip self._total_timeout = total_timeout def connect(self) -> None: # noqa: D401 sock = socket.create_connection((self._pinned_ip, self.port), self.timeout) sock.settimeout(self._total_timeout) self.sock = self._context.wrap_socket(sock, server_hostname=self.host) class HttpClient: def __init__(self, cfg: Config): self.cfg = cfg def _system_ctx(self) -> ssl.SSLContext: return ssl.create_default_context() def _pia_ctx(self) -> ssl.SSLContext: ctx = ssl.create_default_context(cafile=self.cfg.ca_file) ctx.check_hostname = True ctx.verify_mode = ssl.CERT_REQUIRED return ctx def _request(self, conn: http.client.HTTPSConnection, method: str, path: str, body: Optional[bytes], headers: dict) -> tuple[int, bytes]: try: conn.request(method, path, body=body, headers=headers) resp = conn.getresponse() data = resp.read(4_000_000) return resp.status, data except ssl.SSLCertVerificationError as e: raise ApiError("tls", "certificate verification failed") from e except ssl.SSLError as e: raise ApiError("tls", e.__class__.__name__) from e except socket.timeout as e: raise ApiError("timeout", "request timed out") from e except (OSError, http.client.HTTPException) as e: raise ApiError("network", e.__class__.__name__) from e finally: with contextlib.suppress(Exception): conn.close() def get(self, url: str) -> tuple[int, bytes]: u = urllib.parse.urlsplit(url) conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx()) return self._request(conn, "GET", u.path or "/", None, {"User-Agent": "pia-gateway"}) def post_multipart(self, url: str, fields: dict[str, str]) -> tuple[int, bytes]: boundary = "----pia-gateway-" + secrets.token_hex(12) parts = [] for k, v in fields.items(): parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n") body = ("".join(parts) + f"--{boundary}--\r\n").encode("utf-8") u = urllib.parse.urlsplit(url) conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx()) headers = { "User-Agent": "pia-gateway", "Content-Type": f"multipart/form-data; boundary={boundary}", "Content-Length": str(len(body)), } return self._request(conn, "POST", u.path, body, headers) def get_pinned(self, hostname: str, ip: str, port: int, path: str, query: dict[str, str]) -> tuple[int, bytes]: conn = _PinnedHTTPSConnection( hostname, ip, port, self._pia_ctx(), self.cfg.http_connect_timeout_sec, self.cfg.http_total_timeout_sec ) full = path + "?" + urllib.parse.urlencode(query) return self._request(conn, "GET", full, None, {"User-Agent": "pia-gateway"}) @dataclasses.dataclass class Candidate: region_id: str region_name: str cn: str ip: str @dataclasses.dataclass class PeerInfo: server_key: str peer_ip: str server_port: int server_ip: str region_id: str region_name: str cn: str _B64KEY = re.compile(r"^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$") _IPV4 = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$") def valid_wg_key(k: str) -> bool: return bool(_B64KEY.match(k or "")) def valid_ipv4(s: str) -> bool: if not _IPV4.match(s or ""): return False return all(0 <= int(o) <= 255 for o in s.split(".")) class PiaApi: def __init__(self, cfg: Config, http: HttpClient, log: Log): self.cfg = cfg self.http = http self.log = log def fetch_serverlist(self) -> dict: status, data = self.http.get(SERVERLIST_URL) if status != 200: raise ApiError("http", f"serverlist HTTP {status}") first = data.split(b"\n", 1)[0] try: doc = json.loads(first.decode("utf-8")) except (ValueError, UnicodeDecodeError) as e: raise ApiError("malformed", "serverlist is not JSON") from e if not isinstance(doc, dict) or not isinstance(doc.get("regions"), list): raise ApiError("malformed", "serverlist lacks a regions list") return doc def build_candidates(self, doc: dict, regions: Iterable[str]) -> list[Candidate]: by_id = {r.get("id"): r for r in doc["regions"] if isinstance(r, dict)} out: list[Candidate] = [] missing, empty = [], [] for rid in regions: r = by_id.get(rid) if r is None: missing.append(rid) continue servers = (r.get("servers") or {}).get("wg") or [] if not servers: empty.append(rid) continue for s in servers: cn, ip = s.get("cn"), s.get("ip") if not cn or not valid_ipv4(ip or ""): continue out.append(Candidate(rid, r.get("name") or rid, cn, ip)) if missing: raise ApiError("malformed", f"requested regions not in serverlist: {missing}") if empty: # A configured fallback region with no WireGuard servers right now # must not block recovery through the regions that do have them. self.log.warn("regions without WireGuard servers skipped", regions=empty) if not out: raise ApiError("malformed", "no usable WireGuard servers in requested regions") return out def get_token(self, creds: Credentials) -> str: last_status = None for attempt in range(3): try: status, data = self.http.post_multipart(TOKEN_URL, {"username": creds.pia_user, "password": creds.pia_password}) except ApiError as e: if e.kind in ("tls",): raise last_status = e.kind if attempt < 2: continue raise last_status = status if status == 200: try: token = json.loads(data.decode("utf-8")).get("token") except (ValueError, AttributeError, UnicodeDecodeError) as e: raise ApiError("malformed", "token response is not JSON") from e if not token or not isinstance(token, str): raise ApiError("malformed", "token response has no token") return token if status in (401, 403): raise ApiError("auth", f"token HTTP {status}") if attempt < 2 and status >= 500: continue break raise ApiError("http", f"token HTTP {last_status}") def add_key(self, cand: Candidate, token: str, pubkey: str) -> PeerInfo: status, data = self.http.get_pinned(cand.cn, cand.ip, ADDKEY_PORT, "/addKey", {"pt": token, "pubkey": pubkey}) if status != 200: raise ApiError("http", f"addKey HTTP {status}") if not data.strip(): raise ApiError("malformed", "addKey empty body") try: doc = json.loads(data.decode("utf-8")) except (ValueError, UnicodeDecodeError) as e: raise ApiError("malformed", "addKey body is not JSON") from e if not isinstance(doc, dict) or doc.get("status") != "OK": raise ApiError("status", f"addKey status={doc.get('status') if isinstance(doc, dict) else '?'}") server_key = doc.get("server_key") peer_ip = doc.get("peer_ip") port = doc.get("server_port") if not valid_wg_key(server_key or ""): raise ApiError("malformed", "addKey server_key invalid") if not valid_ipv4(peer_ip or ""): raise ApiError("malformed", "addKey peer_ip invalid") try: port = int(port) except (TypeError, ValueError): raise ApiError("malformed", "addKey server_port invalid") if not (1 <= port <= 65535): raise ApiError("malformed", "addKey server_port out of range") return PeerInfo(server_key, peer_ip, port, cand.ip, cand.region_id, cand.region_name, cand.cn) # -------------------------------------------------------------------------- # WireGuard config rendering / validation # -------------------------------------------------------------------------- def render_wg_conf(cfg: Config, private_key: str, peer: PeerInfo) -> str: i = cfg.interface t = cfg.route_table s = cfg.vlan50_subnet c = cfg.iptables_chain lines = [ "# Managed by pia-gateway (ansible/roles/pia-gateway). Do not edit by hand.", "# Table = off: routing for this interface lives only in table " f"{t}, reached only by traffic sourced from {s}.", "[Interface]", f"PrivateKey = {private_key}", f"Address = {peer.peer_ip}/32", "Table = off", "", f"PostUp = ip route replace default dev %i table {t}", f"PostUp = iptables -I {c} 1 -s {s} -o %i -j ACCEPT", f"PostUp = iptables -I {c} 1 -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT", f"PostUp = iptables -t nat -A POSTROUTING -s {s} -o %i -j MASQUERADE", ] if cfg.mss_clamp_enabled: lines.append( f"PostUp = iptables -t mangle -A FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu" ) lines += [ "", f"PreDown = ip route replace unreachable default table {t}", f"PreDown = iptables -t nat -D POSTROUTING -s {s} -o %i -j MASQUERADE 2>/dev/null || true", f"PreDown = iptables -D {c} -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT 2>/dev/null || true", f"PreDown = iptables -D {c} -s {s} -o %i -j ACCEPT 2>/dev/null || true", ] if cfg.mss_clamp_enabled: lines.append( f"PreDown = iptables -t mangle -D FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null || true" ) lines += [ "", "[Peer]", f"# PIA WireGuard server actually used — {peer.region_name} ({peer.region_id}), {peer.cn}", f"PublicKey = {peer.server_key}", f"Endpoint = {peer.server_ip}:{peer.server_port}", "AllowedIPs = 0.0.0.0/0", "PersistentKeepalive = 25", "", ] return "\n".join(lines) def validate_wg_conf(cfg: Config, text: str) -> list[str]: """Return a list of problems (empty when the staged config is acceptable).""" problems: list[str] = [] section = None kv: dict[str, list[str]] = {} for line in text.splitlines(): s = line.strip() if not s or s.startswith("#"): continue if s.startswith("["): section = s.lower() continue k, _, v = s.partition("=") kv.setdefault(f"{section}:{k.strip()}", []).append(v.strip()) if kv.get("[interface]:Table") != ["off"]: problems.append("Table must be off") pk = kv.get("[interface]:PrivateKey", [""])[0] if not valid_wg_key(pk): problems.append("PrivateKey invalid") addr = kv.get("[interface]:Address", [""])[0] if not addr.endswith("/32") or not valid_ipv4(addr[:-3]): problems.append("Address must be a /32 IPv4") if kv.get("[peer]:AllowedIPs") != ["0.0.0.0/0"]: problems.append("AllowedIPs must be 0.0.0.0/0") if not valid_wg_key(kv.get("[peer]:PublicKey", [""])[0]): problems.append("peer PublicKey invalid") ep = kv.get("[peer]:Endpoint", [""])[0] host, _, port = ep.rpartition(":") if not valid_ipv4(host) or not port.isdigit(): problems.append("Endpoint invalid") postup = kv.get("[interface]:PostUp", []) predown = kv.get("[interface]:PreDown", []) if not any(f"table {cfg.route_table}" in p and "default dev %i" in p for p in postup): problems.append("PostUp lacks table route open") if not any(f"unreachable default table {cfg.route_table}" in p for p in predown): problems.append("PreDown lacks unreachable route") for hook in postup + predown: if "ip route" in hook and "table" not in hook: problems.append("hook touches routing without a table qualifier") if f"-s {cfg.vlan50_subnet}" not in hook and "iptables" in hook and f"-d {cfg.vlan50_subnet}" not in hook: problems.append("iptables hook not scoped to VLAN 50") if "-P FORWARD" in hook: problems.append("hook changes FORWARD policy") return problems # -------------------------------------------------------------------------- # Notifier # -------------------------------------------------------------------------- class Notifier: def __init__(self, cfg: Config, creds: Credentials, http: HttpClient, log: Log): self.cfg = cfg self.creds = creds self.http = http self.log = log def send(self, level: str, message: str) -> bool: if not self.cfg.alert_bridge_url or not self.creds.alert_bridge_api_key: self.log.info("notify skipped (no alert-bridge configured)", level=level, message=message) return True u = urllib.parse.urlsplit(self.cfg.alert_bridge_url) body = json.dumps({"source": self.cfg.alert_source, "message": message, "level": level}).encode("utf-8") try: conn_cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection conn = conn_cls(u.hostname, u.port or (443 if u.scheme == "https" else 80), timeout=5) conn.request( "POST", u.path or "/", body=body, headers={ "Authorization": f"Bearer {self.creds.alert_bridge_api_key}", "Content-Type": "application/json", "Content-Length": str(len(body)), }, ) resp = conn.getresponse() resp.read(4096) conn.close() ok = 200 <= resp.status < 300 if not ok: self.log.warn("notify failed", http_status=resp.status) return ok except (OSError, http.client.HTTPException) as e: self.log.warn("notify failed", error=e.__class__.__name__) return False # -------------------------------------------------------------------------- # Controller # -------------------------------------------------------------------------- @dataclasses.dataclass class RegistrationOutcome: ok: bool stage: str failure_class: Optional[str] = None detail: Optional[str] = None region: Optional[str] = None server_cn: Optional[str] = None any_api_ok: bool = False all_api_failed_same_way: bool = False verified: Optional[Status] = None class Controller: def __init__(self, cfg: Config, system: System, log: Log, http: Optional[HttpClient] = None, creds: Optional[Credentials] = None): self.cfg = cfg self.sys = system self.log = log self.http = http or HttpClient(cfg) self._creds = creds self.store = StateStore(cfg, system) self.classifier = Classifier(cfg, system) self.api = PiaApi(cfg, self.http, log) # -- helpers ---------------------------------------------------------- @property def creds(self) -> Credentials: if self._creds is None: self._creds = load_credentials(self.cfg.credentials_file, require_pia=True) return self._creds def creds_optional(self) -> Credentials: if self._creds is None: try: self._creds = load_credentials(self.cfg.credentials_file, require_pia=False) except GatewayError as e: self.log.warn("credentials unavailable", error=str(e)) self._creds = Credentials("", "", None) return self._creds def notifier(self) -> Notifier: return Notifier(self.cfg, self.creds_optional(), self.http, self.log) def notify(self, state: dict, level: str, message: str) -> None: pending = state.setdefault("pending_notifications", []) pending.append({"level": level, "message": message, "epoch": self.sys.now()}) del pending[:-20] self.flush_notifications(state) def flush_notifications(self, state: dict) -> None: pending = state.get("pending_notifications") or [] n = self.notifier() remaining = [] for item in pending: if not n.send(item["level"], item["message"]): remaining.append(item) state["pending_notifications"] = remaining def lock(self, wait_sec: float = 0) -> Lock: lk = Lock(self.cfg.lock_file, self.sys) lk.acquire(wait_sec) return lk def write_status_outputs(self, st: Status, state: dict) -> None: cfg = self.cfg rec = state.get("recovery", {}) doc = { "status": st.to_dict(), "controller": { "paused": bool(state.get("paused")), "consecutive_failures": state.get("consecutive_failures", 0), "recovery_in_progress": rec.get("in_progress", False), "backoff_index": rec.get("backoff_index", 0), "next_retry_epoch": rec.get("next_retry_epoch"), "last_failure_class": rec.get("last_failure_class"), "last_verified_registration": state.get("last_verified_registration"), "last_recovery": state.get("last_recovery"), "emergency_close_active": state.get("emergency_close_active", False), }, } try: self.sys.makedirs(cfg.state_dir, 0o700) self.sys.write_atomic(cfg.status_file, json.dumps(doc, indent=1, sort_keys=True) + "\n", 0o644) self.sys.write_atomic(cfg.metrics_file, self.render_metrics(st, state), 0o644) except OSError as e: self.log.warn("status/metrics write failed", error=e.__class__.__name__) def render_metrics(self, st: Status, state: dict) -> str: rec = state.get("recovery", {}) lvr = state.get("last_verified_registration") or {} lr = state.get("last_recovery") or {} m = [ ("pia_gateway_check_timestamp_seconds", st.timestamp), ("pia_gateway_healthy", 1 if st.cls == CLASS_HEALTHY else 0), ("pia_gateway_tunnel_ok", 1 if st.tunnel_ok else 0), ("pia_gateway_safety_ok", 1 if st.safety_ok else 0), ("pia_gateway_handshake_age_seconds", st.handshake_age_sec if st.handshake_age_sec is not None else -1), ("pia_gateway_consecutive_failures", state.get("consecutive_failures", 0)), ("pia_gateway_paused", 1 if state.get("paused") else 0), ("pia_gateway_recovery_in_backoff", 1 if rec.get("next_retry_epoch") else 0), ("pia_gateway_recovery_backoff_index", rec.get("backoff_index", 0)), ("pia_gateway_last_verified_registration_timestamp_seconds", lvr.get("epoch") or 0), ("pia_gateway_last_recovery_timestamp_seconds", lr.get("epoch") or 0), ("pia_gateway_emergency_close_active", 1 if state.get("emergency_close_active") else 0), ] out = [f"# pia-gateway class={st.cls} tunnel_class={st.tunnel_class}"] for name, val in m: out.append(f"# TYPE {name} gauge") out.append(f"{name} {val}") return "\n".join(out) + "\n" def refresh_boot_grace(self, state: dict) -> None: bid = self.sys.boot_id() if state.get("boot_id") != bid: state["boot_id"] = bid state["boot_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec state["restart_grace_until"] = 0 self.log.info("new boot detected; startup grace applied", grace_sec=self.cfg.startup_grace_sec) self.guard_clock_jump(state) def guard_clock_jump(self, state: dict) -> None: """Persisted wall-clock deadlines further out than they could ever have been scheduled mean the clock went backwards; clear them rather than stall.""" now = self.sys.now() horizon = self.cfg.backoff_schedule_sec[-1] * 2 + 300 for section in ("recovery", "renewal"): nxt = state.get(section, {}).get("next_retry_epoch") if nxt and nxt - now > horizon: self.log.warn("clock jump detected; clearing stale retry deadline", section=section, was_in_sec=nxt - now) state[section]["next_retry_epoch"] = None for key in ("boot_grace_until", "restart_grace_until"): if (state.get(key) or 0) - now > self.cfg.startup_grace_sec * 2 + 300: state[key] = 0 def current_status(self, state: dict) -> Status: return self.classifier.classify(state) def log_status(self, st: Status) -> None: self.log.info( "status", **{"class": st.cls}, tunnel_class=st.tunnel_class, handshake_age_sec=st.handshake_age_sec, route=st.route_state, safety_ok=st.safety_ok, tunnel_ok=st.tunnel_ok, endpoint=st.peer_endpoint, reasons=st.reasons, ) if st.configured_peer_line: self.log.info("configured-peer", line=st.configured_peer_line) # -- emergency closing ------------------------------------------------ def emergency_rule_argv(self, op: str) -> list[str]: return ["iptables", op, "FORWARD"] + (["1"] if op == "-I" else []) + [ "-s", self.cfg.vlan50_subnet, "-m", "comment", "--comment", EMERGENCY_COMMENT, "-j", "DROP", ] def emergency_close(self, state: dict, st: Status) -> bool: """Insert an emergency DROP first in FORWARD and close table pia. Returns effectiveness.""" cfg = self.cfg fwd0 = self.sys.run(["iptables", "-S", "FORWARD"]) rules0 = [l for l in fwd0.out.splitlines() if l.startswith("-A FORWARD")] if fwd0.rc == 0 else [] already_first = bool(rules0) and _is_emergency_rule(rules0[0]) if not already_first: # `-C` (existence) is not enough here: something could have # prepended a rule ahead of ours (e.g. a `ufw reload` outside # this role's own handler ordering) without removing it, in # which case it still exists but is no longer FIRST — and a # rule that isn't first can be pre-empted by whatever now sits # above it. Reposition explicitly: delete wherever it is (`-D` # matches by rule spec, not position), then insert at 1. if any(_is_emergency_rule(r) for r in rules0): d = self.sys.run(self.emergency_rule_argv("-D")) if d.rc != 0: self.log.error("emergency DROP removal (repositioning) failed", rc=d.rc) ins = self.sys.run(self.emergency_rule_argv("-I")) if ins.rc != 0: self.log.error("emergency DROP insert failed", rc=ins.rc) r2 = self.sys.run(["ip", "route", "replace", "unreachable", "default", "table", cfg.route_table]) if r2.rc != 0: self.log.error("emergency unreachable route failed", rc=r2.rc) fwd = self.sys.run(["iptables", "-S", "FORWARD"]) rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")] effective = fwd.rc == 0 and bool(rules) and _is_emergency_rule(rules[0]) and r2.rc == 0 state["emergency_close_active"] = True if effective: self.log.error("SAFETY DRIFT: emergency close applied", reasons=st.reasons) self.notify(state, "error", f"pia-gateway SAFETY DRIFT: kill switch invariants failed ({'; '.join(st.reasons)}). " "Emergency VLAN 50 DROP applied; automatic recovery halted until repaired.") else: self.log.error("SAFETY DRIFT: emergency close NOT verified effective", reasons=st.reasons) self.notify(state, "error", "pia-gateway CRITICAL: safety drift and the emergency close could not be verified " f"effective ({'; '.join(st.reasons)}). Manual intervention required NOW.") return effective def emergency_clear_if_repaired(self, state: dict, st: Status) -> None: if not state.get("emergency_close_active"): return if not st.safety_ok: return r = self.sys.run(self.emergency_rule_argv("-C")) if r.rc == 0: d = self.sys.run(self.emergency_rule_argv("-D")) if d.rc != 0: self.log.error("emergency DROP removal failed", rc=d.rc) return state["emergency_close_active"] = False self.log.info("kill switch repaired; emergency DROP removed") self.notify(state, "info", "pia-gateway: kill switch invariants restored; emergency DROP removed.") # -- key handling ----------------------------------------------------- def _read_private_key(self) -> str: key = self.sys.read_text(self.cfg.wg_key).strip() if not valid_wg_key(key): raise GatewayError("existing private key file is not a valid WireGuard key") return key def _generate_key(self) -> str: r = self.sys.run(["wg", "genkey"]) key = r.out.strip() if r.rc != 0 or not valid_wg_key(key): raise GatewayError("wg genkey failed") return key def _pubkey(self, private_key: str) -> str: r = self.sys.run(["wg", "pubkey"], stdin=private_key + "\n") pub = r.out.strip() if r.rc != 0 or not valid_wg_key(pub): raise GatewayError("wg pubkey failed") return pub def clean_abandoned_stages(self) -> int: cfg = self.cfg n = 0 prefix_key = f"{cfg.interface}.key.new" prefix_conf = f"{cfg.interface}.conf.stage" for name in self.sys.listdir(cfg.config_dir): if name.startswith(prefix_key) or name.startswith(prefix_conf): self.sys.unlink(os.path.join(cfg.config_dir, name)) n += 1 if n: self.log.info("removed abandoned staged files", count=n) return n def prune_backups(self) -> None: cfg = self.cfg for base in (f"{cfg.interface}.conf.bak-", f"{cfg.interface}.key.bak-"): names = [n for n in self.sys.listdir(cfg.config_dir) if n.startswith(base)] names.sort() for old in names[: max(0, len(names) - cfg.backup_keep)]: self.sys.unlink(os.path.join(cfg.config_dir, old)) def _ts(self) -> str: return _dt.datetime.fromtimestamp(self.sys.now(), _dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") # -- activation and verification --------------------------------------- def assert_killswitch_active(self) -> None: r = self.sys.run(["systemctl", "is-active", self.cfg.killswitch_unit]) if r.out.strip() != "active": raise SafetyError(f"{self.cfg.killswitch_unit} is not active; refusing to activate the tunnel") def restart_interface(self, state: dict) -> bool: state["restart_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec r = self.sys.run(["systemctl", "restart", self.cfg.wg_unit], timeout=120) if r.rc != 0: self.log.error("restart failed", unit=self.cfg.wg_unit, rc=r.rc) if "already exists" in (r.err or ""): self.log.warn("orphaned interface detected; deleting and retrying start") self.sys.run(["ip", "link", "delete", "dev", self.cfg.interface]) r = self.sys.run(["systemctl", "start", self.cfg.wg_unit], timeout=120) return r.rc == 0 return False return True def wait_for_verified(self, state: dict, since_epoch: int, timeout_sec: int, expected_peer: Optional[str] = None) -> Optional[Status]: """Poll until two consecutive healthy classifications with a handshake newer than since_epoch.""" start = self.sys.monotonic() healthy_streak = 0 last: Optional[Status] = None while True: st = self.classifier.classify({**state, "restart_grace_until": 0, "boot_grace_until": 0, "paused": None}) last = st # Strictly greater, not >=: `wg show`/`since_epoch` are both # whole seconds, so a handshake reported at the exact same # second as `since_epoch` is indistinguishable from one that # predates activation and just happens to share that second # due to truncation. Treating it as fresh would let a stale, # pre-activation handshake pass verification — exactly the # "addKey said OK but nothing actually changed" failure mode # this whole check exists to rule out. A handshake that is # genuinely new but lands in the same second as `since_epoch` # is simply not counted yet; the next 5s poll picks it up. fresh = st.handshake_epoch is not None and st.handshake_epoch > since_epoch peer_ok = expected_peer is None or st.expected_peer == expected_peer if st.tunnel_class == CLASS_HEALTHY and fresh and peer_ok: healthy_streak += 1 if healthy_streak >= 2: return st else: healthy_streak = 0 if st.tunnel_class == CLASS_SAFETY_DRIFT: return None if self.sys.monotonic() - start >= timeout_sec: return None self.sys.sleep(5) def backup_current_config(self) -> Optional[str]: cfg = self.cfg if not self.sys.exists(cfg.wg_conf): return None dst = os.path.join(cfg.config_dir, f"{cfg.interface}.conf.bak-{self._ts()}-{os.getpid()}") self.sys.copy_file(cfg.wg_conf, dst, 0o600) return dst def activate_config(self, state: dict, staged_path: str, expected_peer: str, since_epoch: int) -> Optional[Status]: cfg = self.cfg self.assert_killswitch_active() self.sys.rename(staged_path, cfg.wg_conf) if not self.restart_interface(state): return None return self.wait_for_verified(state, since_epoch, cfg.activation_wait_sec, expected_peer) def stage_conf(self, text: str) -> str: problems = validate_wg_conf(self.cfg, text) if problems: raise GatewayError("staged config rejected: " + "; ".join(problems)) path = os.path.join(self.cfg.config_dir, f"{self.cfg.interface}.conf.stage-{self._ts()}-{os.getpid()}") self.sys.write_atomic(path, text, 0o600) return path def restore_previous_config(self, state: dict, backup_path: Optional[str]) -> None: cfg = self.cfg if not backup_path or not self.sys.exists(backup_path): self.log.warn("no previous config to restore") return self.sys.copy_file(backup_path, cfg.wg_conf, 0o600) self.log.warn("restored previous config (best effort)", backup=os.path.basename(backup_path)) self.restart_interface(state) # -- registration transaction ---------------------------------------- def registration_transaction( self, state: dict, mode: str, regions: Optional[list[str]], via: str, max_candidates: Optional[int] = None, deadline_mono: Optional[float] = None, ) -> RegistrationOutcome: """Same-key or new-key registration: API -> stage -> activate -> verify. Never opens the gateway unless verification passes. On total failure the previous config is restored (best effort) and the outcome says so. """ cfg = self.cfg self.clean_abandoned_stages() if mode not in ("same-key", "new-key"): raise GatewayError(f"unknown registration mode {mode}") # key material staged_key_path: Optional[str] = None if mode == "new-key": private_key = self._generate_key() staged_key_path = os.path.join(cfg.config_dir, f"{cfg.interface}.key.new.{self._ts()}.{os.getpid()}") self.sys.write_atomic(staged_key_path, private_key + "\n", 0o600) elif self.sys.exists(cfg.wg_key): private_key = self._read_private_key() else: private_key = self._generate_key() self.sys.write_atomic(cfg.wg_key, private_key + "\n", 0o600) self.log.info("generated initial private key") pubkey = self._pubkey(private_key) # API path try: doc = self.api.fetch_serverlist() except ApiError as e: fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION return RegistrationOutcome(False, "serverlist", fc, str(e)) try: cands = self.api.build_candidates(doc, regions or cfg.region_candidates) except ApiError as e: return RegistrationOutcome(False, "serverlist", FAIL_REGISTRATION, str(e)) if max_candidates: cands = cands[:max_candidates] try: token = self.api.get_token(self.creds) except ApiError as e: if e.kind == "auth": return RegistrationOutcome(False, "token", FAIL_AUTH, str(e)) fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION return RegistrationOutcome(False, "token", fc, str(e)) any_api_ok = False api_fail_kinds: list[str] = [] last_detail = None original_backup = self.backup_current_config() for cand in cands: if deadline_mono is not None and self.sys.monotonic() >= deadline_mono: last_detail = "recovery time budget exhausted" break try: peer = self.api.add_key(cand, token, pubkey) except ApiError as e: api_fail_kinds.append(e.kind) last_detail = f"{cand.region_id}/{cand.cn}: {e}" self.log.warn("addKey failed", region=cand.region_id, server=cand.cn, kind=e.kind, detail=e.detail) continue any_api_ok = True self.log.info("addKey accepted", region=cand.region_id, server=cand.cn) text = render_wg_conf(cfg, private_key, peer) try: staged = self.stage_conf(text) except GatewayError as e: last_detail = str(e) self.log.warn("staged config rejected", region=cand.region_id, server=cand.cn, detail=str(e)) continue since = self.sys.now() try: verified = self.activate_config(state, staged, peer.server_key, since) except SafetyError as e: self.sys.unlink(staged) return RegistrationOutcome(False, "activate", FAIL_SAFETY, str(e)) if verified is not None: if staged_key_path: if self.sys.exists(cfg.wg_key): self.sys.copy_file(cfg.wg_key, os.path.join(cfg.config_dir, f"{cfg.interface}.key.bak-{self._ts()}"), 0o600) self.sys.rename(staged_key_path, cfg.wg_key) self.prune_backups() epoch = self.sys.now() reg = { "epoch": epoch, "region": peer.region_id, "region_name": peer.region_name, "server_cn": peer.cn, "server_ip": peer.server_ip, "server_port": peer.server_port, "server_key": peer.server_key, "peer_ip": peer.peer_ip, "pubkey": pubkey, "mode": mode, "via": via, } state["last_verified_registration"] = {k: reg[k] for k in ("epoch", "region", "region_name", "server_cn", "server_ip", "pubkey", "mode", "via")} state["active_registration"] = reg state["renewal"] = {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None} self.log.info("registration verified", region=peer.region_id, server=peer.cn, mode=mode, via=via, handshake_age_sec=verified.handshake_age_sec) return RegistrationOutcome(True, "register", None, None, peer.region_id, peer.cn, True, verified=verified) last_detail = f"{cand.region_id}/{cand.cn}: activated but no verified handshake" self.log.warn("activation not verified; trying next candidate", region=cand.region_id, server=cand.cn) # all candidates failed if staged_key_path: self.sys.unlink(staged_key_path) self.restore_previous_config(state, original_backup) uniform = bool(api_fail_kinds) and not any_api_ok and len(set(api_fail_kinds)) == 1 fc = FAIL_REGISTRATION if uniform and api_fail_kinds[0] in ("network", "timeout"): fc = FAIL_HOST_CONNECTIVITY return RegistrationOutcome(False, "register", fc, last_detail, any_api_ok=any_api_ok, all_api_failed_same_way=uniform) # -- recovery cycle --------------------------------------------------- def schedule_backoff(self, state: dict, section: str, failure_class: str, detail: Optional[str]) -> int: cfg = self.cfg sec = state[section] idx = min(sec.get("backoff_index", 0), len(cfg.backoff_schedule_sec) - 1) base = cfg.backoff_schedule_sec[idx] jitter = int(base * cfg.backoff_jitter_fraction * (2 * self.sys.random_jitter() - 1)) delay = max(60, min(cfg.backoff_schedule_sec[-1], base + jitter)) sec["backoff_index"] = min(idx + 1, len(cfg.backoff_schedule_sec) - 1) sec["next_retry_epoch"] = self.sys.now() + delay sec["last_failure_epoch"] = self.sys.now() if section == "recovery": sec["last_failure_class"] = failure_class sec["last_failure_detail"] = detail return delay def recovery_cycle(self, state: dict, st: Status) -> bool: cfg = self.cfg rec = state["recovery"] start_mono = self.sys.monotonic() deadline = start_mono + cfg.recovery_max_duration_sec pre_age = st.handshake_age_sec evidence = { "pre_fix_class": st.tunnel_class, "pre_fix_handshake_age_sec": pre_age, "pre_fix_endpoint": st.peer_endpoint, "pre_fix_reasons": st.reasons, } first_cycle = not rec.get("in_progress") rec["in_progress"] = True rec["cycles"] = rec.get("cycles", 0) + 1 self.log.warn("recovery started", **evidence) if first_cycle: self.notify(state, "warn", f"pia-gateway: tunnel {st.tunnel_class} (handshake age {pre_age}s); automatic recovery started.") def finish_ok(stage: str, region: Optional[str], server: Optional[str], verified: Status) -> bool: elapsed = int(self.sys.monotonic() - start_mono) state["last_recovery"] = { "epoch": self.sys.now(), "stage": stage, "region": region, "server_cn": server, "elapsed_sec": elapsed, **evidence, } state["recovery"] = {**default_state()["recovery"]} state["consecutive_failures"] = 0 state["last_healthy_epoch"] = self.sys.now() self.log.info("recovered", stage=stage, region=region, server=server, elapsed_sec=elapsed, handshake_age_sec=verified.handshake_age_sec) self.notify(state, "info", f"pia-gateway: recovered via {stage} in {elapsed}s" + (f" ({region}/{server})" if server else "") + ".") return True def finish_fail(failure_class: str, detail: Optional[str]) -> bool: delay = self.schedule_backoff(state, "recovery", failure_class, detail) self.log.error("recovery failed", failure_class=failure_class, detail=detail, retry_in_sec=delay) level = "error" if failure_class in (FAIL_AUTH, FAIL_SAFETY) else "warn" self.notify(state, level, f"pia-gateway: recovery failed ({failure_class}: {detail}); gateway stays closed, retry in {delay // 60} min.") return False # stage 1: restart since = self.sys.now() if not self.restart_interface(state): self.log.warn("restart stage failed; escalating to registration") else: verified = self.wait_for_verified(state, since, cfg.restart_wait_sec) if verified is not None: return finish_ok("restart", None, None, verified) self.log.warn("restart did not restore a fresh handshake; escalating to registration") if self.sys.monotonic() >= deadline: return finish_fail(FAIL_REGISTRATION, "time budget exhausted after restart stage") try: # stage 2: same-key registration out = self.registration_transaction(state, "same-key", None, "recover", cfg.max_candidates_per_cycle, deadline) if out.ok: return finish_ok("reregister", out.region, out.server_cn, out.verified) if out.failure_class in (FAIL_AUTH, FAIL_HOST_CONNECTIVITY, FAIL_SAFETY): return finish_fail(out.failure_class, out.detail) if self.sys.monotonic() >= deadline: return finish_fail(out.failure_class or FAIL_REGISTRATION, "time budget exhausted after same-key stage") # stage 3: one staged fresh-key attempt, only when the API accepted # the old key but no activated candidate handshook (evidence of # server-side key rejection), never on uniform API failures. if out.any_api_ok and not out.all_api_failed_same_way: self.log.warn("same-key registration accepted but never handshook; attempting one staged key rotation") out2 = self.registration_transaction(state, "new-key", None, "recover", cfg.max_rotation_candidates, deadline) if out2.ok: return finish_ok("rotate", out2.region, out2.server_cn, out2.verified) return finish_fail(out2.failure_class or FAIL_REGISTRATION, out2.detail) return finish_fail(out.failure_class or FAIL_REGISTRATION, out.detail) except GatewayError as e: # e.g. credentials file missing/unreadable, wg tooling failure — # still enter backoff so the restart stage is not re-run every 60s. return finish_fail(FAIL_SAFETY if isinstance(e, SafetyError) else FAIL_REGISTRATION, str(e)) # -- commands --------------------------------------------------------- def cmd_check(self, as_json: bool = False) -> int: try: state = self.store.load() except StateCorruptError as e: self.log.error("state corrupt", error=str(e)) state = default_state() st = self.current_status(state) if as_json: print(json.dumps(st.to_dict(), sort_keys=True)) else: self.log_status(st) self.write_status_outputs(st, state) return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY def cmd_recover(self, force: bool = False) -> int: cfg = self.cfg try: lk = self.lock(0) except LockBusyError as e: self.log.info("skipping; another action holds the lock", detail=str(e)) return EXIT_OK state = None try: # The lock is acquired BEFORE state is read, not after — every # mutating command follows this order now (see cmd_pause for # the race it closes: a load taken before the lock can go # stale while this process waits, so saving it back afterwards # would silently clobber whatever the lock-holder just wrote). try: state = self.store.load() except StateCorruptError as e: self.log.error("state corrupt; recovery halted (fail closed)", error=str(e)) tmp = default_state() self.notify(tmp, "error", f"pia-gateway: state file corrupt ({e}); automatic recovery halted until `state-reset --confirm`.") return EXIT_STATE_ERROR self.refresh_boot_grace(state) self.flush_notifications(state) st = self.current_status(state) self.log_status(st) state["last_check"] = {"epoch": st.timestamp, "class": st.cls, "reasons": st.reasons, "handshake_age_sec": st.handshake_age_sec} cycles_before = state["recovery"].get("cycles", 0) emergency_before = state.get("emergency_close_active", False) rc = self._decide(state, st, force) acted = state["recovery"].get("cycles", 0) != cycles_before or state.get("emergency_close_active") != emergency_before self.write_status_outputs(self.current_status(state) if acted else st, state) return rc finally: if state is not None: self.store.save(state) lk.release() def _decide(self, state: dict, st: Status, force: bool) -> int: cfg = self.cfg rec = state["recovery"] if st.tunnel_class == CLASS_SAFETY_DRIFT: state["consecutive_failures"] = 0 self.emergency_close(state, st) return EXIT_SAFETY self.emergency_clear_if_repaired(state, st) if st.tunnel_class == CLASS_UNKNOWN: state["consecutive_unknown"] = state.get("consecutive_unknown", 0) + 1 if state["consecutive_unknown"] == cfg.failure_threshold: self.notify(state, "warn", f"pia-gateway: status unknown for {cfg.failure_threshold} checks ({'; '.join(st.reasons)}); no action taken.") return EXIT_UNHEALTHY state["consecutive_unknown"] = 0 if st.tunnel_class == CLASS_HEALTHY: if rec.get("in_progress") or rec.get("next_retry_epoch"): self.log.info("tunnel healthy again without a completed recovery cycle; clearing backoff") self.notify(state, "info", "pia-gateway: tunnel healthy again; backoff cleared.") state["recovery"] = {**default_state()["recovery"]} state["consecutive_failures"] = 0 state["last_healthy_epoch"] = st.timestamp return EXIT_OK if state.get("paused"): self.log.warn("paused; no recovery action", since=state["paused"].get("since"), reason=state["paused"].get("reason")) return EXIT_UNHEALTHY if st.cls == CLASS_BOOT_GRACE and not force: self.log.info("within startup/restart grace; not counting as a failure") return EXIT_UNHEALTHY state["consecutive_failures"] = state.get("consecutive_failures", 0) + 1 if state["consecutive_failures"] < cfg.failure_threshold and not force: self.log.warn("unhealthy; below threshold", consecutive=state["consecutive_failures"], threshold=cfg.failure_threshold) return EXIT_UNHEALTHY if rec.get("next_retry_epoch") and st.timestamp < rec["next_retry_epoch"] and not force: self.log.warn("in backoff; not retrying yet", retry_in_sec=rec["next_retry_epoch"] - st.timestamp, last_failure=rec.get("last_failure_class")) return EXIT_UNHEALTHY if force: rec["next_retry_epoch"] = None ok = self.recovery_cycle(state, st) return EXIT_OK if ok else EXIT_UNHEALTHY def cmd_renew(self, force_due: bool = False) -> int: cfg = self.cfg try: lk = self.lock(0) except LockBusyError: self.log.info("renewal skipped; another action holds the lock") return EXIT_OK state = None try: try: state = self.store.load() except StateCorruptError as e: self.log.error("state corrupt; renewal halted", error=str(e)) return EXIT_STATE_ERROR if state.get("paused"): self.log.info("paused; renewal skipped") return EXIT_OK st = self.current_status(state) if st.tunnel_class != CLASS_HEALTHY: self.log.info("tunnel not healthy; renewal defers to recovery", **{"class": st.tunnel_class}) return EXIT_OK ren = state["renewal"] now = self.sys.now() if ren.get("next_retry_epoch") and now < ren["next_retry_epoch"] and not force_due: self.log.info("renewal in backoff", retry_in_sec=ren["next_retry_epoch"] - now) return EXIT_OK lvr = state.get("last_verified_registration") due = force_due or lvr is None or (now - int(lvr.get("epoch", 0))) >= cfg.renewal_after_days * 86400 if not due: self.log.info("renewal not due", age_days=(now - int(lvr["epoch"])) // 86400, after_days=cfg.renewal_after_days) return EXIT_OK self.log.info("renewal due; re-registering with the existing key", baseline_missing=lvr is None) out = self.registration_transaction(state, "same-key", None, "renew", cfg.max_candidates_per_cycle) if out.ok: self.log.info("renewal verified", region=out.region, server=out.server_cn) return EXIT_OK delay = self.schedule_backoff(state, "renewal", out.failure_class or FAIL_REGISTRATION, out.detail) self.log.error("renewal failed; previous registration timestamp unchanged", failure_class=out.failure_class, detail=out.detail, retry_in_sec=delay) self.notify(state, "warn", f"pia-gateway: scheduled renewal failed ({out.failure_class}: {out.detail}); retry in {delay // 60} min.") return EXIT_UNHEALTHY finally: if state is not None: self.store.save(state) lk.release() def cmd_register(self, mode: str, region: Optional[str], via: str) -> int: cfg = self.cfg try: lk = self.lock(cfg.lock_wait_sec) except LockBusyError as e: self.log.error("could not acquire lock", detail=str(e)) return EXIT_LOCKED state = None try: try: state = self.store.load() except StateCorruptError as e: self.log.error("state corrupt; refusing to register", error=str(e)) return EXIT_STATE_ERROR self.refresh_boot_grace(state) st = self.current_status(state) if not st.safety_ok: self.log.error("safety invariants failing; refusing to register", reasons=st.reasons) return EXIT_SAFETY regions = [region] if region else None out = self.registration_transaction(state, mode, regions, via) summary = {"ok": out.ok, "stage": out.stage, "failure_class": out.failure_class, "detail": out.detail, "region": out.region, "server_cn": out.server_cn} print(json.dumps(summary, sort_keys=True)) if out.ok: state["consecutive_failures"] = 0 state["recovery"] = {**default_state()["recovery"]} return EXIT_OK return EXIT_UNHEALTHY finally: if state is not None: self.store.save(state) lk.release() def cmd_rewrite_config(self) -> int: cfg = self.cfg try: lk = self.lock(cfg.lock_wait_sec) except LockBusyError as e: self.log.error("could not acquire lock", detail=str(e)) return EXIT_LOCKED state = None try: try: state = self.store.load() except StateCorruptError as e: self.log.error("state corrupt; refusing to rewrite", error=str(e)) return EXIT_STATE_ERROR reg = state.get("active_registration") if not reg: self.log.error("no verified registration recorded; run `pia-gateway register` instead") return EXIT_USAGE st = self.current_status(state) if not st.safety_ok: self.log.error("safety invariants failing; refusing to rewrite", reasons=st.reasons) return EXIT_SAFETY peer = PeerInfo(reg["server_key"], reg["peer_ip"], int(reg["server_port"]), reg["server_ip"], reg["region"], reg.get("region_name", reg["region"]), reg["server_cn"]) private_key = self._read_private_key() if self._pubkey(private_key) != reg.get("pubkey"): self.log.error("on-disk key does not match the recorded registration; run register") return EXIT_USAGE staged = self.stage_conf(render_wg_conf(cfg, private_key, peer)) backup = self.backup_current_config() since = self.sys.now() verified = self.activate_config(state, staged, peer.server_key, since) if verified is None: self.log.error("rewritten config did not verify; restoring previous") self.restore_previous_config(state, backup) return EXIT_UNHEALTHY self.prune_backups() self.log.info("config rewritten and verified") return EXIT_OK finally: if state is not None: self.store.save(state) lk.release() def cmd_status(self) -> int: try: state = self.store.load() except StateCorruptError as e: print(f"STATE CORRUPT: {e}") return EXIT_STATE_ERROR st = self.current_status(state) rec = state["recovery"] now = self.sys.now() lines = [ f"class: {st.cls} (tunnel: {st.tunnel_class})", f"reasons: {'; '.join(st.reasons) or '-'}", f"handshake age: {st.handshake_age_sec if st.handshake_age_sec is not None else '-'}s (max {self.cfg.handshake_max_age_sec}s)", f"endpoint: {st.peer_endpoint or '-'}", f"route table {self.cfg.route_table}: {st.route_state}", f"safety ok: {st.safety_ok}", f"paused: {state.get('paused') or False}", f"consecutive failures: {state.get('consecutive_failures', 0)}", f"recovery in progress: {rec.get('in_progress')}", f"backoff next retry: {_rel(rec.get('next_retry_epoch'), now)} (last failure: {rec.get('last_failure_class') or '-'})", f"last verified reg.: {_rel((state.get('last_verified_registration') or {}).get('epoch'), now)} " f"{_reg_summary(state.get('last_verified_registration'))}", f"last recovery: {_rel((state.get('last_recovery') or {}).get('epoch'), now)} " f"{(state.get('last_recovery') or {}).get('stage') or ''}", f"emergency close active: {state.get('emergency_close_active', False)}", f"last check: {_rel((state.get('last_check') or {}).get('epoch'), now)}", ] if st.configured_peer_line: lines.append(f"configured peer: {st.configured_peer_line}") print("\n".join(lines)) return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY def cmd_pause(self, reason: str) -> int: # Same lock as recover/register/renew/rewrite-config, taken BEFORE # the state load: pause has to wait for any in-flight action to # finish (and save) before it reads state itself, or a recovery # pass that read state before pause ran could save its own # (unpaused) copy back afterwards and silently erase the pause. try: lk = self.lock(self.cfg.lock_wait_sec) except LockBusyError as e: self.log.error("could not acquire lock", detail=str(e)) return EXIT_LOCKED try: try: state = self.store.load() except StateCorruptError as e: self.log.error("state corrupt; pause not recorded", error=str(e)) return EXIT_STATE_ERROR state["paused"] = {"since": self.sys.now(), "reason": reason or "operator"} self.store.save(state) self.log.info("paused: recovery and renewal suppressed; kill switch unchanged", reason=reason) return EXIT_OK finally: lk.release() def cmd_resume(self) -> int: try: lk = self.lock(self.cfg.lock_wait_sec) except LockBusyError as e: self.log.error("could not acquire lock", detail=str(e)) return EXIT_LOCKED try: try: state = self.store.load() except StateCorruptError as e: self.log.error("state corrupt; resume not recorded", error=str(e)) return EXIT_STATE_ERROR state["paused"] = None state["consecutive_failures"] = 0 state["recovery"]["next_retry_epoch"] = None self.store.save(state) self.log.info("resumed") return EXIT_OK finally: lk.release() def cmd_retry_now(self) -> int: return self.cmd_recover(force=True) def cmd_state_reset(self, confirm: bool) -> int: if not confirm: self.log.error("refusing without --confirm") return EXIT_USAGE try: lk = self.lock(self.cfg.lock_wait_sec) except LockBusyError as e: self.log.error("could not acquire lock", detail=str(e)) return EXIT_LOCKED try: old = None if self.sys.exists(self.cfg.state_file): old = self.cfg.state_file + f".corrupt-{self._ts()}" self.sys.rename(self.cfg.state_file, old) self.store.save(default_state()) self.log.warn("state reset", previous_moved_to=old) return EXIT_OK finally: lk.release() def cmd_notify(self, level: str, message: str) -> int: ok = self.notifier().send(level, message) return EXIT_OK if ok else EXIT_UNHEALTHY def _rel(epoch: Optional[int], now: int) -> str: if not epoch: return "never" d = now - int(epoch) if d < 0: return f"in {-d}s" if d < 3600: return f"{d // 60}m ago" if d < 86400: return f"{d // 3600}h ago" return f"{d // 86400}d ago" def _reg_summary(reg: Optional[dict]) -> str: if not reg: return "" return f"({reg.get('region')}/{reg.get('server_cn')}, {reg.get('mode')}, via {reg.get('via')})" # -------------------------------------------------------------------------- # CLI # -------------------------------------------------------------------------- def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="pia-gateway", description=__doc__.split("\n\n")[0]) p.add_argument("--config", default="/etc/pia-gateway/config.json") p.add_argument("--json-log", action="store_true", help="emit JSON log lines") sub = p.add_subparsers(dest="cmd", required=True) c = sub.add_parser("check") c.add_argument("--json", action="store_true") sub.add_parser("recover") r = sub.add_parser("renew") r.add_argument("--force-due", action="store_true") g = sub.add_parser("register") g.add_argument("--mode", choices=["same-key", "new-key"], default="same-key") g.add_argument("--region", default=None) g.add_argument("--via", default="operator") sub.add_parser("rewrite-config") sub.add_parser("status") pa = sub.add_parser("pause") pa.add_argument("--reason", default="operator") sub.add_parser("resume") sub.add_parser("retry-now") sr = sub.add_parser("state-reset") sr.add_argument("--confirm", action="store_true") n = sub.add_parser("notify") n.add_argument("--level", choices=["info", "warn", "error"], default="error") n.add_argument("--message", required=True) return p def main(argv: Optional[list[str]] = None, system: Optional[System] = None, http: Optional[HttpClient] = None, stream=None) -> int: args = build_parser().parse_args(argv) log = Log(stream=stream, json_mode=args.json_log) try: cfg = Config.load(args.config) except (OSError, ValueError, GatewayError) as e: log.error("config load failed", error=str(e)) return EXIT_USAGE ctl = Controller(cfg, system or System(), log, http) try: if args.cmd == "check": return ctl.cmd_check(as_json=args.json) if args.cmd == "recover": return ctl.cmd_recover() if args.cmd == "renew": return ctl.cmd_renew(force_due=args.force_due) if args.cmd == "register": return ctl.cmd_register(args.mode, args.region, args.via) if args.cmd == "rewrite-config": return ctl.cmd_rewrite_config() if args.cmd == "status": return ctl.cmd_status() if args.cmd == "pause": return ctl.cmd_pause(args.reason) if args.cmd == "resume": return ctl.cmd_resume() if args.cmd == "retry-now": return ctl.cmd_retry_now() if args.cmd == "state-reset": return ctl.cmd_state_reset(args.confirm) if args.cmd == "notify": return ctl.cmd_notify(args.level, args.message) except GatewayError as e: log.error("failed", error=str(e)) return EXIT_SAFETY if isinstance(e, SafetyError) else EXIT_UNHEALTHY except Exception as e: # noqa: BLE001 - never let a traceback (with source context) reach the journal log.error("unexpected failure", error=e.__class__.__name__) return EXIT_UNHEALTHY return EXIT_USAGE if __name__ == "__main__": sys.exit(main())