Nik Afiq 99f6ef84f0 fix: use curl for addKey — PIA's CA cert fails modern OpenSSL by default
Found live, 2026-09-22, on the addKey step of the first successful
kill-switch-check pass: every candidate failed with
kind=tls detail="certificate verification failed". Root-caused on
minisforum: PIA's own CA certificate (files/pia-ca.crt) has a CA:TRUE
Basic Constraints extension that isn't marked critical - a real RFC
5280 non-conformance, confirmed not a stale vendored copy (a fresh
fetch from pia-foss/manual-connections is byte-identical, same
sha256 already recorded in the file's own header, same defect).
Recent OpenSSL (3.2+; confirmed via a direct repro against
minisforum's 3.5.7 - SSLCertVerificationError, verify_message
"Basic Constraints of CA cert not marked critical") rejects this by
default, a check promoted from X509_STRICT-only into the baseline
chain-verification path. openssl s_client (LibreSSL, this dev
machine) and curl both accept the identical cert/chain/CA file - only
Python's ssl module, via this host's system OpenSSL, rejects it. This
went unnoticed until now because the three prior successful
registrations all went through curl --cacert (the legacy Ansible
path); this was the first time the Python ssl code path ran for real.

Python's stdlib ssl module has no custom verify-callback hook, so
there's no clean way to tolerate just this one known,
third-party-controlled defect while keeping full validation for
everything else. Rather than weakening verify_mode or adding a new
dependency (pyOpenSSL, for a real callback - considered, rejected as
a bigger lift than necessary) to route around a gap in a library we
don't control, addKey now shells out to curl
(HttpClient.get_pinned_curl, curl added to this role's apt list) -
the same tool PIA's own reference client already uses for this exact
endpoint, with --cacert and never -k/--insecure, so this is a
different, already-proven-correct TLS backend for the one endpoint
that needs it, not a downgrade. serverlist/token are unaffected
(public WebPKI certs) and still use the stdlib ssl path.

Verified: a real subprocess curl call against the actual failing
candidate (Server-11669-10a / 69.33.198.97) gets a genuine HTTP
response from PIA, proving TLS validation succeeds via this path.
Also found and handled: curl's own exit code for a --fail-triggered
rejection isn't fully portable (macOS/SecureTransport curl reports 56
where GNU/OpenSSL curl reports 22 for the same case) - classification
now also matches curl's own error text as a second signal, not just
the exit code.

Added 8 tests (94 total, all passing) covering the curl invocation
shape, stdin token delivery (no trailing newline, never in argv), and
exit-code + text-based error classification including the
cross-platform exit-code case found live.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-22 18:48:48 +09:00

2139 lines
92 KiB
Python

#!/usr/bin/env python3
"""pia-gateway: PIA WireGuard egress gateway controller for minisforum.
Installed by ansible/roles/pia-gateway. One program, several subcommands:
check read-only status classification (journal + status/metrics files)
recover one bounded recovery pass (timer-driven, every 60s)
renew proactive same-key re-registration when the last verified
registration is older than the configured age (daily timer)
register explicit registration transaction (used by Ansible and operators)
rewrite-config
re-render pia-wg.conf from the last verified registration
(no PIA API call), activate and verify
status / pause / resume / retry-now / state-reset / notify
Invariants this program must never break (see the role README):
* The host's main default route and the global FORWARD policy are never
touched. Only table `pia`, the `PIA-VLAN50` chain, and (in an
emergency) one extra FORWARD rule scoped to the VLAN 50 source.
* Every failure path leaves VLAN 50 closed. No path opens the gateway
except a verified WireGuard activation.
* No PIA credential, token, or private key is ever written to stdout,
stderr, the journal, argv, a notification, or the metrics file.
Only the Python standard library is used so the same file runs on the
gateway (python3.13) and under the repo's local unittest suite.
"""
from __future__ import annotations
import argparse
import contextlib
import dataclasses
import datetime as _dt
import fcntl
import http.client
import io
import json
import os
import random
import re
import secrets
import socket
import ssl
import subprocess
import sys
import tempfile
import time
import urllib.parse
from typing import Any, Iterable, Optional
STATE_SCHEMA_VERSION = 1
EMERGENCY_COMMENT = "pia-gateway-emergency"
SERVERLIST_URL = "https://serverlist.piaservers.net/vpninfo/servers/v6"
TOKEN_URL = "https://www.privateinternetaccess.com/api/client/v2/token"
ADDKEY_PORT = 1337
CLASS_HEALTHY = "healthy"
CLASS_BOOT_GRACE = "boot_grace"
CLASS_PAUSED = "paused"
CLASS_TUNNEL_STALE = "tunnel_stale"
CLASS_TUNNEL_ABSENT = "tunnel_absent"
CLASS_REGISTRATION_FAILURE = "registration_or_endpoint_failure"
CLASS_SAFETY_DRIFT = "safety_drift"
CLASS_HOST_CONNECTIVITY = "host_connectivity_failure"
CLASS_UNKNOWN = "unknown"
FAIL_AUTH = "auth_failure"
FAIL_HOST_CONNECTIVITY = CLASS_HOST_CONNECTIVITY
FAIL_REGISTRATION = CLASS_REGISTRATION_FAILURE
FAIL_SAFETY = CLASS_SAFETY_DRIFT
EXIT_OK = 0
EXIT_UNHEALTHY = 1
EXIT_STATE_ERROR = 2
EXIT_SAFETY = 3
EXIT_LOCKED = 4
EXIT_USAGE = 5
class GatewayError(Exception):
"""Base class; message must never contain secrets."""
class StateCorruptError(GatewayError):
pass
class SafetyError(GatewayError):
pass
class LockBusyError(GatewayError):
pass
class ApiError(GatewayError):
def __init__(self, kind: str, detail: str):
super().__init__(f"{kind}: {detail}")
self.kind = kind
self.detail = detail
# --------------------------------------------------------------------------
# Configuration and credentials
# --------------------------------------------------------------------------
@dataclasses.dataclass
class Config:
interface: str = "pia-wg"
config_dir: str = "/etc/wireguard"
state_dir: str = "/var/lib/pia-gateway"
credentials_file: str = "/etc/pia-gateway/credentials.env"
ca_file: str = "/etc/wireguard/pia-ca.crt"
lock_file: str = "/run/lock/pia-gateway.lock"
route_table: str = "pia"
rule_priority: int = 100
vlan50_subnet: str = "10.10.50.0/24"
iptables_chain: str = "PIA-VLAN50"
lan_interface: str = "enp1s0"
mss_clamp_enabled: bool = True
region_candidates: list = dataclasses.field(default_factory=lambda: ["hk", "taiwan", "japan"])
handshake_max_age_sec: int = 180
failure_threshold: int = 3
startup_grace_sec: int = 120
restart_wait_sec: int = 90
activation_wait_sec: int = 90
recovery_max_duration_sec: int = 600
backoff_schedule_sec: list = dataclasses.field(default_factory=lambda: [300, 900, 1800, 3600])
backoff_jitter_fraction: float = 0.1
max_candidates_per_cycle: int = 6
max_rotation_candidates: int = 3
renewal_after_days: int = 21
http_connect_timeout_sec: int = 10
http_total_timeout_sec: int = 30
clock_skew_allowance_sec: int = 60
backup_keep: int = 5
lock_wait_sec: int = 600
alert_bridge_url: Optional[str] = None
alert_source: str = "pia-gateway"
metrics_file: str = "/var/lib/pia-gateway/metrics.prom"
status_file: str = "/var/lib/pia-gateway/status.json"
killswitch_unit: str = "pia-killswitch.service"
@classmethod
def load(cls, path: str) -> "Config":
with open(path, "r", encoding="utf-8") as f:
raw = json.load(f)
known = {f.name for f in dataclasses.fields(cls)}
unknown = set(raw) - known
if unknown:
raise GatewayError(f"config: unknown keys {sorted(unknown)}")
cfg = cls(**raw)
if not cfg.region_candidates:
raise GatewayError("config: region_candidates must be non-empty")
if cfg.failure_threshold < 1:
raise GatewayError("config: failure_threshold must be >= 1")
return cfg
@property
def wg_conf(self) -> str:
return os.path.join(self.config_dir, f"{self.interface}.conf")
@property
def wg_key(self) -> str:
return os.path.join(self.config_dir, f"{self.interface}.key")
@property
def state_file(self) -> str:
return os.path.join(self.state_dir, "state.json")
@property
def wg_unit(self) -> str:
return f"wg-quick@{self.interface}.service"
@dataclasses.dataclass
class Credentials:
pia_user: str
pia_password: str
alert_bridge_api_key: Optional[str] = None
def __repr__(self) -> str: # never leak via repr/str
return "Credentials(<redacted>)"
__str__ = __repr__
def load_credentials(path: str, require_pia: bool = True) -> Credentials:
values: dict[str, str] = {}
try:
st = os.stat(path)
except FileNotFoundError:
if require_pia:
raise GatewayError(f"credentials file {path} is missing")
return Credentials("", "", None)
if st.st_mode & 0o077:
raise GatewayError(f"credentials file {path} must be mode 0600 (group/other bits set)")
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, v = line.split("=", 1)
values[k.strip()] = v.strip()
user = values.get("PIA_USER", "")
password = values.get("PIA_PASSWORD", "")
if require_pia and (not user or not password):
raise GatewayError(f"credentials file {path} lacks PIA_USER/PIA_PASSWORD")
return Credentials(user, password, values.get("ALERT_BRIDGE_API_KEY") or None)
# --------------------------------------------------------------------------
# System boundary (commands, clock, files) — replaced by fakes in tests
# --------------------------------------------------------------------------
@dataclasses.dataclass
class CmdResult:
rc: int
out: str
err: str
class System:
def run(self, argv: list[str], stdin: Optional[str] = None, timeout: int = 60) -> CmdResult:
try:
p = subprocess.run(
argv,
input=stdin,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except FileNotFoundError:
return CmdResult(127, "", f"{argv[0]}: not found")
except subprocess.TimeoutExpired:
return CmdResult(124, "", f"{argv[0]}: timeout after {timeout}s")
return CmdResult(p.returncode, p.stdout, p.stderr)
def now(self) -> int:
return int(time.time())
def monotonic(self) -> float:
return time.monotonic()
def sleep(self, sec: float) -> None:
time.sleep(sec)
def boot_id(self) -> str:
try:
with open("/proc/sys/kernel/random/boot_id", "r", encoding="utf-8") as f:
return f.read().strip()
except OSError:
return "unknown"
def read_text(self, path: str) -> str:
with open(path, "r", encoding="utf-8") as f:
return f.read()
def exists(self, path: str) -> bool:
return os.path.exists(path)
def listdir(self, path: str) -> list[str]:
try:
return sorted(os.listdir(path))
except FileNotFoundError:
return []
def unlink(self, path: str) -> None:
with contextlib.suppress(FileNotFoundError):
os.unlink(path)
def rename(self, src: str, dst: str) -> None:
os.replace(src, dst)
def copy_file(self, src: str, dst: str, mode: int = 0o600) -> None:
data = self.read_text(src)
self.write_atomic(dst, data, mode)
def mtime(self, path: str) -> float:
return os.stat(path).st_mtime
def makedirs(self, path: str, mode: int = 0o700) -> None:
os.makedirs(path, mode=mode, exist_ok=True)
os.chmod(path, mode)
def write_atomic(self, path: str, data: str, mode: int = 0o600) -> None:
directory = os.path.dirname(path) or "."
fd, tmp = tempfile.mkstemp(dir=directory, prefix=f".{os.path.basename(path)}.", suffix=".tmp")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write(data)
f.flush()
os.fsync(f.fileno())
os.chmod(tmp, mode)
os.replace(tmp, path)
dfd = os.open(directory, os.O_RDONLY)
try:
os.fsync(dfd)
finally:
os.close(dfd)
except BaseException:
with contextlib.suppress(FileNotFoundError):
os.unlink(tmp)
raise
def random_jitter(self) -> float:
return random.random()
# --------------------------------------------------------------------------
# Logging (journal-friendly, secret-free by construction)
# --------------------------------------------------------------------------
class Log:
def __init__(self, stream: io.TextIOBase = None, json_mode: bool = False):
self.stream = stream or sys.stdout
self.json_mode = json_mode
self.lines: list[str] = []
def emit(self, level: str, msg: str, **fields: Any) -> None:
if self.json_mode:
rec = {"level": level, "msg": msg}
rec.update(fields)
line = json.dumps(rec, sort_keys=True)
else:
extra = " ".join(f"{k}={_fmt(v)}" for k, v in fields.items())
line = f"{level} {msg}" + (f" {extra}" if extra else "")
self.lines.append(line)
print(line, file=self.stream, flush=True)
def info(self, msg: str, **f: Any) -> None:
self.emit("INFO", msg, **f)
def warn(self, msg: str, **f: Any) -> None:
self.emit("WARN", msg, **f)
def error(self, msg: str, **f: Any) -> None:
self.emit("ERROR", msg, **f)
def _fmt(v: Any) -> str:
if isinstance(v, (list, dict)):
return json.dumps(v, separators=(",", ":"))
return str(v)
# --------------------------------------------------------------------------
# Persistent state
# --------------------------------------------------------------------------
def default_state() -> dict:
return {
"schema_version": STATE_SCHEMA_VERSION,
"boot_id": None,
"boot_grace_until": 0,
"restart_grace_until": 0,
"consecutive_failures": 0,
"consecutive_unknown": 0,
"last_check": None,
"last_healthy_epoch": None,
"recovery": {
"in_progress": False,
"backoff_index": 0,
"next_retry_epoch": None,
"last_failure_class": None,
"last_failure_epoch": None,
"last_failure_detail": None,
"cycles": 0,
},
"last_recovery": None,
"last_verified_registration": None,
"active_registration": None,
"renewal": {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None},
"paused": None,
"emergency_close_active": False,
"pending_notifications": [],
}
class StateStore:
def __init__(self, cfg: Config, system: System):
self.cfg = cfg
self.sys = system
def load(self) -> dict:
path = self.cfg.state_file
if not self.sys.exists(path):
return default_state()
try:
raw = json.loads(self.sys.read_text(path))
except (ValueError, OSError) as e:
raise StateCorruptError(f"state file {path} is unreadable or not JSON ({e.__class__.__name__})")
if not isinstance(raw, dict) or raw.get("schema_version") != STATE_SCHEMA_VERSION:
raise StateCorruptError(
f"state file {path} has schema_version={raw.get('schema_version') if isinstance(raw, dict) else '?'}, "
f"expected {STATE_SCHEMA_VERSION}; run `pia-gateway state-reset --confirm` after review"
)
merged = default_state()
for k, v in raw.items():
if isinstance(merged.get(k), dict) and isinstance(v, dict):
merged[k].update(v)
else:
merged[k] = v
return merged
def save(self, state: dict) -> None:
self.sys.makedirs(self.cfg.state_dir, 0o700)
self.sys.write_atomic(self.cfg.state_file, json.dumps(state, indent=1, sort_keys=True) + "\n", 0o600)
# --------------------------------------------------------------------------
# Lock
# --------------------------------------------------------------------------
class Lock:
"""flock-based; released automatically by the kernel if the process dies."""
def __init__(self, path: str, system: System):
self.path = path
self.sys = system
self.fd: Optional[int] = None
def acquire(self, wait_sec: float = 0) -> None:
directory = os.path.dirname(self.path)
if directory and not os.path.isdir(directory):
os.makedirs(directory, mode=0o755, exist_ok=True)
fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600)
start = self.sys.monotonic()
while True:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
self.fd = fd
return
except BlockingIOError:
if self.sys.monotonic() - start >= wait_sec:
os.close(fd)
raise LockBusyError(f"lock {self.path} is held by another pia-gateway action")
self.sys.sleep(1)
def release(self) -> None:
if self.fd is not None:
with contextlib.suppress(OSError):
fcntl.flock(self.fd, fcntl.LOCK_UN)
os.close(self.fd)
self.fd = None
# --------------------------------------------------------------------------
# Status classification
# --------------------------------------------------------------------------
@dataclasses.dataclass
class Status:
timestamp: int
tunnel_class: str
cls: str
reasons: list[str]
safety_ok: bool
tunnel_ok: bool
interface_up: bool
peer_present: bool
handshake_epoch: Optional[int]
handshake_age_sec: Optional[int]
peer_endpoint: Optional[str]
expected_peer: Optional[str]
route_state: str # open | closed | missing | unknown
configured_peer_line: Optional[str]
checks: dict
def to_dict(self) -> dict:
d = dataclasses.asdict(self)
d["class"] = d.pop("cls")
return d
def parse_expected_peer(conf_text: str) -> Optional[str]:
in_peer = False
for line in conf_text.splitlines():
s = line.strip()
if s.startswith("["):
in_peer = s.lower() == "[peer]"
continue
if in_peer and s.startswith("PublicKey"):
_, _, v = s.partition("=")
return v.strip()
return None
def parse_configured_peer_line(conf_text: str) -> Optional[str]:
for line in conf_text.splitlines():
if line.startswith("# PIA WireGuard server actually used"):
return line[2:]
return None
def _split_rule_tokens(rule: str) -> list[str]:
return rule.split()
def _is_emergency_rule(rule: str) -> bool:
return f"--comment {EMERGENCY_COMMENT}" in rule or f'--comment "{EMERGENCY_COMMENT}"' in rule
class Classifier:
def __init__(self, cfg: Config, system: System):
self.cfg = cfg
self.sys = system
def _rc(self, argv: list[str]) -> CmdResult:
return self.sys.run(argv, timeout=20)
def classify(self, state: dict) -> Status:
cfg = self.cfg
now = self.sys.now()
reasons: list[str] = []
safety_ok = True
checks: dict[str, Any] = {}
unknown = False
# -- interface
link = self._rc(["ip", "-o", "link", "show", "dev", cfg.interface])
interface_up = False
if link.rc == 0 and link.out.strip():
m = re.search(r"<([^>]*)>", link.out)
flags = m.group(1).split(",") if m else []
interface_up = "UP" in flags
checks["interface"] = "up" if interface_up else "down"
if not interface_up:
reasons.append("interface is administratively down")
else:
checks["interface"] = "absent"
reasons.append("interface absent")
# -- expected peer from config
expected_peer = None
configured_line = None
if self.sys.exists(cfg.wg_conf):
try:
conf_text = self.sys.read_text(cfg.wg_conf)
expected_peer = parse_expected_peer(conf_text)
configured_line = parse_configured_peer_line(conf_text)
except OSError:
reasons.append("config unreadable")
else:
reasons.append("config missing")
checks["expected_peer_present_in_config"] = expected_peer is not None
# -- handshake
handshake_epoch: Optional[int] = None
handshake_age: Optional[int] = None
peer_present = False
endpoint = None
handshake_ok = False
if interface_up:
hs = self._rc(["wg", "show", cfg.interface, "latest-handshakes"])
ep = self._rc(["wg", "show", cfg.interface, "endpoints"])
if hs.rc != 0:
reasons.append("wg show failed")
else:
peers = {}
for line in hs.out.splitlines():
parts = line.split("\t")
if len(parts) == 2 and parts[1].strip().lstrip("-").isdigit():
peers[parts[0].strip()] = int(parts[1].strip())
elif line.strip():
reasons.append("malformed wg output")
if expected_peer is None:
reasons.append("no expected peer to compare against")
elif expected_peer not in peers:
reasons.append("expected peer not present on interface")
else:
peer_present = True
handshake_epoch = peers[expected_peer]
if ep.rc == 0:
for line in ep.out.splitlines():
parts = line.split("\t")
if len(parts) == 2 and parts[0].strip() == expected_peer:
endpoint = parts[1].strip()
if endpoint in (None, "", "(none)"):
reasons.append("peer has no endpoint")
if handshake_epoch <= 0:
reasons.append("no handshake recorded")
elif handshake_epoch > now + cfg.clock_skew_allowance_sec:
reasons.append(f"handshake timestamp {handshake_epoch - now}s in the future (clock jump?)")
else:
handshake_age = max(0, now - handshake_epoch)
if handshake_age > cfg.handshake_max_age_sec:
reasons.append(f"handshake {handshake_age}s old exceeds {cfg.handshake_max_age_sec}s")
else:
handshake_ok = True
checks["handshake_age_sec"] = handshake_age
# -- policy rule
rule = self._rc(["ip", "rule", "show"])
rule_ok = False
if rule.rc == 0:
pat = re.compile(rf"^{cfg.rule_priority}:\s+from {re.escape(cfg.vlan50_subnet)} lookup {re.escape(cfg.route_table)}\s*$", re.M)
rule_ok = bool(pat.search(rule.out))
else:
unknown = True
checks["policy_rule"] = rule_ok
if not rule_ok:
safety_ok = False
reasons.append("policy rule missing")
# -- table default
route = self._rc(["ip", "route", "show", "table", cfg.route_table])
route_state = "unknown"
if route.rc == 0:
open_default = any(
l.startswith("default") and f"dev {cfg.interface}" in l for l in route.out.splitlines()
)
unreachable_default = any(l.strip().startswith("unreachable default") for l in route.out.splitlines())
if open_default:
route_state = "open"
elif unreachable_default:
route_state = "closed"
else:
route_state = "missing"
else:
unknown = True
checks["route_table_default"] = route_state
if route_state in ("missing", "unknown"):
safety_ok = False
reasons.append(f"table {cfg.route_table} default route {route_state}")
# -- FORWARD ordering
fwd = self._rc(["iptables", "-S", "FORWARD"])
jump_first = False
emergency_present = False
if fwd.rc == 0:
rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")]
effective = []
for r in rules:
if _is_emergency_rule(r):
emergency_present = True
continue
effective.append(r)
expected_jump = f"-A FORWARD -s {cfg.vlan50_subnet} -j {cfg.iptables_chain}"
jump_first = bool(effective) and effective[0].strip() == expected_jump
else:
unknown = True
checks["forward_jump_first"] = jump_first
checks["emergency_rule_present"] = emergency_present
if not jump_first:
safety_ok = False
reasons.append("FORWARD jump to kill-switch chain is not first")
# -- chain contents
chain = self._rc(["iptables", "-S", cfg.iptables_chain])
chain_ok = False
accepts_ok = False
if chain.rc == 0:
rules = [l.strip() for l in chain.out.splitlines() if l.startswith(f"-A {cfg.iptables_chain}")]
drop_idx = next((i for i, r in enumerate(rules) if r == f"-A {cfg.iptables_chain} -j DROP"), None)
if drop_idx is not None and drop_idx == len(rules) - 1:
chain_ok = True
else:
reasons.append("kill-switch chain lacks a terminal unconditional DROP")
before_drop = rules[:drop_idx] if drop_idx is not None else rules
out_accept = f"-s {cfg.vlan50_subnet} -o {cfg.interface} -j ACCEPT"
in_accept_prefix = f"-d {cfg.vlan50_subnet} -i {cfg.interface}"
has_out = any(r.endswith(out_accept) for r in before_drop)
has_in = any(in_accept_prefix in r and r.endswith("-j ACCEPT") for r in before_drop)
for r in before_drop:
if r.endswith("-j ACCEPT") and f"-o {cfg.interface}" not in r and f"-i {cfg.interface}" not in r:
chain_ok = False
reasons.append("kill-switch chain has an ACCEPT not scoped to the tunnel interface")
break
accepts_ok = has_out and has_in
else:
unknown = True
reasons.append("kill-switch chain missing")
checks["chain_terminal_drop"] = chain_ok
checks["chain_tunnel_accepts"] = accepts_ok
if not chain_ok:
safety_ok = False
# -- host default route
main = self._rc(["ip", "-4", "route", "show", "default"])
main_ok = False
if main.rc == 0:
main_ok = f"dev {cfg.lan_interface}" in main.out and f"dev {cfg.interface}" not in main.out
else:
unknown = True
checks["host_default_route_ok"] = main_ok
if not main_ok:
safety_ok = False
reasons.append("host default route is not via the LAN interface")
tunnel_ok = interface_up and peer_present and handshake_ok and route_state == "open" and accepts_ok
if interface_up and peer_present and handshake_ok and route_state != "open":
reasons.append("handshake fresh but table route is not open")
if interface_up and peer_present and handshake_ok and route_state == "open" and not accepts_ok:
reasons.append("tunnel ACCEPT rules missing from kill-switch chain")
# -- classify
if not safety_ok:
tunnel_class = CLASS_SAFETY_DRIFT
elif unknown:
tunnel_class = CLASS_UNKNOWN
elif tunnel_ok:
tunnel_class = CLASS_HEALTHY
elif not interface_up or not peer_present or (handshake_ok and not tunnel_ok):
tunnel_class = CLASS_TUNNEL_ABSENT
else:
tunnel_class = CLASS_TUNNEL_STALE
cls = tunnel_class
rec = state.get("recovery", {})
if tunnel_class in (CLASS_TUNNEL_STALE, CLASS_TUNNEL_ABSENT):
grace = max(state.get("boot_grace_until") or 0, state.get("restart_grace_until") or 0)
if now < grace:
cls = CLASS_BOOT_GRACE
elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_HOST_CONNECTIVITY,):
cls = CLASS_HOST_CONNECTIVITY
elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_REGISTRATION, FAIL_AUTH):
cls = CLASS_REGISTRATION_FAILURE
if state.get("paused") and cls not in (CLASS_HEALTHY, CLASS_SAFETY_DRIFT):
cls = CLASS_PAUSED
return Status(
timestamp=now,
tunnel_class=tunnel_class,
cls=cls,
reasons=reasons,
safety_ok=safety_ok,
tunnel_ok=tunnel_ok,
interface_up=interface_up,
peer_present=peer_present,
handshake_epoch=handshake_epoch,
handshake_age_sec=handshake_age,
peer_endpoint=endpoint,
expected_peer=expected_peer,
route_state=route_state,
configured_peer_line=configured_line,
checks=checks,
)
# --------------------------------------------------------------------------
# PIA API client (stdlib only)
# --------------------------------------------------------------------------
class HttpClient:
def __init__(self, cfg: Config, system: Optional[System] = None):
self.cfg = cfg
self.system = system or System()
def _system_ctx(self) -> ssl.SSLContext:
return ssl.create_default_context()
def _request(self, conn: http.client.HTTPSConnection, method: str, path: str, body: Optional[bytes], headers: dict) -> tuple[int, bytes]:
try:
conn.request(method, path, body=body, headers=headers)
resp = conn.getresponse()
data = resp.read(4_000_000)
return resp.status, data
except ssl.SSLCertVerificationError as e:
# verify_message/reason are standard OpenSSL diagnostic text
# (e.g. "unable to get local issuer certificate", "certificate
# has expired", a hostname mismatch) — never a secret, and
# exactly what's needed to tell "wrong CA", "expired cert",
# "clock skew", and "hostname mismatch" apart instead of
# collapsing all of them into one opaque string.
detail = e.verify_message or e.reason or str(e) or "certificate verification failed"
raise ApiError("tls", detail) from e
except ssl.SSLError as e:
detail = getattr(e, "reason", None) or str(e) or e.__class__.__name__
raise ApiError("tls", f"{e.__class__.__name__}: {detail}") from e
except socket.timeout as e:
raise ApiError("timeout", "request timed out") from e
except (OSError, http.client.HTTPException) as e:
detail = str(e) or e.__class__.__name__
raise ApiError("network", f"{e.__class__.__name__}: {detail}") from e
finally:
with contextlib.suppress(Exception):
conn.close()
def get(self, url: str) -> tuple[int, bytes]:
u = urllib.parse.urlsplit(url)
conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx())
return self._request(conn, "GET", u.path or "/", None, {"User-Agent": "pia-gateway"})
def post_multipart(self, url: str, fields: dict[str, str]) -> tuple[int, bytes]:
boundary = "----pia-gateway-" + secrets.token_hex(12)
parts = []
for k, v in fields.items():
parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n")
body = ("".join(parts) + f"--{boundary}--\r\n").encode("utf-8")
u = urllib.parse.urlsplit(url)
conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx())
headers = {
"User-Agent": "pia-gateway",
"Content-Type": f"multipart/form-data; boundary={boundary}",
"Content-Length": str(len(body)),
}
return self._request(conn, "POST", u.path, body, headers)
def get_pinned_curl(self, hostname: str, ip: str, port: int, path: str, token: str, pubkey: str) -> tuple[int, bytes]:
"""Pinned-IP, SNI-validated, CA-pinned GET — for PIA's addKey only.
This shells out to curl instead of using the stdlib `ssl` module.
Confirmed live (2026-09-22): PIA's own CA certificate (byte-
identical to a fresh fetch from PIA's own repo — not a stale
vendored copy) has a non-critical Basic Constraints extension on
its CA cert, a real RFC 5280 non-conformance in a CA we don't
control. Recent OpenSSL (3.2+; confirmed on minisforum's 3.5.7 via
a direct repro — SSLCertVerificationError, verify_message
"Basic Constraints of CA cert not marked critical") rejects that
in its default chain verification, a check promoted from
X509_STRICT-only to the default path in recent OpenSSL releases.
Python's stdlib `ssl` module has no custom verify-callback hook to
selectively tolerate just this one known, third-party-controlled
defect while still fully validating everything else (chain,
expiry, hostname/SAN) — that gap is long-standing and
deliberate upstream, not something to route around by weakening
verify_mode. curl's TLS stack on this host does not enforce this
specific check and is what PIA's own reference implementation
(connect_to_wireguard_with_token.sh) already uses for this exact
endpoint — reusing it here with --cacert (never -k/--insecure) is
not a security downgrade, just a different, already-proven-correct
TLS backend for the one endpoint that needs it. serverlist and
token still use the stdlib ssl path above against public WebPKI
certs, which do not have this problem.
"""
argv = [
"curl", "--silent", "--show-error", "--fail", "--location", "-G",
"--connect-timeout", str(self.cfg.http_connect_timeout_sec),
"--max-time", str(self.cfg.http_total_timeout_sec),
"--cacert", self.cfg.ca_file,
"--connect-to", f"{hostname}::{ip}:",
"--data-urlencode", "pt@-",
"--data-urlencode", f"pubkey={pubkey}",
f"https://{hostname}:{port}{path}",
]
r = self.system.run(argv, stdin=token, timeout=self.cfg.http_total_timeout_sec + 10)
if r.rc == 0:
return 200, r.out.encode("utf-8")
err = (r.err or "").strip()
# Exit codes are the primary signal, but not fully portable across
# curl builds/TLS backends — confirmed live: a curl built against
# SecureTransport reported exit 56 (CURLE_RECV_ERROR) for the exact
# same --fail-triggered HTTP rejection GNU/OpenSSL curl reports as
# 22. curl's own error text ("returned error: NNN" for --fail,
# "timed out" for CURLE_OPERATION_TIMEDOUT, TLS-related wording for
# cert/handshake problems) is a second, more portable signal — used
# here as a fallback classifier alongside the exit code, not a
# replacement for it.
if r.rc == 127:
raise ApiError("network", "curl not found on PATH")
if r.rc == 28 or "timed out" in err.lower():
raise ApiError("timeout", f"curl: connection or transfer timed out: {err}")
if r.rc in (35, 53, 54, 58, 59, 60, 66, 77, 80, 82, 83) or "ssl" in err.lower() or "certificate" in err.lower():
raise ApiError("tls", f"curl TLS error (exit {r.rc}): {err}")
if r.rc == 22 or "returned error:" in err:
# --fail suppresses the body on a non-2xx response, so there is
# no status code or body to parse here — same limitation the
# legacy curl-based Ansible path already had.
raise ApiError("http", f"curl: non-2xx HTTP response (exit {r.rc}): {err}")
raise ApiError("network", f"curl exit {r.rc}: {err}")
@dataclasses.dataclass
class Candidate:
region_id: str
region_name: str
cn: str
ip: str
@dataclasses.dataclass
class PeerInfo:
server_key: str
peer_ip: str
server_port: int
server_ip: str
region_id: str
region_name: str
cn: str
_B64KEY = re.compile(r"^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$")
_IPV4 = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$")
def valid_wg_key(k: str) -> bool:
return bool(_B64KEY.match(k or ""))
def valid_ipv4(s: str) -> bool:
if not _IPV4.match(s or ""):
return False
return all(0 <= int(o) <= 255 for o in s.split("."))
class PiaApi:
def __init__(self, cfg: Config, http: HttpClient, log: Log):
self.cfg = cfg
self.http = http
self.log = log
def fetch_serverlist(self) -> dict:
status, data = self.http.get(SERVERLIST_URL)
if status != 200:
raise ApiError("http", f"serverlist HTTP {status}")
first = data.split(b"\n", 1)[0]
try:
doc = json.loads(first.decode("utf-8"))
except (ValueError, UnicodeDecodeError) as e:
raise ApiError("malformed", "serverlist is not JSON") from e
if not isinstance(doc, dict) or not isinstance(doc.get("regions"), list):
raise ApiError("malformed", "serverlist lacks a regions list")
return doc
def build_candidates(self, doc: dict, regions: Iterable[str]) -> list[Candidate]:
by_id = {r.get("id"): r for r in doc["regions"] if isinstance(r, dict)}
out: list[Candidate] = []
missing, empty = [], []
for rid in regions:
r = by_id.get(rid)
if r is None:
missing.append(rid)
continue
servers = (r.get("servers") or {}).get("wg") or []
if not servers:
empty.append(rid)
continue
for s in servers:
cn, ip = s.get("cn"), s.get("ip")
if not cn or not valid_ipv4(ip or ""):
continue
out.append(Candidate(rid, r.get("name") or rid, cn, ip))
if missing:
raise ApiError("malformed", f"requested regions not in serverlist: {missing}")
if empty:
# A configured fallback region with no WireGuard servers right now
# must not block recovery through the regions that do have them.
self.log.warn("regions without WireGuard servers skipped", regions=empty)
if not out:
raise ApiError("malformed", "no usable WireGuard servers in requested regions")
return out
def get_token(self, creds: Credentials) -> str:
last_status = None
for attempt in range(3):
try:
status, data = self.http.post_multipart(TOKEN_URL, {"username": creds.pia_user, "password": creds.pia_password})
except ApiError as e:
if e.kind in ("tls",):
raise
last_status = e.kind
if attempt < 2:
continue
raise
last_status = status
if status == 200:
try:
token = json.loads(data.decode("utf-8")).get("token")
except (ValueError, AttributeError, UnicodeDecodeError) as e:
raise ApiError("malformed", "token response is not JSON") from e
if not token or not isinstance(token, str):
raise ApiError("malformed", "token response has no token")
return token
if status in (401, 403):
raise ApiError("auth", f"token HTTP {status}")
if attempt < 2 and status >= 500:
continue
break
raise ApiError("http", f"token HTTP {last_status}")
def add_key(self, cand: Candidate, token: str, pubkey: str) -> PeerInfo:
status, data = self.http.get_pinned_curl(cand.cn, cand.ip, ADDKEY_PORT, "/addKey", token, pubkey)
if status != 200:
raise ApiError("http", f"addKey HTTP {status}")
if not data.strip():
raise ApiError("malformed", "addKey empty body")
try:
doc = json.loads(data.decode("utf-8"))
except (ValueError, UnicodeDecodeError) as e:
raise ApiError("malformed", "addKey body is not JSON") from e
if not isinstance(doc, dict) or doc.get("status") != "OK":
raise ApiError("status", f"addKey status={doc.get('status') if isinstance(doc, dict) else '?'}")
server_key = doc.get("server_key")
peer_ip = doc.get("peer_ip")
port = doc.get("server_port")
if not valid_wg_key(server_key or ""):
raise ApiError("malformed", "addKey server_key invalid")
if not valid_ipv4(peer_ip or ""):
raise ApiError("malformed", "addKey peer_ip invalid")
try:
port = int(port)
except (TypeError, ValueError):
raise ApiError("malformed", "addKey server_port invalid")
if not (1 <= port <= 65535):
raise ApiError("malformed", "addKey server_port out of range")
return PeerInfo(server_key, peer_ip, port, cand.ip, cand.region_id, cand.region_name, cand.cn)
# --------------------------------------------------------------------------
# WireGuard config rendering / validation
# --------------------------------------------------------------------------
def render_wg_conf(cfg: Config, private_key: str, peer: PeerInfo) -> str:
i = cfg.interface
t = cfg.route_table
s = cfg.vlan50_subnet
c = cfg.iptables_chain
lines = [
"# Managed by pia-gateway (ansible/roles/pia-gateway). Do not edit by hand.",
"# Table = off: routing for this interface lives only in table "
f"{t}, reached only by traffic sourced from {s}.",
"[Interface]",
f"PrivateKey = {private_key}",
f"Address = {peer.peer_ip}/32",
"Table = off",
"",
f"PostUp = ip route replace default dev %i table {t}",
f"PostUp = iptables -I {c} 1 -s {s} -o %i -j ACCEPT",
f"PostUp = iptables -I {c} 1 -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT",
f"PostUp = iptables -t nat -A POSTROUTING -s {s} -o %i -j MASQUERADE",
]
if cfg.mss_clamp_enabled:
lines.append(
f"PostUp = iptables -t mangle -A FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu"
)
lines += [
"",
f"PreDown = ip route replace unreachable default table {t}",
f"PreDown = iptables -t nat -D POSTROUTING -s {s} -o %i -j MASQUERADE 2>/dev/null || true",
f"PreDown = iptables -D {c} -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT 2>/dev/null || true",
f"PreDown = iptables -D {c} -s {s} -o %i -j ACCEPT 2>/dev/null || true",
]
if cfg.mss_clamp_enabled:
lines.append(
f"PreDown = iptables -t mangle -D FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null || true"
)
lines += [
"",
"[Peer]",
f"# PIA WireGuard server actually used — {peer.region_name} ({peer.region_id}), {peer.cn}",
f"PublicKey = {peer.server_key}",
f"Endpoint = {peer.server_ip}:{peer.server_port}",
"AllowedIPs = 0.0.0.0/0",
"PersistentKeepalive = 25",
"",
]
return "\n".join(lines)
def validate_wg_conf(cfg: Config, text: str) -> list[str]:
"""Return a list of problems (empty when the staged config is acceptable)."""
problems: list[str] = []
section = None
kv: dict[str, list[str]] = {}
for line in text.splitlines():
s = line.strip()
if not s or s.startswith("#"):
continue
if s.startswith("["):
section = s.lower()
continue
k, _, v = s.partition("=")
kv.setdefault(f"{section}:{k.strip()}", []).append(v.strip())
if kv.get("[interface]:Table") != ["off"]:
problems.append("Table must be off")
pk = kv.get("[interface]:PrivateKey", [""])[0]
if not valid_wg_key(pk):
problems.append("PrivateKey invalid")
addr = kv.get("[interface]:Address", [""])[0]
if not addr.endswith("/32") or not valid_ipv4(addr[:-3]):
problems.append("Address must be a /32 IPv4")
if kv.get("[peer]:AllowedIPs") != ["0.0.0.0/0"]:
problems.append("AllowedIPs must be 0.0.0.0/0")
if not valid_wg_key(kv.get("[peer]:PublicKey", [""])[0]):
problems.append("peer PublicKey invalid")
ep = kv.get("[peer]:Endpoint", [""])[0]
host, _, port = ep.rpartition(":")
if not valid_ipv4(host) or not port.isdigit():
problems.append("Endpoint invalid")
postup = kv.get("[interface]:PostUp", [])
predown = kv.get("[interface]:PreDown", [])
if not any(f"table {cfg.route_table}" in p and "default dev %i" in p for p in postup):
problems.append("PostUp lacks table route open")
if not any(f"unreachable default table {cfg.route_table}" in p for p in predown):
problems.append("PreDown lacks unreachable route")
for hook in postup + predown:
if "ip route" in hook and "table" not in hook:
problems.append("hook touches routing without a table qualifier")
if f"-s {cfg.vlan50_subnet}" not in hook and "iptables" in hook and f"-d {cfg.vlan50_subnet}" not in hook:
problems.append("iptables hook not scoped to VLAN 50")
if "-P FORWARD" in hook:
problems.append("hook changes FORWARD policy")
return problems
# --------------------------------------------------------------------------
# Notifier
# --------------------------------------------------------------------------
class Notifier:
def __init__(self, cfg: Config, creds: Credentials, http: HttpClient, log: Log):
self.cfg = cfg
self.creds = creds
self.http = http
self.log = log
def send(self, level: str, message: str) -> bool:
if not self.cfg.alert_bridge_url or not self.creds.alert_bridge_api_key:
self.log.info("notify skipped (no alert-bridge configured)", level=level, message=message)
return True
u = urllib.parse.urlsplit(self.cfg.alert_bridge_url)
body = json.dumps({"source": self.cfg.alert_source, "message": message, "level": level}).encode("utf-8")
try:
conn_cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection
conn = conn_cls(u.hostname, u.port or (443 if u.scheme == "https" else 80), timeout=5)
conn.request(
"POST",
u.path or "/",
body=body,
headers={
"Authorization": f"Bearer {self.creds.alert_bridge_api_key}",
"Content-Type": "application/json",
"Content-Length": str(len(body)),
},
)
resp = conn.getresponse()
resp.read(4096)
conn.close()
ok = 200 <= resp.status < 300
if not ok:
self.log.warn("notify failed", http_status=resp.status)
return ok
except (OSError, http.client.HTTPException) as e:
self.log.warn("notify failed", error=e.__class__.__name__)
return False
# --------------------------------------------------------------------------
# Controller
# --------------------------------------------------------------------------
@dataclasses.dataclass
class RegistrationOutcome:
ok: bool
stage: str
failure_class: Optional[str] = None
detail: Optional[str] = None
region: Optional[str] = None
server_cn: Optional[str] = None
any_api_ok: bool = False
all_api_failed_same_way: bool = False
verified: Optional[Status] = None
class Controller:
def __init__(self, cfg: Config, system: System, log: Log, http: Optional[HttpClient] = None, creds: Optional[Credentials] = None):
self.cfg = cfg
self.sys = system
self.log = log
self.http = http or HttpClient(cfg, system)
self._creds = creds
self.store = StateStore(cfg, system)
self.classifier = Classifier(cfg, system)
self.api = PiaApi(cfg, self.http, log)
# -- helpers ----------------------------------------------------------
@property
def creds(self) -> Credentials:
if self._creds is None:
self._creds = load_credentials(self.cfg.credentials_file, require_pia=True)
return self._creds
def creds_optional(self) -> Credentials:
if self._creds is None:
try:
self._creds = load_credentials(self.cfg.credentials_file, require_pia=False)
except GatewayError as e:
self.log.warn("credentials unavailable", error=str(e))
self._creds = Credentials("", "", None)
return self._creds
def notifier(self) -> Notifier:
return Notifier(self.cfg, self.creds_optional(), self.http, self.log)
def notify(self, state: dict, level: str, message: str) -> None:
pending = state.setdefault("pending_notifications", [])
pending.append({"level": level, "message": message, "epoch": self.sys.now()})
del pending[:-20]
self.flush_notifications(state)
def flush_notifications(self, state: dict) -> None:
pending = state.get("pending_notifications") or []
n = self.notifier()
remaining = []
for item in pending:
if not n.send(item["level"], item["message"]):
remaining.append(item)
state["pending_notifications"] = remaining
def lock(self, wait_sec: float = 0) -> Lock:
lk = Lock(self.cfg.lock_file, self.sys)
lk.acquire(wait_sec)
return lk
def write_status_outputs(self, st: Status, state: dict) -> None:
cfg = self.cfg
rec = state.get("recovery", {})
doc = {
"status": st.to_dict(),
"controller": {
"paused": bool(state.get("paused")),
"consecutive_failures": state.get("consecutive_failures", 0),
"recovery_in_progress": rec.get("in_progress", False),
"backoff_index": rec.get("backoff_index", 0),
"next_retry_epoch": rec.get("next_retry_epoch"),
"last_failure_class": rec.get("last_failure_class"),
"last_verified_registration": state.get("last_verified_registration"),
"last_recovery": state.get("last_recovery"),
"emergency_close_active": state.get("emergency_close_active", False),
},
}
try:
self.sys.makedirs(cfg.state_dir, 0o700)
self.sys.write_atomic(cfg.status_file, json.dumps(doc, indent=1, sort_keys=True) + "\n", 0o644)
self.sys.write_atomic(cfg.metrics_file, self.render_metrics(st, state), 0o644)
except OSError as e:
self.log.warn("status/metrics write failed", error=e.__class__.__name__)
def render_metrics(self, st: Status, state: dict) -> str:
rec = state.get("recovery", {})
lvr = state.get("last_verified_registration") or {}
lr = state.get("last_recovery") or {}
m = [
("pia_gateway_check_timestamp_seconds", st.timestamp),
("pia_gateway_healthy", 1 if st.cls == CLASS_HEALTHY else 0),
("pia_gateway_tunnel_ok", 1 if st.tunnel_ok else 0),
("pia_gateway_safety_ok", 1 if st.safety_ok else 0),
("pia_gateway_handshake_age_seconds", st.handshake_age_sec if st.handshake_age_sec is not None else -1),
("pia_gateway_consecutive_failures", state.get("consecutive_failures", 0)),
("pia_gateway_paused", 1 if state.get("paused") else 0),
("pia_gateway_recovery_in_backoff", 1 if rec.get("next_retry_epoch") else 0),
("pia_gateway_recovery_backoff_index", rec.get("backoff_index", 0)),
("pia_gateway_last_verified_registration_timestamp_seconds", lvr.get("epoch") or 0),
("pia_gateway_last_recovery_timestamp_seconds", lr.get("epoch") or 0),
("pia_gateway_emergency_close_active", 1 if state.get("emergency_close_active") else 0),
]
out = [f"# pia-gateway class={st.cls} tunnel_class={st.tunnel_class}"]
for name, val in m:
out.append(f"# TYPE {name} gauge")
out.append(f"{name} {val}")
return "\n".join(out) + "\n"
def refresh_boot_grace(self, state: dict) -> None:
bid = self.sys.boot_id()
if state.get("boot_id") != bid:
state["boot_id"] = bid
state["boot_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec
state["restart_grace_until"] = 0
self.log.info("new boot detected; startup grace applied", grace_sec=self.cfg.startup_grace_sec)
self.guard_clock_jump(state)
def guard_clock_jump(self, state: dict) -> None:
"""Persisted wall-clock deadlines further out than they could ever have been
scheduled mean the clock went backwards; clear them rather than stall."""
now = self.sys.now()
horizon = self.cfg.backoff_schedule_sec[-1] * 2 + 300
for section in ("recovery", "renewal"):
nxt = state.get(section, {}).get("next_retry_epoch")
if nxt and nxt - now > horizon:
self.log.warn("clock jump detected; clearing stale retry deadline", section=section, was_in_sec=nxt - now)
state[section]["next_retry_epoch"] = None
for key in ("boot_grace_until", "restart_grace_until"):
if (state.get(key) or 0) - now > self.cfg.startup_grace_sec * 2 + 300:
state[key] = 0
def current_status(self, state: dict) -> Status:
return self.classifier.classify(state)
def log_status(self, st: Status) -> None:
self.log.info(
"status",
**{"class": st.cls},
tunnel_class=st.tunnel_class,
handshake_age_sec=st.handshake_age_sec,
route=st.route_state,
safety_ok=st.safety_ok,
tunnel_ok=st.tunnel_ok,
endpoint=st.peer_endpoint,
reasons=st.reasons,
)
if st.configured_peer_line:
self.log.info("configured-peer", line=st.configured_peer_line)
# -- emergency closing ------------------------------------------------
def emergency_rule_argv(self, op: str) -> list[str]:
return ["iptables", op, "FORWARD"] + (["1"] if op == "-I" else []) + [
"-s", self.cfg.vlan50_subnet, "-m", "comment", "--comment", EMERGENCY_COMMENT, "-j", "DROP",
]
def emergency_close(self, state: dict, st: Status) -> bool:
"""Insert an emergency DROP first in FORWARD and close table pia. Returns effectiveness."""
cfg = self.cfg
fwd0 = self.sys.run(["iptables", "-S", "FORWARD"])
rules0 = [l for l in fwd0.out.splitlines() if l.startswith("-A FORWARD")] if fwd0.rc == 0 else []
already_first = bool(rules0) and _is_emergency_rule(rules0[0])
if not already_first:
# `-C` (existence) is not enough here: something could have
# prepended a rule ahead of ours (e.g. a `ufw reload` outside
# this role's own handler ordering) without removing it, in
# which case it still exists but is no longer FIRST — and a
# rule that isn't first can be pre-empted by whatever now sits
# above it. Reposition explicitly: delete wherever it is (`-D`
# matches by rule spec, not position), then insert at 1.
if any(_is_emergency_rule(r) for r in rules0):
d = self.sys.run(self.emergency_rule_argv("-D"))
if d.rc != 0:
self.log.error("emergency DROP removal (repositioning) failed", rc=d.rc)
ins = self.sys.run(self.emergency_rule_argv("-I"))
if ins.rc != 0:
self.log.error("emergency DROP insert failed", rc=ins.rc)
r2 = self.sys.run(["ip", "route", "replace", "unreachable", "default", "table", cfg.route_table])
if r2.rc != 0:
self.log.error("emergency unreachable route failed", rc=r2.rc)
fwd = self.sys.run(["iptables", "-S", "FORWARD"])
rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")]
effective = fwd.rc == 0 and bool(rules) and _is_emergency_rule(rules[0]) and r2.rc == 0
state["emergency_close_active"] = True
if effective:
self.log.error("SAFETY DRIFT: emergency close applied", reasons=st.reasons)
self.notify(state, "error", f"pia-gateway SAFETY DRIFT: kill switch invariants failed ({'; '.join(st.reasons)}). "
"Emergency VLAN 50 DROP applied; automatic recovery halted until repaired.")
else:
self.log.error("SAFETY DRIFT: emergency close NOT verified effective", reasons=st.reasons)
self.notify(state, "error", "pia-gateway CRITICAL: safety drift and the emergency close could not be verified "
f"effective ({'; '.join(st.reasons)}). Manual intervention required NOW.")
return effective
def emergency_clear_if_repaired(self, state: dict, st: Status) -> None:
if not state.get("emergency_close_active"):
return
if not st.safety_ok:
return
r = self.sys.run(self.emergency_rule_argv("-C"))
if r.rc == 0:
d = self.sys.run(self.emergency_rule_argv("-D"))
if d.rc != 0:
self.log.error("emergency DROP removal failed", rc=d.rc)
return
state["emergency_close_active"] = False
self.log.info("kill switch repaired; emergency DROP removed")
self.notify(state, "info", "pia-gateway: kill switch invariants restored; emergency DROP removed.")
# -- key handling -----------------------------------------------------
def _read_private_key(self) -> str:
key = self.sys.read_text(self.cfg.wg_key).strip()
if not valid_wg_key(key):
raise GatewayError("existing private key file is not a valid WireGuard key")
return key
def _generate_key(self) -> str:
r = self.sys.run(["wg", "genkey"])
key = r.out.strip()
if r.rc != 0 or not valid_wg_key(key):
raise GatewayError("wg genkey failed")
return key
def _pubkey(self, private_key: str) -> str:
r = self.sys.run(["wg", "pubkey"], stdin=private_key + "\n")
pub = r.out.strip()
if r.rc != 0 or not valid_wg_key(pub):
raise GatewayError("wg pubkey failed")
return pub
def clean_abandoned_stages(self) -> int:
cfg = self.cfg
n = 0
prefix_key = f"{cfg.interface}.key.new"
prefix_conf = f"{cfg.interface}.conf.stage"
for name in self.sys.listdir(cfg.config_dir):
if name.startswith(prefix_key) or name.startswith(prefix_conf):
self.sys.unlink(os.path.join(cfg.config_dir, name))
n += 1
if n:
self.log.info("removed abandoned staged files", count=n)
return n
def prune_backups(self) -> None:
cfg = self.cfg
for base in (f"{cfg.interface}.conf.bak-", f"{cfg.interface}.key.bak-"):
names = [n for n in self.sys.listdir(cfg.config_dir) if n.startswith(base)]
names.sort()
for old in names[: max(0, len(names) - cfg.backup_keep)]:
self.sys.unlink(os.path.join(cfg.config_dir, old))
def _ts(self) -> str:
return _dt.datetime.fromtimestamp(self.sys.now(), _dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
# -- activation and verification ---------------------------------------
def assert_killswitch_active(self) -> None:
r = self.sys.run(["systemctl", "is-active", self.cfg.killswitch_unit])
if r.out.strip() != "active":
raise SafetyError(f"{self.cfg.killswitch_unit} is not active; refusing to activate the tunnel")
def restart_interface(self, state: dict) -> bool:
state["restart_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec
r = self.sys.run(["systemctl", "restart", self.cfg.wg_unit], timeout=120)
if r.rc != 0:
self.log.error("restart failed", unit=self.cfg.wg_unit, rc=r.rc)
if "already exists" in (r.err or ""):
self.log.warn("orphaned interface detected; deleting and retrying start")
self.sys.run(["ip", "link", "delete", "dev", self.cfg.interface])
r = self.sys.run(["systemctl", "start", self.cfg.wg_unit], timeout=120)
return r.rc == 0
return False
return True
def wait_for_verified(self, state: dict, since_epoch: int, timeout_sec: int, expected_peer: Optional[str] = None) -> Optional[Status]:
"""Poll until two consecutive healthy classifications with a handshake newer than since_epoch."""
start = self.sys.monotonic()
healthy_streak = 0
last: Optional[Status] = None
while True:
st = self.classifier.classify({**state, "restart_grace_until": 0, "boot_grace_until": 0, "paused": None})
last = st
# Strictly greater, not >=: `wg show`/`since_epoch` are both
# whole seconds, so a handshake reported at the exact same
# second as `since_epoch` is indistinguishable from one that
# predates activation and just happens to share that second
# due to truncation. Treating it as fresh would let a stale,
# pre-activation handshake pass verification — exactly the
# "addKey said OK but nothing actually changed" failure mode
# this whole check exists to rule out. A handshake that is
# genuinely new but lands in the same second as `since_epoch`
# is simply not counted yet; the next 5s poll picks it up.
fresh = st.handshake_epoch is not None and st.handshake_epoch > since_epoch
peer_ok = expected_peer is None or st.expected_peer == expected_peer
if st.tunnel_class == CLASS_HEALTHY and fresh and peer_ok:
healthy_streak += 1
if healthy_streak >= 2:
return st
else:
healthy_streak = 0
if st.tunnel_class == CLASS_SAFETY_DRIFT:
return None
if self.sys.monotonic() - start >= timeout_sec:
return None
self.sys.sleep(5)
def backup_current_config(self) -> Optional[str]:
cfg = self.cfg
if not self.sys.exists(cfg.wg_conf):
return None
dst = os.path.join(cfg.config_dir, f"{cfg.interface}.conf.bak-{self._ts()}-{os.getpid()}")
self.sys.copy_file(cfg.wg_conf, dst, 0o600)
return dst
def activate_config(self, state: dict, staged_path: str, expected_peer: str, since_epoch: int) -> Optional[Status]:
cfg = self.cfg
self.assert_killswitch_active()
self.sys.rename(staged_path, cfg.wg_conf)
if not self.restart_interface(state):
return None
return self.wait_for_verified(state, since_epoch, cfg.activation_wait_sec, expected_peer)
def stage_conf(self, text: str) -> str:
problems = validate_wg_conf(self.cfg, text)
if problems:
raise GatewayError("staged config rejected: " + "; ".join(problems))
path = os.path.join(self.cfg.config_dir, f"{self.cfg.interface}.conf.stage-{self._ts()}-{os.getpid()}")
self.sys.write_atomic(path, text, 0o600)
return path
def restore_previous_config(self, state: dict, backup_path: Optional[str]) -> None:
cfg = self.cfg
if not backup_path or not self.sys.exists(backup_path):
self.log.warn("no previous config to restore")
return
self.sys.copy_file(backup_path, cfg.wg_conf, 0o600)
self.log.warn("restored previous config (best effort)", backup=os.path.basename(backup_path))
self.restart_interface(state)
# -- registration transaction ----------------------------------------
def registration_transaction(
self,
state: dict,
mode: str,
regions: Optional[list[str]],
via: str,
max_candidates: Optional[int] = None,
deadline_mono: Optional[float] = None,
) -> RegistrationOutcome:
"""Same-key or new-key registration: API -> stage -> activate -> verify.
Never opens the gateway unless verification passes. On total failure the
previous config is restored (best effort) and the outcome says so.
"""
cfg = self.cfg
self.clean_abandoned_stages()
if mode not in ("same-key", "new-key"):
raise GatewayError(f"unknown registration mode {mode}")
# key material
staged_key_path: Optional[str] = None
if mode == "new-key":
private_key = self._generate_key()
staged_key_path = os.path.join(cfg.config_dir, f"{cfg.interface}.key.new.{self._ts()}.{os.getpid()}")
self.sys.write_atomic(staged_key_path, private_key + "\n", 0o600)
elif self.sys.exists(cfg.wg_key):
private_key = self._read_private_key()
else:
private_key = self._generate_key()
self.sys.write_atomic(cfg.wg_key, private_key + "\n", 0o600)
self.log.info("generated initial private key")
pubkey = self._pubkey(private_key)
# API path
try:
doc = self.api.fetch_serverlist()
except ApiError as e:
fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION
return RegistrationOutcome(False, "serverlist", fc, str(e))
try:
cands = self.api.build_candidates(doc, regions or cfg.region_candidates)
except ApiError as e:
return RegistrationOutcome(False, "serverlist", FAIL_REGISTRATION, str(e))
if max_candidates:
cands = cands[:max_candidates]
try:
token = self.api.get_token(self.creds)
except ApiError as e:
if e.kind == "auth":
return RegistrationOutcome(False, "token", FAIL_AUTH, str(e))
fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION
return RegistrationOutcome(False, "token", fc, str(e))
any_api_ok = False
api_fail_kinds: list[str] = []
last_detail = None
original_backup = self.backup_current_config()
for cand in cands:
if deadline_mono is not None and self.sys.monotonic() >= deadline_mono:
last_detail = "recovery time budget exhausted"
break
try:
peer = self.api.add_key(cand, token, pubkey)
except ApiError as e:
api_fail_kinds.append(e.kind)
last_detail = f"{cand.region_id}/{cand.cn}: {e}"
self.log.warn("addKey failed", region=cand.region_id, server=cand.cn, kind=e.kind, detail=e.detail)
continue
any_api_ok = True
self.log.info("addKey accepted", region=cand.region_id, server=cand.cn)
text = render_wg_conf(cfg, private_key, peer)
try:
staged = self.stage_conf(text)
except GatewayError as e:
last_detail = str(e)
self.log.warn("staged config rejected", region=cand.region_id, server=cand.cn, detail=str(e))
continue
since = self.sys.now()
try:
verified = self.activate_config(state, staged, peer.server_key, since)
except SafetyError as e:
self.sys.unlink(staged)
return RegistrationOutcome(False, "activate", FAIL_SAFETY, str(e))
if verified is not None:
if staged_key_path:
if self.sys.exists(cfg.wg_key):
self.sys.copy_file(cfg.wg_key, os.path.join(cfg.config_dir, f"{cfg.interface}.key.bak-{self._ts()}"), 0o600)
self.sys.rename(staged_key_path, cfg.wg_key)
self.prune_backups()
epoch = self.sys.now()
reg = {
"epoch": epoch,
"region": peer.region_id,
"region_name": peer.region_name,
"server_cn": peer.cn,
"server_ip": peer.server_ip,
"server_port": peer.server_port,
"server_key": peer.server_key,
"peer_ip": peer.peer_ip,
"pubkey": pubkey,
"mode": mode,
"via": via,
}
state["last_verified_registration"] = {k: reg[k] for k in ("epoch", "region", "region_name", "server_cn", "server_ip", "pubkey", "mode", "via")}
state["active_registration"] = reg
state["renewal"] = {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None}
self.log.info("registration verified", region=peer.region_id, server=peer.cn, mode=mode, via=via, handshake_age_sec=verified.handshake_age_sec)
return RegistrationOutcome(True, "register", None, None, peer.region_id, peer.cn, True, verified=verified)
last_detail = f"{cand.region_id}/{cand.cn}: activated but no verified handshake"
self.log.warn("activation not verified; trying next candidate", region=cand.region_id, server=cand.cn)
# all candidates failed
if staged_key_path:
self.sys.unlink(staged_key_path)
self.restore_previous_config(state, original_backup)
uniform = bool(api_fail_kinds) and not any_api_ok and len(set(api_fail_kinds)) == 1
fc = FAIL_REGISTRATION
if uniform and api_fail_kinds[0] in ("network", "timeout"):
fc = FAIL_HOST_CONNECTIVITY
return RegistrationOutcome(False, "register", fc, last_detail, any_api_ok=any_api_ok, all_api_failed_same_way=uniform)
# -- recovery cycle ---------------------------------------------------
def schedule_backoff(self, state: dict, section: str, failure_class: str, detail: Optional[str]) -> int:
cfg = self.cfg
sec = state[section]
idx = min(sec.get("backoff_index", 0), len(cfg.backoff_schedule_sec) - 1)
base = cfg.backoff_schedule_sec[idx]
jitter = int(base * cfg.backoff_jitter_fraction * (2 * self.sys.random_jitter() - 1))
delay = max(60, min(cfg.backoff_schedule_sec[-1], base + jitter))
sec["backoff_index"] = min(idx + 1, len(cfg.backoff_schedule_sec) - 1)
sec["next_retry_epoch"] = self.sys.now() + delay
sec["last_failure_epoch"] = self.sys.now()
if section == "recovery":
sec["last_failure_class"] = failure_class
sec["last_failure_detail"] = detail
return delay
def recovery_cycle(self, state: dict, st: Status) -> bool:
cfg = self.cfg
rec = state["recovery"]
start_mono = self.sys.monotonic()
deadline = start_mono + cfg.recovery_max_duration_sec
pre_age = st.handshake_age_sec
evidence = {
"pre_fix_class": st.tunnel_class,
"pre_fix_handshake_age_sec": pre_age,
"pre_fix_endpoint": st.peer_endpoint,
"pre_fix_reasons": st.reasons,
}
first_cycle = not rec.get("in_progress")
rec["in_progress"] = True
rec["cycles"] = rec.get("cycles", 0) + 1
self.log.warn("recovery started", **evidence)
if first_cycle:
self.notify(state, "warn", f"pia-gateway: tunnel {st.tunnel_class} (handshake age {pre_age}s); automatic recovery started.")
def finish_ok(stage: str, region: Optional[str], server: Optional[str], verified: Status) -> bool:
elapsed = int(self.sys.monotonic() - start_mono)
state["last_recovery"] = {
"epoch": self.sys.now(),
"stage": stage,
"region": region,
"server_cn": server,
"elapsed_sec": elapsed,
**evidence,
}
state["recovery"] = {**default_state()["recovery"]}
state["consecutive_failures"] = 0
state["last_healthy_epoch"] = self.sys.now()
self.log.info("recovered", stage=stage, region=region, server=server, elapsed_sec=elapsed, handshake_age_sec=verified.handshake_age_sec)
self.notify(state, "info", f"pia-gateway: recovered via {stage} in {elapsed}s" + (f" ({region}/{server})" if server else "") + ".")
return True
def finish_fail(failure_class: str, detail: Optional[str]) -> bool:
delay = self.schedule_backoff(state, "recovery", failure_class, detail)
self.log.error("recovery failed", failure_class=failure_class, detail=detail, retry_in_sec=delay)
level = "error" if failure_class in (FAIL_AUTH, FAIL_SAFETY) else "warn"
self.notify(state, level, f"pia-gateway: recovery failed ({failure_class}: {detail}); gateway stays closed, retry in {delay // 60} min.")
return False
# stage 1: restart
since = self.sys.now()
if not self.restart_interface(state):
self.log.warn("restart stage failed; escalating to registration")
else:
verified = self.wait_for_verified(state, since, cfg.restart_wait_sec)
if verified is not None:
return finish_ok("restart", None, None, verified)
self.log.warn("restart did not restore a fresh handshake; escalating to registration")
if self.sys.monotonic() >= deadline:
return finish_fail(FAIL_REGISTRATION, "time budget exhausted after restart stage")
try:
# stage 2: same-key registration
out = self.registration_transaction(state, "same-key", None, "recover", cfg.max_candidates_per_cycle, deadline)
if out.ok:
return finish_ok("reregister", out.region, out.server_cn, out.verified)
if out.failure_class in (FAIL_AUTH, FAIL_HOST_CONNECTIVITY, FAIL_SAFETY):
return finish_fail(out.failure_class, out.detail)
if self.sys.monotonic() >= deadline:
return finish_fail(out.failure_class or FAIL_REGISTRATION, "time budget exhausted after same-key stage")
# stage 3: one staged fresh-key attempt, only when the API accepted
# the old key but no activated candidate handshook (evidence of
# server-side key rejection), never on uniform API failures.
if out.any_api_ok and not out.all_api_failed_same_way:
self.log.warn("same-key registration accepted but never handshook; attempting one staged key rotation")
out2 = self.registration_transaction(state, "new-key", None, "recover", cfg.max_rotation_candidates, deadline)
if out2.ok:
return finish_ok("rotate", out2.region, out2.server_cn, out2.verified)
return finish_fail(out2.failure_class or FAIL_REGISTRATION, out2.detail)
return finish_fail(out.failure_class or FAIL_REGISTRATION, out.detail)
except GatewayError as e:
# e.g. credentials file missing/unreadable, wg tooling failure —
# still enter backoff so the restart stage is not re-run every 60s.
return finish_fail(FAIL_SAFETY if isinstance(e, SafetyError) else FAIL_REGISTRATION, str(e))
# -- commands ---------------------------------------------------------
def cmd_check(self, as_json: bool = False) -> int:
try:
state = self.store.load()
except StateCorruptError as e:
self.log.error("state corrupt", error=str(e))
state = default_state()
st = self.current_status(state)
if as_json:
print(json.dumps(st.to_dict(), sort_keys=True))
else:
self.log_status(st)
self.write_status_outputs(st, state)
return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY
def cmd_recover(self, force: bool = False) -> int:
cfg = self.cfg
try:
lk = self.lock(0)
except LockBusyError as e:
self.log.info("skipping; another action holds the lock", detail=str(e))
return EXIT_OK
state = None
try:
# The lock is acquired BEFORE state is read, not after — every
# mutating command follows this order now (see cmd_pause for
# the race it closes: a load taken before the lock can go
# stale while this process waits, so saving it back afterwards
# would silently clobber whatever the lock-holder just wrote).
try:
state = self.store.load()
except StateCorruptError as e:
self.log.error("state corrupt; recovery halted (fail closed)", error=str(e))
tmp = default_state()
self.notify(tmp, "error", f"pia-gateway: state file corrupt ({e}); automatic recovery halted until `state-reset --confirm`.")
return EXIT_STATE_ERROR
self.refresh_boot_grace(state)
self.flush_notifications(state)
st = self.current_status(state)
self.log_status(st)
state["last_check"] = {"epoch": st.timestamp, "class": st.cls, "reasons": st.reasons, "handshake_age_sec": st.handshake_age_sec}
cycles_before = state["recovery"].get("cycles", 0)
emergency_before = state.get("emergency_close_active", False)
rc = self._decide(state, st, force)
acted = state["recovery"].get("cycles", 0) != cycles_before or state.get("emergency_close_active") != emergency_before
self.write_status_outputs(self.current_status(state) if acted else st, state)
return rc
finally:
if state is not None:
self.store.save(state)
lk.release()
def _decide(self, state: dict, st: Status, force: bool) -> int:
cfg = self.cfg
rec = state["recovery"]
if st.tunnel_class == CLASS_SAFETY_DRIFT:
state["consecutive_failures"] = 0
self.emergency_close(state, st)
return EXIT_SAFETY
self.emergency_clear_if_repaired(state, st)
if st.tunnel_class == CLASS_UNKNOWN:
state["consecutive_unknown"] = state.get("consecutive_unknown", 0) + 1
if state["consecutive_unknown"] == cfg.failure_threshold:
self.notify(state, "warn", f"pia-gateway: status unknown for {cfg.failure_threshold} checks ({'; '.join(st.reasons)}); no action taken.")
return EXIT_UNHEALTHY
state["consecutive_unknown"] = 0
if st.tunnel_class == CLASS_HEALTHY:
if rec.get("in_progress") or rec.get("next_retry_epoch"):
self.log.info("tunnel healthy again without a completed recovery cycle; clearing backoff")
self.notify(state, "info", "pia-gateway: tunnel healthy again; backoff cleared.")
state["recovery"] = {**default_state()["recovery"]}
state["consecutive_failures"] = 0
state["last_healthy_epoch"] = st.timestamp
return EXIT_OK
if state.get("paused"):
self.log.warn("paused; no recovery action", since=state["paused"].get("since"), reason=state["paused"].get("reason"))
return EXIT_UNHEALTHY
if st.cls == CLASS_BOOT_GRACE and not force:
self.log.info("within startup/restart grace; not counting as a failure")
return EXIT_UNHEALTHY
state["consecutive_failures"] = state.get("consecutive_failures", 0) + 1
if state["consecutive_failures"] < cfg.failure_threshold and not force:
self.log.warn("unhealthy; below threshold", consecutive=state["consecutive_failures"], threshold=cfg.failure_threshold)
return EXIT_UNHEALTHY
if rec.get("next_retry_epoch") and st.timestamp < rec["next_retry_epoch"] and not force:
self.log.warn("in backoff; not retrying yet", retry_in_sec=rec["next_retry_epoch"] - st.timestamp, last_failure=rec.get("last_failure_class"))
return EXIT_UNHEALTHY
if force:
rec["next_retry_epoch"] = None
ok = self.recovery_cycle(state, st)
return EXIT_OK if ok else EXIT_UNHEALTHY
def cmd_renew(self, force_due: bool = False) -> int:
cfg = self.cfg
try:
lk = self.lock(0)
except LockBusyError:
self.log.info("renewal skipped; another action holds the lock")
return EXIT_OK
state = None
try:
try:
state = self.store.load()
except StateCorruptError as e:
self.log.error("state corrupt; renewal halted", error=str(e))
return EXIT_STATE_ERROR
if state.get("paused"):
self.log.info("paused; renewal skipped")
return EXIT_OK
st = self.current_status(state)
if st.tunnel_class != CLASS_HEALTHY:
self.log.info("tunnel not healthy; renewal defers to recovery", **{"class": st.tunnel_class})
return EXIT_OK
ren = state["renewal"]
now = self.sys.now()
if ren.get("next_retry_epoch") and now < ren["next_retry_epoch"] and not force_due:
self.log.info("renewal in backoff", retry_in_sec=ren["next_retry_epoch"] - now)
return EXIT_OK
lvr = state.get("last_verified_registration")
due = force_due or lvr is None or (now - int(lvr.get("epoch", 0))) >= cfg.renewal_after_days * 86400
if not due:
self.log.info("renewal not due", age_days=(now - int(lvr["epoch"])) // 86400, after_days=cfg.renewal_after_days)
return EXIT_OK
self.log.info("renewal due; re-registering with the existing key", baseline_missing=lvr is None)
out = self.registration_transaction(state, "same-key", None, "renew", cfg.max_candidates_per_cycle)
if out.ok:
self.log.info("renewal verified", region=out.region, server=out.server_cn)
return EXIT_OK
delay = self.schedule_backoff(state, "renewal", out.failure_class or FAIL_REGISTRATION, out.detail)
self.log.error("renewal failed; previous registration timestamp unchanged", failure_class=out.failure_class, detail=out.detail, retry_in_sec=delay)
self.notify(state, "warn", f"pia-gateway: scheduled renewal failed ({out.failure_class}: {out.detail}); retry in {delay // 60} min.")
return EXIT_UNHEALTHY
finally:
if state is not None:
self.store.save(state)
lk.release()
def cmd_register(self, mode: str, region: Optional[str], via: str) -> int:
cfg = self.cfg
try:
lk = self.lock(cfg.lock_wait_sec)
except LockBusyError as e:
self.log.error("could not acquire lock", detail=str(e))
return EXIT_LOCKED
state = None
try:
try:
state = self.store.load()
except StateCorruptError as e:
self.log.error("state corrupt; refusing to register", error=str(e))
return EXIT_STATE_ERROR
self.refresh_boot_grace(state)
st = self.current_status(state)
if not st.safety_ok:
self.log.error("safety invariants failing; refusing to register", reasons=st.reasons)
return EXIT_SAFETY
regions = [region] if region else None
out = self.registration_transaction(state, mode, regions, via)
summary = {"ok": out.ok, "stage": out.stage, "failure_class": out.failure_class, "detail": out.detail, "region": out.region, "server_cn": out.server_cn}
print(json.dumps(summary, sort_keys=True))
if out.ok:
state["consecutive_failures"] = 0
state["recovery"] = {**default_state()["recovery"]}
return EXIT_OK
return EXIT_UNHEALTHY
finally:
if state is not None:
self.store.save(state)
lk.release()
def cmd_rewrite_config(self) -> int:
cfg = self.cfg
try:
lk = self.lock(cfg.lock_wait_sec)
except LockBusyError as e:
self.log.error("could not acquire lock", detail=str(e))
return EXIT_LOCKED
state = None
try:
try:
state = self.store.load()
except StateCorruptError as e:
self.log.error("state corrupt; refusing to rewrite", error=str(e))
return EXIT_STATE_ERROR
reg = state.get("active_registration")
if not reg:
self.log.error("no verified registration recorded; run `pia-gateway register` instead")
return EXIT_USAGE
st = self.current_status(state)
if not st.safety_ok:
self.log.error("safety invariants failing; refusing to rewrite", reasons=st.reasons)
return EXIT_SAFETY
peer = PeerInfo(reg["server_key"], reg["peer_ip"], int(reg["server_port"]), reg["server_ip"], reg["region"], reg.get("region_name", reg["region"]), reg["server_cn"])
private_key = self._read_private_key()
if self._pubkey(private_key) != reg.get("pubkey"):
self.log.error("on-disk key does not match the recorded registration; run register")
return EXIT_USAGE
staged = self.stage_conf(render_wg_conf(cfg, private_key, peer))
backup = self.backup_current_config()
since = self.sys.now()
verified = self.activate_config(state, staged, peer.server_key, since)
if verified is None:
self.log.error("rewritten config did not verify; restoring previous")
self.restore_previous_config(state, backup)
return EXIT_UNHEALTHY
self.prune_backups()
self.log.info("config rewritten and verified")
return EXIT_OK
finally:
if state is not None:
self.store.save(state)
lk.release()
def cmd_status(self) -> int:
try:
state = self.store.load()
except StateCorruptError as e:
print(f"STATE CORRUPT: {e}")
return EXIT_STATE_ERROR
st = self.current_status(state)
rec = state["recovery"]
now = self.sys.now()
lines = [
f"class: {st.cls} (tunnel: {st.tunnel_class})",
f"reasons: {'; '.join(st.reasons) or '-'}",
f"handshake age: {st.handshake_age_sec if st.handshake_age_sec is not None else '-'}s (max {self.cfg.handshake_max_age_sec}s)",
f"endpoint: {st.peer_endpoint or '-'}",
f"route table {self.cfg.route_table}: {st.route_state}",
f"safety ok: {st.safety_ok}",
f"paused: {state.get('paused') or False}",
f"consecutive failures: {state.get('consecutive_failures', 0)}",
f"recovery in progress: {rec.get('in_progress')}",
f"backoff next retry: {_rel(rec.get('next_retry_epoch'), now)} (last failure: {rec.get('last_failure_class') or '-'})",
f"last verified reg.: {_rel((state.get('last_verified_registration') or {}).get('epoch'), now)} "
f"{_reg_summary(state.get('last_verified_registration'))}",
f"last recovery: {_rel((state.get('last_recovery') or {}).get('epoch'), now)} "
f"{(state.get('last_recovery') or {}).get('stage') or ''}",
f"emergency close active: {state.get('emergency_close_active', False)}",
f"last check: {_rel((state.get('last_check') or {}).get('epoch'), now)}",
]
if st.configured_peer_line:
lines.append(f"configured peer: {st.configured_peer_line}")
print("\n".join(lines))
return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY
def cmd_pause(self, reason: str) -> int:
# Same lock as recover/register/renew/rewrite-config, taken BEFORE
# the state load: pause has to wait for any in-flight action to
# finish (and save) before it reads state itself, or a recovery
# pass that read state before pause ran could save its own
# (unpaused) copy back afterwards and silently erase the pause.
try:
lk = self.lock(self.cfg.lock_wait_sec)
except LockBusyError as e:
self.log.error("could not acquire lock", detail=str(e))
return EXIT_LOCKED
try:
try:
state = self.store.load()
except StateCorruptError as e:
self.log.error("state corrupt; pause not recorded", error=str(e))
return EXIT_STATE_ERROR
state["paused"] = {"since": self.sys.now(), "reason": reason or "operator"}
self.store.save(state)
self.log.info("paused: recovery and renewal suppressed; kill switch unchanged", reason=reason)
return EXIT_OK
finally:
lk.release()
def cmd_resume(self) -> int:
try:
lk = self.lock(self.cfg.lock_wait_sec)
except LockBusyError as e:
self.log.error("could not acquire lock", detail=str(e))
return EXIT_LOCKED
try:
try:
state = self.store.load()
except StateCorruptError as e:
self.log.error("state corrupt; resume not recorded", error=str(e))
return EXIT_STATE_ERROR
state["paused"] = None
state["consecutive_failures"] = 0
state["recovery"]["next_retry_epoch"] = None
self.store.save(state)
self.log.info("resumed")
return EXIT_OK
finally:
lk.release()
def cmd_retry_now(self) -> int:
return self.cmd_recover(force=True)
def cmd_state_reset(self, confirm: bool) -> int:
if not confirm:
self.log.error("refusing without --confirm")
return EXIT_USAGE
try:
lk = self.lock(self.cfg.lock_wait_sec)
except LockBusyError as e:
self.log.error("could not acquire lock", detail=str(e))
return EXIT_LOCKED
try:
old = None
if self.sys.exists(self.cfg.state_file):
old = self.cfg.state_file + f".corrupt-{self._ts()}"
self.sys.rename(self.cfg.state_file, old)
self.store.save(default_state())
self.log.warn("state reset", previous_moved_to=old)
return EXIT_OK
finally:
lk.release()
def cmd_notify(self, level: str, message: str) -> int:
ok = self.notifier().send(level, message)
return EXIT_OK if ok else EXIT_UNHEALTHY
def _rel(epoch: Optional[int], now: int) -> str:
if not epoch:
return "never"
d = now - int(epoch)
if d < 0:
return f"in {-d}s"
if d < 3600:
return f"{d // 60}m ago"
if d < 86400:
return f"{d // 3600}h ago"
return f"{d // 86400}d ago"
def _reg_summary(reg: Optional[dict]) -> str:
if not reg:
return ""
return f"({reg.get('region')}/{reg.get('server_cn')}, {reg.get('mode')}, via {reg.get('via')})"
# --------------------------------------------------------------------------
# CLI
# --------------------------------------------------------------------------
def build_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(prog="pia-gateway", description=__doc__.split("\n\n")[0])
p.add_argument("--config", default="/etc/pia-gateway/config.json")
p.add_argument("--json-log", action="store_true", help="emit JSON log lines")
sub = p.add_subparsers(dest="cmd", required=True)
c = sub.add_parser("check")
c.add_argument("--json", action="store_true")
sub.add_parser("recover")
r = sub.add_parser("renew")
r.add_argument("--force-due", action="store_true")
g = sub.add_parser("register")
g.add_argument("--mode", choices=["same-key", "new-key"], default="same-key")
g.add_argument("--region", default=None)
g.add_argument("--via", default="operator")
sub.add_parser("rewrite-config")
sub.add_parser("status")
pa = sub.add_parser("pause")
pa.add_argument("--reason", default="operator")
sub.add_parser("resume")
sub.add_parser("retry-now")
sr = sub.add_parser("state-reset")
sr.add_argument("--confirm", action="store_true")
n = sub.add_parser("notify")
n.add_argument("--level", choices=["info", "warn", "error"], default="error")
n.add_argument("--message", required=True)
return p
def main(argv: Optional[list[str]] = None, system: Optional[System] = None, http: Optional[HttpClient] = None, stream=None) -> int:
args = build_parser().parse_args(argv)
log = Log(stream=stream, json_mode=args.json_log)
try:
cfg = Config.load(args.config)
except (OSError, ValueError, GatewayError) as e:
log.error("config load failed", error=str(e))
return EXIT_USAGE
ctl = Controller(cfg, system or System(), log, http)
try:
if args.cmd == "check":
return ctl.cmd_check(as_json=args.json)
if args.cmd == "recover":
return ctl.cmd_recover()
if args.cmd == "renew":
return ctl.cmd_renew(force_due=args.force_due)
if args.cmd == "register":
return ctl.cmd_register(args.mode, args.region, args.via)
if args.cmd == "rewrite-config":
return ctl.cmd_rewrite_config()
if args.cmd == "status":
return ctl.cmd_status()
if args.cmd == "pause":
return ctl.cmd_pause(args.reason)
if args.cmd == "resume":
return ctl.cmd_resume()
if args.cmd == "retry-now":
return ctl.cmd_retry_now()
if args.cmd == "state-reset":
return ctl.cmd_state_reset(args.confirm)
if args.cmd == "notify":
return ctl.cmd_notify(args.level, args.message)
except GatewayError as e:
log.error("failed", error=str(e))
return EXIT_SAFETY if isinstance(e, SafetyError) else EXIT_UNHEALTHY
except Exception as e: # noqa: BLE001 - never let a traceback (with source context) reach the journal
log.error("unexpected failure", error=e.__class__.__name__)
return EXIT_UNHEALTHY
return EXIT_USAGE
if __name__ == "__main__":
sys.exit(main())