Found live, 2026-09-22, on the addKey step of the first successful kill-switch-check pass: every candidate failed with kind=tls detail="certificate verification failed". Root-caused on minisforum: PIA's own CA certificate (files/pia-ca.crt) has a CA:TRUE Basic Constraints extension that isn't marked critical - a real RFC 5280 non-conformance, confirmed not a stale vendored copy (a fresh fetch from pia-foss/manual-connections is byte-identical, same sha256 already recorded in the file's own header, same defect). Recent OpenSSL (3.2+; confirmed via a direct repro against minisforum's 3.5.7 - SSLCertVerificationError, verify_message "Basic Constraints of CA cert not marked critical") rejects this by default, a check promoted from X509_STRICT-only into the baseline chain-verification path. openssl s_client (LibreSSL, this dev machine) and curl both accept the identical cert/chain/CA file - only Python's ssl module, via this host's system OpenSSL, rejects it. This went unnoticed until now because the three prior successful registrations all went through curl --cacert (the legacy Ansible path); this was the first time the Python ssl code path ran for real. Python's stdlib ssl module has no custom verify-callback hook, so there's no clean way to tolerate just this one known, third-party-controlled defect while keeping full validation for everything else. Rather than weakening verify_mode or adding a new dependency (pyOpenSSL, for a real callback - considered, rejected as a bigger lift than necessary) to route around a gap in a library we don't control, addKey now shells out to curl (HttpClient.get_pinned_curl, curl added to this role's apt list) - the same tool PIA's own reference client already uses for this exact endpoint, with --cacert and never -k/--insecure, so this is a different, already-proven-correct TLS backend for the one endpoint that needs it, not a downgrade. serverlist/token are unaffected (public WebPKI certs) and still use the stdlib ssl path. Verified: a real subprocess curl call against the actual failing candidate (Server-11669-10a / 69.33.198.97) gets a genuine HTTP response from PIA, proving TLS validation succeeds via this path. Also found and handled: curl's own exit code for a --fail-triggered rejection isn't fully portable (macOS/SecureTransport curl reports 56 where GNU/OpenSSL curl reports 22 for the same case) - classification now also matches curl's own error text as a second signal, not just the exit code. Added 8 tests (94 total, all passing) covering the curl invocation shape, stdin token delivery (no trailing newline, never in argv), and exit-code + text-based error classification including the cross-platform exit-code case found live. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2139 lines
92 KiB
Python
2139 lines
92 KiB
Python
#!/usr/bin/env python3
|
|
"""pia-gateway: PIA WireGuard egress gateway controller for minisforum.
|
|
|
|
Installed by ansible/roles/pia-gateway. One program, several subcommands:
|
|
|
|
check read-only status classification (journal + status/metrics files)
|
|
recover one bounded recovery pass (timer-driven, every 60s)
|
|
renew proactive same-key re-registration when the last verified
|
|
registration is older than the configured age (daily timer)
|
|
register explicit registration transaction (used by Ansible and operators)
|
|
rewrite-config
|
|
re-render pia-wg.conf from the last verified registration
|
|
(no PIA API call), activate and verify
|
|
status / pause / resume / retry-now / state-reset / notify
|
|
|
|
Invariants this program must never break (see the role README):
|
|
* The host's main default route and the global FORWARD policy are never
|
|
touched. Only table `pia`, the `PIA-VLAN50` chain, and (in an
|
|
emergency) one extra FORWARD rule scoped to the VLAN 50 source.
|
|
* Every failure path leaves VLAN 50 closed. No path opens the gateway
|
|
except a verified WireGuard activation.
|
|
* No PIA credential, token, or private key is ever written to stdout,
|
|
stderr, the journal, argv, a notification, or the metrics file.
|
|
|
|
Only the Python standard library is used so the same file runs on the
|
|
gateway (python3.13) and under the repo's local unittest suite.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import dataclasses
|
|
import datetime as _dt
|
|
import fcntl
|
|
import http.client
|
|
import io
|
|
import json
|
|
import os
|
|
import random
|
|
import re
|
|
import secrets
|
|
import socket
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib.parse
|
|
from typing import Any, Iterable, Optional
|
|
|
|
STATE_SCHEMA_VERSION = 1
|
|
EMERGENCY_COMMENT = "pia-gateway-emergency"
|
|
SERVERLIST_URL = "https://serverlist.piaservers.net/vpninfo/servers/v6"
|
|
TOKEN_URL = "https://www.privateinternetaccess.com/api/client/v2/token"
|
|
ADDKEY_PORT = 1337
|
|
|
|
CLASS_HEALTHY = "healthy"
|
|
CLASS_BOOT_GRACE = "boot_grace"
|
|
CLASS_PAUSED = "paused"
|
|
CLASS_TUNNEL_STALE = "tunnel_stale"
|
|
CLASS_TUNNEL_ABSENT = "tunnel_absent"
|
|
CLASS_REGISTRATION_FAILURE = "registration_or_endpoint_failure"
|
|
CLASS_SAFETY_DRIFT = "safety_drift"
|
|
CLASS_HOST_CONNECTIVITY = "host_connectivity_failure"
|
|
CLASS_UNKNOWN = "unknown"
|
|
|
|
FAIL_AUTH = "auth_failure"
|
|
FAIL_HOST_CONNECTIVITY = CLASS_HOST_CONNECTIVITY
|
|
FAIL_REGISTRATION = CLASS_REGISTRATION_FAILURE
|
|
FAIL_SAFETY = CLASS_SAFETY_DRIFT
|
|
|
|
EXIT_OK = 0
|
|
EXIT_UNHEALTHY = 1
|
|
EXIT_STATE_ERROR = 2
|
|
EXIT_SAFETY = 3
|
|
EXIT_LOCKED = 4
|
|
EXIT_USAGE = 5
|
|
|
|
|
|
class GatewayError(Exception):
|
|
"""Base class; message must never contain secrets."""
|
|
|
|
|
|
class StateCorruptError(GatewayError):
|
|
pass
|
|
|
|
|
|
class SafetyError(GatewayError):
|
|
pass
|
|
|
|
|
|
class LockBusyError(GatewayError):
|
|
pass
|
|
|
|
|
|
class ApiError(GatewayError):
|
|
def __init__(self, kind: str, detail: str):
|
|
super().__init__(f"{kind}: {detail}")
|
|
self.kind = kind
|
|
self.detail = detail
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Configuration and credentials
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Config:
|
|
interface: str = "pia-wg"
|
|
config_dir: str = "/etc/wireguard"
|
|
state_dir: str = "/var/lib/pia-gateway"
|
|
credentials_file: str = "/etc/pia-gateway/credentials.env"
|
|
ca_file: str = "/etc/wireguard/pia-ca.crt"
|
|
lock_file: str = "/run/lock/pia-gateway.lock"
|
|
route_table: str = "pia"
|
|
rule_priority: int = 100
|
|
vlan50_subnet: str = "10.10.50.0/24"
|
|
iptables_chain: str = "PIA-VLAN50"
|
|
lan_interface: str = "enp1s0"
|
|
mss_clamp_enabled: bool = True
|
|
region_candidates: list = dataclasses.field(default_factory=lambda: ["hk", "taiwan", "japan"])
|
|
handshake_max_age_sec: int = 180
|
|
failure_threshold: int = 3
|
|
startup_grace_sec: int = 120
|
|
restart_wait_sec: int = 90
|
|
activation_wait_sec: int = 90
|
|
recovery_max_duration_sec: int = 600
|
|
backoff_schedule_sec: list = dataclasses.field(default_factory=lambda: [300, 900, 1800, 3600])
|
|
backoff_jitter_fraction: float = 0.1
|
|
max_candidates_per_cycle: int = 6
|
|
max_rotation_candidates: int = 3
|
|
renewal_after_days: int = 21
|
|
http_connect_timeout_sec: int = 10
|
|
http_total_timeout_sec: int = 30
|
|
clock_skew_allowance_sec: int = 60
|
|
backup_keep: int = 5
|
|
lock_wait_sec: int = 600
|
|
alert_bridge_url: Optional[str] = None
|
|
alert_source: str = "pia-gateway"
|
|
metrics_file: str = "/var/lib/pia-gateway/metrics.prom"
|
|
status_file: str = "/var/lib/pia-gateway/status.json"
|
|
killswitch_unit: str = "pia-killswitch.service"
|
|
|
|
@classmethod
|
|
def load(cls, path: str) -> "Config":
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
raw = json.load(f)
|
|
known = {f.name for f in dataclasses.fields(cls)}
|
|
unknown = set(raw) - known
|
|
if unknown:
|
|
raise GatewayError(f"config: unknown keys {sorted(unknown)}")
|
|
cfg = cls(**raw)
|
|
if not cfg.region_candidates:
|
|
raise GatewayError("config: region_candidates must be non-empty")
|
|
if cfg.failure_threshold < 1:
|
|
raise GatewayError("config: failure_threshold must be >= 1")
|
|
return cfg
|
|
|
|
@property
|
|
def wg_conf(self) -> str:
|
|
return os.path.join(self.config_dir, f"{self.interface}.conf")
|
|
|
|
@property
|
|
def wg_key(self) -> str:
|
|
return os.path.join(self.config_dir, f"{self.interface}.key")
|
|
|
|
@property
|
|
def state_file(self) -> str:
|
|
return os.path.join(self.state_dir, "state.json")
|
|
|
|
@property
|
|
def wg_unit(self) -> str:
|
|
return f"wg-quick@{self.interface}.service"
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Credentials:
|
|
pia_user: str
|
|
pia_password: str
|
|
alert_bridge_api_key: Optional[str] = None
|
|
|
|
def __repr__(self) -> str: # never leak via repr/str
|
|
return "Credentials(<redacted>)"
|
|
|
|
__str__ = __repr__
|
|
|
|
|
|
def load_credentials(path: str, require_pia: bool = True) -> Credentials:
|
|
values: dict[str, str] = {}
|
|
try:
|
|
st = os.stat(path)
|
|
except FileNotFoundError:
|
|
if require_pia:
|
|
raise GatewayError(f"credentials file {path} is missing")
|
|
return Credentials("", "", None)
|
|
if st.st_mode & 0o077:
|
|
raise GatewayError(f"credentials file {path} must be mode 0600 (group/other bits set)")
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
k, v = line.split("=", 1)
|
|
values[k.strip()] = v.strip()
|
|
user = values.get("PIA_USER", "")
|
|
password = values.get("PIA_PASSWORD", "")
|
|
if require_pia and (not user or not password):
|
|
raise GatewayError(f"credentials file {path} lacks PIA_USER/PIA_PASSWORD")
|
|
return Credentials(user, password, values.get("ALERT_BRIDGE_API_KEY") or None)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# System boundary (commands, clock, files) — replaced by fakes in tests
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class CmdResult:
|
|
rc: int
|
|
out: str
|
|
err: str
|
|
|
|
|
|
class System:
|
|
def run(self, argv: list[str], stdin: Optional[str] = None, timeout: int = 60) -> CmdResult:
|
|
try:
|
|
p = subprocess.run(
|
|
argv,
|
|
input=stdin,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
except FileNotFoundError:
|
|
return CmdResult(127, "", f"{argv[0]}: not found")
|
|
except subprocess.TimeoutExpired:
|
|
return CmdResult(124, "", f"{argv[0]}: timeout after {timeout}s")
|
|
return CmdResult(p.returncode, p.stdout, p.stderr)
|
|
|
|
def now(self) -> int:
|
|
return int(time.time())
|
|
|
|
def monotonic(self) -> float:
|
|
return time.monotonic()
|
|
|
|
def sleep(self, sec: float) -> None:
|
|
time.sleep(sec)
|
|
|
|
def boot_id(self) -> str:
|
|
try:
|
|
with open("/proc/sys/kernel/random/boot_id", "r", encoding="utf-8") as f:
|
|
return f.read().strip()
|
|
except OSError:
|
|
return "unknown"
|
|
|
|
def read_text(self, path: str) -> str:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
def exists(self, path: str) -> bool:
|
|
return os.path.exists(path)
|
|
|
|
def listdir(self, path: str) -> list[str]:
|
|
try:
|
|
return sorted(os.listdir(path))
|
|
except FileNotFoundError:
|
|
return []
|
|
|
|
def unlink(self, path: str) -> None:
|
|
with contextlib.suppress(FileNotFoundError):
|
|
os.unlink(path)
|
|
|
|
def rename(self, src: str, dst: str) -> None:
|
|
os.replace(src, dst)
|
|
|
|
def copy_file(self, src: str, dst: str, mode: int = 0o600) -> None:
|
|
data = self.read_text(src)
|
|
self.write_atomic(dst, data, mode)
|
|
|
|
def mtime(self, path: str) -> float:
|
|
return os.stat(path).st_mtime
|
|
|
|
def makedirs(self, path: str, mode: int = 0o700) -> None:
|
|
os.makedirs(path, mode=mode, exist_ok=True)
|
|
os.chmod(path, mode)
|
|
|
|
def write_atomic(self, path: str, data: str, mode: int = 0o600) -> None:
|
|
directory = os.path.dirname(path) or "."
|
|
fd, tmp = tempfile.mkstemp(dir=directory, prefix=f".{os.path.basename(path)}.", suffix=".tmp")
|
|
try:
|
|
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
f.write(data)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
os.chmod(tmp, mode)
|
|
os.replace(tmp, path)
|
|
dfd = os.open(directory, os.O_RDONLY)
|
|
try:
|
|
os.fsync(dfd)
|
|
finally:
|
|
os.close(dfd)
|
|
except BaseException:
|
|
with contextlib.suppress(FileNotFoundError):
|
|
os.unlink(tmp)
|
|
raise
|
|
|
|
def random_jitter(self) -> float:
|
|
return random.random()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Logging (journal-friendly, secret-free by construction)
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class Log:
|
|
def __init__(self, stream: io.TextIOBase = None, json_mode: bool = False):
|
|
self.stream = stream or sys.stdout
|
|
self.json_mode = json_mode
|
|
self.lines: list[str] = []
|
|
|
|
def emit(self, level: str, msg: str, **fields: Any) -> None:
|
|
if self.json_mode:
|
|
rec = {"level": level, "msg": msg}
|
|
rec.update(fields)
|
|
line = json.dumps(rec, sort_keys=True)
|
|
else:
|
|
extra = " ".join(f"{k}={_fmt(v)}" for k, v in fields.items())
|
|
line = f"{level} {msg}" + (f" {extra}" if extra else "")
|
|
self.lines.append(line)
|
|
print(line, file=self.stream, flush=True)
|
|
|
|
def info(self, msg: str, **f: Any) -> None:
|
|
self.emit("INFO", msg, **f)
|
|
|
|
def warn(self, msg: str, **f: Any) -> None:
|
|
self.emit("WARN", msg, **f)
|
|
|
|
def error(self, msg: str, **f: Any) -> None:
|
|
self.emit("ERROR", msg, **f)
|
|
|
|
|
|
def _fmt(v: Any) -> str:
|
|
if isinstance(v, (list, dict)):
|
|
return json.dumps(v, separators=(",", ":"))
|
|
return str(v)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Persistent state
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def default_state() -> dict:
|
|
return {
|
|
"schema_version": STATE_SCHEMA_VERSION,
|
|
"boot_id": None,
|
|
"boot_grace_until": 0,
|
|
"restart_grace_until": 0,
|
|
"consecutive_failures": 0,
|
|
"consecutive_unknown": 0,
|
|
"last_check": None,
|
|
"last_healthy_epoch": None,
|
|
"recovery": {
|
|
"in_progress": False,
|
|
"backoff_index": 0,
|
|
"next_retry_epoch": None,
|
|
"last_failure_class": None,
|
|
"last_failure_epoch": None,
|
|
"last_failure_detail": None,
|
|
"cycles": 0,
|
|
},
|
|
"last_recovery": None,
|
|
"last_verified_registration": None,
|
|
"active_registration": None,
|
|
"renewal": {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None},
|
|
"paused": None,
|
|
"emergency_close_active": False,
|
|
"pending_notifications": [],
|
|
}
|
|
|
|
|
|
class StateStore:
|
|
def __init__(self, cfg: Config, system: System):
|
|
self.cfg = cfg
|
|
self.sys = system
|
|
|
|
def load(self) -> dict:
|
|
path = self.cfg.state_file
|
|
if not self.sys.exists(path):
|
|
return default_state()
|
|
try:
|
|
raw = json.loads(self.sys.read_text(path))
|
|
except (ValueError, OSError) as e:
|
|
raise StateCorruptError(f"state file {path} is unreadable or not JSON ({e.__class__.__name__})")
|
|
if not isinstance(raw, dict) or raw.get("schema_version") != STATE_SCHEMA_VERSION:
|
|
raise StateCorruptError(
|
|
f"state file {path} has schema_version={raw.get('schema_version') if isinstance(raw, dict) else '?'}, "
|
|
f"expected {STATE_SCHEMA_VERSION}; run `pia-gateway state-reset --confirm` after review"
|
|
)
|
|
merged = default_state()
|
|
for k, v in raw.items():
|
|
if isinstance(merged.get(k), dict) and isinstance(v, dict):
|
|
merged[k].update(v)
|
|
else:
|
|
merged[k] = v
|
|
return merged
|
|
|
|
def save(self, state: dict) -> None:
|
|
self.sys.makedirs(self.cfg.state_dir, 0o700)
|
|
self.sys.write_atomic(self.cfg.state_file, json.dumps(state, indent=1, sort_keys=True) + "\n", 0o600)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Lock
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class Lock:
|
|
"""flock-based; released automatically by the kernel if the process dies."""
|
|
|
|
def __init__(self, path: str, system: System):
|
|
self.path = path
|
|
self.sys = system
|
|
self.fd: Optional[int] = None
|
|
|
|
def acquire(self, wait_sec: float = 0) -> None:
|
|
directory = os.path.dirname(self.path)
|
|
if directory and not os.path.isdir(directory):
|
|
os.makedirs(directory, mode=0o755, exist_ok=True)
|
|
fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600)
|
|
start = self.sys.monotonic()
|
|
while True:
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
self.fd = fd
|
|
return
|
|
except BlockingIOError:
|
|
if self.sys.monotonic() - start >= wait_sec:
|
|
os.close(fd)
|
|
raise LockBusyError(f"lock {self.path} is held by another pia-gateway action")
|
|
self.sys.sleep(1)
|
|
|
|
def release(self) -> None:
|
|
if self.fd is not None:
|
|
with contextlib.suppress(OSError):
|
|
fcntl.flock(self.fd, fcntl.LOCK_UN)
|
|
os.close(self.fd)
|
|
self.fd = None
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Status classification
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Status:
|
|
timestamp: int
|
|
tunnel_class: str
|
|
cls: str
|
|
reasons: list[str]
|
|
safety_ok: bool
|
|
tunnel_ok: bool
|
|
interface_up: bool
|
|
peer_present: bool
|
|
handshake_epoch: Optional[int]
|
|
handshake_age_sec: Optional[int]
|
|
peer_endpoint: Optional[str]
|
|
expected_peer: Optional[str]
|
|
route_state: str # open | closed | missing | unknown
|
|
configured_peer_line: Optional[str]
|
|
checks: dict
|
|
|
|
def to_dict(self) -> dict:
|
|
d = dataclasses.asdict(self)
|
|
d["class"] = d.pop("cls")
|
|
return d
|
|
|
|
|
|
def parse_expected_peer(conf_text: str) -> Optional[str]:
|
|
in_peer = False
|
|
for line in conf_text.splitlines():
|
|
s = line.strip()
|
|
if s.startswith("["):
|
|
in_peer = s.lower() == "[peer]"
|
|
continue
|
|
if in_peer and s.startswith("PublicKey"):
|
|
_, _, v = s.partition("=")
|
|
return v.strip()
|
|
return None
|
|
|
|
|
|
def parse_configured_peer_line(conf_text: str) -> Optional[str]:
|
|
for line in conf_text.splitlines():
|
|
if line.startswith("# PIA WireGuard server actually used"):
|
|
return line[2:]
|
|
return None
|
|
|
|
|
|
def _split_rule_tokens(rule: str) -> list[str]:
|
|
return rule.split()
|
|
|
|
|
|
def _is_emergency_rule(rule: str) -> bool:
|
|
return f"--comment {EMERGENCY_COMMENT}" in rule or f'--comment "{EMERGENCY_COMMENT}"' in rule
|
|
|
|
|
|
class Classifier:
|
|
def __init__(self, cfg: Config, system: System):
|
|
self.cfg = cfg
|
|
self.sys = system
|
|
|
|
def _rc(self, argv: list[str]) -> CmdResult:
|
|
return self.sys.run(argv, timeout=20)
|
|
|
|
def classify(self, state: dict) -> Status:
|
|
cfg = self.cfg
|
|
now = self.sys.now()
|
|
reasons: list[str] = []
|
|
safety_ok = True
|
|
checks: dict[str, Any] = {}
|
|
unknown = False
|
|
|
|
# -- interface
|
|
link = self._rc(["ip", "-o", "link", "show", "dev", cfg.interface])
|
|
interface_up = False
|
|
if link.rc == 0 and link.out.strip():
|
|
m = re.search(r"<([^>]*)>", link.out)
|
|
flags = m.group(1).split(",") if m else []
|
|
interface_up = "UP" in flags
|
|
checks["interface"] = "up" if interface_up else "down"
|
|
if not interface_up:
|
|
reasons.append("interface is administratively down")
|
|
else:
|
|
checks["interface"] = "absent"
|
|
reasons.append("interface absent")
|
|
|
|
# -- expected peer from config
|
|
expected_peer = None
|
|
configured_line = None
|
|
if self.sys.exists(cfg.wg_conf):
|
|
try:
|
|
conf_text = self.sys.read_text(cfg.wg_conf)
|
|
expected_peer = parse_expected_peer(conf_text)
|
|
configured_line = parse_configured_peer_line(conf_text)
|
|
except OSError:
|
|
reasons.append("config unreadable")
|
|
else:
|
|
reasons.append("config missing")
|
|
checks["expected_peer_present_in_config"] = expected_peer is not None
|
|
|
|
# -- handshake
|
|
handshake_epoch: Optional[int] = None
|
|
handshake_age: Optional[int] = None
|
|
peer_present = False
|
|
endpoint = None
|
|
handshake_ok = False
|
|
if interface_up:
|
|
hs = self._rc(["wg", "show", cfg.interface, "latest-handshakes"])
|
|
ep = self._rc(["wg", "show", cfg.interface, "endpoints"])
|
|
if hs.rc != 0:
|
|
reasons.append("wg show failed")
|
|
else:
|
|
peers = {}
|
|
for line in hs.out.splitlines():
|
|
parts = line.split("\t")
|
|
if len(parts) == 2 and parts[1].strip().lstrip("-").isdigit():
|
|
peers[parts[0].strip()] = int(parts[1].strip())
|
|
elif line.strip():
|
|
reasons.append("malformed wg output")
|
|
if expected_peer is None:
|
|
reasons.append("no expected peer to compare against")
|
|
elif expected_peer not in peers:
|
|
reasons.append("expected peer not present on interface")
|
|
else:
|
|
peer_present = True
|
|
handshake_epoch = peers[expected_peer]
|
|
if ep.rc == 0:
|
|
for line in ep.out.splitlines():
|
|
parts = line.split("\t")
|
|
if len(parts) == 2 and parts[0].strip() == expected_peer:
|
|
endpoint = parts[1].strip()
|
|
if endpoint in (None, "", "(none)"):
|
|
reasons.append("peer has no endpoint")
|
|
if handshake_epoch <= 0:
|
|
reasons.append("no handshake recorded")
|
|
elif handshake_epoch > now + cfg.clock_skew_allowance_sec:
|
|
reasons.append(f"handshake timestamp {handshake_epoch - now}s in the future (clock jump?)")
|
|
else:
|
|
handshake_age = max(0, now - handshake_epoch)
|
|
if handshake_age > cfg.handshake_max_age_sec:
|
|
reasons.append(f"handshake {handshake_age}s old exceeds {cfg.handshake_max_age_sec}s")
|
|
else:
|
|
handshake_ok = True
|
|
checks["handshake_age_sec"] = handshake_age
|
|
|
|
# -- policy rule
|
|
rule = self._rc(["ip", "rule", "show"])
|
|
rule_ok = False
|
|
if rule.rc == 0:
|
|
pat = re.compile(rf"^{cfg.rule_priority}:\s+from {re.escape(cfg.vlan50_subnet)} lookup {re.escape(cfg.route_table)}\s*$", re.M)
|
|
rule_ok = bool(pat.search(rule.out))
|
|
else:
|
|
unknown = True
|
|
checks["policy_rule"] = rule_ok
|
|
if not rule_ok:
|
|
safety_ok = False
|
|
reasons.append("policy rule missing")
|
|
|
|
# -- table default
|
|
route = self._rc(["ip", "route", "show", "table", cfg.route_table])
|
|
route_state = "unknown"
|
|
if route.rc == 0:
|
|
open_default = any(
|
|
l.startswith("default") and f"dev {cfg.interface}" in l for l in route.out.splitlines()
|
|
)
|
|
unreachable_default = any(l.strip().startswith("unreachable default") for l in route.out.splitlines())
|
|
if open_default:
|
|
route_state = "open"
|
|
elif unreachable_default:
|
|
route_state = "closed"
|
|
else:
|
|
route_state = "missing"
|
|
else:
|
|
unknown = True
|
|
checks["route_table_default"] = route_state
|
|
if route_state in ("missing", "unknown"):
|
|
safety_ok = False
|
|
reasons.append(f"table {cfg.route_table} default route {route_state}")
|
|
|
|
# -- FORWARD ordering
|
|
fwd = self._rc(["iptables", "-S", "FORWARD"])
|
|
jump_first = False
|
|
emergency_present = False
|
|
if fwd.rc == 0:
|
|
rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")]
|
|
effective = []
|
|
for r in rules:
|
|
if _is_emergency_rule(r):
|
|
emergency_present = True
|
|
continue
|
|
effective.append(r)
|
|
expected_jump = f"-A FORWARD -s {cfg.vlan50_subnet} -j {cfg.iptables_chain}"
|
|
jump_first = bool(effective) and effective[0].strip() == expected_jump
|
|
else:
|
|
unknown = True
|
|
checks["forward_jump_first"] = jump_first
|
|
checks["emergency_rule_present"] = emergency_present
|
|
if not jump_first:
|
|
safety_ok = False
|
|
reasons.append("FORWARD jump to kill-switch chain is not first")
|
|
|
|
# -- chain contents
|
|
chain = self._rc(["iptables", "-S", cfg.iptables_chain])
|
|
chain_ok = False
|
|
accepts_ok = False
|
|
if chain.rc == 0:
|
|
rules = [l.strip() for l in chain.out.splitlines() if l.startswith(f"-A {cfg.iptables_chain}")]
|
|
drop_idx = next((i for i, r in enumerate(rules) if r == f"-A {cfg.iptables_chain} -j DROP"), None)
|
|
if drop_idx is not None and drop_idx == len(rules) - 1:
|
|
chain_ok = True
|
|
else:
|
|
reasons.append("kill-switch chain lacks a terminal unconditional DROP")
|
|
before_drop = rules[:drop_idx] if drop_idx is not None else rules
|
|
out_accept = f"-s {cfg.vlan50_subnet} -o {cfg.interface} -j ACCEPT"
|
|
in_accept_prefix = f"-d {cfg.vlan50_subnet} -i {cfg.interface}"
|
|
has_out = any(r.endswith(out_accept) for r in before_drop)
|
|
has_in = any(in_accept_prefix in r and r.endswith("-j ACCEPT") for r in before_drop)
|
|
for r in before_drop:
|
|
if r.endswith("-j ACCEPT") and f"-o {cfg.interface}" not in r and f"-i {cfg.interface}" not in r:
|
|
chain_ok = False
|
|
reasons.append("kill-switch chain has an ACCEPT not scoped to the tunnel interface")
|
|
break
|
|
accepts_ok = has_out and has_in
|
|
else:
|
|
unknown = True
|
|
reasons.append("kill-switch chain missing")
|
|
checks["chain_terminal_drop"] = chain_ok
|
|
checks["chain_tunnel_accepts"] = accepts_ok
|
|
if not chain_ok:
|
|
safety_ok = False
|
|
|
|
# -- host default route
|
|
main = self._rc(["ip", "-4", "route", "show", "default"])
|
|
main_ok = False
|
|
if main.rc == 0:
|
|
main_ok = f"dev {cfg.lan_interface}" in main.out and f"dev {cfg.interface}" not in main.out
|
|
else:
|
|
unknown = True
|
|
checks["host_default_route_ok"] = main_ok
|
|
if not main_ok:
|
|
safety_ok = False
|
|
reasons.append("host default route is not via the LAN interface")
|
|
|
|
tunnel_ok = interface_up and peer_present and handshake_ok and route_state == "open" and accepts_ok
|
|
if interface_up and peer_present and handshake_ok and route_state != "open":
|
|
reasons.append("handshake fresh but table route is not open")
|
|
if interface_up and peer_present and handshake_ok and route_state == "open" and not accepts_ok:
|
|
reasons.append("tunnel ACCEPT rules missing from kill-switch chain")
|
|
|
|
# -- classify
|
|
if not safety_ok:
|
|
tunnel_class = CLASS_SAFETY_DRIFT
|
|
elif unknown:
|
|
tunnel_class = CLASS_UNKNOWN
|
|
elif tunnel_ok:
|
|
tunnel_class = CLASS_HEALTHY
|
|
elif not interface_up or not peer_present or (handshake_ok and not tunnel_ok):
|
|
tunnel_class = CLASS_TUNNEL_ABSENT
|
|
else:
|
|
tunnel_class = CLASS_TUNNEL_STALE
|
|
|
|
cls = tunnel_class
|
|
rec = state.get("recovery", {})
|
|
if tunnel_class in (CLASS_TUNNEL_STALE, CLASS_TUNNEL_ABSENT):
|
|
grace = max(state.get("boot_grace_until") or 0, state.get("restart_grace_until") or 0)
|
|
if now < grace:
|
|
cls = CLASS_BOOT_GRACE
|
|
elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_HOST_CONNECTIVITY,):
|
|
cls = CLASS_HOST_CONNECTIVITY
|
|
elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_REGISTRATION, FAIL_AUTH):
|
|
cls = CLASS_REGISTRATION_FAILURE
|
|
if state.get("paused") and cls not in (CLASS_HEALTHY, CLASS_SAFETY_DRIFT):
|
|
cls = CLASS_PAUSED
|
|
|
|
return Status(
|
|
timestamp=now,
|
|
tunnel_class=tunnel_class,
|
|
cls=cls,
|
|
reasons=reasons,
|
|
safety_ok=safety_ok,
|
|
tunnel_ok=tunnel_ok,
|
|
interface_up=interface_up,
|
|
peer_present=peer_present,
|
|
handshake_epoch=handshake_epoch,
|
|
handshake_age_sec=handshake_age,
|
|
peer_endpoint=endpoint,
|
|
expected_peer=expected_peer,
|
|
route_state=route_state,
|
|
configured_peer_line=configured_line,
|
|
checks=checks,
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# PIA API client (stdlib only)
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class HttpClient:
|
|
def __init__(self, cfg: Config, system: Optional[System] = None):
|
|
self.cfg = cfg
|
|
self.system = system or System()
|
|
|
|
def _system_ctx(self) -> ssl.SSLContext:
|
|
return ssl.create_default_context()
|
|
|
|
def _request(self, conn: http.client.HTTPSConnection, method: str, path: str, body: Optional[bytes], headers: dict) -> tuple[int, bytes]:
|
|
try:
|
|
conn.request(method, path, body=body, headers=headers)
|
|
resp = conn.getresponse()
|
|
data = resp.read(4_000_000)
|
|
return resp.status, data
|
|
except ssl.SSLCertVerificationError as e:
|
|
# verify_message/reason are standard OpenSSL diagnostic text
|
|
# (e.g. "unable to get local issuer certificate", "certificate
|
|
# has expired", a hostname mismatch) — never a secret, and
|
|
# exactly what's needed to tell "wrong CA", "expired cert",
|
|
# "clock skew", and "hostname mismatch" apart instead of
|
|
# collapsing all of them into one opaque string.
|
|
detail = e.verify_message or e.reason or str(e) or "certificate verification failed"
|
|
raise ApiError("tls", detail) from e
|
|
except ssl.SSLError as e:
|
|
detail = getattr(e, "reason", None) or str(e) or e.__class__.__name__
|
|
raise ApiError("tls", f"{e.__class__.__name__}: {detail}") from e
|
|
except socket.timeout as e:
|
|
raise ApiError("timeout", "request timed out") from e
|
|
except (OSError, http.client.HTTPException) as e:
|
|
detail = str(e) or e.__class__.__name__
|
|
raise ApiError("network", f"{e.__class__.__name__}: {detail}") from e
|
|
finally:
|
|
with contextlib.suppress(Exception):
|
|
conn.close()
|
|
|
|
def get(self, url: str) -> tuple[int, bytes]:
|
|
u = urllib.parse.urlsplit(url)
|
|
conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx())
|
|
return self._request(conn, "GET", u.path or "/", None, {"User-Agent": "pia-gateway"})
|
|
|
|
def post_multipart(self, url: str, fields: dict[str, str]) -> tuple[int, bytes]:
|
|
boundary = "----pia-gateway-" + secrets.token_hex(12)
|
|
parts = []
|
|
for k, v in fields.items():
|
|
parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n")
|
|
body = ("".join(parts) + f"--{boundary}--\r\n").encode("utf-8")
|
|
u = urllib.parse.urlsplit(url)
|
|
conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx())
|
|
headers = {
|
|
"User-Agent": "pia-gateway",
|
|
"Content-Type": f"multipart/form-data; boundary={boundary}",
|
|
"Content-Length": str(len(body)),
|
|
}
|
|
return self._request(conn, "POST", u.path, body, headers)
|
|
|
|
def get_pinned_curl(self, hostname: str, ip: str, port: int, path: str, token: str, pubkey: str) -> tuple[int, bytes]:
|
|
"""Pinned-IP, SNI-validated, CA-pinned GET — for PIA's addKey only.
|
|
|
|
This shells out to curl instead of using the stdlib `ssl` module.
|
|
Confirmed live (2026-09-22): PIA's own CA certificate (byte-
|
|
identical to a fresh fetch from PIA's own repo — not a stale
|
|
vendored copy) has a non-critical Basic Constraints extension on
|
|
its CA cert, a real RFC 5280 non-conformance in a CA we don't
|
|
control. Recent OpenSSL (3.2+; confirmed on minisforum's 3.5.7 via
|
|
a direct repro — SSLCertVerificationError, verify_message
|
|
"Basic Constraints of CA cert not marked critical") rejects that
|
|
in its default chain verification, a check promoted from
|
|
X509_STRICT-only to the default path in recent OpenSSL releases.
|
|
Python's stdlib `ssl` module has no custom verify-callback hook to
|
|
selectively tolerate just this one known, third-party-controlled
|
|
defect while still fully validating everything else (chain,
|
|
expiry, hostname/SAN) — that gap is long-standing and
|
|
deliberate upstream, not something to route around by weakening
|
|
verify_mode. curl's TLS stack on this host does not enforce this
|
|
specific check and is what PIA's own reference implementation
|
|
(connect_to_wireguard_with_token.sh) already uses for this exact
|
|
endpoint — reusing it here with --cacert (never -k/--insecure) is
|
|
not a security downgrade, just a different, already-proven-correct
|
|
TLS backend for the one endpoint that needs it. serverlist and
|
|
token still use the stdlib ssl path above against public WebPKI
|
|
certs, which do not have this problem.
|
|
"""
|
|
argv = [
|
|
"curl", "--silent", "--show-error", "--fail", "--location", "-G",
|
|
"--connect-timeout", str(self.cfg.http_connect_timeout_sec),
|
|
"--max-time", str(self.cfg.http_total_timeout_sec),
|
|
"--cacert", self.cfg.ca_file,
|
|
"--connect-to", f"{hostname}::{ip}:",
|
|
"--data-urlencode", "pt@-",
|
|
"--data-urlencode", f"pubkey={pubkey}",
|
|
f"https://{hostname}:{port}{path}",
|
|
]
|
|
r = self.system.run(argv, stdin=token, timeout=self.cfg.http_total_timeout_sec + 10)
|
|
if r.rc == 0:
|
|
return 200, r.out.encode("utf-8")
|
|
err = (r.err or "").strip()
|
|
# Exit codes are the primary signal, but not fully portable across
|
|
# curl builds/TLS backends — confirmed live: a curl built against
|
|
# SecureTransport reported exit 56 (CURLE_RECV_ERROR) for the exact
|
|
# same --fail-triggered HTTP rejection GNU/OpenSSL curl reports as
|
|
# 22. curl's own error text ("returned error: NNN" for --fail,
|
|
# "timed out" for CURLE_OPERATION_TIMEDOUT, TLS-related wording for
|
|
# cert/handshake problems) is a second, more portable signal — used
|
|
# here as a fallback classifier alongside the exit code, not a
|
|
# replacement for it.
|
|
if r.rc == 127:
|
|
raise ApiError("network", "curl not found on PATH")
|
|
if r.rc == 28 or "timed out" in err.lower():
|
|
raise ApiError("timeout", f"curl: connection or transfer timed out: {err}")
|
|
if r.rc in (35, 53, 54, 58, 59, 60, 66, 77, 80, 82, 83) or "ssl" in err.lower() or "certificate" in err.lower():
|
|
raise ApiError("tls", f"curl TLS error (exit {r.rc}): {err}")
|
|
if r.rc == 22 or "returned error:" in err:
|
|
# --fail suppresses the body on a non-2xx response, so there is
|
|
# no status code or body to parse here — same limitation the
|
|
# legacy curl-based Ansible path already had.
|
|
raise ApiError("http", f"curl: non-2xx HTTP response (exit {r.rc}): {err}")
|
|
raise ApiError("network", f"curl exit {r.rc}: {err}")
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Candidate:
|
|
region_id: str
|
|
region_name: str
|
|
cn: str
|
|
ip: str
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class PeerInfo:
|
|
server_key: str
|
|
peer_ip: str
|
|
server_port: int
|
|
server_ip: str
|
|
region_id: str
|
|
region_name: str
|
|
cn: str
|
|
|
|
|
|
_B64KEY = re.compile(r"^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$")
|
|
_IPV4 = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$")
|
|
|
|
|
|
def valid_wg_key(k: str) -> bool:
|
|
return bool(_B64KEY.match(k or ""))
|
|
|
|
|
|
def valid_ipv4(s: str) -> bool:
|
|
if not _IPV4.match(s or ""):
|
|
return False
|
|
return all(0 <= int(o) <= 255 for o in s.split("."))
|
|
|
|
|
|
class PiaApi:
|
|
def __init__(self, cfg: Config, http: HttpClient, log: Log):
|
|
self.cfg = cfg
|
|
self.http = http
|
|
self.log = log
|
|
|
|
def fetch_serverlist(self) -> dict:
|
|
status, data = self.http.get(SERVERLIST_URL)
|
|
if status != 200:
|
|
raise ApiError("http", f"serverlist HTTP {status}")
|
|
first = data.split(b"\n", 1)[0]
|
|
try:
|
|
doc = json.loads(first.decode("utf-8"))
|
|
except (ValueError, UnicodeDecodeError) as e:
|
|
raise ApiError("malformed", "serverlist is not JSON") from e
|
|
if not isinstance(doc, dict) or not isinstance(doc.get("regions"), list):
|
|
raise ApiError("malformed", "serverlist lacks a regions list")
|
|
return doc
|
|
|
|
def build_candidates(self, doc: dict, regions: Iterable[str]) -> list[Candidate]:
|
|
by_id = {r.get("id"): r for r in doc["regions"] if isinstance(r, dict)}
|
|
out: list[Candidate] = []
|
|
missing, empty = [], []
|
|
for rid in regions:
|
|
r = by_id.get(rid)
|
|
if r is None:
|
|
missing.append(rid)
|
|
continue
|
|
servers = (r.get("servers") or {}).get("wg") or []
|
|
if not servers:
|
|
empty.append(rid)
|
|
continue
|
|
for s in servers:
|
|
cn, ip = s.get("cn"), s.get("ip")
|
|
if not cn or not valid_ipv4(ip or ""):
|
|
continue
|
|
out.append(Candidate(rid, r.get("name") or rid, cn, ip))
|
|
if missing:
|
|
raise ApiError("malformed", f"requested regions not in serverlist: {missing}")
|
|
if empty:
|
|
# A configured fallback region with no WireGuard servers right now
|
|
# must not block recovery through the regions that do have them.
|
|
self.log.warn("regions without WireGuard servers skipped", regions=empty)
|
|
if not out:
|
|
raise ApiError("malformed", "no usable WireGuard servers in requested regions")
|
|
return out
|
|
|
|
def get_token(self, creds: Credentials) -> str:
|
|
last_status = None
|
|
for attempt in range(3):
|
|
try:
|
|
status, data = self.http.post_multipart(TOKEN_URL, {"username": creds.pia_user, "password": creds.pia_password})
|
|
except ApiError as e:
|
|
if e.kind in ("tls",):
|
|
raise
|
|
last_status = e.kind
|
|
if attempt < 2:
|
|
continue
|
|
raise
|
|
last_status = status
|
|
if status == 200:
|
|
try:
|
|
token = json.loads(data.decode("utf-8")).get("token")
|
|
except (ValueError, AttributeError, UnicodeDecodeError) as e:
|
|
raise ApiError("malformed", "token response is not JSON") from e
|
|
if not token or not isinstance(token, str):
|
|
raise ApiError("malformed", "token response has no token")
|
|
return token
|
|
if status in (401, 403):
|
|
raise ApiError("auth", f"token HTTP {status}")
|
|
if attempt < 2 and status >= 500:
|
|
continue
|
|
break
|
|
raise ApiError("http", f"token HTTP {last_status}")
|
|
|
|
def add_key(self, cand: Candidate, token: str, pubkey: str) -> PeerInfo:
|
|
status, data = self.http.get_pinned_curl(cand.cn, cand.ip, ADDKEY_PORT, "/addKey", token, pubkey)
|
|
if status != 200:
|
|
raise ApiError("http", f"addKey HTTP {status}")
|
|
if not data.strip():
|
|
raise ApiError("malformed", "addKey empty body")
|
|
try:
|
|
doc = json.loads(data.decode("utf-8"))
|
|
except (ValueError, UnicodeDecodeError) as e:
|
|
raise ApiError("malformed", "addKey body is not JSON") from e
|
|
if not isinstance(doc, dict) or doc.get("status") != "OK":
|
|
raise ApiError("status", f"addKey status={doc.get('status') if isinstance(doc, dict) else '?'}")
|
|
server_key = doc.get("server_key")
|
|
peer_ip = doc.get("peer_ip")
|
|
port = doc.get("server_port")
|
|
if not valid_wg_key(server_key or ""):
|
|
raise ApiError("malformed", "addKey server_key invalid")
|
|
if not valid_ipv4(peer_ip or ""):
|
|
raise ApiError("malformed", "addKey peer_ip invalid")
|
|
try:
|
|
port = int(port)
|
|
except (TypeError, ValueError):
|
|
raise ApiError("malformed", "addKey server_port invalid")
|
|
if not (1 <= port <= 65535):
|
|
raise ApiError("malformed", "addKey server_port out of range")
|
|
return PeerInfo(server_key, peer_ip, port, cand.ip, cand.region_id, cand.region_name, cand.cn)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# WireGuard config rendering / validation
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def render_wg_conf(cfg: Config, private_key: str, peer: PeerInfo) -> str:
|
|
i = cfg.interface
|
|
t = cfg.route_table
|
|
s = cfg.vlan50_subnet
|
|
c = cfg.iptables_chain
|
|
lines = [
|
|
"# Managed by pia-gateway (ansible/roles/pia-gateway). Do not edit by hand.",
|
|
"# Table = off: routing for this interface lives only in table "
|
|
f"{t}, reached only by traffic sourced from {s}.",
|
|
"[Interface]",
|
|
f"PrivateKey = {private_key}",
|
|
f"Address = {peer.peer_ip}/32",
|
|
"Table = off",
|
|
"",
|
|
f"PostUp = ip route replace default dev %i table {t}",
|
|
f"PostUp = iptables -I {c} 1 -s {s} -o %i -j ACCEPT",
|
|
f"PostUp = iptables -I {c} 1 -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT",
|
|
f"PostUp = iptables -t nat -A POSTROUTING -s {s} -o %i -j MASQUERADE",
|
|
]
|
|
if cfg.mss_clamp_enabled:
|
|
lines.append(
|
|
f"PostUp = iptables -t mangle -A FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu"
|
|
)
|
|
lines += [
|
|
"",
|
|
f"PreDown = ip route replace unreachable default table {t}",
|
|
f"PreDown = iptables -t nat -D POSTROUTING -s {s} -o %i -j MASQUERADE 2>/dev/null || true",
|
|
f"PreDown = iptables -D {c} -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT 2>/dev/null || true",
|
|
f"PreDown = iptables -D {c} -s {s} -o %i -j ACCEPT 2>/dev/null || true",
|
|
]
|
|
if cfg.mss_clamp_enabled:
|
|
lines.append(
|
|
f"PreDown = iptables -t mangle -D FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null || true"
|
|
)
|
|
lines += [
|
|
"",
|
|
"[Peer]",
|
|
f"# PIA WireGuard server actually used — {peer.region_name} ({peer.region_id}), {peer.cn}",
|
|
f"PublicKey = {peer.server_key}",
|
|
f"Endpoint = {peer.server_ip}:{peer.server_port}",
|
|
"AllowedIPs = 0.0.0.0/0",
|
|
"PersistentKeepalive = 25",
|
|
"",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def validate_wg_conf(cfg: Config, text: str) -> list[str]:
|
|
"""Return a list of problems (empty when the staged config is acceptable)."""
|
|
problems: list[str] = []
|
|
section = None
|
|
kv: dict[str, list[str]] = {}
|
|
for line in text.splitlines():
|
|
s = line.strip()
|
|
if not s or s.startswith("#"):
|
|
continue
|
|
if s.startswith("["):
|
|
section = s.lower()
|
|
continue
|
|
k, _, v = s.partition("=")
|
|
kv.setdefault(f"{section}:{k.strip()}", []).append(v.strip())
|
|
if kv.get("[interface]:Table") != ["off"]:
|
|
problems.append("Table must be off")
|
|
pk = kv.get("[interface]:PrivateKey", [""])[0]
|
|
if not valid_wg_key(pk):
|
|
problems.append("PrivateKey invalid")
|
|
addr = kv.get("[interface]:Address", [""])[0]
|
|
if not addr.endswith("/32") or not valid_ipv4(addr[:-3]):
|
|
problems.append("Address must be a /32 IPv4")
|
|
if kv.get("[peer]:AllowedIPs") != ["0.0.0.0/0"]:
|
|
problems.append("AllowedIPs must be 0.0.0.0/0")
|
|
if not valid_wg_key(kv.get("[peer]:PublicKey", [""])[0]):
|
|
problems.append("peer PublicKey invalid")
|
|
ep = kv.get("[peer]:Endpoint", [""])[0]
|
|
host, _, port = ep.rpartition(":")
|
|
if not valid_ipv4(host) or not port.isdigit():
|
|
problems.append("Endpoint invalid")
|
|
postup = kv.get("[interface]:PostUp", [])
|
|
predown = kv.get("[interface]:PreDown", [])
|
|
if not any(f"table {cfg.route_table}" in p and "default dev %i" in p for p in postup):
|
|
problems.append("PostUp lacks table route open")
|
|
if not any(f"unreachable default table {cfg.route_table}" in p for p in predown):
|
|
problems.append("PreDown lacks unreachable route")
|
|
for hook in postup + predown:
|
|
if "ip route" in hook and "table" not in hook:
|
|
problems.append("hook touches routing without a table qualifier")
|
|
if f"-s {cfg.vlan50_subnet}" not in hook and "iptables" in hook and f"-d {cfg.vlan50_subnet}" not in hook:
|
|
problems.append("iptables hook not scoped to VLAN 50")
|
|
if "-P FORWARD" in hook:
|
|
problems.append("hook changes FORWARD policy")
|
|
return problems
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Notifier
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
class Notifier:
|
|
def __init__(self, cfg: Config, creds: Credentials, http: HttpClient, log: Log):
|
|
self.cfg = cfg
|
|
self.creds = creds
|
|
self.http = http
|
|
self.log = log
|
|
|
|
def send(self, level: str, message: str) -> bool:
|
|
if not self.cfg.alert_bridge_url or not self.creds.alert_bridge_api_key:
|
|
self.log.info("notify skipped (no alert-bridge configured)", level=level, message=message)
|
|
return True
|
|
u = urllib.parse.urlsplit(self.cfg.alert_bridge_url)
|
|
body = json.dumps({"source": self.cfg.alert_source, "message": message, "level": level}).encode("utf-8")
|
|
try:
|
|
conn_cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection
|
|
conn = conn_cls(u.hostname, u.port or (443 if u.scheme == "https" else 80), timeout=5)
|
|
conn.request(
|
|
"POST",
|
|
u.path or "/",
|
|
body=body,
|
|
headers={
|
|
"Authorization": f"Bearer {self.creds.alert_bridge_api_key}",
|
|
"Content-Type": "application/json",
|
|
"Content-Length": str(len(body)),
|
|
},
|
|
)
|
|
resp = conn.getresponse()
|
|
resp.read(4096)
|
|
conn.close()
|
|
ok = 200 <= resp.status < 300
|
|
if not ok:
|
|
self.log.warn("notify failed", http_status=resp.status)
|
|
return ok
|
|
except (OSError, http.client.HTTPException) as e:
|
|
self.log.warn("notify failed", error=e.__class__.__name__)
|
|
return False
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Controller
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class RegistrationOutcome:
|
|
ok: bool
|
|
stage: str
|
|
failure_class: Optional[str] = None
|
|
detail: Optional[str] = None
|
|
region: Optional[str] = None
|
|
server_cn: Optional[str] = None
|
|
any_api_ok: bool = False
|
|
all_api_failed_same_way: bool = False
|
|
verified: Optional[Status] = None
|
|
|
|
|
|
class Controller:
|
|
def __init__(self, cfg: Config, system: System, log: Log, http: Optional[HttpClient] = None, creds: Optional[Credentials] = None):
|
|
self.cfg = cfg
|
|
self.sys = system
|
|
self.log = log
|
|
self.http = http or HttpClient(cfg, system)
|
|
self._creds = creds
|
|
self.store = StateStore(cfg, system)
|
|
self.classifier = Classifier(cfg, system)
|
|
self.api = PiaApi(cfg, self.http, log)
|
|
|
|
# -- helpers ----------------------------------------------------------
|
|
|
|
@property
|
|
def creds(self) -> Credentials:
|
|
if self._creds is None:
|
|
self._creds = load_credentials(self.cfg.credentials_file, require_pia=True)
|
|
return self._creds
|
|
|
|
def creds_optional(self) -> Credentials:
|
|
if self._creds is None:
|
|
try:
|
|
self._creds = load_credentials(self.cfg.credentials_file, require_pia=False)
|
|
except GatewayError as e:
|
|
self.log.warn("credentials unavailable", error=str(e))
|
|
self._creds = Credentials("", "", None)
|
|
return self._creds
|
|
|
|
def notifier(self) -> Notifier:
|
|
return Notifier(self.cfg, self.creds_optional(), self.http, self.log)
|
|
|
|
def notify(self, state: dict, level: str, message: str) -> None:
|
|
pending = state.setdefault("pending_notifications", [])
|
|
pending.append({"level": level, "message": message, "epoch": self.sys.now()})
|
|
del pending[:-20]
|
|
self.flush_notifications(state)
|
|
|
|
def flush_notifications(self, state: dict) -> None:
|
|
pending = state.get("pending_notifications") or []
|
|
n = self.notifier()
|
|
remaining = []
|
|
for item in pending:
|
|
if not n.send(item["level"], item["message"]):
|
|
remaining.append(item)
|
|
state["pending_notifications"] = remaining
|
|
|
|
def lock(self, wait_sec: float = 0) -> Lock:
|
|
lk = Lock(self.cfg.lock_file, self.sys)
|
|
lk.acquire(wait_sec)
|
|
return lk
|
|
|
|
def write_status_outputs(self, st: Status, state: dict) -> None:
|
|
cfg = self.cfg
|
|
rec = state.get("recovery", {})
|
|
doc = {
|
|
"status": st.to_dict(),
|
|
"controller": {
|
|
"paused": bool(state.get("paused")),
|
|
"consecutive_failures": state.get("consecutive_failures", 0),
|
|
"recovery_in_progress": rec.get("in_progress", False),
|
|
"backoff_index": rec.get("backoff_index", 0),
|
|
"next_retry_epoch": rec.get("next_retry_epoch"),
|
|
"last_failure_class": rec.get("last_failure_class"),
|
|
"last_verified_registration": state.get("last_verified_registration"),
|
|
"last_recovery": state.get("last_recovery"),
|
|
"emergency_close_active": state.get("emergency_close_active", False),
|
|
},
|
|
}
|
|
try:
|
|
self.sys.makedirs(cfg.state_dir, 0o700)
|
|
self.sys.write_atomic(cfg.status_file, json.dumps(doc, indent=1, sort_keys=True) + "\n", 0o644)
|
|
self.sys.write_atomic(cfg.metrics_file, self.render_metrics(st, state), 0o644)
|
|
except OSError as e:
|
|
self.log.warn("status/metrics write failed", error=e.__class__.__name__)
|
|
|
|
def render_metrics(self, st: Status, state: dict) -> str:
|
|
rec = state.get("recovery", {})
|
|
lvr = state.get("last_verified_registration") or {}
|
|
lr = state.get("last_recovery") or {}
|
|
m = [
|
|
("pia_gateway_check_timestamp_seconds", st.timestamp),
|
|
("pia_gateway_healthy", 1 if st.cls == CLASS_HEALTHY else 0),
|
|
("pia_gateway_tunnel_ok", 1 if st.tunnel_ok else 0),
|
|
("pia_gateway_safety_ok", 1 if st.safety_ok else 0),
|
|
("pia_gateway_handshake_age_seconds", st.handshake_age_sec if st.handshake_age_sec is not None else -1),
|
|
("pia_gateway_consecutive_failures", state.get("consecutive_failures", 0)),
|
|
("pia_gateway_paused", 1 if state.get("paused") else 0),
|
|
("pia_gateway_recovery_in_backoff", 1 if rec.get("next_retry_epoch") else 0),
|
|
("pia_gateway_recovery_backoff_index", rec.get("backoff_index", 0)),
|
|
("pia_gateway_last_verified_registration_timestamp_seconds", lvr.get("epoch") or 0),
|
|
("pia_gateway_last_recovery_timestamp_seconds", lr.get("epoch") or 0),
|
|
("pia_gateway_emergency_close_active", 1 if state.get("emergency_close_active") else 0),
|
|
]
|
|
out = [f"# pia-gateway class={st.cls} tunnel_class={st.tunnel_class}"]
|
|
for name, val in m:
|
|
out.append(f"# TYPE {name} gauge")
|
|
out.append(f"{name} {val}")
|
|
return "\n".join(out) + "\n"
|
|
|
|
def refresh_boot_grace(self, state: dict) -> None:
|
|
bid = self.sys.boot_id()
|
|
if state.get("boot_id") != bid:
|
|
state["boot_id"] = bid
|
|
state["boot_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec
|
|
state["restart_grace_until"] = 0
|
|
self.log.info("new boot detected; startup grace applied", grace_sec=self.cfg.startup_grace_sec)
|
|
self.guard_clock_jump(state)
|
|
|
|
def guard_clock_jump(self, state: dict) -> None:
|
|
"""Persisted wall-clock deadlines further out than they could ever have been
|
|
scheduled mean the clock went backwards; clear them rather than stall."""
|
|
now = self.sys.now()
|
|
horizon = self.cfg.backoff_schedule_sec[-1] * 2 + 300
|
|
for section in ("recovery", "renewal"):
|
|
nxt = state.get(section, {}).get("next_retry_epoch")
|
|
if nxt and nxt - now > horizon:
|
|
self.log.warn("clock jump detected; clearing stale retry deadline", section=section, was_in_sec=nxt - now)
|
|
state[section]["next_retry_epoch"] = None
|
|
for key in ("boot_grace_until", "restart_grace_until"):
|
|
if (state.get(key) or 0) - now > self.cfg.startup_grace_sec * 2 + 300:
|
|
state[key] = 0
|
|
|
|
def current_status(self, state: dict) -> Status:
|
|
return self.classifier.classify(state)
|
|
|
|
def log_status(self, st: Status) -> None:
|
|
self.log.info(
|
|
"status",
|
|
**{"class": st.cls},
|
|
tunnel_class=st.tunnel_class,
|
|
handshake_age_sec=st.handshake_age_sec,
|
|
route=st.route_state,
|
|
safety_ok=st.safety_ok,
|
|
tunnel_ok=st.tunnel_ok,
|
|
endpoint=st.peer_endpoint,
|
|
reasons=st.reasons,
|
|
)
|
|
if st.configured_peer_line:
|
|
self.log.info("configured-peer", line=st.configured_peer_line)
|
|
|
|
# -- emergency closing ------------------------------------------------
|
|
|
|
def emergency_rule_argv(self, op: str) -> list[str]:
|
|
return ["iptables", op, "FORWARD"] + (["1"] if op == "-I" else []) + [
|
|
"-s", self.cfg.vlan50_subnet, "-m", "comment", "--comment", EMERGENCY_COMMENT, "-j", "DROP",
|
|
]
|
|
|
|
def emergency_close(self, state: dict, st: Status) -> bool:
|
|
"""Insert an emergency DROP first in FORWARD and close table pia. Returns effectiveness."""
|
|
cfg = self.cfg
|
|
fwd0 = self.sys.run(["iptables", "-S", "FORWARD"])
|
|
rules0 = [l for l in fwd0.out.splitlines() if l.startswith("-A FORWARD")] if fwd0.rc == 0 else []
|
|
already_first = bool(rules0) and _is_emergency_rule(rules0[0])
|
|
if not already_first:
|
|
# `-C` (existence) is not enough here: something could have
|
|
# prepended a rule ahead of ours (e.g. a `ufw reload` outside
|
|
# this role's own handler ordering) without removing it, in
|
|
# which case it still exists but is no longer FIRST — and a
|
|
# rule that isn't first can be pre-empted by whatever now sits
|
|
# above it. Reposition explicitly: delete wherever it is (`-D`
|
|
# matches by rule spec, not position), then insert at 1.
|
|
if any(_is_emergency_rule(r) for r in rules0):
|
|
d = self.sys.run(self.emergency_rule_argv("-D"))
|
|
if d.rc != 0:
|
|
self.log.error("emergency DROP removal (repositioning) failed", rc=d.rc)
|
|
ins = self.sys.run(self.emergency_rule_argv("-I"))
|
|
if ins.rc != 0:
|
|
self.log.error("emergency DROP insert failed", rc=ins.rc)
|
|
r2 = self.sys.run(["ip", "route", "replace", "unreachable", "default", "table", cfg.route_table])
|
|
if r2.rc != 0:
|
|
self.log.error("emergency unreachable route failed", rc=r2.rc)
|
|
fwd = self.sys.run(["iptables", "-S", "FORWARD"])
|
|
rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")]
|
|
effective = fwd.rc == 0 and bool(rules) and _is_emergency_rule(rules[0]) and r2.rc == 0
|
|
state["emergency_close_active"] = True
|
|
if effective:
|
|
self.log.error("SAFETY DRIFT: emergency close applied", reasons=st.reasons)
|
|
self.notify(state, "error", f"pia-gateway SAFETY DRIFT: kill switch invariants failed ({'; '.join(st.reasons)}). "
|
|
"Emergency VLAN 50 DROP applied; automatic recovery halted until repaired.")
|
|
else:
|
|
self.log.error("SAFETY DRIFT: emergency close NOT verified effective", reasons=st.reasons)
|
|
self.notify(state, "error", "pia-gateway CRITICAL: safety drift and the emergency close could not be verified "
|
|
f"effective ({'; '.join(st.reasons)}). Manual intervention required NOW.")
|
|
return effective
|
|
|
|
def emergency_clear_if_repaired(self, state: dict, st: Status) -> None:
|
|
if not state.get("emergency_close_active"):
|
|
return
|
|
if not st.safety_ok:
|
|
return
|
|
r = self.sys.run(self.emergency_rule_argv("-C"))
|
|
if r.rc == 0:
|
|
d = self.sys.run(self.emergency_rule_argv("-D"))
|
|
if d.rc != 0:
|
|
self.log.error("emergency DROP removal failed", rc=d.rc)
|
|
return
|
|
state["emergency_close_active"] = False
|
|
self.log.info("kill switch repaired; emergency DROP removed")
|
|
self.notify(state, "info", "pia-gateway: kill switch invariants restored; emergency DROP removed.")
|
|
|
|
# -- key handling -----------------------------------------------------
|
|
|
|
def _read_private_key(self) -> str:
|
|
key = self.sys.read_text(self.cfg.wg_key).strip()
|
|
if not valid_wg_key(key):
|
|
raise GatewayError("existing private key file is not a valid WireGuard key")
|
|
return key
|
|
|
|
def _generate_key(self) -> str:
|
|
r = self.sys.run(["wg", "genkey"])
|
|
key = r.out.strip()
|
|
if r.rc != 0 or not valid_wg_key(key):
|
|
raise GatewayError("wg genkey failed")
|
|
return key
|
|
|
|
def _pubkey(self, private_key: str) -> str:
|
|
r = self.sys.run(["wg", "pubkey"], stdin=private_key + "\n")
|
|
pub = r.out.strip()
|
|
if r.rc != 0 or not valid_wg_key(pub):
|
|
raise GatewayError("wg pubkey failed")
|
|
return pub
|
|
|
|
def clean_abandoned_stages(self) -> int:
|
|
cfg = self.cfg
|
|
n = 0
|
|
prefix_key = f"{cfg.interface}.key.new"
|
|
prefix_conf = f"{cfg.interface}.conf.stage"
|
|
for name in self.sys.listdir(cfg.config_dir):
|
|
if name.startswith(prefix_key) or name.startswith(prefix_conf):
|
|
self.sys.unlink(os.path.join(cfg.config_dir, name))
|
|
n += 1
|
|
if n:
|
|
self.log.info("removed abandoned staged files", count=n)
|
|
return n
|
|
|
|
def prune_backups(self) -> None:
|
|
cfg = self.cfg
|
|
for base in (f"{cfg.interface}.conf.bak-", f"{cfg.interface}.key.bak-"):
|
|
names = [n for n in self.sys.listdir(cfg.config_dir) if n.startswith(base)]
|
|
names.sort()
|
|
for old in names[: max(0, len(names) - cfg.backup_keep)]:
|
|
self.sys.unlink(os.path.join(cfg.config_dir, old))
|
|
|
|
def _ts(self) -> str:
|
|
return _dt.datetime.fromtimestamp(self.sys.now(), _dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
|
|
# -- activation and verification ---------------------------------------
|
|
|
|
def assert_killswitch_active(self) -> None:
|
|
r = self.sys.run(["systemctl", "is-active", self.cfg.killswitch_unit])
|
|
if r.out.strip() != "active":
|
|
raise SafetyError(f"{self.cfg.killswitch_unit} is not active; refusing to activate the tunnel")
|
|
|
|
def restart_interface(self, state: dict) -> bool:
|
|
state["restart_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec
|
|
r = self.sys.run(["systemctl", "restart", self.cfg.wg_unit], timeout=120)
|
|
if r.rc != 0:
|
|
self.log.error("restart failed", unit=self.cfg.wg_unit, rc=r.rc)
|
|
if "already exists" in (r.err or ""):
|
|
self.log.warn("orphaned interface detected; deleting and retrying start")
|
|
self.sys.run(["ip", "link", "delete", "dev", self.cfg.interface])
|
|
r = self.sys.run(["systemctl", "start", self.cfg.wg_unit], timeout=120)
|
|
return r.rc == 0
|
|
return False
|
|
return True
|
|
|
|
def wait_for_verified(self, state: dict, since_epoch: int, timeout_sec: int, expected_peer: Optional[str] = None) -> Optional[Status]:
|
|
"""Poll until two consecutive healthy classifications with a handshake newer than since_epoch."""
|
|
start = self.sys.monotonic()
|
|
healthy_streak = 0
|
|
last: Optional[Status] = None
|
|
while True:
|
|
st = self.classifier.classify({**state, "restart_grace_until": 0, "boot_grace_until": 0, "paused": None})
|
|
last = st
|
|
# Strictly greater, not >=: `wg show`/`since_epoch` are both
|
|
# whole seconds, so a handshake reported at the exact same
|
|
# second as `since_epoch` is indistinguishable from one that
|
|
# predates activation and just happens to share that second
|
|
# due to truncation. Treating it as fresh would let a stale,
|
|
# pre-activation handshake pass verification — exactly the
|
|
# "addKey said OK but nothing actually changed" failure mode
|
|
# this whole check exists to rule out. A handshake that is
|
|
# genuinely new but lands in the same second as `since_epoch`
|
|
# is simply not counted yet; the next 5s poll picks it up.
|
|
fresh = st.handshake_epoch is not None and st.handshake_epoch > since_epoch
|
|
peer_ok = expected_peer is None or st.expected_peer == expected_peer
|
|
if st.tunnel_class == CLASS_HEALTHY and fresh and peer_ok:
|
|
healthy_streak += 1
|
|
if healthy_streak >= 2:
|
|
return st
|
|
else:
|
|
healthy_streak = 0
|
|
if st.tunnel_class == CLASS_SAFETY_DRIFT:
|
|
return None
|
|
if self.sys.monotonic() - start >= timeout_sec:
|
|
return None
|
|
self.sys.sleep(5)
|
|
|
|
def backup_current_config(self) -> Optional[str]:
|
|
cfg = self.cfg
|
|
if not self.sys.exists(cfg.wg_conf):
|
|
return None
|
|
dst = os.path.join(cfg.config_dir, f"{cfg.interface}.conf.bak-{self._ts()}-{os.getpid()}")
|
|
self.sys.copy_file(cfg.wg_conf, dst, 0o600)
|
|
return dst
|
|
|
|
def activate_config(self, state: dict, staged_path: str, expected_peer: str, since_epoch: int) -> Optional[Status]:
|
|
cfg = self.cfg
|
|
self.assert_killswitch_active()
|
|
self.sys.rename(staged_path, cfg.wg_conf)
|
|
if not self.restart_interface(state):
|
|
return None
|
|
return self.wait_for_verified(state, since_epoch, cfg.activation_wait_sec, expected_peer)
|
|
|
|
def stage_conf(self, text: str) -> str:
|
|
problems = validate_wg_conf(self.cfg, text)
|
|
if problems:
|
|
raise GatewayError("staged config rejected: " + "; ".join(problems))
|
|
path = os.path.join(self.cfg.config_dir, f"{self.cfg.interface}.conf.stage-{self._ts()}-{os.getpid()}")
|
|
self.sys.write_atomic(path, text, 0o600)
|
|
return path
|
|
|
|
def restore_previous_config(self, state: dict, backup_path: Optional[str]) -> None:
|
|
cfg = self.cfg
|
|
if not backup_path or not self.sys.exists(backup_path):
|
|
self.log.warn("no previous config to restore")
|
|
return
|
|
self.sys.copy_file(backup_path, cfg.wg_conf, 0o600)
|
|
self.log.warn("restored previous config (best effort)", backup=os.path.basename(backup_path))
|
|
self.restart_interface(state)
|
|
|
|
# -- registration transaction ----------------------------------------
|
|
|
|
def registration_transaction(
|
|
self,
|
|
state: dict,
|
|
mode: str,
|
|
regions: Optional[list[str]],
|
|
via: str,
|
|
max_candidates: Optional[int] = None,
|
|
deadline_mono: Optional[float] = None,
|
|
) -> RegistrationOutcome:
|
|
"""Same-key or new-key registration: API -> stage -> activate -> verify.
|
|
|
|
Never opens the gateway unless verification passes. On total failure the
|
|
previous config is restored (best effort) and the outcome says so.
|
|
"""
|
|
cfg = self.cfg
|
|
self.clean_abandoned_stages()
|
|
if mode not in ("same-key", "new-key"):
|
|
raise GatewayError(f"unknown registration mode {mode}")
|
|
|
|
# key material
|
|
staged_key_path: Optional[str] = None
|
|
if mode == "new-key":
|
|
private_key = self._generate_key()
|
|
staged_key_path = os.path.join(cfg.config_dir, f"{cfg.interface}.key.new.{self._ts()}.{os.getpid()}")
|
|
self.sys.write_atomic(staged_key_path, private_key + "\n", 0o600)
|
|
elif self.sys.exists(cfg.wg_key):
|
|
private_key = self._read_private_key()
|
|
else:
|
|
private_key = self._generate_key()
|
|
self.sys.write_atomic(cfg.wg_key, private_key + "\n", 0o600)
|
|
self.log.info("generated initial private key")
|
|
pubkey = self._pubkey(private_key)
|
|
|
|
# API path
|
|
try:
|
|
doc = self.api.fetch_serverlist()
|
|
except ApiError as e:
|
|
fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION
|
|
return RegistrationOutcome(False, "serverlist", fc, str(e))
|
|
try:
|
|
cands = self.api.build_candidates(doc, regions or cfg.region_candidates)
|
|
except ApiError as e:
|
|
return RegistrationOutcome(False, "serverlist", FAIL_REGISTRATION, str(e))
|
|
if max_candidates:
|
|
cands = cands[:max_candidates]
|
|
try:
|
|
token = self.api.get_token(self.creds)
|
|
except ApiError as e:
|
|
if e.kind == "auth":
|
|
return RegistrationOutcome(False, "token", FAIL_AUTH, str(e))
|
|
fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION
|
|
return RegistrationOutcome(False, "token", fc, str(e))
|
|
|
|
any_api_ok = False
|
|
api_fail_kinds: list[str] = []
|
|
last_detail = None
|
|
original_backup = self.backup_current_config()
|
|
for cand in cands:
|
|
if deadline_mono is not None and self.sys.monotonic() >= deadline_mono:
|
|
last_detail = "recovery time budget exhausted"
|
|
break
|
|
try:
|
|
peer = self.api.add_key(cand, token, pubkey)
|
|
except ApiError as e:
|
|
api_fail_kinds.append(e.kind)
|
|
last_detail = f"{cand.region_id}/{cand.cn}: {e}"
|
|
self.log.warn("addKey failed", region=cand.region_id, server=cand.cn, kind=e.kind, detail=e.detail)
|
|
continue
|
|
any_api_ok = True
|
|
self.log.info("addKey accepted", region=cand.region_id, server=cand.cn)
|
|
text = render_wg_conf(cfg, private_key, peer)
|
|
try:
|
|
staged = self.stage_conf(text)
|
|
except GatewayError as e:
|
|
last_detail = str(e)
|
|
self.log.warn("staged config rejected", region=cand.region_id, server=cand.cn, detail=str(e))
|
|
continue
|
|
since = self.sys.now()
|
|
try:
|
|
verified = self.activate_config(state, staged, peer.server_key, since)
|
|
except SafetyError as e:
|
|
self.sys.unlink(staged)
|
|
return RegistrationOutcome(False, "activate", FAIL_SAFETY, str(e))
|
|
if verified is not None:
|
|
if staged_key_path:
|
|
if self.sys.exists(cfg.wg_key):
|
|
self.sys.copy_file(cfg.wg_key, os.path.join(cfg.config_dir, f"{cfg.interface}.key.bak-{self._ts()}"), 0o600)
|
|
self.sys.rename(staged_key_path, cfg.wg_key)
|
|
self.prune_backups()
|
|
epoch = self.sys.now()
|
|
reg = {
|
|
"epoch": epoch,
|
|
"region": peer.region_id,
|
|
"region_name": peer.region_name,
|
|
"server_cn": peer.cn,
|
|
"server_ip": peer.server_ip,
|
|
"server_port": peer.server_port,
|
|
"server_key": peer.server_key,
|
|
"peer_ip": peer.peer_ip,
|
|
"pubkey": pubkey,
|
|
"mode": mode,
|
|
"via": via,
|
|
}
|
|
state["last_verified_registration"] = {k: reg[k] for k in ("epoch", "region", "region_name", "server_cn", "server_ip", "pubkey", "mode", "via")}
|
|
state["active_registration"] = reg
|
|
state["renewal"] = {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None}
|
|
self.log.info("registration verified", region=peer.region_id, server=peer.cn, mode=mode, via=via, handshake_age_sec=verified.handshake_age_sec)
|
|
return RegistrationOutcome(True, "register", None, None, peer.region_id, peer.cn, True, verified=verified)
|
|
last_detail = f"{cand.region_id}/{cand.cn}: activated but no verified handshake"
|
|
self.log.warn("activation not verified; trying next candidate", region=cand.region_id, server=cand.cn)
|
|
|
|
# all candidates failed
|
|
if staged_key_path:
|
|
self.sys.unlink(staged_key_path)
|
|
self.restore_previous_config(state, original_backup)
|
|
uniform = bool(api_fail_kinds) and not any_api_ok and len(set(api_fail_kinds)) == 1
|
|
fc = FAIL_REGISTRATION
|
|
if uniform and api_fail_kinds[0] in ("network", "timeout"):
|
|
fc = FAIL_HOST_CONNECTIVITY
|
|
return RegistrationOutcome(False, "register", fc, last_detail, any_api_ok=any_api_ok, all_api_failed_same_way=uniform)
|
|
|
|
# -- recovery cycle ---------------------------------------------------
|
|
|
|
def schedule_backoff(self, state: dict, section: str, failure_class: str, detail: Optional[str]) -> int:
|
|
cfg = self.cfg
|
|
sec = state[section]
|
|
idx = min(sec.get("backoff_index", 0), len(cfg.backoff_schedule_sec) - 1)
|
|
base = cfg.backoff_schedule_sec[idx]
|
|
jitter = int(base * cfg.backoff_jitter_fraction * (2 * self.sys.random_jitter() - 1))
|
|
delay = max(60, min(cfg.backoff_schedule_sec[-1], base + jitter))
|
|
sec["backoff_index"] = min(idx + 1, len(cfg.backoff_schedule_sec) - 1)
|
|
sec["next_retry_epoch"] = self.sys.now() + delay
|
|
sec["last_failure_epoch"] = self.sys.now()
|
|
if section == "recovery":
|
|
sec["last_failure_class"] = failure_class
|
|
sec["last_failure_detail"] = detail
|
|
return delay
|
|
|
|
def recovery_cycle(self, state: dict, st: Status) -> bool:
|
|
cfg = self.cfg
|
|
rec = state["recovery"]
|
|
start_mono = self.sys.monotonic()
|
|
deadline = start_mono + cfg.recovery_max_duration_sec
|
|
pre_age = st.handshake_age_sec
|
|
evidence = {
|
|
"pre_fix_class": st.tunnel_class,
|
|
"pre_fix_handshake_age_sec": pre_age,
|
|
"pre_fix_endpoint": st.peer_endpoint,
|
|
"pre_fix_reasons": st.reasons,
|
|
}
|
|
first_cycle = not rec.get("in_progress")
|
|
rec["in_progress"] = True
|
|
rec["cycles"] = rec.get("cycles", 0) + 1
|
|
self.log.warn("recovery started", **evidence)
|
|
if first_cycle:
|
|
self.notify(state, "warn", f"pia-gateway: tunnel {st.tunnel_class} (handshake age {pre_age}s); automatic recovery started.")
|
|
|
|
def finish_ok(stage: str, region: Optional[str], server: Optional[str], verified: Status) -> bool:
|
|
elapsed = int(self.sys.monotonic() - start_mono)
|
|
state["last_recovery"] = {
|
|
"epoch": self.sys.now(),
|
|
"stage": stage,
|
|
"region": region,
|
|
"server_cn": server,
|
|
"elapsed_sec": elapsed,
|
|
**evidence,
|
|
}
|
|
state["recovery"] = {**default_state()["recovery"]}
|
|
state["consecutive_failures"] = 0
|
|
state["last_healthy_epoch"] = self.sys.now()
|
|
self.log.info("recovered", stage=stage, region=region, server=server, elapsed_sec=elapsed, handshake_age_sec=verified.handshake_age_sec)
|
|
self.notify(state, "info", f"pia-gateway: recovered via {stage} in {elapsed}s" + (f" ({region}/{server})" if server else "") + ".")
|
|
return True
|
|
|
|
def finish_fail(failure_class: str, detail: Optional[str]) -> bool:
|
|
delay = self.schedule_backoff(state, "recovery", failure_class, detail)
|
|
self.log.error("recovery failed", failure_class=failure_class, detail=detail, retry_in_sec=delay)
|
|
level = "error" if failure_class in (FAIL_AUTH, FAIL_SAFETY) else "warn"
|
|
self.notify(state, level, f"pia-gateway: recovery failed ({failure_class}: {detail}); gateway stays closed, retry in {delay // 60} min.")
|
|
return False
|
|
|
|
# stage 1: restart
|
|
since = self.sys.now()
|
|
if not self.restart_interface(state):
|
|
self.log.warn("restart stage failed; escalating to registration")
|
|
else:
|
|
verified = self.wait_for_verified(state, since, cfg.restart_wait_sec)
|
|
if verified is not None:
|
|
return finish_ok("restart", None, None, verified)
|
|
self.log.warn("restart did not restore a fresh handshake; escalating to registration")
|
|
|
|
if self.sys.monotonic() >= deadline:
|
|
return finish_fail(FAIL_REGISTRATION, "time budget exhausted after restart stage")
|
|
|
|
try:
|
|
# stage 2: same-key registration
|
|
out = self.registration_transaction(state, "same-key", None, "recover", cfg.max_candidates_per_cycle, deadline)
|
|
if out.ok:
|
|
return finish_ok("reregister", out.region, out.server_cn, out.verified)
|
|
if out.failure_class in (FAIL_AUTH, FAIL_HOST_CONNECTIVITY, FAIL_SAFETY):
|
|
return finish_fail(out.failure_class, out.detail)
|
|
if self.sys.monotonic() >= deadline:
|
|
return finish_fail(out.failure_class or FAIL_REGISTRATION, "time budget exhausted after same-key stage")
|
|
|
|
# stage 3: one staged fresh-key attempt, only when the API accepted
|
|
# the old key but no activated candidate handshook (evidence of
|
|
# server-side key rejection), never on uniform API failures.
|
|
if out.any_api_ok and not out.all_api_failed_same_way:
|
|
self.log.warn("same-key registration accepted but never handshook; attempting one staged key rotation")
|
|
out2 = self.registration_transaction(state, "new-key", None, "recover", cfg.max_rotation_candidates, deadline)
|
|
if out2.ok:
|
|
return finish_ok("rotate", out2.region, out2.server_cn, out2.verified)
|
|
return finish_fail(out2.failure_class or FAIL_REGISTRATION, out2.detail)
|
|
return finish_fail(out.failure_class or FAIL_REGISTRATION, out.detail)
|
|
except GatewayError as e:
|
|
# e.g. credentials file missing/unreadable, wg tooling failure —
|
|
# still enter backoff so the restart stage is not re-run every 60s.
|
|
return finish_fail(FAIL_SAFETY if isinstance(e, SafetyError) else FAIL_REGISTRATION, str(e))
|
|
|
|
# -- commands ---------------------------------------------------------
|
|
|
|
def cmd_check(self, as_json: bool = False) -> int:
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
self.log.error("state corrupt", error=str(e))
|
|
state = default_state()
|
|
st = self.current_status(state)
|
|
if as_json:
|
|
print(json.dumps(st.to_dict(), sort_keys=True))
|
|
else:
|
|
self.log_status(st)
|
|
self.write_status_outputs(st, state)
|
|
return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY
|
|
|
|
def cmd_recover(self, force: bool = False) -> int:
|
|
cfg = self.cfg
|
|
try:
|
|
lk = self.lock(0)
|
|
except LockBusyError as e:
|
|
self.log.info("skipping; another action holds the lock", detail=str(e))
|
|
return EXIT_OK
|
|
state = None
|
|
try:
|
|
# The lock is acquired BEFORE state is read, not after — every
|
|
# mutating command follows this order now (see cmd_pause for
|
|
# the race it closes: a load taken before the lock can go
|
|
# stale while this process waits, so saving it back afterwards
|
|
# would silently clobber whatever the lock-holder just wrote).
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
self.log.error("state corrupt; recovery halted (fail closed)", error=str(e))
|
|
tmp = default_state()
|
|
self.notify(tmp, "error", f"pia-gateway: state file corrupt ({e}); automatic recovery halted until `state-reset --confirm`.")
|
|
return EXIT_STATE_ERROR
|
|
self.refresh_boot_grace(state)
|
|
self.flush_notifications(state)
|
|
st = self.current_status(state)
|
|
self.log_status(st)
|
|
state["last_check"] = {"epoch": st.timestamp, "class": st.cls, "reasons": st.reasons, "handshake_age_sec": st.handshake_age_sec}
|
|
cycles_before = state["recovery"].get("cycles", 0)
|
|
emergency_before = state.get("emergency_close_active", False)
|
|
rc = self._decide(state, st, force)
|
|
acted = state["recovery"].get("cycles", 0) != cycles_before or state.get("emergency_close_active") != emergency_before
|
|
self.write_status_outputs(self.current_status(state) if acted else st, state)
|
|
return rc
|
|
finally:
|
|
if state is not None:
|
|
self.store.save(state)
|
|
lk.release()
|
|
|
|
def _decide(self, state: dict, st: Status, force: bool) -> int:
|
|
cfg = self.cfg
|
|
rec = state["recovery"]
|
|
if st.tunnel_class == CLASS_SAFETY_DRIFT:
|
|
state["consecutive_failures"] = 0
|
|
self.emergency_close(state, st)
|
|
return EXIT_SAFETY
|
|
self.emergency_clear_if_repaired(state, st)
|
|
|
|
if st.tunnel_class == CLASS_UNKNOWN:
|
|
state["consecutive_unknown"] = state.get("consecutive_unknown", 0) + 1
|
|
if state["consecutive_unknown"] == cfg.failure_threshold:
|
|
self.notify(state, "warn", f"pia-gateway: status unknown for {cfg.failure_threshold} checks ({'; '.join(st.reasons)}); no action taken.")
|
|
return EXIT_UNHEALTHY
|
|
state["consecutive_unknown"] = 0
|
|
|
|
if st.tunnel_class == CLASS_HEALTHY:
|
|
if rec.get("in_progress") or rec.get("next_retry_epoch"):
|
|
self.log.info("tunnel healthy again without a completed recovery cycle; clearing backoff")
|
|
self.notify(state, "info", "pia-gateway: tunnel healthy again; backoff cleared.")
|
|
state["recovery"] = {**default_state()["recovery"]}
|
|
state["consecutive_failures"] = 0
|
|
state["last_healthy_epoch"] = st.timestamp
|
|
return EXIT_OK
|
|
|
|
if state.get("paused"):
|
|
self.log.warn("paused; no recovery action", since=state["paused"].get("since"), reason=state["paused"].get("reason"))
|
|
return EXIT_UNHEALTHY
|
|
|
|
if st.cls == CLASS_BOOT_GRACE and not force:
|
|
self.log.info("within startup/restart grace; not counting as a failure")
|
|
return EXIT_UNHEALTHY
|
|
|
|
state["consecutive_failures"] = state.get("consecutive_failures", 0) + 1
|
|
if state["consecutive_failures"] < cfg.failure_threshold and not force:
|
|
self.log.warn("unhealthy; below threshold", consecutive=state["consecutive_failures"], threshold=cfg.failure_threshold)
|
|
return EXIT_UNHEALTHY
|
|
if rec.get("next_retry_epoch") and st.timestamp < rec["next_retry_epoch"] and not force:
|
|
self.log.warn("in backoff; not retrying yet", retry_in_sec=rec["next_retry_epoch"] - st.timestamp, last_failure=rec.get("last_failure_class"))
|
|
return EXIT_UNHEALTHY
|
|
if force:
|
|
rec["next_retry_epoch"] = None
|
|
ok = self.recovery_cycle(state, st)
|
|
return EXIT_OK if ok else EXIT_UNHEALTHY
|
|
|
|
def cmd_renew(self, force_due: bool = False) -> int:
|
|
cfg = self.cfg
|
|
try:
|
|
lk = self.lock(0)
|
|
except LockBusyError:
|
|
self.log.info("renewal skipped; another action holds the lock")
|
|
return EXIT_OK
|
|
state = None
|
|
try:
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
self.log.error("state corrupt; renewal halted", error=str(e))
|
|
return EXIT_STATE_ERROR
|
|
if state.get("paused"):
|
|
self.log.info("paused; renewal skipped")
|
|
return EXIT_OK
|
|
st = self.current_status(state)
|
|
if st.tunnel_class != CLASS_HEALTHY:
|
|
self.log.info("tunnel not healthy; renewal defers to recovery", **{"class": st.tunnel_class})
|
|
return EXIT_OK
|
|
ren = state["renewal"]
|
|
now = self.sys.now()
|
|
if ren.get("next_retry_epoch") and now < ren["next_retry_epoch"] and not force_due:
|
|
self.log.info("renewal in backoff", retry_in_sec=ren["next_retry_epoch"] - now)
|
|
return EXIT_OK
|
|
lvr = state.get("last_verified_registration")
|
|
due = force_due or lvr is None or (now - int(lvr.get("epoch", 0))) >= cfg.renewal_after_days * 86400
|
|
if not due:
|
|
self.log.info("renewal not due", age_days=(now - int(lvr["epoch"])) // 86400, after_days=cfg.renewal_after_days)
|
|
return EXIT_OK
|
|
self.log.info("renewal due; re-registering with the existing key", baseline_missing=lvr is None)
|
|
out = self.registration_transaction(state, "same-key", None, "renew", cfg.max_candidates_per_cycle)
|
|
if out.ok:
|
|
self.log.info("renewal verified", region=out.region, server=out.server_cn)
|
|
return EXIT_OK
|
|
delay = self.schedule_backoff(state, "renewal", out.failure_class or FAIL_REGISTRATION, out.detail)
|
|
self.log.error("renewal failed; previous registration timestamp unchanged", failure_class=out.failure_class, detail=out.detail, retry_in_sec=delay)
|
|
self.notify(state, "warn", f"pia-gateway: scheduled renewal failed ({out.failure_class}: {out.detail}); retry in {delay // 60} min.")
|
|
return EXIT_UNHEALTHY
|
|
finally:
|
|
if state is not None:
|
|
self.store.save(state)
|
|
lk.release()
|
|
|
|
def cmd_register(self, mode: str, region: Optional[str], via: str) -> int:
|
|
cfg = self.cfg
|
|
try:
|
|
lk = self.lock(cfg.lock_wait_sec)
|
|
except LockBusyError as e:
|
|
self.log.error("could not acquire lock", detail=str(e))
|
|
return EXIT_LOCKED
|
|
state = None
|
|
try:
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
self.log.error("state corrupt; refusing to register", error=str(e))
|
|
return EXIT_STATE_ERROR
|
|
self.refresh_boot_grace(state)
|
|
st = self.current_status(state)
|
|
if not st.safety_ok:
|
|
self.log.error("safety invariants failing; refusing to register", reasons=st.reasons)
|
|
return EXIT_SAFETY
|
|
regions = [region] if region else None
|
|
out = self.registration_transaction(state, mode, regions, via)
|
|
summary = {"ok": out.ok, "stage": out.stage, "failure_class": out.failure_class, "detail": out.detail, "region": out.region, "server_cn": out.server_cn}
|
|
print(json.dumps(summary, sort_keys=True))
|
|
if out.ok:
|
|
state["consecutive_failures"] = 0
|
|
state["recovery"] = {**default_state()["recovery"]}
|
|
return EXIT_OK
|
|
return EXIT_UNHEALTHY
|
|
finally:
|
|
if state is not None:
|
|
self.store.save(state)
|
|
lk.release()
|
|
|
|
def cmd_rewrite_config(self) -> int:
|
|
cfg = self.cfg
|
|
try:
|
|
lk = self.lock(cfg.lock_wait_sec)
|
|
except LockBusyError as e:
|
|
self.log.error("could not acquire lock", detail=str(e))
|
|
return EXIT_LOCKED
|
|
state = None
|
|
try:
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
self.log.error("state corrupt; refusing to rewrite", error=str(e))
|
|
return EXIT_STATE_ERROR
|
|
reg = state.get("active_registration")
|
|
if not reg:
|
|
self.log.error("no verified registration recorded; run `pia-gateway register` instead")
|
|
return EXIT_USAGE
|
|
st = self.current_status(state)
|
|
if not st.safety_ok:
|
|
self.log.error("safety invariants failing; refusing to rewrite", reasons=st.reasons)
|
|
return EXIT_SAFETY
|
|
peer = PeerInfo(reg["server_key"], reg["peer_ip"], int(reg["server_port"]), reg["server_ip"], reg["region"], reg.get("region_name", reg["region"]), reg["server_cn"])
|
|
private_key = self._read_private_key()
|
|
if self._pubkey(private_key) != reg.get("pubkey"):
|
|
self.log.error("on-disk key does not match the recorded registration; run register")
|
|
return EXIT_USAGE
|
|
staged = self.stage_conf(render_wg_conf(cfg, private_key, peer))
|
|
backup = self.backup_current_config()
|
|
since = self.sys.now()
|
|
verified = self.activate_config(state, staged, peer.server_key, since)
|
|
if verified is None:
|
|
self.log.error("rewritten config did not verify; restoring previous")
|
|
self.restore_previous_config(state, backup)
|
|
return EXIT_UNHEALTHY
|
|
self.prune_backups()
|
|
self.log.info("config rewritten and verified")
|
|
return EXIT_OK
|
|
finally:
|
|
if state is not None:
|
|
self.store.save(state)
|
|
lk.release()
|
|
|
|
def cmd_status(self) -> int:
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
print(f"STATE CORRUPT: {e}")
|
|
return EXIT_STATE_ERROR
|
|
st = self.current_status(state)
|
|
rec = state["recovery"]
|
|
now = self.sys.now()
|
|
lines = [
|
|
f"class: {st.cls} (tunnel: {st.tunnel_class})",
|
|
f"reasons: {'; '.join(st.reasons) or '-'}",
|
|
f"handshake age: {st.handshake_age_sec if st.handshake_age_sec is not None else '-'}s (max {self.cfg.handshake_max_age_sec}s)",
|
|
f"endpoint: {st.peer_endpoint or '-'}",
|
|
f"route table {self.cfg.route_table}: {st.route_state}",
|
|
f"safety ok: {st.safety_ok}",
|
|
f"paused: {state.get('paused') or False}",
|
|
f"consecutive failures: {state.get('consecutive_failures', 0)}",
|
|
f"recovery in progress: {rec.get('in_progress')}",
|
|
f"backoff next retry: {_rel(rec.get('next_retry_epoch'), now)} (last failure: {rec.get('last_failure_class') or '-'})",
|
|
f"last verified reg.: {_rel((state.get('last_verified_registration') or {}).get('epoch'), now)} "
|
|
f"{_reg_summary(state.get('last_verified_registration'))}",
|
|
f"last recovery: {_rel((state.get('last_recovery') or {}).get('epoch'), now)} "
|
|
f"{(state.get('last_recovery') or {}).get('stage') or ''}",
|
|
f"emergency close active: {state.get('emergency_close_active', False)}",
|
|
f"last check: {_rel((state.get('last_check') or {}).get('epoch'), now)}",
|
|
]
|
|
if st.configured_peer_line:
|
|
lines.append(f"configured peer: {st.configured_peer_line}")
|
|
print("\n".join(lines))
|
|
return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY
|
|
|
|
def cmd_pause(self, reason: str) -> int:
|
|
# Same lock as recover/register/renew/rewrite-config, taken BEFORE
|
|
# the state load: pause has to wait for any in-flight action to
|
|
# finish (and save) before it reads state itself, or a recovery
|
|
# pass that read state before pause ran could save its own
|
|
# (unpaused) copy back afterwards and silently erase the pause.
|
|
try:
|
|
lk = self.lock(self.cfg.lock_wait_sec)
|
|
except LockBusyError as e:
|
|
self.log.error("could not acquire lock", detail=str(e))
|
|
return EXIT_LOCKED
|
|
try:
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
self.log.error("state corrupt; pause not recorded", error=str(e))
|
|
return EXIT_STATE_ERROR
|
|
state["paused"] = {"since": self.sys.now(), "reason": reason or "operator"}
|
|
self.store.save(state)
|
|
self.log.info("paused: recovery and renewal suppressed; kill switch unchanged", reason=reason)
|
|
return EXIT_OK
|
|
finally:
|
|
lk.release()
|
|
|
|
def cmd_resume(self) -> int:
|
|
try:
|
|
lk = self.lock(self.cfg.lock_wait_sec)
|
|
except LockBusyError as e:
|
|
self.log.error("could not acquire lock", detail=str(e))
|
|
return EXIT_LOCKED
|
|
try:
|
|
try:
|
|
state = self.store.load()
|
|
except StateCorruptError as e:
|
|
self.log.error("state corrupt; resume not recorded", error=str(e))
|
|
return EXIT_STATE_ERROR
|
|
state["paused"] = None
|
|
state["consecutive_failures"] = 0
|
|
state["recovery"]["next_retry_epoch"] = None
|
|
self.store.save(state)
|
|
self.log.info("resumed")
|
|
return EXIT_OK
|
|
finally:
|
|
lk.release()
|
|
|
|
def cmd_retry_now(self) -> int:
|
|
return self.cmd_recover(force=True)
|
|
|
|
def cmd_state_reset(self, confirm: bool) -> int:
|
|
if not confirm:
|
|
self.log.error("refusing without --confirm")
|
|
return EXIT_USAGE
|
|
try:
|
|
lk = self.lock(self.cfg.lock_wait_sec)
|
|
except LockBusyError as e:
|
|
self.log.error("could not acquire lock", detail=str(e))
|
|
return EXIT_LOCKED
|
|
try:
|
|
old = None
|
|
if self.sys.exists(self.cfg.state_file):
|
|
old = self.cfg.state_file + f".corrupt-{self._ts()}"
|
|
self.sys.rename(self.cfg.state_file, old)
|
|
self.store.save(default_state())
|
|
self.log.warn("state reset", previous_moved_to=old)
|
|
return EXIT_OK
|
|
finally:
|
|
lk.release()
|
|
|
|
def cmd_notify(self, level: str, message: str) -> int:
|
|
ok = self.notifier().send(level, message)
|
|
return EXIT_OK if ok else EXIT_UNHEALTHY
|
|
|
|
|
|
def _rel(epoch: Optional[int], now: int) -> str:
|
|
if not epoch:
|
|
return "never"
|
|
d = now - int(epoch)
|
|
if d < 0:
|
|
return f"in {-d}s"
|
|
if d < 3600:
|
|
return f"{d // 60}m ago"
|
|
if d < 86400:
|
|
return f"{d // 3600}h ago"
|
|
return f"{d // 86400}d ago"
|
|
|
|
|
|
def _reg_summary(reg: Optional[dict]) -> str:
|
|
if not reg:
|
|
return ""
|
|
return f"({reg.get('region')}/{reg.get('server_cn')}, {reg.get('mode')}, via {reg.get('via')})"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# CLI
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser(prog="pia-gateway", description=__doc__.split("\n\n")[0])
|
|
p.add_argument("--config", default="/etc/pia-gateway/config.json")
|
|
p.add_argument("--json-log", action="store_true", help="emit JSON log lines")
|
|
sub = p.add_subparsers(dest="cmd", required=True)
|
|
c = sub.add_parser("check")
|
|
c.add_argument("--json", action="store_true")
|
|
sub.add_parser("recover")
|
|
r = sub.add_parser("renew")
|
|
r.add_argument("--force-due", action="store_true")
|
|
g = sub.add_parser("register")
|
|
g.add_argument("--mode", choices=["same-key", "new-key"], default="same-key")
|
|
g.add_argument("--region", default=None)
|
|
g.add_argument("--via", default="operator")
|
|
sub.add_parser("rewrite-config")
|
|
sub.add_parser("status")
|
|
pa = sub.add_parser("pause")
|
|
pa.add_argument("--reason", default="operator")
|
|
sub.add_parser("resume")
|
|
sub.add_parser("retry-now")
|
|
sr = sub.add_parser("state-reset")
|
|
sr.add_argument("--confirm", action="store_true")
|
|
n = sub.add_parser("notify")
|
|
n.add_argument("--level", choices=["info", "warn", "error"], default="error")
|
|
n.add_argument("--message", required=True)
|
|
return p
|
|
|
|
|
|
def main(argv: Optional[list[str]] = None, system: Optional[System] = None, http: Optional[HttpClient] = None, stream=None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
log = Log(stream=stream, json_mode=args.json_log)
|
|
try:
|
|
cfg = Config.load(args.config)
|
|
except (OSError, ValueError, GatewayError) as e:
|
|
log.error("config load failed", error=str(e))
|
|
return EXIT_USAGE
|
|
ctl = Controller(cfg, system or System(), log, http)
|
|
try:
|
|
if args.cmd == "check":
|
|
return ctl.cmd_check(as_json=args.json)
|
|
if args.cmd == "recover":
|
|
return ctl.cmd_recover()
|
|
if args.cmd == "renew":
|
|
return ctl.cmd_renew(force_due=args.force_due)
|
|
if args.cmd == "register":
|
|
return ctl.cmd_register(args.mode, args.region, args.via)
|
|
if args.cmd == "rewrite-config":
|
|
return ctl.cmd_rewrite_config()
|
|
if args.cmd == "status":
|
|
return ctl.cmd_status()
|
|
if args.cmd == "pause":
|
|
return ctl.cmd_pause(args.reason)
|
|
if args.cmd == "resume":
|
|
return ctl.cmd_resume()
|
|
if args.cmd == "retry-now":
|
|
return ctl.cmd_retry_now()
|
|
if args.cmd == "state-reset":
|
|
return ctl.cmd_state_reset(args.confirm)
|
|
if args.cmd == "notify":
|
|
return ctl.cmd_notify(args.level, args.message)
|
|
except GatewayError as e:
|
|
log.error("failed", error=str(e))
|
|
return EXIT_SAFETY if isinstance(e, SafetyError) else EXIT_UNHEALTHY
|
|
except Exception as e: # noqa: BLE001 - never let a traceback (with source context) reach the journal
|
|
log.error("unexpected failure", error=e.__class__.__name__)
|
|
return EXIT_UNHEALTHY
|
|
return EXIT_USAGE
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|