homelab/ansible/roles/pia-gateway/tests/test_pia_gateway.py
Nik Afiq 99f6ef84f0 fix: use curl for addKey — PIA's CA cert fails modern OpenSSL by default
Found live, 2026-09-22, on the addKey step of the first successful
kill-switch-check pass: every candidate failed with
kind=tls detail="certificate verification failed". Root-caused on
minisforum: PIA's own CA certificate (files/pia-ca.crt) has a CA:TRUE
Basic Constraints extension that isn't marked critical - a real RFC
5280 non-conformance, confirmed not a stale vendored copy (a fresh
fetch from pia-foss/manual-connections is byte-identical, same
sha256 already recorded in the file's own header, same defect).
Recent OpenSSL (3.2+; confirmed via a direct repro against
minisforum's 3.5.7 - SSLCertVerificationError, verify_message
"Basic Constraints of CA cert not marked critical") rejects this by
default, a check promoted from X509_STRICT-only into the baseline
chain-verification path. openssl s_client (LibreSSL, this dev
machine) and curl both accept the identical cert/chain/CA file - only
Python's ssl module, via this host's system OpenSSL, rejects it. This
went unnoticed until now because the three prior successful
registrations all went through curl --cacert (the legacy Ansible
path); this was the first time the Python ssl code path ran for real.

Python's stdlib ssl module has no custom verify-callback hook, so
there's no clean way to tolerate just this one known,
third-party-controlled defect while keeping full validation for
everything else. Rather than weakening verify_mode or adding a new
dependency (pyOpenSSL, for a real callback - considered, rejected as
a bigger lift than necessary) to route around a gap in a library we
don't control, addKey now shells out to curl
(HttpClient.get_pinned_curl, curl added to this role's apt list) -
the same tool PIA's own reference client already uses for this exact
endpoint, with --cacert and never -k/--insecure, so this is a
different, already-proven-correct TLS backend for the one endpoint
that needs it, not a downgrade. serverlist/token are unaffected
(public WebPKI certs) and still use the stdlib ssl path.

Verified: a real subprocess curl call against the actual failing
candidate (Server-11669-10a / 69.33.198.97) gets a genuine HTTP
response from PIA, proving TLS validation succeeds via this path.
Also found and handled: curl's own exit code for a --fail-triggered
rejection isn't fully portable (macOS/SecureTransport curl reports 56
where GNU/OpenSSL curl reports 22 for the same case) - classification
now also matches curl's own error text as a second signal, not just
the exit code.

Added 8 tests (94 total, all passing) covering the curl invocation
shape, stdin token delivery (no trailing newline, never in argv), and
exit-code + text-based error classification including the
cross-platform exit-code case found live.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-22 18:48:48 +09:00

1380 lines
62 KiB
Python

"""Offline tests for ansible/roles/pia-gateway/files/pia_gateway.py.
Run from the repo root: python3 -m unittest discover -s ansible/roles/pia-gateway/tests -v
Everything that touches the host (ip/wg/iptables/systemctl, clock, HTTP) is
faked; real files are used inside a temporary directory so atomic writes,
modes, staging and backups are exercised for real.
"""
from __future__ import annotations
import importlib.util
import io
import json
import os
import re
import shutil
import stat
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
HERE = Path(__file__).resolve().parent
SRC = HERE.parent / "files" / "pia_gateway.py"
spec = importlib.util.spec_from_file_location("pia_gateway", SRC)
pg = importlib.util.module_from_spec(spec)
sys.modules["pia_gateway"] = pg
spec.loader.exec_module(pg)
SUBNET = "10.10.50.0/24"
CHAIN = "PIA-VLAN50"
IFACE = "pia-wg"
# Sentinels that must never appear in any output.
SECRET_USER = "sentinel-user-9f1c"
SECRET_PASS = "sentinel-pass-77aa"
SECRET_TOKEN = "sentinel-token-13bd"
SECRET_ALERT_KEY = "sentinel-alertkey-55ee"
# Valid-looking 32-byte base64 keys (44 chars, last char from the allowed set).
PRIV_A = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
PRIV_B = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA="
PUB_A = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaA="
PUB_B = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbA="
SRV_HK1 = "hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1A="
SRV_HK2 = "hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2A="
SRV_TW1 = "tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1A="
SRV_OLD = "oldoldoldoldoldoldoldoldoldoldoldoldoldoldA="
PUBKEYS = {PRIV_A: PUB_A, PRIV_B: PUB_B}
SERVER_KEYS = {"hk1.example": SRV_HK1, "hk2.example": SRV_HK2, "tw1.example": SRV_TW1}
def serverlist_doc():
return {
"regions": [
{"id": "hk", "name": "Hong Kong", "servers": {"wg": [{"cn": "hk1.example", "ip": "10.99.0.1"}, {"cn": "hk2.example", "ip": "10.99.0.2"}]}},
{"id": "taiwan", "name": "Taiwan", "servers": {"wg": [{"cn": "tw1.example", "ip": "10.99.1.1"}]}},
{"id": "japan", "name": "JP Tokyo", "servers": {"wg": []}},
]
}
class FakeSystem(pg.System):
"""Models just enough of the gateway host for the controller."""
def __init__(self, tmp: Path):
self.tmp = tmp
self.t = 1_700_000_000
self.mono = 1000.0
self.boot = "boot-1"
self.iface_exists = True
self.iface_up = True
self.peers: dict[str, int] = {}
self.endpoints: dict[str, str] = {}
self.rule_present = True
self.table_default = "open" # open | closed | missing
self.forward_rules = [f"-A FORWARD -s {SUBNET} -j {CHAIN}", "-A FORWARD -j ufw-before-forward"]
self.chain_rules = self.open_chain()
self.main_default = "default via 10.10.40.1 dev enp1s0 proto static"
self.killswitch_active = True
self.restart_rc = 0
self.restart_count = 0
self.handshake_on_restart = "fresh" # fresh | stale | callable(server_key)->epoch|None
self.commands: list[list[str]] = []
self.stdins: list[str] = []
self.genkey_queue = [PRIV_B]
self.jitter = 0.5
@staticmethod
def open_chain():
return [
f"-A {CHAIN} -d {SUBNET} -i {IFACE} -m state --state RELATED,ESTABLISHED -j ACCEPT",
f"-A {CHAIN} -s {SUBNET} -o {IFACE} -j ACCEPT",
f"-A {CHAIN} -m limit --limit 5/min --limit-burst 5 -j LOG --log-prefix \"PIA-KILLSWITCH-DROP: \"",
f"-A {CHAIN} -j DROP",
]
@staticmethod
def closed_chain():
return [
f"-A {CHAIN} -m limit --limit 5/min --limit-burst 5 -j LOG --log-prefix \"PIA-KILLSWITCH-DROP: \"",
f"-A {CHAIN} -j DROP",
]
# clock
def now(self):
return self.t
def monotonic(self):
return self.mono
def sleep(self, sec):
self.t += int(sec)
self.mono += sec
def boot_id(self):
return self.boot
def random_jitter(self):
return self.jitter
# helpers
def set_peer(self, key, age, endpoint="10.99.0.1:1337"):
self.peers = {key: self.t - age if age is not None else 0}
self.endpoints = {key: endpoint}
def conf_peer(self, conf_path):
if os.path.exists(conf_path):
return pg.parse_expected_peer(open(conf_path).read())
return None
def _apply_restart(self, conf_path):
self.restart_count += 1
if self.restart_rc != 0:
return pg.CmdResult(self.restart_rc, "", "Job for wg-quick@pia-wg.service failed")
self.t += 1 # a real restart takes measurable wall-clock time; keeps
# any "fresh" handshake strictly after whatever `since_epoch` the
# caller captured just before invoking this restart (see the
# `>` — not `>=` — comparison in wait_for_verified).
key = self.conf_peer(conf_path)
self.iface_exists = True
self.iface_up = True
self.table_default = "open"
self.chain_rules = self.open_chain()
beh = self.handshake_on_restart
if callable(beh):
epoch = beh(key)
elif beh == "fresh":
epoch = self.t
else:
epoch = self.t - 100_000
self.peers = {key: epoch if epoch is not None else 0}
self.endpoints = {key: "10.99.0.9:1337"}
return pg.CmdResult(0, "", "")
# command dispatch
def run(self, argv, stdin=None, timeout=60):
self.commands.append(list(argv))
if stdin is not None:
self.stdins.append(stdin)
a = argv
if a[:2] == ["wg", "genkey"]:
return pg.CmdResult(0, self.genkey_queue.pop(0) + "\n", "")
if a[:2] == ["wg", "pubkey"]:
return pg.CmdResult(0, PUBKEYS[stdin.strip()] + "\n", "")
if a[:2] == ["wg", "show"]:
if not self.iface_exists:
return pg.CmdResult(1, "", "Unable to access interface: No such device")
if a[3] == "latest-handshakes":
return pg.CmdResult(0, "".join(f"{k}\t{v}\n" for k, v in self.peers.items()), "")
if a[3] == "endpoints":
return pg.CmdResult(0, "".join(f"{k}\t{v}\n" for k, v in self.endpoints.items()), "")
if a[:4] == ["ip", "-o", "link", "show"]:
if not self.iface_exists:
return pg.CmdResult(1, "", 'Device "pia-wg" does not exist.')
flags = "POINTOPOINT,NOARP,UP,LOWER_UP" if self.iface_up else "POINTOPOINT,NOARP"
return pg.CmdResult(0, f"5: {IFACE}: <{flags}> mtu 1420 qdisc noqueue state UNKNOWN\n", "")
if a[:3] == ["ip", "rule", "show"]:
lines = ["0:\tfrom all lookup local"]
if self.rule_present:
lines.append(f"100:\tfrom {SUBNET} lookup pia")
lines += ["32766:\tfrom all lookup main", "32767:\tfrom all lookup default"]
return pg.CmdResult(0, "\n".join(lines) + "\n", "")
if a[:3] == ["ip", "route", "show"] and "table" in a:
if self.table_default == "open":
return pg.CmdResult(0, f"default dev {IFACE} scope link\n", "")
if self.table_default == "closed":
return pg.CmdResult(0, "unreachable default\n", "")
return pg.CmdResult(0, "", "")
if a[:4] == ["ip", "-4", "route", "show"]:
return pg.CmdResult(0, self.main_default + "\n", "")
if a[:3] == ["ip", "route", "replace"]:
if "unreachable" in a:
self.table_default = "closed"
return pg.CmdResult(0, "", "")
if a[:4] == ["ip", "link", "delete", "dev"]:
self.iface_exists = False
return pg.CmdResult(0, "", "")
if a[:2] == ["iptables", "-S"]:
if a[2] == "FORWARD":
return pg.CmdResult(0, "-P FORWARD ACCEPT\n" + "\n".join(self.forward_rules) + "\n", "")
if a[2] == CHAIN:
if self.chain_rules is None:
return pg.CmdResult(1, "", "iptables: No chain/target/match by that name.")
return pg.CmdResult(0, f"-N {CHAIN}\n" + "\n".join(self.chain_rules) + "\n", "")
if a[:2] == ["iptables", "-C"]:
rule = "-A " + " ".join(a[2:])
return pg.CmdResult(0 if rule in self.forward_rules else 1, "", "")
if a[:2] == ["iptables", "-I"]:
rule = "-A FORWARD " + " ".join(a[4:])
self.forward_rules.insert(0, rule)
return pg.CmdResult(0, "", "")
if a[:2] == ["iptables", "-D"]:
rule = "-A " + " ".join(a[2:])
if rule in self.forward_rules:
self.forward_rules.remove(rule)
return pg.CmdResult(0, "", "")
return pg.CmdResult(1, "", "")
if a[:2] == ["systemctl", "is-active"]:
return pg.CmdResult(0 if self.killswitch_active else 3, "active\n" if self.killswitch_active else "inactive\n", "")
if a[:2] == ["systemctl", "restart"] or a[:2] == ["systemctl", "start"]:
return self._apply_restart(str(self.tmp / "wg" / f"{IFACE}.conf"))
raise AssertionError(f"unexpected command {argv}")
class FakeHttp:
def __init__(self):
self.serverlist = ("ok", serverlist_doc())
self.token = ("ok", SECRET_TOKEN)
self.addkey = {} # cn -> ("ok", {...}) | ("err", kind) | ("http", status) | ("raw", bytes)
self.calls = []
def get(self, url):
self.calls.append(("get", url))
kind, val = self.serverlist
if kind == "err":
raise pg.ApiError(val, "fake")
if kind == "http":
return val, b""
if kind == "raw":
return 200, val
return 200, (json.dumps(val) + "\n-----BEGIN SIG-----\nxxx\n").encode()
def post_multipart(self, url, fields):
self.calls.append(("post", url, dict(fields)))
kind, val = self.token
if kind == "err":
raise pg.ApiError(val, "fake")
if kind == "http":
return val, b"{}"
if kind == "raw":
return 200, val
return 200, json.dumps({"token": val}).encode()
def get_pinned_curl(self, hostname, ip, port, path, token, pubkey):
self.calls.append(("addkey", hostname, ip, port, path, {"pt": token, "pubkey": pubkey}))
kind, val = self.addkey.get(hostname, ("http", 502))
if kind == "err":
raise pg.ApiError(val, "fake")
if kind == "http":
return val, b""
if kind == "raw":
return 200, val
body = {"status": "OK", "server_key": SERVER_KEYS[hostname], "peer_ip": "10.5.5.5", "server_port": 1337, "server_ip": ip}
body.update(val or {})
return 200, json.dumps(body).encode()
class Base(unittest.TestCase):
def setUp(self):
self.tmp = Path(tempfile.mkdtemp(prefix="pia-gateway-test-"))
(self.tmp / "wg").mkdir(mode=0o700)
(self.tmp / "state").mkdir(mode=0o700)
(self.tmp / "etc").mkdir(mode=0o700)
creds = self.tmp / "etc" / "credentials.env"
creds.write_text(f"PIA_USER={SECRET_USER}\nPIA_PASSWORD={SECRET_PASS}\nALERT_BRIDGE_API_KEY={SECRET_ALERT_KEY}\n")
creds.chmod(0o600)
(self.tmp / "wg" / "pia-ca.crt").write_text("dummy")
self.cfg = pg.Config(
config_dir=str(self.tmp / "wg"),
state_dir=str(self.tmp / "state"),
credentials_file=str(creds),
ca_file=str(self.tmp / "wg" / "pia-ca.crt"),
lock_file=str(self.tmp / "lock"),
metrics_file=str(self.tmp / "state" / "metrics.prom"),
status_file=str(self.tmp / "state" / "status.json"),
alert_bridge_url="http://alert-bridge.test/alerts",
)
self.sysm = FakeSystem(self.tmp)
self.http = FakeHttp()
self.out = io.StringIO()
self.log = pg.Log(stream=self.out)
self.notified: list[tuple[str, str]] = []
def fake_send(_self, level, message):
self.notified.append((level, message))
return True
self._send_patch = mock.patch.object(pg.Notifier, "send", fake_send)
self._send_patch.start()
self.write_conf(PRIV_A, SRV_OLD, "10.99.0.1")
(self.tmp / "wg" / "pia-wg.key").write_text(PRIV_A + "\n")
(self.tmp / "wg" / "pia-wg.key").chmod(0o600)
self.sysm.set_peer(SRV_OLD, 30)
self.ctl = pg.Controller(self.cfg, self.sysm, self.log, http=self.http)
# Simulate a controller that has already run once on this boot, so the
# startup grace does not absorb the first checks of every test. The
# fresh-boot behaviour has its own test.
seeded = pg.default_state()
seeded["boot_id"] = self.sysm.boot
self.ctl.store.save(seeded)
def tearDown(self):
self._send_patch.stop()
shutil.rmtree(self.tmp, ignore_errors=True)
def write_conf(self, priv, server_key, ip):
peer = pg.PeerInfo(server_key, "10.5.5.5", 1337, ip, "hk", "Hong Kong", "old.example")
p = self.tmp / "wg" / "pia-wg.conf"
p.write_text(pg.render_wg_conf(self.cfg, priv, peer))
p.chmod(0o600)
def state(self):
return json.loads((self.tmp / "state" / "state.json").read_text())
def run_recover_n(self, n, force=False):
rcs = []
for _ in range(n):
rcs.append(self.ctl.cmd_recover(force=force))
self.sysm.sleep(60)
return rcs
def go_stale(self):
self.sysm.set_peer(SRV_OLD, 5000)
def all_output(self):
s = self.out.getvalue()
for p in (self.tmp / "state" / "status.json", self.tmp / "state" / "metrics.prom", self.tmp / "state" / "state.json"):
if p.exists():
s += p.read_text()
s += json.dumps(self.notified)
s += json.dumps(self.sysm.commands)
return s
def assert_no_secrets(self):
blob = self.all_output()
for sec in (SECRET_USER, SECRET_PASS, SECRET_TOKEN, SECRET_ALERT_KEY, PRIV_A, PRIV_B):
self.assertNotIn(sec, blob, f"secret {sec[:12]}... leaked into output")
class TestClassification(Base):
def test_healthy(self):
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_HEALTHY)
self.assertTrue(st.safety_ok and st.tunnel_ok)
self.assertEqual(st.handshake_age_sec, 30)
def test_stale_handshake(self):
self.go_stale()
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE)
self.assertIn("handshake 5000s old exceeds 180s", st.reasons)
def test_zero_handshake(self):
self.sysm.set_peer(SRV_OLD, None)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE)
self.assertIn("no handshake recorded", st.reasons)
def test_future_handshake_is_not_healthy(self):
self.sysm.set_peer(SRV_OLD, -5000)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE)
self.assertTrue(any("future" in r for r in st.reasons))
def test_small_clock_skew_tolerated(self):
self.sysm.set_peer(SRV_OLD, -10)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_HEALTHY)
def test_malformed_wg_output(self):
self.sysm.peers = {}
orig = self.sysm.run
def run(argv, stdin=None, timeout=60):
if argv[:2] == ["wg", "show"] and argv[3] == "latest-handshakes":
self.sysm.commands.append(argv)
return pg.CmdResult(0, "garbage without tab\n", "")
return orig(argv, stdin, timeout)
self.sysm.run = run
st = self.ctl.current_status(pg.default_state())
self.assertNotEqual(st.cls, pg.CLASS_HEALTHY)
self.assertIn("malformed wg output", st.reasons)
def test_wrong_peer_on_interface(self):
self.sysm.set_peer(SRV_HK1, 10)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT)
def test_interface_absent(self):
self.sysm.iface_exists = False
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT)
def test_fresh_handshake_but_route_closed_is_not_ok(self):
self.sysm.table_default = "closed"
st = self.ctl.current_status(pg.default_state())
self.assertTrue(st.safety_ok)
self.assertFalse(st.tunnel_ok)
self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT)
def test_boot_grace_class(self):
self.go_stale()
state = pg.default_state()
state["boot_grace_until"] = self.sysm.t + 60
self.assertEqual(self.ctl.current_status(state).cls, pg.CLASS_BOOT_GRACE)
def test_paused_class(self):
self.go_stale()
state = pg.default_state()
state["paused"] = {"since": 1, "reason": "x"}
st = self.ctl.current_status(state)
self.assertEqual(st.cls, pg.CLASS_PAUSED)
self.assertEqual(st.tunnel_class, pg.CLASS_TUNNEL_STALE)
class TestSafetyDrift(Base):
def drift_cases(self):
yield "missing rule", lambda: setattr(self.sysm, "rule_present", False)
yield "missing route", lambda: setattr(self.sysm, "table_default", "missing")
yield "jump below accept", lambda: setattr(self.sysm, "forward_rules", ["-A FORWARD -j ACCEPT", f"-A FORWARD -s {SUBNET} -j {CHAIN}"])
yield "no terminal drop", lambda: setattr(self.sysm, "chain_rules", self.sysm.open_chain()[:-1])
yield "wrong iface accept", lambda: setattr(self.sysm, "chain_rules", [f"-A {CHAIN} -s {SUBNET} -o enp1s0 -j ACCEPT"] + self.sysm.open_chain())
yield "chain missing", lambda: setattr(self.sysm, "chain_rules", None)
yield "host default via tunnel", lambda: setattr(self.sysm, "main_default", "default dev pia-wg scope link")
def test_each_drift_triggers_emergency_close_and_no_restart(self):
for name, apply in self.drift_cases():
with self.subTest(name):
self.setUp()
apply()
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_SAFETY, name)
self.assertEqual(self.sysm.restart_count, 0, name)
self.assertTrue(self.sysm.forward_rules[0].startswith(f"-A FORWARD -s {SUBNET} -m comment --comment pia-gateway-emergency -j DROP"), name)
self.assertEqual(self.sysm.table_default, "closed", name)
self.assertTrue(self.state()["emergency_close_active"], name)
self.assertTrue(any(lvl == "error" and "SAFETY DRIFT" in msg for lvl, msg in self.notified), name)
self.assert_no_secrets()
self.tearDown()
def test_emergency_rule_removed_after_repair(self):
self.sysm.rule_present = False
self.ctl.cmd_recover()
self.assertTrue(self.state()["emergency_close_active"])
# operator repairs the kill switch and the tunnel
self.sysm.rule_present = True
self.sysm.table_default = "open"
self.sysm.set_peer(SRV_OLD, 5)
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_OK)
self.assertFalse(self.state()["emergency_close_active"])
self.assertFalse(any("emergency" in r for r in self.sysm.forward_rules))
def test_emergency_close_is_idempotent(self):
self.sysm.rule_present = False
self.ctl.cmd_recover()
self.ctl.cmd_recover()
self.assertEqual(sum("emergency" in r for r in self.sysm.forward_rules), 1)
def test_emergency_close_repositions_when_pushed_below_a_preceding_accept(self):
self.sysm.rule_present = False
self.ctl.cmd_recover()
self.assertTrue(pg._is_emergency_rule(self.sysm.forward_rules[0]))
# Something outside this role's own handler ordering (e.g. a
# manual `ufw reload`) prepends a broader ACCEPT ahead of our
# rule without removing it — it still exists, but is no longer
# first, so it could be pre-empted by whatever now sits above it.
self.sysm.forward_rules.insert(0, "-A FORWARD -j ACCEPT")
self.assertFalse(pg._is_emergency_rule(self.sysm.forward_rules[0]))
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_SAFETY)
self.assertTrue(pg._is_emergency_rule(self.sysm.forward_rules[0]))
self.assertEqual(sum(1 for r in self.sysm.forward_rules if pg._is_emergency_rule(r)), 1)
self.assertIn("-A FORWARD -j ACCEPT", self.sysm.forward_rules)
def test_register_refuses_on_drift(self):
self.sysm.rule_present = False
rc = self.ctl.cmd_register("same-key", None, "ansible")
self.assertEqual(rc, pg.EXIT_SAFETY)
self.assertEqual(self.http.calls, [])
class TestRecoveryStateMachine(Base):
def test_healthy_runs_are_idempotent_and_do_not_mutate(self):
self.run_recover_n(3)
mutating = [
c for c in self.sysm.commands
if c[:2] in (["systemctl", "restart"], ["systemctl", "start"], ["wg", "genkey"], ["iptables", "-I"], ["iptables", "-D"], ["ip", "link"])
or c[:3] == ["ip", "route", "replace"]
]
self.assertEqual(mutating, [])
self.assertEqual(self.state()["consecutive_failures"], 0)
def test_threshold_before_action(self):
self.go_stale()
self.run_recover_n(2)
self.assertEqual(self.sysm.restart_count, 0)
self.assertEqual(self.state()["consecutive_failures"], 2)
def test_boot_grace_not_counted(self):
self.go_stale()
self.sysm.boot = "boot-2"
self.run_recover_n(2)
self.assertEqual(self.state()["consecutive_failures"], 0)
self.assertEqual(self.sysm.restart_count, 0)
self.sysm.sleep(120)
self.run_recover_n(3)
self.assertEqual(self.sysm.restart_count, 1)
def test_restart_success_recovers(self):
self.go_stale()
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_OK)
self.assertEqual(self.sysm.restart_count, 1)
s = self.state()
self.assertEqual(s["last_recovery"]["stage"], "restart")
self.assertEqual(s["last_recovery"]["pre_fix_handshake_age_sec"], 5000 + 120)
self.assertEqual(s["consecutive_failures"], 0)
self.assertFalse(s["recovery"]["in_progress"])
self.assertEqual(self.http.calls, [])
levels = [l for l, _ in self.notified]
self.assertEqual(levels, ["warn", "info"])
self.assert_no_secrets()
def test_restart_failure_escalates_to_same_key_registration(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK1 else None)
self.http.addkey = {"hk1.example": ("ok", {})}
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_OK)
s = self.state()
self.assertEqual(s["last_recovery"]["stage"], "reregister")
self.assertEqual(s["last_verified_registration"]["server_cn"], "hk1.example")
self.assertEqual(s["last_verified_registration"]["mode"], "same-key")
self.assertEqual(s["last_verified_registration"]["pubkey"], PUB_A)
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
self.assertIn(f"PublicKey = {SRV_HK1}", conf)
self.assertIn("Endpoint = 10.99.0.1:1337", conf)
self.assertIn("Table = off", conf)
self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.conf").stat().st_mode), 0o600)
addkeys = [c for c in self.http.calls if c[0] == "addkey"]
self.assertEqual(addkeys[0][5]["pubkey"], PUB_A)
self.assertEqual(addkeys[0][5]["pt"], SECRET_TOKEN)
self.assertTrue(any(n.startswith("pia-wg.conf.bak-") for n in os.listdir(self.tmp / "wg")))
self.assert_no_secrets()
def test_later_candidate_after_activation_failure(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK2 else None)
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {})}
self.run_recover_n(3)
s = self.state()
self.assertEqual(s["last_verified_registration"]["server_cn"], "hk2.example")
order = [c[1] for c in self.http.calls if c[0] == "addkey"]
self.assertEqual(order, ["hk1.example", "hk2.example"])
def test_region_order_hk_then_taiwan(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_TW1 else None)
self.http.addkey = {"hk1.example": ("http", 500), "hk2.example": ("err", "timeout"), "tw1.example": ("ok", {})}
self.run_recover_n(3)
order = [c[1] for c in self.http.calls if c[0] == "addkey"]
self.assertEqual(order, ["hk1.example", "hk2.example", "tw1.example"])
self.assertEqual(self.state()["last_verified_registration"]["region"], "taiwan")
def test_rotation_after_same_key_accepted_but_no_handshake(self):
self.go_stale()
def beh(key):
# Only a config carrying the NEW key handshakes.
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
return self.sysm.t if f"PrivateKey = {PRIV_B}" in conf else None
self.sysm.handshake_on_restart = beh
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})}
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_OK)
s = self.state()
self.assertEqual(s["last_recovery"]["stage"], "rotate")
self.assertEqual(s["last_verified_registration"]["mode"], "new-key")
self.assertEqual(s["last_verified_registration"]["pubkey"], PUB_B)
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_B)
names = os.listdir(self.tmp / "wg")
self.assertTrue(any(n.startswith("pia-wg.key.bak-") for n in names))
self.assertFalse(any(n.startswith("pia-wg.key.new") for n in names))
self.assertFalse(any(".conf.stage" in n for n in names))
self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.key").stat().st_mode), 0o600)
self.assert_no_secrets()
def test_no_rotation_on_uniform_api_failure(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("err", "timeout"), "hk2.example": ("err", "timeout"), "tw1.example": ("err", "timeout")}
self.run_recover_n(3)
s = self.state()
self.assertEqual(s["recovery"]["last_failure_class"], pg.FAIL_HOST_CONNECTIVITY)
self.assertIsNotNone(s["recovery"]["next_retry_epoch"])
self.assertEqual(len(self.sysm.genkey_queue), 1) # no key generated
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
def test_all_candidates_fail_restores_previous_and_backs_off(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("http", 500), "tw1.example": ("ok", {})}
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_UNHEALTHY)
s = self.state()
self.assertTrue(s["recovery"]["in_progress"])
self.assertIsNotNone(s["recovery"]["next_retry_epoch"])
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
self.assertIn(f"PublicKey = {SRV_OLD}", conf)
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
self.assertFalse(any(n.startswith("pia-wg.key.new") for n in os.listdir(self.tmp / "wg")))
# kill switch untouched: chain still ends in DROP, jump still first
self.assertEqual(self.sysm.forward_rules[0], f"-A FORWARD -s {SUBNET} -j {CHAIN}")
self.assertEqual(self.sysm.chain_rules[-1], f"-A {CHAIN} -j DROP")
self.assertTrue(any("recovery failed" in m for _, m in self.notified))
self.assert_no_secrets()
def test_auth_failure_no_rotation_error_alert(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.token = ("http", 401)
self.run_recover_n(3)
s = self.state()
self.assertEqual(s["recovery"]["last_failure_class"], pg.FAIL_AUTH)
self.assertFalse(any(c[0] == "addkey" for c in self.http.calls))
self.assertTrue(any(l == "error" for l, _ in self.notified))
self.assertEqual(len(self.sysm.genkey_queue), 1)
def test_serverlist_unreachable_is_host_connectivity(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_HOST_CONNECTIVITY)
self.assertFalse(any(c[0] in ("post", "addkey") for c in self.http.calls))
def test_malformed_serverlist(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("raw", b"not json\n")
self.run_recover_n(3)
self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_REGISTRATION)
def test_malformed_and_invalid_addkey_responses_skip_to_next(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_TW1 else None)
self.http.addkey = {
"hk1.example": ("raw", b"<html>"),
"hk2.example": ("ok", {"peer_ip": "999.1.1.1"}),
"tw1.example": ("ok", {}),
}
self.run_recover_n(3)
self.assertEqual(self.state()["last_verified_registration"]["server_cn"], "tw1.example")
def test_tls_failure_kind_recorded(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("err", "tls"), "hk2.example": ("err", "tls"), "tw1.example": ("err", "tls")}
self.run_recover_n(3)
self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_REGISTRATION)
self.assertIn("kind=tls", self.out.getvalue())
def test_backoff_sequence_bounded(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
delays = []
self.run_recover_n(3)
for _ in range(5):
s = self.state()
delays.append(s["recovery"]["next_retry_epoch"] - s["recovery"]["last_failure_epoch"])
self.sysm.t = s["recovery"]["next_retry_epoch"] + 1
self.ctl.cmd_recover()
self.assertEqual(delays, [300, 900, 1800, 3600, 3600])
def test_backoff_prevents_hammering(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
before = self.sysm.restart_count
self.run_recover_n(4)
self.assertEqual(self.sysm.restart_count, before)
def test_retry_now_bypasses_backoff(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
before = self.sysm.restart_count
self.sysm.handshake_on_restart = "fresh"
rc = self.ctl.cmd_retry_now()
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(self.sysm.restart_count, before + 1)
def test_credentials_missing_still_backs_off(self):
os.unlink(self.cfg.credentials_file)
self.ctl._creds = None
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.run_recover_n(3)
s = self.state()
self.assertIsNotNone(s["recovery"]["next_retry_epoch"])
before = self.sysm.restart_count
self.run_recover_n(2)
self.assertEqual(self.sysm.restart_count, before)
def test_time_budget_bounds_a_cycle(self):
self.cfg.recovery_max_duration_sec = 100
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})}
self.run_recover_n(3)
addkeys = [c for c in self.http.calls if c[0] == "addkey"]
self.assertLess(len(addkeys), 3)
self.assertIn("time budget", self.out.getvalue())
def test_recovered_without_our_action_clears_backoff(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.sysm.set_peer(SRV_OLD, 5)
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_OK)
self.assertIsNone(self.state()["recovery"]["next_retry_epoch"])
self.assertFalse(self.state()["recovery"]["in_progress"])
def test_backward_clock_jump_clears_stale_deadline(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.assertIsNotNone(self.state()["recovery"]["next_retry_epoch"])
self.sysm.t -= 6 * 3600 # NTP steps the clock back six hours
self.sysm.handshake_on_restart = "fresh"
self.http.serverlist = ("ok", serverlist_doc())
self.sysm.set_peer(SRV_OLD, 5000)
rc = self.ctl.cmd_recover()
self.assertIn("clock jump detected", self.out.getvalue())
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(self.state()["last_recovery"]["stage"], "restart")
def test_normal_backoff_is_not_mistaken_for_clock_jump(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.ctl.cmd_recover()
self.assertNotIn("clock jump detected", self.out.getvalue())
self.assertIsNotNone(self.state()["recovery"]["next_retry_epoch"])
def test_orphaned_interface_recovery_on_restart(self):
self.go_stale()
calls = {"n": 0}
orig = self.sysm._apply_restart
def restart(conf_path):
calls["n"] += 1
if calls["n"] == 1:
self.sysm.restart_count += 1
return pg.CmdResult(1, "", "RTNETLINK answers: File exists\n`pia-wg' already exists")
return orig(conf_path)
self.sysm._apply_restart = restart
self.run_recover_n(3)
self.assertIn(["ip", "link", "delete", "dev", IFACE], self.sysm.commands)
self.assertEqual(self.state()["last_recovery"]["stage"], "restart")
class TestHandshakeVerificationBoundary(Base):
"""wait_for_verified's freshness check, at the exact one-second boundary.
`wg show`'s handshake epoch and `since_epoch` are both whole seconds,
so a handshake reported in the SAME second as `since_epoch` is
indistinguishable from one that predates activation and merely shares
that truncated second. It must never count as proof of a NEW
handshake — that's the whole point of comparing against `since_epoch`
at all (an `addKey` returning `status: OK` was already established
elsewhere as necessary-but-not-sufficient; the same logic applies one
layer down, to the handshake timestamp itself).
"""
def test_handshake_at_exactly_since_epoch_is_rejected(self):
since_epoch = self.sysm.t
self.sysm.set_peer(SRV_OLD, 0) # handshake_epoch == self.sysm.t == since_epoch, exactly
verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD)
self.assertIsNone(verified)
def test_handshake_one_second_after_since_epoch_is_accepted(self):
since_epoch = self.sysm.t
self.sysm.sleep(1)
self.sysm.set_peer(SRV_OLD, 0) # handshake_epoch == self.sysm.t == since_epoch + 1
verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD)
self.assertIsNotNone(verified)
def test_handshake_one_second_before_since_epoch_is_rejected(self):
self.sysm.sleep(1)
since_epoch = self.sysm.t
self.sysm.set_peer(SRV_OLD, 1) # handshake_epoch == self.sysm.t - 1 == since_epoch - 1
verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD)
self.assertIsNone(verified)
class TestPersistenceAndConcurrency(Base):
def test_state_survives_new_controller(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
s1 = self.state()
ctl2 = pg.Controller(self.cfg, self.sysm, self.log, http=self.http)
self.sysm.boot = "boot-2"
ctl2.cmd_recover()
s2 = self.state()
self.assertEqual(s1["recovery"]["next_retry_epoch"], s2["recovery"]["next_retry_epoch"])
self.assertEqual(s2["boot_id"], "boot-2")
def test_corrupt_state_fails_closed(self):
(self.tmp / "state" / "state.json").write_text("{not json")
self.go_stale()
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_STATE_ERROR)
self.assertEqual(self.sysm.restart_count, 0)
self.assertTrue(any("corrupt" in m for _, m in self.notified))
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_register("same-key", None, "operator"), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_rewrite_config(), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_pause("x"), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_resume(), pg.EXIT_STATE_ERROR)
def test_wrong_schema_version_fails_closed(self):
s = pg.default_state()
s["schema_version"] = 99
(self.tmp / "state" / "state.json").write_text(json.dumps(s))
self.assertEqual(self.ctl.cmd_recover(), pg.EXIT_STATE_ERROR)
def test_state_reset_requires_confirm_and_preserves_old(self):
(self.tmp / "state" / "state.json").write_text("{bad")
self.assertEqual(self.ctl.cmd_state_reset(confirm=False), pg.EXIT_USAGE)
self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_OK)
names = os.listdir(self.tmp / "state")
self.assertTrue(any(n.startswith("state.json.corrupt-") for n in names))
self.assertEqual(self.state()["schema_version"], pg.STATE_SCHEMA_VERSION)
def test_atomic_write_no_temp_left_and_mode(self):
p = self.tmp / "state" / "x.json"
self.sysm.write_atomic(str(p), "abc", 0o600)
self.assertEqual(p.read_text(), "abc")
self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o600)
self.assertEqual([n for n in os.listdir(self.tmp / "state") if n.endswith(".tmp")], [])
def test_interrupted_write_keeps_original(self):
p = self.tmp / "state" / "state.json"
p.write_text("original")
with mock.patch.object(pg.os, "replace", side_effect=OSError("boom")):
with self.assertRaises(OSError):
self.sysm.write_atomic(str(p), "new", 0o600)
self.assertEqual(p.read_text(), "original")
self.assertEqual([n for n in os.listdir(self.tmp / "state") if n.endswith(".tmp")], [])
def test_lock_contention_skips_without_acting(self):
self.go_stale()
self.run_recover_n(2)
holder = pg.Lock(self.cfg.lock_file, self.sysm)
holder.acquire(0)
try:
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(self.sysm.restart_count, 0)
self.assertIn("another action holds the lock", self.out.getvalue())
self.assertEqual(self.ctl.cmd_register("same-key", None, "operator"), pg.EXIT_LOCKED)
finally:
holder.release()
rc = self.ctl.cmd_recover()
self.assertEqual(self.sysm.restart_count, 1)
def test_lock_released_on_exception(self):
with mock.patch.object(self.ctl, "_decide", side_effect=RuntimeError("x")):
with self.assertRaises(RuntimeError):
self.ctl.cmd_recover()
lk = pg.Lock(self.cfg.lock_file, self.sysm)
lk.acquire(0)
lk.release()
def test_pause_and_resume(self):
self.ctl.cmd_pause("maintenance")
self.go_stale()
self.run_recover_n(5)
self.assertEqual(self.sysm.restart_count, 0)
self.assertTrue(self.state()["paused"])
self.assertEqual(self.ctl.cmd_renew(force_due=True), pg.EXIT_OK)
self.assertEqual(self.http.calls, [])
self.ctl.cmd_resume()
self.run_recover_n(3)
self.assertEqual(self.sysm.restart_count, 1)
def test_pause_does_not_open_gateway(self):
self.sysm.table_default = "closed"
self.sysm.chain_rules = self.sysm.closed_chain()
self.ctl.cmd_pause("x")
self.ctl.cmd_recover()
self.assertEqual(self.sysm.table_default, "closed")
self.assertEqual(self.sysm.restart_count, 0)
def test_pause_and_resume_serialize_with_an_inflight_recovery_pass(self):
# Simulate a recovery pass that is mid-flight: it already holds the
# SAME lock pause/resume now also take, and has not yet saved its
# (unpaused) state back to disk — this is exactly the window in
# which the old, unlocked pause/resume could silently lose the
# race and have their write clobbered by the pass finishing later.
self.cfg.lock_wait_sec = 3
holder = pg.Lock(self.cfg.lock_file, self.sysm)
holder.acquire(0)
try:
self.assertEqual(self.ctl.cmd_pause("maintenance"), pg.EXIT_LOCKED)
self.assertEqual(self.ctl.cmd_resume(), pg.EXIT_LOCKED)
# blocked, not silently skipped-and-lost: state is untouched
self.assertIsNone(self.state().get("paused"))
finally:
# the in-flight pass "finishes": saves its own state (as
# cmd_recover's finally block does) and releases the lock.
self.ctl.store.save(pg.default_state())
holder.release()
# now that the lock is free, pause proceeds against the CURRENT
# on-disk state rather than a copy read before it waited
self.assertEqual(self.ctl.cmd_pause("maintenance"), pg.EXIT_OK)
self.assertTrue(self.state()["paused"])
# a recovery pass started afterwards must also acquire the lock
# and load state fresh before it can act, so it observes the
# pause and never clears or overwrites it
self.go_stale()
self.run_recover_n(5)
self.assertEqual(self.sysm.restart_count, 0)
self.assertTrue(self.state()["paused"])
def test_state_reset_serializes_with_lock(self):
holder = pg.Lock(self.cfg.lock_file, self.sysm)
holder.acquire(0)
try:
self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_LOCKED)
finally:
holder.release()
self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_OK)
class TestRenewal(Base):
def seed_registration(self, age_days):
s = pg.default_state()
s["last_verified_registration"] = {"epoch": self.sysm.t - age_days * 86400, "region": "hk", "server_cn": "old.example", "pubkey": PUB_A, "mode": "same-key", "via": "ansible"}
self.ctl.store.save(s)
def test_not_due(self):
self.seed_registration(10)
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK)
self.assertEqual(self.http.calls, [])
def test_due_and_succeeds_advances_timestamp(self):
self.seed_registration(22)
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK)
s = self.state()
self.assertEqual(s["last_verified_registration"]["via"], "renew")
self.assertEqual(s["last_verified_registration"]["mode"], "same-key")
self.assertGreaterEqual(s["last_verified_registration"]["epoch"], self.sysm.t - 60)
self.assertEqual(len(self.sysm.genkey_queue), 1)
def test_due_and_fails_keeps_timestamp_and_backs_off(self):
self.seed_registration(22)
old = self.state()["last_verified_registration"]["epoch"]
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_OLD else None)
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})}
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_UNHEALTHY)
s = self.state()
self.assertEqual(s["last_verified_registration"]["epoch"], old)
self.assertIsNotNone(s["renewal"]["next_retry_epoch"])
self.assertIn(f"PublicKey = {SRV_OLD}", (self.tmp / "wg" / "pia-wg.conf").read_text())
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK) # in backoff, no call
self.assertTrue(any("renewal failed" in m for _, m in self.notified))
def test_force_due(self):
self.seed_registration(1)
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.assertEqual(self.ctl.cmd_renew(force_due=True), pg.EXIT_OK)
self.assertEqual(self.state()["last_verified_registration"]["via"], "renew")
def test_defers_to_recovery_when_unhealthy(self):
self.seed_registration(30)
self.go_stale()
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK)
self.assertEqual(self.http.calls, [])
class TestRegisterCommand(Base):
def test_fresh_install_generates_key_and_registers(self):
os.unlink(self.tmp / "wg" / "pia-wg.conf")
os.unlink(self.tmp / "wg" / "pia-wg.key")
self.sysm.iface_exists = False
self.sysm.table_default = "closed"
self.sysm.chain_rules = self.sysm.closed_chain()
self.sysm.genkey_queue = [PRIV_A]
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
rc = self.ctl.cmd_register("same-key", None, "ansible")
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.key").stat().st_mode), 0o600)
s = self.state()
self.assertEqual(s["last_verified_registration"]["via"], "ansible")
self.assertEqual(s["active_registration"]["server_key"], SRV_HK1)
self.assert_no_secrets()
def test_register_refuses_when_killswitch_inactive(self):
self.sysm.killswitch_active = False
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
rc = self.ctl.cmd_register("same-key", None, "ansible")
self.assertEqual(rc, pg.EXIT_UNHEALTHY)
self.assertEqual(self.sysm.restart_count, 0)
self.assertIn(f"PublicKey = {SRV_OLD}", (self.tmp / "wg" / "pia-wg.conf").read_text())
def test_region_override(self):
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"tw1.example": ("ok", {})}
rc = self.ctl.cmd_register("same-key", "taiwan", "operator")
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual([c[1] for c in self.http.calls if c[0] == "addkey"], ["tw1.example"])
def test_new_key_mode_with_abandoned_stage_cleanup(self):
(self.tmp / "wg" / "pia-wg.key.new").write_text("abandoned\n")
(self.tmp / "wg" / "pia-wg.key.new.20260101T000000Z.1").write_text("abandoned2\n")
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
rc = self.ctl.cmd_register("new-key", None, "operator")
self.assertEqual(rc, pg.EXIT_OK)
names = os.listdir(self.tmp / "wg")
self.assertFalse(any(n.startswith("pia-wg.key.new") for n in names))
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_B)
self.assertIn(f"pubkey", json.dumps(self.state()["last_verified_registration"]))
self.assertEqual(self.state()["last_verified_registration"]["pubkey"], PUB_B)
def test_same_key_does_not_rotate(self):
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.ctl.cmd_register("same-key", None, "operator")
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
self.assertEqual(len(self.sysm.genkey_queue), 1)
def test_backups_bounded(self):
self.cfg.backup_keep = 2
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
for i in range(5):
self.sysm.sleep(1)
self.ctl.cmd_register("same-key", None, "operator")
baks = [n for n in os.listdir(self.tmp / "wg") if n.startswith("pia-wg.conf.bak-")]
self.assertEqual(len(baks), 2)
def test_rewrite_config_uses_recorded_registration(self):
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.ctl.cmd_register("same-key", None, "operator")
self.cfg.mss_clamp_enabled = False
calls_before = len(self.http.calls)
rc = self.ctl.cmd_rewrite_config()
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(len(self.http.calls), calls_before)
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
self.assertNotIn("TCPMSS", conf)
self.assertIn(f"PublicKey = {SRV_HK1}", conf)
class TestConfigValidation(Base):
def render(self, **kw):
peer = pg.PeerInfo(SRV_HK1, "10.5.5.5", 1337, "10.99.0.1", "hk", "Hong Kong", "hk1.example")
return pg.render_wg_conf(self.cfg, PRIV_A, peer)
def test_rendered_config_is_valid(self):
self.assertEqual(pg.validate_wg_conf(self.cfg, self.render()), [])
def test_rejects_table_not_off(self):
text = self.render().replace("Table = off", "Table = auto")
self.assertIn("Table must be off", pg.validate_wg_conf(self.cfg, text))
def test_rejects_bad_keys_and_endpoint(self):
text = self.render().replace(f"PublicKey = {SRV_HK1}", "PublicKey = nope").replace("Endpoint = 10.99.0.1:1337", "Endpoint = bad")
problems = pg.validate_wg_conf(self.cfg, text)
self.assertIn("peer PublicKey invalid", problems)
self.assertIn("Endpoint invalid", problems)
def test_rejects_unscoped_hook_and_policy_change(self):
text = self.render().replace(
f"PostUp = ip route replace default dev %i table pia",
"PostUp = ip route replace default dev %i\nPostUp = iptables -P FORWARD ACCEPT",
)
problems = pg.validate_wg_conf(self.cfg, text)
self.assertTrue(any("without a table" in p for p in problems))
self.assertIn("hook changes FORWARD policy", problems)
self.assertIn("PostUp lacks table route open", problems)
def test_rejects_wide_allowed_ips_change(self):
text = self.render().replace("AllowedIPs = 0.0.0.0/0", "AllowedIPs = ::/0")
self.assertIn("AllowedIPs must be 0.0.0.0/0", pg.validate_wg_conf(self.cfg, text))
def test_expected_peer_parse(self):
self.assertEqual(pg.parse_expected_peer(self.render()), SRV_HK1)
self.assertIn("Hong Kong (hk), hk1.example", pg.parse_configured_peer_line(self.render()))
class TestSecretsAndModes(Base):
def test_credentials_mode_enforced(self):
os.chmod(self.cfg.credentials_file, 0o644)
with self.assertRaises(pg.GatewayError):
pg.load_credentials(self.cfg.credentials_file)
def test_credentials_repr_redacted(self):
c = pg.load_credentials(self.cfg.credentials_file)
self.assertNotIn(SECRET_PASS, repr(c) + str(c))
def test_no_secret_in_argv_stdin_logs_or_files_across_full_cycle(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK2 else None)
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {})}
self.run_recover_n(3)
self.ctl.cmd_check()
self.ctl.cmd_status()
self.assert_no_secrets()
for argv in self.sysm.commands:
for tok in argv:
self.assertNotIn(SECRET_TOKEN, tok)
self.assertNotIn(PRIV_A, tok)
# the private key goes to `wg pubkey` via stdin only (never argv) and nowhere else
for s in self.sysm.stdins:
self.assertIn(s.strip(), (PRIV_A, PRIV_B))
def test_notification_payloads_are_secret_free_and_failures_do_not_block(self):
self._send_patch.stop()
sent = []
def flaky(_self, level, message):
sent.append((level, message))
return False
with mock.patch.object(pg.Notifier, "send", flaky):
self.go_stale()
self.run_recover_n(3)
self._send_patch.start()
self.assertEqual(self.state()["last_recovery"]["stage"], "restart")
self.assertGreaterEqual(len(self.state()["pending_notifications"]), 1)
for _, m in sent:
for sec in (SECRET_PASS, SECRET_TOKEN, PRIV_A):
self.assertNotIn(sec, m)
def test_pending_notifications_flushed_later_and_bounded(self):
s = pg.default_state()
s["pending_notifications"] = [{"level": "info", "message": f"m{i}", "epoch": 1} for i in range(30)]
self.ctl.store.save(s)
self.ctl.cmd_recover()
self.assertEqual(self.state()["pending_notifications"], [])
self.assertEqual(len([m for _, m in self.notified if m.startswith("m")]), 30)
def test_state_and_status_file_modes(self):
self.ctl.cmd_recover()
self.assertEqual(stat.S_IMODE((self.tmp / "state" / "state.json").stat().st_mode), 0o600)
self.assertTrue((self.tmp / "state" / "metrics.prom").exists())
m = (self.tmp / "state" / "metrics.prom").read_text()
self.assertIn("pia_gateway_healthy 1", m)
self.assertIn("pia_gateway_check_timestamp_seconds", m)
def test_check_is_read_only(self):
self.go_stale()
before = (self.tmp / "state" / "state.json").read_text()
rc = self.ctl.cmd_check()
self.assertEqual(rc, pg.EXIT_UNHEALTHY)
self.assertEqual(self.sysm.restart_count, 0)
self.assertEqual((self.tmp / "state" / "state.json").read_text(), before)
self.assertIn("class=tunnel_stale", self.out.getvalue())
def test_main_never_prints_traceback(self):
cfgp = self.tmp / "config.json"
cfgp.write_text(json.dumps(dataclass_dict(self.cfg)))
sysm = self.sysm
class Boom(FakeSystem):
def run(self, argv, stdin=None, timeout=60):
raise RuntimeError("kaboom " + SECRET_PASS)
out = io.StringIO()
rc = pg.main(["--config", str(cfgp), "recover"], system=Boom(self.tmp), http=self.http, stream=out)
self.assertEqual(rc, pg.EXIT_UNHEALTHY)
self.assertNotIn("Traceback", out.getvalue())
self.assertNotIn(SECRET_PASS, out.getvalue())
self.assertIn("RuntimeError", out.getvalue())
def test_config_rejects_unknown_keys(self):
cfgp = self.tmp / "config.json"
d = dataclass_dict(self.cfg)
d["typo_key"] = 1
cfgp.write_text(json.dumps(d))
out = io.StringIO()
rc = pg.main(["--config", str(cfgp), "status"], system=self.sysm, http=self.http, stream=out)
self.assertEqual(rc, pg.EXIT_USAGE)
def dataclass_dict(cfg):
import dataclasses
return dataclasses.asdict(cfg)
class FakeCurlSystem(pg.System):
"""Fakes just enough of System.run to exercise get_pinned_curl's argv
construction, stdin handling, and exit-code classification, without a
real curl binary or network access."""
def __init__(self, result=None):
self.result = result or pg.CmdResult(0, '{"status":"OK"}', "")
self.calls = []
self.stdins = []
def run(self, argv, stdin=None, timeout=60):
self.calls.append(list(argv))
if stdin is not None:
self.stdins.append(stdin)
return self.result
class TestPinnedCurlAddKey(unittest.TestCase):
def setUp(self):
self.cfg = pg.Config(ca_file="/etc/wireguard/pia-ca.crt", http_connect_timeout_sec=10, http_total_timeout_sec=30)
def test_argv_shape_matches_the_proven_legacy_curl_invocation(self):
sysm = FakeCurlSystem()
http_client = pg.HttpClient(self.cfg, sysm)
status, data = http_client.get_pinned_curl("Server-11669-10a", "69.33.198.97", 1337, "/addKey", SECRET_TOKEN, PUB_A)
self.assertEqual(status, 200)
self.assertEqual(data, b'{"status":"OK"}')
argv = sysm.calls[0]
self.assertEqual(argv[0], "curl")
self.assertIn("--fail", argv)
self.assertIn("-G", argv)
self.assertIn("--cacert", argv)
self.assertEqual(argv[argv.index("--cacert") + 1], "/etc/wireguard/pia-ca.crt")
self.assertIn("--connect-to", argv)
self.assertEqual(argv[argv.index("--connect-to") + 1], "Server-11669-10a::69.33.198.97:")
self.assertIn("pt@-", argv)
self.assertIn(f"pubkey={PUB_A}", argv)
self.assertEqual(argv[-1], "https://Server-11669-10a:1337/addKey")
# never -k/--insecure — this must still be a real, validating request
self.assertNotIn("-k", argv)
self.assertNotIn("--insecure", argv)
def test_token_passed_via_stdin_with_no_trailing_newline_and_never_in_argv(self):
sysm = FakeCurlSystem()
http_client = pg.HttpClient(self.cfg, sysm)
http_client.get_pinned_curl("Server-11669-10a", "69.33.198.97", 1337, "/addKey", SECRET_TOKEN, PUB_A)
self.assertEqual(sysm.stdins, [SECRET_TOKEN]) # exact match: no appended "\n"
for arg in sysm.calls[0]:
self.assertNotIn(SECRET_TOKEN, arg)
def test_timeout_exit_code_classified(self):
sysm = FakeCurlSystem(pg.CmdResult(28, "", "curl: (28) Connection timed out"))
http_client = pg.HttpClient(self.cfg, sysm)
with self.assertRaises(pg.ApiError) as cm:
http_client.get_pinned_curl("h", "1.2.3.4", 1337, "/addKey", "t", "p")
self.assertEqual(cm.exception.kind, "timeout")
def test_tls_exit_codes_classified_with_stderr_detail(self):
for rc in (35, 53, 54, 58, 59, 60, 66, 77, 80, 82, 83):
with self.subTest(rc=rc):
sysm = FakeCurlSystem(pg.CmdResult(rc, "", "curl: (60) SSL certificate problem"))
http_client = pg.HttpClient(self.cfg, sysm)
with self.assertRaises(pg.ApiError) as cm:
http_client.get_pinned_curl("h", "1.2.3.4", 1337, "/addKey", "t", "p")
self.assertEqual(cm.exception.kind, "tls")
self.assertIn("SSL certificate problem", cm.exception.detail)
def test_http_failure_exit_code_classified(self):
sysm = FakeCurlSystem(pg.CmdResult(22, "", "curl: (22) The requested URL returned error: 401"))
http_client = pg.HttpClient(self.cfg, sysm)
with self.assertRaises(pg.ApiError) as cm:
http_client.get_pinned_curl("h", "1.2.3.4", 1337, "/addKey", "t", "p")
self.assertEqual(cm.exception.kind, "http")
def test_nonstandard_exit_code_with_fail_text_still_classified_as_http(self):
# Observed live: curl built against SecureTransport reports exit 56
# (CURLE_RECV_ERROR), not the "usual" 22, for the same --fail
# -triggered HTTP rejection — the error text is the portable signal.
sysm = FakeCurlSystem(pg.CmdResult(56, "", "curl: (56) The requested URL returned error: 401"))
http_client = pg.HttpClient(self.cfg, sysm)
with self.assertRaises(pg.ApiError) as cm:
http_client.get_pinned_curl("h", "1.2.3.4", 1337, "/addKey", "t", "p")
self.assertEqual(cm.exception.kind, "http")
def test_curl_missing_classified_as_network_not_a_crash(self):
sysm = FakeCurlSystem(pg.CmdResult(127, "", "curl: not found"))
http_client = pg.HttpClient(self.cfg, sysm)
with self.assertRaises(pg.ApiError) as cm:
http_client.get_pinned_curl("h", "1.2.3.4", 1337, "/addKey", "t", "p")
self.assertEqual(cm.exception.kind, "network")
def test_unclassified_exit_code_falls_back_to_network_with_stderr(self):
sysm = FakeCurlSystem(pg.CmdResult(7, "", "curl: (7) Couldn't connect to server"))
http_client = pg.HttpClient(self.cfg, sysm)
with self.assertRaises(pg.ApiError) as cm:
http_client.get_pinned_curl("h", "1.2.3.4", 1337, "/addKey", "t", "p")
self.assertEqual(cm.exception.kind, "network")
self.assertIn("Couldn't connect", cm.exception.detail)
class TestHttpHelpers(unittest.TestCase):
def test_key_and_ip_validators(self):
self.assertTrue(pg.valid_wg_key(PUB_A))
self.assertFalse(pg.valid_wg_key("short"))
self.assertFalse(pg.valid_wg_key(PUB_A[:-1] + "B")) # bad final char
self.assertTrue(pg.valid_ipv4("10.0.0.1"))
self.assertFalse(pg.valid_ipv4("300.0.0.1"))
self.assertFalse(pg.valid_ipv4("::1"))
def test_candidates_order_and_missing_region(self):
api = pg.PiaApi(pg.Config(), None, pg.Log(stream=io.StringIO()))
cands = api.build_candidates(serverlist_doc(), ["hk", "taiwan"])
self.assertEqual([c.cn for c in cands], ["hk1.example", "hk2.example", "tw1.example"])
with self.assertRaises(pg.ApiError):
api.build_candidates(serverlist_doc(), ["hk", "nowhere"])
with self.assertRaises(pg.ApiError):
api.build_candidates(serverlist_doc(), ["japan"]) # exists but no wg servers -> nothing usable
# an empty fallback region is skipped, not fatal
self.assertEqual([c.cn for c in api.build_candidates(serverlist_doc(), ["japan", "hk"])], ["hk1.example", "hk2.example"])
def test_token_retries_then_auth(self):
http = FakeHttp()
api = pg.PiaApi(pg.Config(), http, pg.Log(stream=io.StringIO()))
http.token = ("http", 401)
with self.assertRaises(pg.ApiError) as cm:
api.get_token(pg.Credentials("u", "p"))
self.assertEqual(cm.exception.kind, "auth")
http.token = ("err", "network")
with self.assertRaises(pg.ApiError):
api.get_token(pg.Credentials("u", "p"))
self.assertEqual(len([c for c in http.calls if c[0] == "post"]), 4)
def test_multipart_body_contains_fields_once(self):
captured = {}
class C(pg.HttpClient):
def _request(self, conn, method, path, body, headers):
captured["body"] = body
captured["headers"] = headers
return 200, b'{"token":"t"}'
c = C(pg.Config())
c.post_multipart(pg.TOKEN_URL, {"username": "u1", "password": "p1"})
self.assertIn(b'name="username"\r\n\r\nu1\r\n', captured["body"])
self.assertIn(b'name="password"\r\n\r\np1\r\n', captured["body"])
self.assertTrue(captured["headers"]["Content-Type"].startswith("multipart/form-data; boundary="))
if __name__ == "__main__":
unittest.main()