homelab/ansible/roles/pia-gateway/tests/test_pia_gateway.py
Nik Afiq 87a9303cc3
Some checks failed
validate / lint (push) Failing after 2s
fix: address code review on pia-gateway auto-recovery controller
- Serialize every mutating command (recover, renew, register,
  rewrite-config, pause, resume, state-reset) on the shared lock,
  acquired BEFORE state is read rather than after: reading first and
  locking only around the write let a slower command silently clobber
  whatever a faster one wrote in between — in particular, an operator
  pause could be lost to an in-flight recovery pass that read state
  before the pause ran and only saved its own copy back afterwards.
- Fix emergency_close() to reposition its VLAN 50 DROP to the first
  effective FORWARD position when it exists but has been pushed down
  by something else (e.g. an out-of-band ufw reload) — a bare `-C`
  existence check said nothing about position.
- Tighten wait_for_verified()'s handshake freshness check from >= to >
  since_epoch: wg show and since_epoch are both whole seconds, so a
  handshake at exactly the same second as since_epoch is
  indistinguishable from a stale, pre-activation one and must not
  count as proof of a new one.
- Fix the Ansible config-validation task to fail on any exit code
  outside status's three documented health exits (0/1/2), not just on
  5 — verified live against the real program (no wg/ip/iptables on
  this dev machine reproduces the "nothing configured yet" first-run
  case; status correctly returns 1, and broken/missing config.json
  correctly returns 5 and is now caught).
- Correct the live fault-injection drill in the role README: port
  1337 is only the addKey registration API port, not the tunnel's own
  WireGuard UDP endpoint (derive that from `wg show pia-wg endpoints`
  instead), and make the packet-capture check direction-aware
  (`tcpdump -Q out`) so it can't conflate a real egress leak with
  VLAN 50's normal inbound arrival on enp1s0.

Added 6 tests (86 total, all passing): emergency-close repositioning
past a preceding ACCEPT, pause/resume blocking on an in-flight
lock holder and surviving into a subsequent recovery pass, and the
handshake boundary at exactly/one-second-either-side of since_epoch.

yamllint, ansible-playbook --syntax-check, and ansible-lint (diffed
against main — no new finding categories) all pass. No live changes;
nothing pushed to main.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-22 15:51:02 +09:00

1280 lines
57 KiB
Python

"""Offline tests for ansible/roles/pia-gateway/files/pia_gateway.py.
Run from the repo root: python3 -m unittest discover -s ansible/roles/pia-gateway/tests -v
Everything that touches the host (ip/wg/iptables/systemctl, clock, HTTP) is
faked; real files are used inside a temporary directory so atomic writes,
modes, staging and backups are exercised for real.
"""
from __future__ import annotations
import importlib.util
import io
import json
import os
import re
import shutil
import stat
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
HERE = Path(__file__).resolve().parent
SRC = HERE.parent / "files" / "pia_gateway.py"
spec = importlib.util.spec_from_file_location("pia_gateway", SRC)
pg = importlib.util.module_from_spec(spec)
sys.modules["pia_gateway"] = pg
spec.loader.exec_module(pg)
SUBNET = "10.10.50.0/24"
CHAIN = "PIA-VLAN50"
IFACE = "pia-wg"
# Sentinels that must never appear in any output.
SECRET_USER = "sentinel-user-9f1c"
SECRET_PASS = "sentinel-pass-77aa"
SECRET_TOKEN = "sentinel-token-13bd"
SECRET_ALERT_KEY = "sentinel-alertkey-55ee"
# Valid-looking 32-byte base64 keys (44 chars, last char from the allowed set).
PRIV_A = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
PRIV_B = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA="
PUB_A = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaA="
PUB_B = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbA="
SRV_HK1 = "hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1A="
SRV_HK2 = "hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2A="
SRV_TW1 = "tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1A="
SRV_OLD = "oldoldoldoldoldoldoldoldoldoldoldoldoldoldA="
PUBKEYS = {PRIV_A: PUB_A, PRIV_B: PUB_B}
SERVER_KEYS = {"hk1.example": SRV_HK1, "hk2.example": SRV_HK2, "tw1.example": SRV_TW1}
def serverlist_doc():
return {
"regions": [
{"id": "hk", "name": "Hong Kong", "servers": {"wg": [{"cn": "hk1.example", "ip": "10.99.0.1"}, {"cn": "hk2.example", "ip": "10.99.0.2"}]}},
{"id": "taiwan", "name": "Taiwan", "servers": {"wg": [{"cn": "tw1.example", "ip": "10.99.1.1"}]}},
{"id": "japan", "name": "JP Tokyo", "servers": {"wg": []}},
]
}
class FakeSystem(pg.System):
"""Models just enough of the gateway host for the controller."""
def __init__(self, tmp: Path):
self.tmp = tmp
self.t = 1_700_000_000
self.mono = 1000.0
self.boot = "boot-1"
self.iface_exists = True
self.iface_up = True
self.peers: dict[str, int] = {}
self.endpoints: dict[str, str] = {}
self.rule_present = True
self.table_default = "open" # open | closed | missing
self.forward_rules = [f"-A FORWARD -s {SUBNET} -j {CHAIN}", "-A FORWARD -j ufw-before-forward"]
self.chain_rules = self.open_chain()
self.main_default = "default via 10.10.40.1 dev enp1s0 proto static"
self.killswitch_active = True
self.restart_rc = 0
self.restart_count = 0
self.handshake_on_restart = "fresh" # fresh | stale | callable(server_key)->epoch|None
self.commands: list[list[str]] = []
self.stdins: list[str] = []
self.genkey_queue = [PRIV_B]
self.jitter = 0.5
@staticmethod
def open_chain():
return [
f"-A {CHAIN} -d {SUBNET} -i {IFACE} -m state --state RELATED,ESTABLISHED -j ACCEPT",
f"-A {CHAIN} -s {SUBNET} -o {IFACE} -j ACCEPT",
f"-A {CHAIN} -m limit --limit 5/min --limit-burst 5 -j LOG --log-prefix \"PIA-KILLSWITCH-DROP: \"",
f"-A {CHAIN} -j DROP",
]
@staticmethod
def closed_chain():
return [
f"-A {CHAIN} -m limit --limit 5/min --limit-burst 5 -j LOG --log-prefix \"PIA-KILLSWITCH-DROP: \"",
f"-A {CHAIN} -j DROP",
]
# clock
def now(self):
return self.t
def monotonic(self):
return self.mono
def sleep(self, sec):
self.t += int(sec)
self.mono += sec
def boot_id(self):
return self.boot
def random_jitter(self):
return self.jitter
# helpers
def set_peer(self, key, age, endpoint="10.99.0.1:1337"):
self.peers = {key: self.t - age if age is not None else 0}
self.endpoints = {key: endpoint}
def conf_peer(self, conf_path):
if os.path.exists(conf_path):
return pg.parse_expected_peer(open(conf_path).read())
return None
def _apply_restart(self, conf_path):
self.restart_count += 1
if self.restart_rc != 0:
return pg.CmdResult(self.restart_rc, "", "Job for wg-quick@pia-wg.service failed")
self.t += 1 # a real restart takes measurable wall-clock time; keeps
# any "fresh" handshake strictly after whatever `since_epoch` the
# caller captured just before invoking this restart (see the
# `>` — not `>=` — comparison in wait_for_verified).
key = self.conf_peer(conf_path)
self.iface_exists = True
self.iface_up = True
self.table_default = "open"
self.chain_rules = self.open_chain()
beh = self.handshake_on_restart
if callable(beh):
epoch = beh(key)
elif beh == "fresh":
epoch = self.t
else:
epoch = self.t - 100_000
self.peers = {key: epoch if epoch is not None else 0}
self.endpoints = {key: "10.99.0.9:1337"}
return pg.CmdResult(0, "", "")
# command dispatch
def run(self, argv, stdin=None, timeout=60):
self.commands.append(list(argv))
if stdin is not None:
self.stdins.append(stdin)
a = argv
if a[:2] == ["wg", "genkey"]:
return pg.CmdResult(0, self.genkey_queue.pop(0) + "\n", "")
if a[:2] == ["wg", "pubkey"]:
return pg.CmdResult(0, PUBKEYS[stdin.strip()] + "\n", "")
if a[:2] == ["wg", "show"]:
if not self.iface_exists:
return pg.CmdResult(1, "", "Unable to access interface: No such device")
if a[3] == "latest-handshakes":
return pg.CmdResult(0, "".join(f"{k}\t{v}\n" for k, v in self.peers.items()), "")
if a[3] == "endpoints":
return pg.CmdResult(0, "".join(f"{k}\t{v}\n" for k, v in self.endpoints.items()), "")
if a[:4] == ["ip", "-o", "link", "show"]:
if not self.iface_exists:
return pg.CmdResult(1, "", 'Device "pia-wg" does not exist.')
flags = "POINTOPOINT,NOARP,UP,LOWER_UP" if self.iface_up else "POINTOPOINT,NOARP"
return pg.CmdResult(0, f"5: {IFACE}: <{flags}> mtu 1420 qdisc noqueue state UNKNOWN\n", "")
if a[:3] == ["ip", "rule", "show"]:
lines = ["0:\tfrom all lookup local"]
if self.rule_present:
lines.append(f"100:\tfrom {SUBNET} lookup pia")
lines += ["32766:\tfrom all lookup main", "32767:\tfrom all lookup default"]
return pg.CmdResult(0, "\n".join(lines) + "\n", "")
if a[:3] == ["ip", "route", "show"] and "table" in a:
if self.table_default == "open":
return pg.CmdResult(0, f"default dev {IFACE} scope link\n", "")
if self.table_default == "closed":
return pg.CmdResult(0, "unreachable default\n", "")
return pg.CmdResult(0, "", "")
if a[:4] == ["ip", "-4", "route", "show"]:
return pg.CmdResult(0, self.main_default + "\n", "")
if a[:3] == ["ip", "route", "replace"]:
if "unreachable" in a:
self.table_default = "closed"
return pg.CmdResult(0, "", "")
if a[:4] == ["ip", "link", "delete", "dev"]:
self.iface_exists = False
return pg.CmdResult(0, "", "")
if a[:2] == ["iptables", "-S"]:
if a[2] == "FORWARD":
return pg.CmdResult(0, "-P FORWARD ACCEPT\n" + "\n".join(self.forward_rules) + "\n", "")
if a[2] == CHAIN:
if self.chain_rules is None:
return pg.CmdResult(1, "", "iptables: No chain/target/match by that name.")
return pg.CmdResult(0, f"-N {CHAIN}\n" + "\n".join(self.chain_rules) + "\n", "")
if a[:2] == ["iptables", "-C"]:
rule = "-A " + " ".join(a[2:])
return pg.CmdResult(0 if rule in self.forward_rules else 1, "", "")
if a[:2] == ["iptables", "-I"]:
rule = "-A FORWARD " + " ".join(a[4:])
self.forward_rules.insert(0, rule)
return pg.CmdResult(0, "", "")
if a[:2] == ["iptables", "-D"]:
rule = "-A " + " ".join(a[2:])
if rule in self.forward_rules:
self.forward_rules.remove(rule)
return pg.CmdResult(0, "", "")
return pg.CmdResult(1, "", "")
if a[:2] == ["systemctl", "is-active"]:
return pg.CmdResult(0 if self.killswitch_active else 3, "active\n" if self.killswitch_active else "inactive\n", "")
if a[:2] == ["systemctl", "restart"] or a[:2] == ["systemctl", "start"]:
return self._apply_restart(str(self.tmp / "wg" / f"{IFACE}.conf"))
raise AssertionError(f"unexpected command {argv}")
class FakeHttp:
def __init__(self):
self.serverlist = ("ok", serverlist_doc())
self.token = ("ok", SECRET_TOKEN)
self.addkey = {} # cn -> ("ok", {...}) | ("err", kind) | ("http", status) | ("raw", bytes)
self.calls = []
def get(self, url):
self.calls.append(("get", url))
kind, val = self.serverlist
if kind == "err":
raise pg.ApiError(val, "fake")
if kind == "http":
return val, b""
if kind == "raw":
return 200, val
return 200, (json.dumps(val) + "\n-----BEGIN SIG-----\nxxx\n").encode()
def post_multipart(self, url, fields):
self.calls.append(("post", url, dict(fields)))
kind, val = self.token
if kind == "err":
raise pg.ApiError(val, "fake")
if kind == "http":
return val, b"{}"
if kind == "raw":
return 200, val
return 200, json.dumps({"token": val}).encode()
def get_pinned(self, hostname, ip, port, path, query):
self.calls.append(("addkey", hostname, ip, port, path, dict(query)))
kind, val = self.addkey.get(hostname, ("http", 502))
if kind == "err":
raise pg.ApiError(val, "fake")
if kind == "http":
return val, b""
if kind == "raw":
return 200, val
body = {"status": "OK", "server_key": SERVER_KEYS[hostname], "peer_ip": "10.5.5.5", "server_port": 1337, "server_ip": ip}
body.update(val or {})
return 200, json.dumps(body).encode()
class Base(unittest.TestCase):
def setUp(self):
self.tmp = Path(tempfile.mkdtemp(prefix="pia-gateway-test-"))
(self.tmp / "wg").mkdir(mode=0o700)
(self.tmp / "state").mkdir(mode=0o700)
(self.tmp / "etc").mkdir(mode=0o700)
creds = self.tmp / "etc" / "credentials.env"
creds.write_text(f"PIA_USER={SECRET_USER}\nPIA_PASSWORD={SECRET_PASS}\nALERT_BRIDGE_API_KEY={SECRET_ALERT_KEY}\n")
creds.chmod(0o600)
(self.tmp / "wg" / "pia-ca.crt").write_text("dummy")
self.cfg = pg.Config(
config_dir=str(self.tmp / "wg"),
state_dir=str(self.tmp / "state"),
credentials_file=str(creds),
ca_file=str(self.tmp / "wg" / "pia-ca.crt"),
lock_file=str(self.tmp / "lock"),
metrics_file=str(self.tmp / "state" / "metrics.prom"),
status_file=str(self.tmp / "state" / "status.json"),
alert_bridge_url="http://alert-bridge.test/alerts",
)
self.sysm = FakeSystem(self.tmp)
self.http = FakeHttp()
self.out = io.StringIO()
self.log = pg.Log(stream=self.out)
self.notified: list[tuple[str, str]] = []
def fake_send(_self, level, message):
self.notified.append((level, message))
return True
self._send_patch = mock.patch.object(pg.Notifier, "send", fake_send)
self._send_patch.start()
self.write_conf(PRIV_A, SRV_OLD, "10.99.0.1")
(self.tmp / "wg" / "pia-wg.key").write_text(PRIV_A + "\n")
(self.tmp / "wg" / "pia-wg.key").chmod(0o600)
self.sysm.set_peer(SRV_OLD, 30)
self.ctl = pg.Controller(self.cfg, self.sysm, self.log, http=self.http)
# Simulate a controller that has already run once on this boot, so the
# startup grace does not absorb the first checks of every test. The
# fresh-boot behaviour has its own test.
seeded = pg.default_state()
seeded["boot_id"] = self.sysm.boot
self.ctl.store.save(seeded)
def tearDown(self):
self._send_patch.stop()
shutil.rmtree(self.tmp, ignore_errors=True)
def write_conf(self, priv, server_key, ip):
peer = pg.PeerInfo(server_key, "10.5.5.5", 1337, ip, "hk", "Hong Kong", "old.example")
p = self.tmp / "wg" / "pia-wg.conf"
p.write_text(pg.render_wg_conf(self.cfg, priv, peer))
p.chmod(0o600)
def state(self):
return json.loads((self.tmp / "state" / "state.json").read_text())
def run_recover_n(self, n, force=False):
rcs = []
for _ in range(n):
rcs.append(self.ctl.cmd_recover(force=force))
self.sysm.sleep(60)
return rcs
def go_stale(self):
self.sysm.set_peer(SRV_OLD, 5000)
def all_output(self):
s = self.out.getvalue()
for p in (self.tmp / "state" / "status.json", self.tmp / "state" / "metrics.prom", self.tmp / "state" / "state.json"):
if p.exists():
s += p.read_text()
s += json.dumps(self.notified)
s += json.dumps(self.sysm.commands)
return s
def assert_no_secrets(self):
blob = self.all_output()
for sec in (SECRET_USER, SECRET_PASS, SECRET_TOKEN, SECRET_ALERT_KEY, PRIV_A, PRIV_B):
self.assertNotIn(sec, blob, f"secret {sec[:12]}... leaked into output")
class TestClassification(Base):
def test_healthy(self):
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_HEALTHY)
self.assertTrue(st.safety_ok and st.tunnel_ok)
self.assertEqual(st.handshake_age_sec, 30)
def test_stale_handshake(self):
self.go_stale()
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE)
self.assertIn("handshake 5000s old exceeds 180s", st.reasons)
def test_zero_handshake(self):
self.sysm.set_peer(SRV_OLD, None)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE)
self.assertIn("no handshake recorded", st.reasons)
def test_future_handshake_is_not_healthy(self):
self.sysm.set_peer(SRV_OLD, -5000)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE)
self.assertTrue(any("future" in r for r in st.reasons))
def test_small_clock_skew_tolerated(self):
self.sysm.set_peer(SRV_OLD, -10)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_HEALTHY)
def test_malformed_wg_output(self):
self.sysm.peers = {}
orig = self.sysm.run
def run(argv, stdin=None, timeout=60):
if argv[:2] == ["wg", "show"] and argv[3] == "latest-handshakes":
self.sysm.commands.append(argv)
return pg.CmdResult(0, "garbage without tab\n", "")
return orig(argv, stdin, timeout)
self.sysm.run = run
st = self.ctl.current_status(pg.default_state())
self.assertNotEqual(st.cls, pg.CLASS_HEALTHY)
self.assertIn("malformed wg output", st.reasons)
def test_wrong_peer_on_interface(self):
self.sysm.set_peer(SRV_HK1, 10)
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT)
def test_interface_absent(self):
self.sysm.iface_exists = False
st = self.ctl.current_status(pg.default_state())
self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT)
def test_fresh_handshake_but_route_closed_is_not_ok(self):
self.sysm.table_default = "closed"
st = self.ctl.current_status(pg.default_state())
self.assertTrue(st.safety_ok)
self.assertFalse(st.tunnel_ok)
self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT)
def test_boot_grace_class(self):
self.go_stale()
state = pg.default_state()
state["boot_grace_until"] = self.sysm.t + 60
self.assertEqual(self.ctl.current_status(state).cls, pg.CLASS_BOOT_GRACE)
def test_paused_class(self):
self.go_stale()
state = pg.default_state()
state["paused"] = {"since": 1, "reason": "x"}
st = self.ctl.current_status(state)
self.assertEqual(st.cls, pg.CLASS_PAUSED)
self.assertEqual(st.tunnel_class, pg.CLASS_TUNNEL_STALE)
class TestSafetyDrift(Base):
def drift_cases(self):
yield "missing rule", lambda: setattr(self.sysm, "rule_present", False)
yield "missing route", lambda: setattr(self.sysm, "table_default", "missing")
yield "jump below accept", lambda: setattr(self.sysm, "forward_rules", ["-A FORWARD -j ACCEPT", f"-A FORWARD -s {SUBNET} -j {CHAIN}"])
yield "no terminal drop", lambda: setattr(self.sysm, "chain_rules", self.sysm.open_chain()[:-1])
yield "wrong iface accept", lambda: setattr(self.sysm, "chain_rules", [f"-A {CHAIN} -s {SUBNET} -o enp1s0 -j ACCEPT"] + self.sysm.open_chain())
yield "chain missing", lambda: setattr(self.sysm, "chain_rules", None)
yield "host default via tunnel", lambda: setattr(self.sysm, "main_default", "default dev pia-wg scope link")
def test_each_drift_triggers_emergency_close_and_no_restart(self):
for name, apply in self.drift_cases():
with self.subTest(name):
self.setUp()
apply()
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_SAFETY, name)
self.assertEqual(self.sysm.restart_count, 0, name)
self.assertTrue(self.sysm.forward_rules[0].startswith(f"-A FORWARD -s {SUBNET} -m comment --comment pia-gateway-emergency -j DROP"), name)
self.assertEqual(self.sysm.table_default, "closed", name)
self.assertTrue(self.state()["emergency_close_active"], name)
self.assertTrue(any(lvl == "error" and "SAFETY DRIFT" in msg for lvl, msg in self.notified), name)
self.assert_no_secrets()
self.tearDown()
def test_emergency_rule_removed_after_repair(self):
self.sysm.rule_present = False
self.ctl.cmd_recover()
self.assertTrue(self.state()["emergency_close_active"])
# operator repairs the kill switch and the tunnel
self.sysm.rule_present = True
self.sysm.table_default = "open"
self.sysm.set_peer(SRV_OLD, 5)
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_OK)
self.assertFalse(self.state()["emergency_close_active"])
self.assertFalse(any("emergency" in r for r in self.sysm.forward_rules))
def test_emergency_close_is_idempotent(self):
self.sysm.rule_present = False
self.ctl.cmd_recover()
self.ctl.cmd_recover()
self.assertEqual(sum("emergency" in r for r in self.sysm.forward_rules), 1)
def test_emergency_close_repositions_when_pushed_below_a_preceding_accept(self):
self.sysm.rule_present = False
self.ctl.cmd_recover()
self.assertTrue(pg._is_emergency_rule(self.sysm.forward_rules[0]))
# Something outside this role's own handler ordering (e.g. a
# manual `ufw reload`) prepends a broader ACCEPT ahead of our
# rule without removing it — it still exists, but is no longer
# first, so it could be pre-empted by whatever now sits above it.
self.sysm.forward_rules.insert(0, "-A FORWARD -j ACCEPT")
self.assertFalse(pg._is_emergency_rule(self.sysm.forward_rules[0]))
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_SAFETY)
self.assertTrue(pg._is_emergency_rule(self.sysm.forward_rules[0]))
self.assertEqual(sum(1 for r in self.sysm.forward_rules if pg._is_emergency_rule(r)), 1)
self.assertIn("-A FORWARD -j ACCEPT", self.sysm.forward_rules)
def test_register_refuses_on_drift(self):
self.sysm.rule_present = False
rc = self.ctl.cmd_register("same-key", None, "ansible")
self.assertEqual(rc, pg.EXIT_SAFETY)
self.assertEqual(self.http.calls, [])
class TestRecoveryStateMachine(Base):
def test_healthy_runs_are_idempotent_and_do_not_mutate(self):
self.run_recover_n(3)
mutating = [
c for c in self.sysm.commands
if c[:2] in (["systemctl", "restart"], ["systemctl", "start"], ["wg", "genkey"], ["iptables", "-I"], ["iptables", "-D"], ["ip", "link"])
or c[:3] == ["ip", "route", "replace"]
]
self.assertEqual(mutating, [])
self.assertEqual(self.state()["consecutive_failures"], 0)
def test_threshold_before_action(self):
self.go_stale()
self.run_recover_n(2)
self.assertEqual(self.sysm.restart_count, 0)
self.assertEqual(self.state()["consecutive_failures"], 2)
def test_boot_grace_not_counted(self):
self.go_stale()
self.sysm.boot = "boot-2"
self.run_recover_n(2)
self.assertEqual(self.state()["consecutive_failures"], 0)
self.assertEqual(self.sysm.restart_count, 0)
self.sysm.sleep(120)
self.run_recover_n(3)
self.assertEqual(self.sysm.restart_count, 1)
def test_restart_success_recovers(self):
self.go_stale()
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_OK)
self.assertEqual(self.sysm.restart_count, 1)
s = self.state()
self.assertEqual(s["last_recovery"]["stage"], "restart")
self.assertEqual(s["last_recovery"]["pre_fix_handshake_age_sec"], 5000 + 120)
self.assertEqual(s["consecutive_failures"], 0)
self.assertFalse(s["recovery"]["in_progress"])
self.assertEqual(self.http.calls, [])
levels = [l for l, _ in self.notified]
self.assertEqual(levels, ["warn", "info"])
self.assert_no_secrets()
def test_restart_failure_escalates_to_same_key_registration(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK1 else None)
self.http.addkey = {"hk1.example": ("ok", {})}
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_OK)
s = self.state()
self.assertEqual(s["last_recovery"]["stage"], "reregister")
self.assertEqual(s["last_verified_registration"]["server_cn"], "hk1.example")
self.assertEqual(s["last_verified_registration"]["mode"], "same-key")
self.assertEqual(s["last_verified_registration"]["pubkey"], PUB_A)
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
self.assertIn(f"PublicKey = {SRV_HK1}", conf)
self.assertIn("Endpoint = 10.99.0.1:1337", conf)
self.assertIn("Table = off", conf)
self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.conf").stat().st_mode), 0o600)
addkeys = [c for c in self.http.calls if c[0] == "addkey"]
self.assertEqual(addkeys[0][5]["pubkey"], PUB_A)
self.assertEqual(addkeys[0][5]["pt"], SECRET_TOKEN)
self.assertTrue(any(n.startswith("pia-wg.conf.bak-") for n in os.listdir(self.tmp / "wg")))
self.assert_no_secrets()
def test_later_candidate_after_activation_failure(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK2 else None)
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {})}
self.run_recover_n(3)
s = self.state()
self.assertEqual(s["last_verified_registration"]["server_cn"], "hk2.example")
order = [c[1] for c in self.http.calls if c[0] == "addkey"]
self.assertEqual(order, ["hk1.example", "hk2.example"])
def test_region_order_hk_then_taiwan(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_TW1 else None)
self.http.addkey = {"hk1.example": ("http", 500), "hk2.example": ("err", "timeout"), "tw1.example": ("ok", {})}
self.run_recover_n(3)
order = [c[1] for c in self.http.calls if c[0] == "addkey"]
self.assertEqual(order, ["hk1.example", "hk2.example", "tw1.example"])
self.assertEqual(self.state()["last_verified_registration"]["region"], "taiwan")
def test_rotation_after_same_key_accepted_but_no_handshake(self):
self.go_stale()
def beh(key):
# Only a config carrying the NEW key handshakes.
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
return self.sysm.t if f"PrivateKey = {PRIV_B}" in conf else None
self.sysm.handshake_on_restart = beh
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})}
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_OK)
s = self.state()
self.assertEqual(s["last_recovery"]["stage"], "rotate")
self.assertEqual(s["last_verified_registration"]["mode"], "new-key")
self.assertEqual(s["last_verified_registration"]["pubkey"], PUB_B)
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_B)
names = os.listdir(self.tmp / "wg")
self.assertTrue(any(n.startswith("pia-wg.key.bak-") for n in names))
self.assertFalse(any(n.startswith("pia-wg.key.new") for n in names))
self.assertFalse(any(".conf.stage" in n for n in names))
self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.key").stat().st_mode), 0o600)
self.assert_no_secrets()
def test_no_rotation_on_uniform_api_failure(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("err", "timeout"), "hk2.example": ("err", "timeout"), "tw1.example": ("err", "timeout")}
self.run_recover_n(3)
s = self.state()
self.assertEqual(s["recovery"]["last_failure_class"], pg.FAIL_HOST_CONNECTIVITY)
self.assertIsNotNone(s["recovery"]["next_retry_epoch"])
self.assertEqual(len(self.sysm.genkey_queue), 1) # no key generated
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
def test_all_candidates_fail_restores_previous_and_backs_off(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("http", 500), "tw1.example": ("ok", {})}
rcs = self.run_recover_n(3)
self.assertEqual(rcs[-1], pg.EXIT_UNHEALTHY)
s = self.state()
self.assertTrue(s["recovery"]["in_progress"])
self.assertIsNotNone(s["recovery"]["next_retry_epoch"])
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
self.assertIn(f"PublicKey = {SRV_OLD}", conf)
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
self.assertFalse(any(n.startswith("pia-wg.key.new") for n in os.listdir(self.tmp / "wg")))
# kill switch untouched: chain still ends in DROP, jump still first
self.assertEqual(self.sysm.forward_rules[0], f"-A FORWARD -s {SUBNET} -j {CHAIN}")
self.assertEqual(self.sysm.chain_rules[-1], f"-A {CHAIN} -j DROP")
self.assertTrue(any("recovery failed" in m for _, m in self.notified))
self.assert_no_secrets()
def test_auth_failure_no_rotation_error_alert(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.token = ("http", 401)
self.run_recover_n(3)
s = self.state()
self.assertEqual(s["recovery"]["last_failure_class"], pg.FAIL_AUTH)
self.assertFalse(any(c[0] == "addkey" for c in self.http.calls))
self.assertTrue(any(l == "error" for l, _ in self.notified))
self.assertEqual(len(self.sysm.genkey_queue), 1)
def test_serverlist_unreachable_is_host_connectivity(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_HOST_CONNECTIVITY)
self.assertFalse(any(c[0] in ("post", "addkey") for c in self.http.calls))
def test_malformed_serverlist(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("raw", b"not json\n")
self.run_recover_n(3)
self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_REGISTRATION)
def test_malformed_and_invalid_addkey_responses_skip_to_next(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_TW1 else None)
self.http.addkey = {
"hk1.example": ("raw", b"<html>"),
"hk2.example": ("ok", {"peer_ip": "999.1.1.1"}),
"tw1.example": ("ok", {}),
}
self.run_recover_n(3)
self.assertEqual(self.state()["last_verified_registration"]["server_cn"], "tw1.example")
def test_tls_failure_kind_recorded(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("err", "tls"), "hk2.example": ("err", "tls"), "tw1.example": ("err", "tls")}
self.run_recover_n(3)
self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_REGISTRATION)
self.assertIn("kind=tls", self.out.getvalue())
def test_backoff_sequence_bounded(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
delays = []
self.run_recover_n(3)
for _ in range(5):
s = self.state()
delays.append(s["recovery"]["next_retry_epoch"] - s["recovery"]["last_failure_epoch"])
self.sysm.t = s["recovery"]["next_retry_epoch"] + 1
self.ctl.cmd_recover()
self.assertEqual(delays, [300, 900, 1800, 3600, 3600])
def test_backoff_prevents_hammering(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
before = self.sysm.restart_count
self.run_recover_n(4)
self.assertEqual(self.sysm.restart_count, before)
def test_retry_now_bypasses_backoff(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
before = self.sysm.restart_count
self.sysm.handshake_on_restart = "fresh"
rc = self.ctl.cmd_retry_now()
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(self.sysm.restart_count, before + 1)
def test_credentials_missing_still_backs_off(self):
os.unlink(self.cfg.credentials_file)
self.ctl._creds = None
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.run_recover_n(3)
s = self.state()
self.assertIsNotNone(s["recovery"]["next_retry_epoch"])
before = self.sysm.restart_count
self.run_recover_n(2)
self.assertEqual(self.sysm.restart_count, before)
def test_time_budget_bounds_a_cycle(self):
self.cfg.recovery_max_duration_sec = 100
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})}
self.run_recover_n(3)
addkeys = [c for c in self.http.calls if c[0] == "addkey"]
self.assertLess(len(addkeys), 3)
self.assertIn("time budget", self.out.getvalue())
def test_recovered_without_our_action_clears_backoff(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.sysm.set_peer(SRV_OLD, 5)
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_OK)
self.assertIsNone(self.state()["recovery"]["next_retry_epoch"])
self.assertFalse(self.state()["recovery"]["in_progress"])
def test_backward_clock_jump_clears_stale_deadline(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.assertIsNotNone(self.state()["recovery"]["next_retry_epoch"])
self.sysm.t -= 6 * 3600 # NTP steps the clock back six hours
self.sysm.handshake_on_restart = "fresh"
self.http.serverlist = ("ok", serverlist_doc())
self.sysm.set_peer(SRV_OLD, 5000)
rc = self.ctl.cmd_recover()
self.assertIn("clock jump detected", self.out.getvalue())
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(self.state()["last_recovery"]["stage"], "restart")
def test_normal_backoff_is_not_mistaken_for_clock_jump(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
self.ctl.cmd_recover()
self.assertNotIn("clock jump detected", self.out.getvalue())
self.assertIsNotNone(self.state()["recovery"]["next_retry_epoch"])
def test_orphaned_interface_recovery_on_restart(self):
self.go_stale()
calls = {"n": 0}
orig = self.sysm._apply_restart
def restart(conf_path):
calls["n"] += 1
if calls["n"] == 1:
self.sysm.restart_count += 1
return pg.CmdResult(1, "", "RTNETLINK answers: File exists\n`pia-wg' already exists")
return orig(conf_path)
self.sysm._apply_restart = restart
self.run_recover_n(3)
self.assertIn(["ip", "link", "delete", "dev", IFACE], self.sysm.commands)
self.assertEqual(self.state()["last_recovery"]["stage"], "restart")
class TestHandshakeVerificationBoundary(Base):
"""wait_for_verified's freshness check, at the exact one-second boundary.
`wg show`'s handshake epoch and `since_epoch` are both whole seconds,
so a handshake reported in the SAME second as `since_epoch` is
indistinguishable from one that predates activation and merely shares
that truncated second. It must never count as proof of a NEW
handshake — that's the whole point of comparing against `since_epoch`
at all (an `addKey` returning `status: OK` was already established
elsewhere as necessary-but-not-sufficient; the same logic applies one
layer down, to the handshake timestamp itself).
"""
def test_handshake_at_exactly_since_epoch_is_rejected(self):
since_epoch = self.sysm.t
self.sysm.set_peer(SRV_OLD, 0) # handshake_epoch == self.sysm.t == since_epoch, exactly
verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD)
self.assertIsNone(verified)
def test_handshake_one_second_after_since_epoch_is_accepted(self):
since_epoch = self.sysm.t
self.sysm.sleep(1)
self.sysm.set_peer(SRV_OLD, 0) # handshake_epoch == self.sysm.t == since_epoch + 1
verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD)
self.assertIsNotNone(verified)
def test_handshake_one_second_before_since_epoch_is_rejected(self):
self.sysm.sleep(1)
since_epoch = self.sysm.t
self.sysm.set_peer(SRV_OLD, 1) # handshake_epoch == self.sysm.t - 1 == since_epoch - 1
verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD)
self.assertIsNone(verified)
class TestPersistenceAndConcurrency(Base):
def test_state_survives_new_controller(self):
self.go_stale()
self.sysm.handshake_on_restart = "stale"
self.http.serverlist = ("err", "network")
self.run_recover_n(3)
s1 = self.state()
ctl2 = pg.Controller(self.cfg, self.sysm, self.log, http=self.http)
self.sysm.boot = "boot-2"
ctl2.cmd_recover()
s2 = self.state()
self.assertEqual(s1["recovery"]["next_retry_epoch"], s2["recovery"]["next_retry_epoch"])
self.assertEqual(s2["boot_id"], "boot-2")
def test_corrupt_state_fails_closed(self):
(self.tmp / "state" / "state.json").write_text("{not json")
self.go_stale()
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_STATE_ERROR)
self.assertEqual(self.sysm.restart_count, 0)
self.assertTrue(any("corrupt" in m for _, m in self.notified))
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_register("same-key", None, "operator"), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_rewrite_config(), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_pause("x"), pg.EXIT_STATE_ERROR)
self.assertEqual(self.ctl.cmd_resume(), pg.EXIT_STATE_ERROR)
def test_wrong_schema_version_fails_closed(self):
s = pg.default_state()
s["schema_version"] = 99
(self.tmp / "state" / "state.json").write_text(json.dumps(s))
self.assertEqual(self.ctl.cmd_recover(), pg.EXIT_STATE_ERROR)
def test_state_reset_requires_confirm_and_preserves_old(self):
(self.tmp / "state" / "state.json").write_text("{bad")
self.assertEqual(self.ctl.cmd_state_reset(confirm=False), pg.EXIT_USAGE)
self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_OK)
names = os.listdir(self.tmp / "state")
self.assertTrue(any(n.startswith("state.json.corrupt-") for n in names))
self.assertEqual(self.state()["schema_version"], pg.STATE_SCHEMA_VERSION)
def test_atomic_write_no_temp_left_and_mode(self):
p = self.tmp / "state" / "x.json"
self.sysm.write_atomic(str(p), "abc", 0o600)
self.assertEqual(p.read_text(), "abc")
self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o600)
self.assertEqual([n for n in os.listdir(self.tmp / "state") if n.endswith(".tmp")], [])
def test_interrupted_write_keeps_original(self):
p = self.tmp / "state" / "state.json"
p.write_text("original")
with mock.patch.object(pg.os, "replace", side_effect=OSError("boom")):
with self.assertRaises(OSError):
self.sysm.write_atomic(str(p), "new", 0o600)
self.assertEqual(p.read_text(), "original")
self.assertEqual([n for n in os.listdir(self.tmp / "state") if n.endswith(".tmp")], [])
def test_lock_contention_skips_without_acting(self):
self.go_stale()
self.run_recover_n(2)
holder = pg.Lock(self.cfg.lock_file, self.sysm)
holder.acquire(0)
try:
rc = self.ctl.cmd_recover()
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(self.sysm.restart_count, 0)
self.assertIn("another action holds the lock", self.out.getvalue())
self.assertEqual(self.ctl.cmd_register("same-key", None, "operator"), pg.EXIT_LOCKED)
finally:
holder.release()
rc = self.ctl.cmd_recover()
self.assertEqual(self.sysm.restart_count, 1)
def test_lock_released_on_exception(self):
with mock.patch.object(self.ctl, "_decide", side_effect=RuntimeError("x")):
with self.assertRaises(RuntimeError):
self.ctl.cmd_recover()
lk = pg.Lock(self.cfg.lock_file, self.sysm)
lk.acquire(0)
lk.release()
def test_pause_and_resume(self):
self.ctl.cmd_pause("maintenance")
self.go_stale()
self.run_recover_n(5)
self.assertEqual(self.sysm.restart_count, 0)
self.assertTrue(self.state()["paused"])
self.assertEqual(self.ctl.cmd_renew(force_due=True), pg.EXIT_OK)
self.assertEqual(self.http.calls, [])
self.ctl.cmd_resume()
self.run_recover_n(3)
self.assertEqual(self.sysm.restart_count, 1)
def test_pause_does_not_open_gateway(self):
self.sysm.table_default = "closed"
self.sysm.chain_rules = self.sysm.closed_chain()
self.ctl.cmd_pause("x")
self.ctl.cmd_recover()
self.assertEqual(self.sysm.table_default, "closed")
self.assertEqual(self.sysm.restart_count, 0)
def test_pause_and_resume_serialize_with_an_inflight_recovery_pass(self):
# Simulate a recovery pass that is mid-flight: it already holds the
# SAME lock pause/resume now also take, and has not yet saved its
# (unpaused) state back to disk — this is exactly the window in
# which the old, unlocked pause/resume could silently lose the
# race and have their write clobbered by the pass finishing later.
self.cfg.lock_wait_sec = 3
holder = pg.Lock(self.cfg.lock_file, self.sysm)
holder.acquire(0)
try:
self.assertEqual(self.ctl.cmd_pause("maintenance"), pg.EXIT_LOCKED)
self.assertEqual(self.ctl.cmd_resume(), pg.EXIT_LOCKED)
# blocked, not silently skipped-and-lost: state is untouched
self.assertIsNone(self.state().get("paused"))
finally:
# the in-flight pass "finishes": saves its own state (as
# cmd_recover's finally block does) and releases the lock.
self.ctl.store.save(pg.default_state())
holder.release()
# now that the lock is free, pause proceeds against the CURRENT
# on-disk state rather than a copy read before it waited
self.assertEqual(self.ctl.cmd_pause("maintenance"), pg.EXIT_OK)
self.assertTrue(self.state()["paused"])
# a recovery pass started afterwards must also acquire the lock
# and load state fresh before it can act, so it observes the
# pause and never clears or overwrites it
self.go_stale()
self.run_recover_n(5)
self.assertEqual(self.sysm.restart_count, 0)
self.assertTrue(self.state()["paused"])
def test_state_reset_serializes_with_lock(self):
holder = pg.Lock(self.cfg.lock_file, self.sysm)
holder.acquire(0)
try:
self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_LOCKED)
finally:
holder.release()
self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_OK)
class TestRenewal(Base):
def seed_registration(self, age_days):
s = pg.default_state()
s["last_verified_registration"] = {"epoch": self.sysm.t - age_days * 86400, "region": "hk", "server_cn": "old.example", "pubkey": PUB_A, "mode": "same-key", "via": "ansible"}
self.ctl.store.save(s)
def test_not_due(self):
self.seed_registration(10)
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK)
self.assertEqual(self.http.calls, [])
def test_due_and_succeeds_advances_timestamp(self):
self.seed_registration(22)
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK)
s = self.state()
self.assertEqual(s["last_verified_registration"]["via"], "renew")
self.assertEqual(s["last_verified_registration"]["mode"], "same-key")
self.assertGreaterEqual(s["last_verified_registration"]["epoch"], self.sysm.t - 60)
self.assertEqual(len(self.sysm.genkey_queue), 1)
def test_due_and_fails_keeps_timestamp_and_backs_off(self):
self.seed_registration(22)
old = self.state()["last_verified_registration"]["epoch"]
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_OLD else None)
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})}
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_UNHEALTHY)
s = self.state()
self.assertEqual(s["last_verified_registration"]["epoch"], old)
self.assertIsNotNone(s["renewal"]["next_retry_epoch"])
self.assertIn(f"PublicKey = {SRV_OLD}", (self.tmp / "wg" / "pia-wg.conf").read_text())
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK) # in backoff, no call
self.assertTrue(any("renewal failed" in m for _, m in self.notified))
def test_force_due(self):
self.seed_registration(1)
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.assertEqual(self.ctl.cmd_renew(force_due=True), pg.EXIT_OK)
self.assertEqual(self.state()["last_verified_registration"]["via"], "renew")
def test_defers_to_recovery_when_unhealthy(self):
self.seed_registration(30)
self.go_stale()
self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK)
self.assertEqual(self.http.calls, [])
class TestRegisterCommand(Base):
def test_fresh_install_generates_key_and_registers(self):
os.unlink(self.tmp / "wg" / "pia-wg.conf")
os.unlink(self.tmp / "wg" / "pia-wg.key")
self.sysm.iface_exists = False
self.sysm.table_default = "closed"
self.sysm.chain_rules = self.sysm.closed_chain()
self.sysm.genkey_queue = [PRIV_A]
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
rc = self.ctl.cmd_register("same-key", None, "ansible")
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.key").stat().st_mode), 0o600)
s = self.state()
self.assertEqual(s["last_verified_registration"]["via"], "ansible")
self.assertEqual(s["active_registration"]["server_key"], SRV_HK1)
self.assert_no_secrets()
def test_register_refuses_when_killswitch_inactive(self):
self.sysm.killswitch_active = False
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
rc = self.ctl.cmd_register("same-key", None, "ansible")
self.assertEqual(rc, pg.EXIT_UNHEALTHY)
self.assertEqual(self.sysm.restart_count, 0)
self.assertIn(f"PublicKey = {SRV_OLD}", (self.tmp / "wg" / "pia-wg.conf").read_text())
def test_region_override(self):
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"tw1.example": ("ok", {})}
rc = self.ctl.cmd_register("same-key", "taiwan", "operator")
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual([c[1] for c in self.http.calls if c[0] == "addkey"], ["tw1.example"])
def test_new_key_mode_with_abandoned_stage_cleanup(self):
(self.tmp / "wg" / "pia-wg.key.new").write_text("abandoned\n")
(self.tmp / "wg" / "pia-wg.key.new.20260101T000000Z.1").write_text("abandoned2\n")
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
rc = self.ctl.cmd_register("new-key", None, "operator")
self.assertEqual(rc, pg.EXIT_OK)
names = os.listdir(self.tmp / "wg")
self.assertFalse(any(n.startswith("pia-wg.key.new") for n in names))
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_B)
self.assertIn(f"pubkey", json.dumps(self.state()["last_verified_registration"]))
self.assertEqual(self.state()["last_verified_registration"]["pubkey"], PUB_B)
def test_same_key_does_not_rotate(self):
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.ctl.cmd_register("same-key", None, "operator")
self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A)
self.assertEqual(len(self.sysm.genkey_queue), 1)
def test_backups_bounded(self):
self.cfg.backup_keep = 2
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
for i in range(5):
self.sysm.sleep(1)
self.ctl.cmd_register("same-key", None, "operator")
baks = [n for n in os.listdir(self.tmp / "wg") if n.startswith("pia-wg.conf.bak-")]
self.assertEqual(len(baks), 2)
def test_rewrite_config_uses_recorded_registration(self):
self.sysm.handshake_on_restart = "fresh"
self.http.addkey = {"hk1.example": ("ok", {})}
self.ctl.cmd_register("same-key", None, "operator")
self.cfg.mss_clamp_enabled = False
calls_before = len(self.http.calls)
rc = self.ctl.cmd_rewrite_config()
self.assertEqual(rc, pg.EXIT_OK)
self.assertEqual(len(self.http.calls), calls_before)
conf = (self.tmp / "wg" / "pia-wg.conf").read_text()
self.assertNotIn("TCPMSS", conf)
self.assertIn(f"PublicKey = {SRV_HK1}", conf)
class TestConfigValidation(Base):
def render(self, **kw):
peer = pg.PeerInfo(SRV_HK1, "10.5.5.5", 1337, "10.99.0.1", "hk", "Hong Kong", "hk1.example")
return pg.render_wg_conf(self.cfg, PRIV_A, peer)
def test_rendered_config_is_valid(self):
self.assertEqual(pg.validate_wg_conf(self.cfg, self.render()), [])
def test_rejects_table_not_off(self):
text = self.render().replace("Table = off", "Table = auto")
self.assertIn("Table must be off", pg.validate_wg_conf(self.cfg, text))
def test_rejects_bad_keys_and_endpoint(self):
text = self.render().replace(f"PublicKey = {SRV_HK1}", "PublicKey = nope").replace("Endpoint = 10.99.0.1:1337", "Endpoint = bad")
problems = pg.validate_wg_conf(self.cfg, text)
self.assertIn("peer PublicKey invalid", problems)
self.assertIn("Endpoint invalid", problems)
def test_rejects_unscoped_hook_and_policy_change(self):
text = self.render().replace(
f"PostUp = ip route replace default dev %i table pia",
"PostUp = ip route replace default dev %i\nPostUp = iptables -P FORWARD ACCEPT",
)
problems = pg.validate_wg_conf(self.cfg, text)
self.assertTrue(any("without a table" in p for p in problems))
self.assertIn("hook changes FORWARD policy", problems)
self.assertIn("PostUp lacks table route open", problems)
def test_rejects_wide_allowed_ips_change(self):
text = self.render().replace("AllowedIPs = 0.0.0.0/0", "AllowedIPs = ::/0")
self.assertIn("AllowedIPs must be 0.0.0.0/0", pg.validate_wg_conf(self.cfg, text))
def test_expected_peer_parse(self):
self.assertEqual(pg.parse_expected_peer(self.render()), SRV_HK1)
self.assertIn("Hong Kong (hk), hk1.example", pg.parse_configured_peer_line(self.render()))
class TestSecretsAndModes(Base):
def test_credentials_mode_enforced(self):
os.chmod(self.cfg.credentials_file, 0o644)
with self.assertRaises(pg.GatewayError):
pg.load_credentials(self.cfg.credentials_file)
def test_credentials_repr_redacted(self):
c = pg.load_credentials(self.cfg.credentials_file)
self.assertNotIn(SECRET_PASS, repr(c) + str(c))
def test_no_secret_in_argv_stdin_logs_or_files_across_full_cycle(self):
self.go_stale()
self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK2 else None)
self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {})}
self.run_recover_n(3)
self.ctl.cmd_check()
self.ctl.cmd_status()
self.assert_no_secrets()
for argv in self.sysm.commands:
for tok in argv:
self.assertNotIn(SECRET_TOKEN, tok)
self.assertNotIn(PRIV_A, tok)
# the private key goes to `wg pubkey` via stdin only (never argv) and nowhere else
for s in self.sysm.stdins:
self.assertIn(s.strip(), (PRIV_A, PRIV_B))
def test_notification_payloads_are_secret_free_and_failures_do_not_block(self):
self._send_patch.stop()
sent = []
def flaky(_self, level, message):
sent.append((level, message))
return False
with mock.patch.object(pg.Notifier, "send", flaky):
self.go_stale()
self.run_recover_n(3)
self._send_patch.start()
self.assertEqual(self.state()["last_recovery"]["stage"], "restart")
self.assertGreaterEqual(len(self.state()["pending_notifications"]), 1)
for _, m in sent:
for sec in (SECRET_PASS, SECRET_TOKEN, PRIV_A):
self.assertNotIn(sec, m)
def test_pending_notifications_flushed_later_and_bounded(self):
s = pg.default_state()
s["pending_notifications"] = [{"level": "info", "message": f"m{i}", "epoch": 1} for i in range(30)]
self.ctl.store.save(s)
self.ctl.cmd_recover()
self.assertEqual(self.state()["pending_notifications"], [])
self.assertEqual(len([m for _, m in self.notified if m.startswith("m")]), 30)
def test_state_and_status_file_modes(self):
self.ctl.cmd_recover()
self.assertEqual(stat.S_IMODE((self.tmp / "state" / "state.json").stat().st_mode), 0o600)
self.assertTrue((self.tmp / "state" / "metrics.prom").exists())
m = (self.tmp / "state" / "metrics.prom").read_text()
self.assertIn("pia_gateway_healthy 1", m)
self.assertIn("pia_gateway_check_timestamp_seconds", m)
def test_check_is_read_only(self):
self.go_stale()
before = (self.tmp / "state" / "state.json").read_text()
rc = self.ctl.cmd_check()
self.assertEqual(rc, pg.EXIT_UNHEALTHY)
self.assertEqual(self.sysm.restart_count, 0)
self.assertEqual((self.tmp / "state" / "state.json").read_text(), before)
self.assertIn("class=tunnel_stale", self.out.getvalue())
def test_main_never_prints_traceback(self):
cfgp = self.tmp / "config.json"
cfgp.write_text(json.dumps(dataclass_dict(self.cfg)))
sysm = self.sysm
class Boom(FakeSystem):
def run(self, argv, stdin=None, timeout=60):
raise RuntimeError("kaboom " + SECRET_PASS)
out = io.StringIO()
rc = pg.main(["--config", str(cfgp), "recover"], system=Boom(self.tmp), http=self.http, stream=out)
self.assertEqual(rc, pg.EXIT_UNHEALTHY)
self.assertNotIn("Traceback", out.getvalue())
self.assertNotIn(SECRET_PASS, out.getvalue())
self.assertIn("RuntimeError", out.getvalue())
def test_config_rejects_unknown_keys(self):
cfgp = self.tmp / "config.json"
d = dataclass_dict(self.cfg)
d["typo_key"] = 1
cfgp.write_text(json.dumps(d))
out = io.StringIO()
rc = pg.main(["--config", str(cfgp), "status"], system=self.sysm, http=self.http, stream=out)
self.assertEqual(rc, pg.EXIT_USAGE)
def dataclass_dict(cfg):
import dataclasses
return dataclasses.asdict(cfg)
class TestHttpHelpers(unittest.TestCase):
def test_key_and_ip_validators(self):
self.assertTrue(pg.valid_wg_key(PUB_A))
self.assertFalse(pg.valid_wg_key("short"))
self.assertFalse(pg.valid_wg_key(PUB_A[:-1] + "B")) # bad final char
self.assertTrue(pg.valid_ipv4("10.0.0.1"))
self.assertFalse(pg.valid_ipv4("300.0.0.1"))
self.assertFalse(pg.valid_ipv4("::1"))
def test_candidates_order_and_missing_region(self):
api = pg.PiaApi(pg.Config(), None, pg.Log(stream=io.StringIO()))
cands = api.build_candidates(serverlist_doc(), ["hk", "taiwan"])
self.assertEqual([c.cn for c in cands], ["hk1.example", "hk2.example", "tw1.example"])
with self.assertRaises(pg.ApiError):
api.build_candidates(serverlist_doc(), ["hk", "nowhere"])
with self.assertRaises(pg.ApiError):
api.build_candidates(serverlist_doc(), ["japan"]) # exists but no wg servers -> nothing usable
# an empty fallback region is skipped, not fatal
self.assertEqual([c.cn for c in api.build_candidates(serverlist_doc(), ["japan", "hk"])], ["hk1.example", "hk2.example"])
def test_token_retries_then_auth(self):
http = FakeHttp()
api = pg.PiaApi(pg.Config(), http, pg.Log(stream=io.StringIO()))
http.token = ("http", 401)
with self.assertRaises(pg.ApiError) as cm:
api.get_token(pg.Credentials("u", "p"))
self.assertEqual(cm.exception.kind, "auth")
http.token = ("err", "network")
with self.assertRaises(pg.ApiError):
api.get_token(pg.Credentials("u", "p"))
self.assertEqual(len([c for c in http.calls if c[0] == "post"]), 4)
def test_multipart_body_contains_fields_once(self):
captured = {}
class C(pg.HttpClient):
def _request(self, conn, method, path, body, headers):
captured["body"] = body
captured["headers"] = headers
return 200, b'{"token":"t"}'
c = C(pg.Config())
c.post_multipart(pg.TOKEN_URL, {"username": "u1", "password": "p1"})
self.assertIn(b'name="username"\r\n\r\nu1\r\n', captured["body"])
self.assertIn(b'name="password"\r\n\r\np1\r\n', captured["body"])
self.assertTrue(captured["headers"]["Content-Type"].startswith("multipart/form-data; boundary="))
if __name__ == "__main__":
unittest.main()