From 87a9303cc3dfa427a26de5feeb53926d66525958 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 22 Sep 2026 15:51:02 +0900 Subject: [PATCH] fix: address code review on pia-gateway auto-recovery controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Serialize every mutating command (recover, renew, register, rewrite-config, pause, resume, state-reset) on the shared lock, acquired BEFORE state is read rather than after: reading first and locking only around the write let a slower command silently clobber whatever a faster one wrote in between — in particular, an operator pause could be lost to an in-flight recovery pass that read state before the pause ran and only saved its own copy back afterwards. - Fix emergency_close() to reposition its VLAN 50 DROP to the first effective FORWARD position when it exists but has been pushed down by something else (e.g. an out-of-band ufw reload) — a bare `-C` existence check said nothing about position. - Tighten wait_for_verified()'s handshake freshness check from >= to > since_epoch: wg show and since_epoch are both whole seconds, so a handshake at exactly the same second as since_epoch is indistinguishable from a stale, pre-activation one and must not count as proof of a new one. - Fix the Ansible config-validation task to fail on any exit code outside status's three documented health exits (0/1/2), not just on 5 — verified live against the real program (no wg/ip/iptables on this dev machine reproduces the "nothing configured yet" first-run case; status correctly returns 1, and broken/missing config.json correctly returns 5 and is now caught). - Correct the live fault-injection drill in the role README: port 1337 is only the addKey registration API port, not the tunnel's own WireGuard UDP endpoint (derive that from `wg show pia-wg endpoints` instead), and make the packet-capture check direction-aware (`tcpdump -Q out`) so it can't conflate a real egress leak with VLAN 50's normal inbound arrival on enp1s0. Added 6 tests (86 total, all passing): emergency-close repositioning past a preceding ACCEPT, pause/resume blocking on an in-flight lock holder and surviving into a subsequent recovery pass, and the handshake boundary at exactly/one-second-either-side of since_epoch. yamllint, ansible-playbook --syntax-check, and ansible-lint (diffed against main — no new finding categories) all pass. No live changes; nothing pushed to main. Co-Authored-By: Claude Sonnet 5 --- .gitignore | 3 +- INCIDENT-2026-08-25-vlan50-pia-qbittorrent.md | 15 +- README.md | 79 +- ansible/host_vars/minisforum.yaml | 9 + ansible/roles/pia-gateway/README.md | 356 ++- ansible/roles/pia-gateway/defaults/main.yaml | 61 + .../roles/pia-gateway/files/pia_gateway.py | 2092 +++++++++++++++++ .../roles/pia-gateway/tasks/controller.yaml | 98 + .../roles/pia-gateway/tasks/healthcheck.yaml | 21 +- ansible/roles/pia-gateway/tasks/main.yaml | 127 +- .../{register.yaml => register-legacy.yaml} | 24 +- .../pia-gateway/tasks/register-shared.yaml | 64 + .../templates/pia-gateway-config.json.j2 | 36 + .../templates/pia-gateway-credentials.env.j2 | 7 + .../pia-gateway-healthcheck.service.j2 | 7 +- .../templates/pia-gateway-healthcheck.sh.j2 | 110 - .../pia-gateway-onfailure.service.j2 | 12 + .../templates/pia-gateway-recover.service.j2 | 19 + .../templates/pia-gateway-recover.timer.j2 | 13 + .../templates/pia-gateway-renew.service.j2 | 14 + .../templates/pia-gateway-renew.timer.j2 | 11 + .../pia-gateway/tests/test_pia_gateway.py | 1279 ++++++++++ 22 files changed, 4298 insertions(+), 159 deletions(-) create mode 100644 ansible/roles/pia-gateway/files/pia_gateway.py create mode 100644 ansible/roles/pia-gateway/tasks/controller.yaml rename ansible/roles/pia-gateway/tasks/{register.yaml => register-legacy.yaml} (94%) create mode 100644 ansible/roles/pia-gateway/tasks/register-shared.yaml create mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-config.json.j2 create mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-credentials.env.j2 delete mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.sh.j2 create mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-onfailure.service.j2 create mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-recover.service.j2 create mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-recover.timer.j2 create mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-renew.service.j2 create mode 100644 ansible/roles/pia-gateway/templates/pia-gateway-renew.timer.j2 create mode 100644 ansible/roles/pia-gateway/tests/test_pia_gateway.py diff --git a/.gitignore b/.gitignore index cfe2411..2427c39 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ kubeconfig* *.kubeconfig *.swp *.swo -*~ \ No newline at end of file +*~ +__pycache__/ \ No newline at end of file diff --git a/INCIDENT-2026-08-25-vlan50-pia-qbittorrent.md b/INCIDENT-2026-08-25-vlan50-pia-qbittorrent.md index 5b32be3..8e2eb03 100644 --- a/INCIDENT-2026-08-25-vlan50-pia-qbittorrent.md +++ b/INCIDENT-2026-08-25-vlan50-pia-qbittorrent.md @@ -2,7 +2,7 @@ **Date:** 2026-08-25 to 2026-08-27 (discovered/resolved across this window; underlying triggers occurred 2026-08-24/25) **Systems:** `minisforum` (PIA WireGuard gateway), `nik-debian` (VLAN 50 workloads: `browser-vpn-proxy`, `qbittorrent`, `jdownloader`) -**Status:** All four issues resolved live, with code fixes committed for three of them (qBittorrent, MSS clamping, `wg-quick` restart idempotency). The remaining gap (PIA tunnel handshake-death alerting/remediation) is documented with prevention options, deliberately not yet implemented — see "Open follow-up" below. +**Status:** All four issues resolved live, with code fixes committed for three of them (qBittorrent, MSS clamping, `wg-quick` restart idempotency). The remaining gap (PIA tunnel handshake-death alerting/remediation) is documented with prevention options, deliberately not yet implemented — see "Open follow-up" below. **Incident 1 has since recurred (2026-09-22) — see its "Recurrences" subsection.** ## Summary @@ -89,6 +89,15 @@ ansible-playbook -i ansible/inventory.yaml \ This hit a real, separate, now-fixed bug on the way: `ansible/roles/pia-gateway/tasks/register.yaml`'s `when:` conditions used `pia_force_reregister`/`pia_force_key_rotation` directly in boolean expressions. Passed via the CLI's plain `key=value` form (`-e pia_force_reregister=true`, exactly what the role's own README documents), Ansible sets the variable as the **string** `"true"`, not a real boolean — and recent `ansible-core` rejects using a string directly in a `when:` boolean expression (`Conditional result (True) was derived from value of type 'str'`). Fixed by wrapping every use in `| bool` (commit `e757850`), which correctly coerces both real booleans (the `defaults/main.yaml` case) and `"true"`/`"false"` strings (the CLI-override case). Verified with an isolated local test playbook exercising both cases before trusting it against the live host. +### Recurrences + +| Date | Discovered by | Pre-fix evidence | Fix | Notes | +| --- | --- | --- | --- | --- | +| 2026-08-25 | User: proxy not working after nik-debian reboot | Handshake 7h29m old; healthcheck logging `FAIL handshake` every 60s; bare `wg-quick` restart did **not** recover | `pia_force_reregister=true` | This report | +| 2026-09-22 (~09:50 JST) | User: browser can't use the SOCKS5 proxy | **Not captured** — the re-registration was run before `wg show` was checked, so handshake age and healthcheck journal were not recorded; bare restart not attempted | `pia_force_reregister=true` (~10:06 JST) | Kubernetes side verified healthy first (`browser-vpn-proxy` pod, Service, both `svclb` pods all Running). Post-fix `latest-handshakes` = `1790039182` (10:06:22 JST), ~47s before the check — fresh. Two doc gaps hit on the way: root README's command omitted `-J` (fails with "no vault secrets found") and nothing said the paths are repo-root-relative (fails from `ansible/`). Both fixed in the README. | + +~28 days between the two. Not enough to claim a fixed PIA-side registration lifetime, but enough that the next occurrence should be checked against that hypothesis first — if it lands around 2026-10-20, a scheduled proactive `pia_force_reregister` run (every ~3 weeks) becomes the cheapest prevention, independent of whichever alerting option below is chosen. Next time: **run the diagnostic commands before the fix** (root README, "Tunnel silently dead") so the age and journal are on record. + ### Known gap / prevention (not yet implemented — deferred by request) `ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.sh.j2` (run via `pia-gateway-healthcheck.timer`, every 60s) **already correctly detected this failure the entire time** — every single run logged `FAIL handshake: ...s old, exceeds max 180s`. The script is explicitly designed as "observability only, no remediation" (its own header comment says so), which is a deliberate prior design choice, not an oversight. The actual gap is that nothing consumes that detection — no alert, no dashboard, nothing — so a real, correctly-detected outage sat unnoticed until a user happened to test the affected service directly. @@ -238,7 +247,7 @@ None identified for this specific failure mode going forward — the fix address ## Open follow-up -- **Decide and implement PIA tunnel remediation/alerting** (Incident 1's "Known gap" section above) — explicitly deferred by the user to a later date, options documented above. -- **Consider splitting "rewrite config from template" from "re-register with PIA" in the role** (Incident 4's root cause #1) — currently there's no way to pick up a template-only default change without forcing a full re-registration. +- **Decide and implement PIA tunnel remediation/alerting** (Incident 1's "Known gap" section above) — explicitly deferred by the user to a later date, options documented above. **Recurred 2026-09-22** (see Incident 1's "Recurrences" table). **Implemented 2026-09-22 (option 3 + proactive renewal)** on branch `pia-gateway-auto-recovery`, per `PLAN-pia-gateway-auto-recovery.md`: an on-host controller (`ansible/roles/pia-gateway/files/pia_gateway.py`, `pia-gateway-recover.timer`) that restarts, then re-registers with the existing key, then makes one evidence-gated key rotation, verifying a fresh handshake at each step, with persisted bounded backoff, pause/resume, emergency close on kill-switch drift, and alert-bridge notifications; plus a 21-day renewal timer (off by default). 78 offline tests pass. **Not yet deployed or drilled live** — the controlled rollout is in the role README ("Rollout"); the legacy in-Ansible registration path remains selectable (`pia_registration_backend=legacy`) until the shared path passes a live re-registration. Still open after that: node-exporter textfile scrape + a Prometheus stale-check rule so a stopped timer is distinguishable from a healthy gateway. +- **Consider splitting "rewrite config from template" from "re-register with PIA" in the role** (Incident 4's root cause #1) — closed by the same change: `pia-gateway rewrite-config` re-renders `pia-wg.conf` from the recorded registration without a PIA call, and the role runs it automatically when only the controller config changed. - Confirm whether a newer `lscr.io/linuxserver/qbittorrent` tag already includes the upstream fix for #24164, as a potential alternative/complement to the init-container workaround. - If it recurs, try to catch what actually triggers qBittorrent's unclean shutdown in the first place (Incident 2's root cause section) — nothing in this investigation identified it. diff --git a/README.md b/README.md index e428a00..83908d3 100644 --- a/README.md +++ b/README.md @@ -305,14 +305,79 @@ piece's own README for exact commands and rollback: routing/firewall state and is ordered `Before=` `wg-quick@pia-wg.service`; both are `WantedBy=multi-user.target` so a normal boot orders them correctly with no manual step. `pia-gateway-healthcheck.timer` runs -independently and only observes — see -`ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.sh.j2`. +independently and only observes (`pia-gateway check`). -**PIA endpoint/key rotation**: re-run `ansible/playbooks/pia-gateway.yaml --e pia_force_reregister=true -K` — this regenerates the local keypair and -re-registers with PIA, then rewrites `pia-wg.conf` (handler-triggered -restart). Rotating the PIA account password itself just needs -`ansible-vault edit ansible/group_vars/all/vault.yaml` first. +**Automatic recovery** (`ansible/roles/pia-gateway/README.md` → +"Automatic recovery" — read that for the state machine, alerts, +residual risk and the live-drill rollout): `pia-gateway-recover.timer` +on minisforum runs a bounded pass every 60 s — after 3 stale checks it +restarts `pia-wg`, escalates to a same-key PIA re-registration, then to +one staged key rotation on evidence, verifies a fresh handshake before +calling anything fixed, backs off 5→15→30→60 min on failure, and keeps +VLAN 50 fail-closed throughout. Operator commands on minisforum: + +```bash +sudo pia-gateway status # class, handshake age, backoff, last verified registration, pause +sudo pia-gateway pause --reason "..." # stop recovery+renewal (persists across reboot); kill switch untouched +sudo pia-gateway resume +sudo pia-gateway retry-now # one pass ignoring threshold/backoff +sudo journalctl -u pia-gateway-recover -n 50 +``` + +Status as of 2026-09-22: implemented and tested offline (78 unit tests), +**not yet deployed or drilled live** — see the role README "Rollout". +Renewal (`pia_renewal_enabled`) stays off until the drills pass. + +**PIA re-registration / key rotation** — from the repo root (the paths +are repo-root-relative — running from `ansible/` fails with "playbook +could not be found"): + +```bash +# re-register with the EXISTING key (what fixed both incidents): +ansible-playbook -i ansible/inventory.yaml ansible/playbooks/pia-gateway.yaml \ + -e pia_force_reregister=true -K -J +# stage a NEW keypair, register it, promote it only after a verified handshake: +ansible-playbook -i ansible/inventory.yaml ansible/playbooks/pia-gateway.yaml \ + -e pia_force_key_rotation=true -K -J +``` + +`pia_force_reregister` **reuses** the on-disk key (an earlier version of +this paragraph said it regenerated the keypair — it never did; that is +`pia_force_key_rotation`). Both run the shared on-host transaction +(`pia-gateway register`), which restarts and **verifies a handshake** +even when the rendered config is byte-identical. `-J` is required +because `ansible/group_vars/all/vault.yaml` is vault-encrypted and +loaded for every host; the PIA credentials themselves are **not** in +the vault — `ansible/host_vars/minisforum.yaml` reads `PIA_USER`/ +`PIA_PASSWORD` from the repo-root `.env`, and the role now also +provisions those two values to minisforum's root-only +`/etc/pia-gateway/credentials.env` so recovery works unattended. +Rotating the PIA password: edit `.env`, re-run the playbook, re-register. + +**Tunnel silently dead — recurring, known, now auto-recovered**: the +symptom is "the proxy doesn't work" (browser through `10.10.40.20:1080` +fails, qBittorrent/JDownloader have no egress) while everything in +Kubernetes looks healthy and `pia-wg` on minisforum also looks up +(interface present, `wg-quick@pia-wg` `active (exited)`, routes in +place). The kill switch is doing its job — VLAN 50 fails closed rather +than leaking. With the controller deployed, expect an alert and a +recovery within ~10 minutes; check `sudo pia-gateway status` first. If +it is paused, in backoff, or the controller is not deployed, diagnose +before fixing so the age gets recorded: + +```bash +ssh minisforum "sudo wg show pia-wg latest-handshakes" # epoch seconds; >180s old = dead +ssh minisforum "sudo journalctl -u pia-gateway-healthcheck -n 5" +``` + +then `sudo pia-gateway retry-now` on minisforum, or the re-registration +command above. A bare `systemctl restart wg-quick@pia-wg` was **not** +enough the first time (PIA had stopped recognising the registered key). +Occurrences so far — 2026-08-25 (full write-up in +`INCIDENT-2026-08-25-vlan50-pia-qbittorrent.md`, Incident 1) and +2026-09-22 (~28 days later; pre-fix handshake age not captured, fix +confirmed by a fresh handshake immediately after). The ~28-day cadence +is the hypothesis behind the 21-day proactive renewal timer. **Maintenance / taking the gateway down deliberately**: `sudo systemctl stop wg-quick@pia-wg` on minisforum closes VLAN 50 Internet egress diff --git a/ansible/host_vars/minisforum.yaml b/ansible/host_vars/minisforum.yaml index db6af7d..4e0a383 100644 --- a/ansible/host_vars/minisforum.yaml +++ b/ansible/host_vars/minisforum.yaml @@ -12,3 +12,12 @@ ansible_python_interpreter: /usr/bin/python3.13 # playbook triggers evaluation of these two vars. pia_user: "{{ lookup('ansible.builtin.file', playbook_dir + '/../../.env') | regex_search('^PIA_USER=(.*)$', '\\1', multiline=True) | first }}" pia_password: "{{ lookup('ansible.builtin.file', playbook_dir + '/../../.env') | regex_search('^PIA_PASSWORD=(.*)$', '\\1', multiline=True) | first }}" +# Since the auto-recovery work these two ARE copied to minisforum, into the +# root-only /etc/pia-gateway/credentials.env (tasks/controller.yaml, +# no_log) — the host must be able to re-register with PIA unattended. See +# the role README "Credentials on the host" for the tradeoff and rotation. +# +# Optional: alert-bridge's shared bearer key so the controller can post +# state transitions to Discord. Empty (no ALERT_BRIDGE_API_KEY in .env) +# disables delivery; events still go to the journal. +pia_alert_bridge_api_key: "{{ (lookup('ansible.builtin.file', playbook_dir + '/../../.env') | regex_search('^ALERT_BRIDGE_API_KEY=(.*)$', '\\1', multiline=True) | default([''], true)) | first }}" diff --git a/ansible/roles/pia-gateway/README.md b/ansible/roles/pia-gateway/README.md index cbce449..436492e 100644 --- a/ansible/roles/pia-gateway/README.md +++ b/ansible/roles/pia-gateway/README.md @@ -6,8 +6,11 @@ Phase 2 of `~/repo/homelab/plan.md`. ## Before the first real run -1. **The PIA API calls in `tasks/register.yaml` have gone through three - rounds of correction against real live testing (2026-08-24):** +1. **The PIA API calls have gone through three rounds of correction + against real live testing (2026-08-24). They now live in + `files/pia_gateway.py` (`PiaApi`), with the original in-Ansible flow + kept as `tasks/register-legacy.yaml` until the shared path is proven + live — see "Automatic recovery" below:** - `serverlist.piaservers.net/vpninfo/servers/v6`: works. Region values everywhere in this role are the API's **id** field (e.g. `japan`, `hk`, `taiwan`), never a display name ("JP Tokyo", "Hong Kong", @@ -39,9 +42,14 @@ Phase 2 of `~/repo/homelab/plan.md`. keys `manifests/media/pia-secret.sh` already reads) — `ansible/ host_vars/minisforum.yaml` reads them directly from there via a `lookup('ansible.builtin.file', ...)` + regex, deliberately not - ansible-vault, so this needs no vault password at all. The lookup runs - on the control node, so `.env` never touches minisforum and is never - committed (already gitignored). + ansible-vault. The lookup runs on the control node; `.env` itself is + never copied and never committed (gitignored). **Since the + auto-recovery work, only `PIA_USER`/`PIA_PASSWORD` (and, if present, + `ALERT_BRIDGE_API_KEY`) are provisioned to minisforum's root-only + `/etc/pia-gateway/credentials.env`** so the host can re-register + unattended — see "Credentials on the host" below. The playbook still + needs `-J` because `group_vars/all/vault.yaml` is vault-encrypted and + loaded for every host, not because the PIA credentials are in it. 3. Confirm console/recovery access to `minisforum` (IPMI/physical/other out-of-band path) before applying — this role changes host routing and firewall policy. If SSH becomes unreachable, use that path. @@ -111,12 +119,13 @@ stdin, or a complete response body. If Hong Kong and Taiwan fail exactly the same way Tokyo did, that's a strong signal the problem is shared across all three (the request shape, the account credentials, or minisforum's own network path) rather than one region being down — see -`tasks/register.yaml`'s header comment. +`tasks/register-legacy.yaml`'s header comment (the controller logs the +same per-candidate `kind=`/`detail=` classification). **Worked example (2026-08-24):** exactly that happened — all 7 servers across all 3 regions came back `HTTP failure (non-2xx response)`, curl exit 22, `The requested URL returned error: 401`, uniformly. The cause -was in `addkey-attempt.yaml`'s own curl task, not PIA or any region: +was in `addkey-attempt.yaml`'s own curl task (legacy path), not PIA or any region: `ansible.builtin.command`'s `stdin` argument appends a trailing newline by default (`stdin_add_newline` defaults to `true`), and `--data-urlencode pt@-` does not strip it — every server was receiving @@ -191,10 +200,15 @@ real. config change that alters `PostUp`/`PreDown` content can never break a restart this way again — confirmed live, 2026-08-27. - Installs an observability-only health check (`pia-gateway-healthcheck - .timer`, every `pia_healthcheck_interval_sec`) that logs interface, - handshake age, rule/route, and firewall-policy state to the journal. It - never remediates — see the script's header comment for why, and for what - its "route-decision" check does and does not prove. + .timer`, every `pia_healthcheck_interval_sec`) that runs `pia-gateway + check` — the same status classifier the recovery controller uses — and + logs interface, handshake age, rule/route ordering and firewall state to + the journal plus `/var/lib/pia-gateway/status.json` / `metrics.prom`. + It never remediates. Neither it nor the controller sends a packet + through the tunnel: a "healthy" class is a fresh handshake plus intact + kill-switch invariants, not proof of egress identity. +- Installs the automatic recovery controller and its timers — see + "Automatic recovery" below. ## What it deliberately does not do @@ -205,18 +219,330 @@ real. - Does not modify `ansible/roles/wireguard` (the separate `wg0` home-VPN server role) or its interface. +## Known failure mode: the session dies while the interface looks fine + +Seen twice so far, ~28 days apart — 2026-08-25 (root-caused in +`INCIDENT-2026-08-25-vlan50-pia-qbittorrent.md`, Incident 1) and +2026-09-22. `pia-wg`'s handshake with PIA's server stops renewing; +the interface, `wg-quick@pia-wg` (`active (exited)` is normal for +`wg-quick`), the `pia` routing table and all firewall rules stay exactly +as `PostUp` left them. The kill switch's "interface down → unreachable +route" trigger never fires because the interface never goes down, so +VLAN 50 fails closed — correctly, but silently. + +On 2026-08-25 a plain `systemctl restart wg-quick@pia-wg` did not produce +a new handshake and only a fresh `addKey` registration did — strongly +suggesting PIA had dropped the registered key server-side. The +2026-09-22 occurrence went straight to re-registration (bare restart +untested). The underlying trigger is still not confirmed; the ~4-week gap +is consistent with a fixed registration lifetime on PIA's side, which is +what the proactive renewal timer below is for. This failure mode is now +handled automatically — see the next section. + +## Automatic recovery + +`files/pia_gateway.py` is installed as `/usr/local/sbin/pia-gateway` +(root, `0700`, standard library only — runs on the host's `python3.13`). +It is **one** implementation of PIA registration shared by the unattended +timers and by this role's own registration step, so the two can never +drift. Layout on the host: + +| Path | Purpose | +| --- | --- | +| `/etc/pia-gateway/config.json` (`0600`) | rendered from role defaults (`templates/pia-gateway-config.json.j2`) | +| `/etc/pia-gateway/credentials.env` (`0600`) | `PIA_USER`, `PIA_PASSWORD`, optional `ALERT_BRIDGE_API_KEY` — see "Credentials on the host" | +| `/var/lib/pia-gateway/state.json` (`0600`) | versioned controller state (atomic tmp+fsync+rename); **no secrets** | +| `/var/lib/pia-gateway/status.json`, `metrics.prom` (`0644`) | last classification, for humans/monitoring | +| `/run/lock/pia-gateway.lock` | one `flock` shared by every mutating action (timers, `register`, `rewrite-config`); released by the kernel if a process dies | +| `pia-gateway-recover.timer` → `.service` | every 60 s, one bounded recovery pass (`recover`) | +| `pia-gateway-renew.timer` → `.service` | daily 03:30 ±30 min, proactive same-key renewal (`renew`); **disabled by default** (`pia_renewal_enabled`) | +| `pia-gateway-onfailure.service` | `OnFailure=` of the two above: posts an error alert if a unit crashes or refuses to run | +| `pia-gateway-healthcheck.timer` | unchanged cadence, now runs `pia-gateway check` | + +### Status classes + +Every pass classifies the gateway into exactly one of: `healthy`, +`boot_grace` (stale within 120 s of a boot or of an intentional restart — +not counted), `paused`, `tunnel_stale` (handshake 0 / older than +`pia_healthcheck_handshake_max_age_sec` / in the future beyond 60 s of +skew), `tunnel_absent` (interface down/missing, expected peer missing, +fresh handshake but table route closed), `registration_or_endpoint_failure` +/ `host_connectivity_failure` (stale **and** in backoff after a cycle +failed that way), `safety_drift`, `unknown` (a read command failed — no +action taken, alert after 3). `status.json` carries both the decision +class and the raw `tunnel_class`, so "paused" never hides a dead tunnel. + +Safety invariants checked on every pass, all of which must hold before +any restart/registration is attempted: the priority-100 source rule; a +default in table `pia` that is either `dev pia-wg` (open) or +`unreachable` (closed) — *no* default at all is a failure, not "closed"; +the FORWARD jump to `PIA-VLAN50` is the **first effective** rule (an +emergency rule of our own is the only thing allowed above it); the chain +ends in an unconditional `DROP` and every `ACCEPT` before it is scoped to +`pia-wg`; the host default route is still via `enp1s0`. + +### Recovery state machine (`recover`, every 60 s) + +| State | Action | +| --- | --- | +| healthy | reset counters; if a backoff was pending, log/notify "healthy again" | +| 1–2 consecutive stale/absent | count only | +| 3 consecutive (`pia_recovery_failure_threshold`) | save pre-fix evidence (age, endpoint, reasons) → **stage 1**: `systemctl restart wg-quick@pia-wg` once, wait up to 90 s for two consecutive healthy classifications with a handshake newer than the attempt start | +| stage 1 fails | **stage 2**: serverlist → token → `addKey` with the *existing* key against candidates in `pia_region_candidates` order (max `pia_recovery_max_candidates_per_cycle`), each accepted response staged as a validated config, activated, and verified the same way; a candidate that answers `OK` but never handshakes is skipped for the next one | +| stage 2: API accepted the key but nothing handshook | **stage 3**: exactly one staged fresh-key attempt (`pia-wg.key.new..`, promoted only after verification, previous key backed up). Skipped when the failure was uniform (auth error, host connectivity, every API call failing the same way) | +| any failure | previous config restored (best effort) and re-activated; gateway stays closed; backoff 5 → 15 → 30 → 60 min (±10 % jitter, capped) persisted in state; classified failure recorded; alert | +| verified recovery | stage, region/server, elapsed time and pre-fix age recorded in `last_recovery`; `last_verified_registration` updated when a registration happened; one "recovered" alert | +| safety drift | **no** restart/registration. Insert `-I FORWARD 1 -s 10.10.50.0/24 -m comment --comment pia-gateway-emergency -j DROP`, then `ip route replace unreachable default table pia`, verify the DROP is first, alert `error`. If the rule already exists but isn't first any more (something else prepended a rule ahead of it, e.g. an out-of-band `ufw reload`), it's deleted and reinserted at position 1 rather than left where it drifted to — a mere existence check (`iptables -C`) says nothing about position. Left in place until a later pass sees the invariants intact, then removed under the lock. Automatic repair of the kill switch itself is deliberately **not** attempted (see "Residual risk") | +| paused (`pia-gateway pause`) | checks and notifications continue; never restart/register/renew. Safety drift is **still** contained by the emergency close while paused — pausing suspends recovery, never the fail-closed guarantee | +| corrupt/unknown-schema state file | refuse to act (exit 2, unit failure → alert). `pia-gateway state-reset --confirm` moves the file aside and starts fresh | + +One pass is bounded by `pia_recovery_max_duration_sec` (600 s) inside +the program and `TimeoutStartSec` (900 s) in the unit; systemd never +starts a second instance while one runs, and the lock makes a manual +`register` wait (up to `pia_recovery_lock_wait_sec`) rather than +interrupt an activation. Reboots: the boot id is tracked, a new boot gets +`pia_recovery_startup_grace_sec` before stale counts; retry/backoff state +survives because it is on disk. A backward wall-clock step (NTP) that +leaves a persisted retry deadline further out than the backoff cap is +treated as a clock jump and the deadline is cleared rather than stalling +recovery. + +`restart` returning 0, `active (exited)`, or `addKey` returning `status: +OK` are never treated as success on their own — only an observed fresh +handshake on the activated config is, and specifically one reported +**strictly after** the moment activation started: a handshake in the +same whole second as that moment doesn't count either, since `wg show` +only has one-second resolution and there'd be no way to tell it apart +from a stale, pre-activation handshake that happens to share the second. + +Every mutating command — `recover`, `renew`, `register`, +`rewrite-config`, `pause`, `resume`, `state-reset` — takes the same +`/run/lock/pia-gateway.lock` **before** reading state, not after: reading +state first and only locking around the write would let a slower command +silently overwrite whatever a faster one wrote in between (in particular, +it's how a `pause` could otherwise be lost to an in-flight recovery pass +that read state before the pause ran and only saves its own copy back +once it finishes). `pause`/`resume`/`register`/`rewrite-config`/ +`state-reset` wait up to `pia_recovery_lock_wait_sec` for the lock and +exit 4 if it's still busy; `recover`/`renew` (the timers) never wait — +they just skip that tick if something else currently holds it. + +### Operator commands (all on minisforum, as root) + +```bash +pia-gateway status # human summary incl. backoff, last verified registration, pause +pia-gateway check # what the health-check timer runs (read-only, exit 1 = unhealthy) +pia-gateway check --json # machine-readable +pia-gateway pause --reason "maintenance" # persists across reboot; suppresses recover AND renew; kill switch untouched +pia-gateway resume +pia-gateway retry-now # one recovery pass ignoring threshold and backoff +pia-gateway register [--mode same-key|new-key] [--region hk] # explicit transaction, verified, under the lock +pia-gateway rewrite-config # re-render pia-wg.conf from the recorded registration (hook changes), no API call +pia-gateway renew --force-due # exercise renewal without waiting 21 days +pia-gateway state-reset --confirm # only after reading the corrupt file it moves aside +journalctl -u pia-gateway-recover -u pia-gateway-renew -u pia-gateway-healthcheck -n 100 +cat /var/lib/pia-gateway/status.json +``` + +Through Ansible (repo root, `-K -J`): + +```bash +ansible-playbook -i ansible/inventory.yaml ansible/playbooks/pia-gateway.yaml -K -J # idempotent; registers only if no config or no verified baseline +ansible-playbook ... -e pia_force_reregister=true # re-register with the EXISTING key (same-key) +ansible-playbook ... -e pia_force_key_rotation=true # stage a NEW key, register it, promote only after verification +ansible-playbook ... -e pia_region=hk # single-region override (either of the above) +ansible-playbook ... -e pia_auto_recovery_enabled=false # stop+disable both timers, keep everything else (rollback) +ansible-playbook ... -e pia_renewal_enabled=true # after the live drills — enables the daily renewal timer +ansible-playbook ... -e pia_remove_host_credentials=true # deliberate removal of /etc/pia-gateway/credentials.env +``` + +`pia_force_reregister` **reuses** the on-disk key; only +`pia_force_key_rotation` generates a new one (the root README used to say +otherwise — corrected). A normal run also quiesces both timers, waits for +any in-flight pass, and re-enables them after handlers, so an Ansible +restart never races an automatic one. A run whose only change is +`config.json` (e.g. flipping `pia_mss_clamp_enabled`) re-renders the +WireGuard config via `rewrite-config` without touching PIA — the +"template-only change needs a full re-registration" gap from the +2026-08-25 incident is closed. + +### Proactive renewal (`renew`) + +Off until enabled. When on, once a day it re-registers with the +**existing** key if `last_verified_registration` is older than +`pia_renewal_after_days` (21) — or missing, which is why the baseline +must come from a *verified* run of the shared path, never from a file +mtime. Same transaction, same lock, same verification. A failed renewal +leaves the baseline untouched and enters the same bounded backoff (it +does not silently wait another 21 days); if the tunnel is already +unhealthy it defers to the recovery path. Renewal restarts `pia-wg`, so +qBittorrent/JDownloader connections drop for a few seconds at ~03:30 — +verify they reconnect during the drill. Routine key rotation is not +done: rotation happens only in recovery stage 3, on evidence. + +### Alerts and monitoring + +Events (not every 60 s): recovery started, recovered, recovery failed + +backoff (`warn`; `error` for auth failure), safety drift (`error`), +emergency rule removed, renewal failed, status unknown ×3, state corrupt, +unit crashed (via `OnFailure`). Delivery is `POST` to +`pia_alert_bridge_url` with the shared alert-bridge bearer key +(`{"source": "pia-gateway@minisforum", "message": ..., "level": ...}` — +schema read from `~/repo/home-service/alert-bridge`, which has **no** +per-caller tokens, so the cluster's key is what gets provisioned; +accepted tradeoff). 5 s timeout; failures are queued in state (bounded, +20) and flushed on the next pass; they never block recovery. + +`metrics.prom` is node-exporter textfile format +(`pia_gateway_healthy`, `_tunnel_ok`, `_safety_ok`, +`_handshake_age_seconds`, `_consecutive_failures`, `_paused`, +`_recovery_in_backoff`, `_last_verified_registration_timestamp_seconds`, +`_check_timestamp_seconds`, `_emergency_close_active`). **Not yet +scraped**: `values/kube-prometheus-stack.yaml` has no textfile collector +and node-exporter runs as a pod, so wiring it needs a hostPath mount + +`--collector.textfile.directory` plus a `PrometheusRule` including a +stale-check rule (`time() - pia_gateway_check_timestamp_seconds > 300`) +so a stopped timer alerts even when the last value was healthy. Until +that exists, a dead controller is only visible as silence — check +`systemctl list-timers 'pia-gateway-*'` and `status.json`'s timestamp. + +### Credentials on the host + +`/etc/pia-gateway/credentials.env` holds the PIA account username and +password in plaintext, root-only. This is the explicit tradeoff the +design requires: recovery must work while the laptop is off, and PIA's +token is only valid for 24 h, so an unattended host needs a renewable +credential. The program reads the file at runtime and never places +values in argv, environment, logs, notifications, metrics or state; core +dumps are disabled (`LimitCORE=0`) and unexpected exceptions are logged +by class name only. **Rotation:** change `PIA_PASSWORD` in `.env`, run +the playbook (the file is re-rendered with `no_log`), then `pia-gateway +register` (or wait for the next cycle). **Revocation:** `-e +pia_remove_host_credentials=true` removes the file (recovery then fails +closed with `auth`/credentials errors and alerts); change the PIA +password; disabling `pia_auto_recovery_enabled` alone never deletes it. +If an on-host secret is unacceptable, the alternative is an always-on +external secret broker — not a 24-hour token. + +### Residual risk (read before trusting it unattended) + +- Safety drift is contained by the emergency DROP + unreachable route, + not repaired: the kill switch's own rule/jump can be deleted by + something outside this role (a manual `ufw reload` flushes `FORWARD`, + removing both the jump **and** the emergency rule). The controller + re-inserts the emergency rule on its next pass (≤ 60 s) and every boot + re-seeds the kill switch; that 60 s window after an out-of-band flush + is the residual leak risk, and it predates this work. +- Verification is handshake-based. It proves the tunnel is talking to + PIA, not that a VLAN 50 packet leaves via PIA; the end-to-end check is + the SOCKS-proxy/egress-IP step in the live drill. +- The alert-bridge key on minisforum is the cluster's shared key. +- `unknown` (a read command failing) is logged/alerted but never acted + on, by design. + +### Rollout (nothing below has been run live yet) + +1. Baseline on minisforum: `wg show pia-wg`, `ip rule`, `ip route show + table pia`, `iptables -S FORWARD | head -3`, `iptables -S PIA-VLAN50`, + `journalctl -u pia-gateway-healthcheck -n 5`, `curl --socks5-hostname + 10.10.40.20:1080 https://ifconfig.me` from the LAN, `wg show wg0`, + `kubectl get nodes`. Confirm console access. +2. Deploy with timers off: `ansible-playbook ... -K -J -e + pia_auto_recovery_enabled=false`. Check `ls -l /etc/pia-gateway + /var/lib/pia-gateway`, `pia-gateway status`, `systemctl list-timers + 'pia-gateway-*'` (recover/renew inactive). Because the host has no + verified baseline yet, this run **registers through the shared path** + (same-key, explicit restart + handshake verification) — watch the + task output; if it fails, `-e pia_registration_backend=legacy` + restores the previous behaviour. +3. Enable: `ansible-playbook ... -K -J` (recovery on). Confirm one healthy + pass makes no changes (`journalctl -u pia-gateway-recover -n 5`). +4. Drill the silent death. First get the **live WireGuard UDP endpoint** + from the interface itself — **not** port 1337, which is only the + `addKey` registration API port and has nothing to do with the tunnel's + own data-plane port (PIA WireGuard servers commonly use a different + port per server): + + ```bash + sudo wg show pia-wg endpoints # \t: — this ip:port is the block target + ``` + + Then, on minisforum, leave `pia-wg` administratively up and drop only + that path: + + ```bash + sudo iptables -I OUTPUT -d -p udp --dport -j DROP + ``` + + Expect: 3 stale passes → restart → (still blocked) registration to a + different server → recovery; record times from the journal. To test + backoff, block every candidate server before recovery can reach it — + since each candidate's actual WireGuard port isn't known until its own + `addKey` response arrives (only the registration port, 1337, is fixed + and shared), block by IP alone rather than guessing a port: + + ```bash + sudo iptables -I OUTPUT -d -p udp -j DROP # repeat per candidate IP, or -e pia_region=hk to narrow the set first + ``` + + Remove the rule(s) and confirm autonomous recovery. `iptables -D` + every injected rule afterwards — check `iptables -S OUTPUT` is back to + its pre-drill state, not just that the tunnel recovered. +5. Exercise: a failed first server (block one IP only), invalid + credentials (temporarily wrong `PIA_PASSWORD` → expect `auth_failure`, + no rotation, error alert), alert-bridge down, `pia-gateway register` + while a pass runs (expect it to wait), reboot with a pending backoff, + `pause`/`resume`. +6. During every transition, capture on `enp1s0` with an explicit + **egress-only** direction filter: + + ```bash + sudo tcpdump -ni enp1s0 -Q out src net 10.10.50.0/24 + ``` + + `-Q out` matters: VLAN 50 traffic **arriving** at minisforum on + `enp1s0` (from the Flint router, for policy routing into table `pia`) + is normal and expected — a plain `src net 10.10.50.0/24` filter with + no direction would match that inbound traffic too and could hide a + real leak in the noise, or be misread as one. Only a hit in the `out` + direction — a VLAN 50-sourced packet actually being transmitted out + `enp1s0` toward the LAN/Internet instead of via `pia-wg` — is a leak, + and this must stay completely silent through every failure and + recovery transition. Also capture on `pia-wg` itself to confirm + traffic **is** flowing there while open. Verify a request through + `10.10.40.20:1080` and qBittorrent/JDownloader afterwards; check IPv6 + separately; confirm `wg0`, the host default route, Technitium and k3s. +7. Only then `-e pia_renewal_enabled=true`; `pia-gateway renew + --force-due`; confirm `last_verified_registration.epoch` advances only + on verification. Wire the Prometheus stale-check before calling the + feature done. Record actual times and captures in the incident file. + ## Rollback -No tag-based rollback is defined — this role doesn't have an "absent" -mode. Roll back manually on minisforum (reverses this role without +**Partial rollback (keep the gateway, drop only the automation):** +`ansible-playbook ... -e pia_auto_recovery_enabled=false` stops and +disables both timers and leaves the program, credentials, health check, +kill switch and `pia-wg` in place. Add `-e pia_remove_host_credentials=true` +to also delete the on-host PIA credentials; nothing else deletes them. + +No tag-based full rollback is defined — this role doesn't have an +"absent" mode. Roll back manually on minisforum (reverses this role without touching `wg0`, k3s, or the host default route): ```bash +sudo systemctl disable --now pia-gateway-recover.timer pia-gateway-renew.timer # controller first, so it cannot react to the teardown sudo systemctl disable --now wg-quick@pia-wg pia-killswitch.service pia-gateway-healthcheck.timer sudo rm -f /etc/systemd/system/pia-killswitch.service \ /etc/systemd/system/pia-gateway-healthcheck.service \ - /etc/systemd/system/pia-gateway-healthcheck.timer + /etc/systemd/system/pia-gateway-healthcheck.timer \ + /etc/systemd/system/pia-gateway-recover.service \ + /etc/systemd/system/pia-gateway-recover.timer \ + /etc/systemd/system/pia-gateway-renew.service \ + /etc/systemd/system/pia-gateway-renew.timer \ + /etc/systemd/system/pia-gateway-onfailure.service sudo systemctl daemon-reload +sudo iptables -D FORWARD -s 10.10.50.0/24 -m comment --comment pia-gateway-emergency -j DROP 2>/dev/null || true +sudo rm -rf /etc/pia-gateway /var/lib/pia-gateway /usr/local/sbin/pia-gateway /run/lock/pia-gateway.lock # includes the on-host PIA credentials sudo ip rule del from 10.10.50.0/24 table pia priority 100 sudo ip route flush table pia sudo sed -i '/^[0-9]\+\s\+pia$/d' /etc/iproute2/rt_tables @@ -225,7 +551,7 @@ sudo iptables -F PIA-VLAN50 sudo iptables -X PIA-VLAN50 sudo ufw delete allow from 10.10.50.0/24 to 10.10.40.53 port 53 proto tcp sudo ufw delete allow from 10.10.50.0/24 to 10.10.40.53 port 53 proto udp -sudo rm -f /etc/wireguard/pia-wg.conf /etc/wireguard/pia-wg.key /etc/wireguard/pia-wg.key.new /etc/wireguard/pia-ca.crt +sudo rm -f /etc/wireguard/pia-wg.conf /etc/wireguard/pia-wg.conf.bak-* /etc/wireguard/pia-wg.key /etc/wireguard/pia-wg.key.bak-* /etc/wireguard/pia-wg.key.new* /etc/wireguard/pia-ca.crt ``` Inspection commands to confirm rollback actually took (read-only): diff --git a/ansible/roles/pia-gateway/defaults/main.yaml b/ansible/roles/pia-gateway/defaults/main.yaml index 516bf89..9902d19 100644 --- a/ansible/roles/pia-gateway/defaults/main.yaml +++ b/ansible/roles/pia-gateway/defaults/main.yaml @@ -123,3 +123,64 @@ pia_mss_clamp_enabled: true # see tasks/healthcheck.yaml — it must never itself open the kill switch. pia_healthcheck_interval_sec: 60 pia_healthcheck_handshake_max_age_sec: 180 + +# ── Automatic recovery (files/pia_gateway.py, tasks/controller.yaml) ──────── +# +# Which implementation performs PIA registration when this role runs: +# shared — `pia-gateway register` on the host (the same code the unattended +# recovery/renewal timers use; activates and VERIFIES a handshake +# before reporting success). +# legacy — the original in-Ansible token/addKey flow (tasks/register.yaml). +# Kept only until the shared path has passed a live manual +# re-registration; then it is to be deleted (see README). +pia_registration_backend: shared + +# Master switch for the reactive recovery timer. false stops+disables the +# timer but leaves the program, credentials, kill switch, health check and +# pia-wg in place (controlled rollback). Renewal has its own switch below. +pia_auto_recovery_enabled: true + +# Proactive same-key re-registration when the last VERIFIED registration is +# older than pia_renewal_after_days. Off until the reactive path has passed +# its live drills (README "Rollout"). Needs a verified baseline in state — +# established by the shared registration path, never inferred from file +# mtimes. +pia_renewal_enabled: false +pia_renewal_after_days: 21 +pia_renewal_calendar: "*-*-* 03:30:00" +pia_renewal_jitter_sec: 1800 + +# Recovery state machine tunables (seconds unless noted). +pia_recovery_failure_threshold: 3 +pia_recovery_startup_grace_sec: 120 +pia_recovery_restart_wait_sec: 90 +pia_recovery_activation_wait_sec: 90 +pia_recovery_max_duration_sec: 600 +pia_recovery_backoff_schedule_sec: [300, 900, 1800, 3600] +pia_recovery_max_candidates_per_cycle: 6 +pia_recovery_max_rotation_candidates: 3 +pia_recovery_http_connect_timeout_sec: 10 +pia_recovery_http_total_timeout_sec: 30 +pia_recovery_backup_keep: 5 +pia_recovery_lock_wait_sec: 600 + +# Where the on-host controller keeps things. +pia_gateway_program: /usr/local/sbin/pia-gateway +pia_gateway_etc_dir: /etc/pia-gateway +pia_gateway_state_dir: /var/lib/pia-gateway + +# alert-bridge (manifests/home-services/alert-bridge*.yaml) delivery for +# state-transition events. Its API has ONE shared bearer key (no per-caller +# tokens — confirmed against the alert-bridge source), so the same key the +# cluster uses is provisioned to minisforum's root-only credentials file +# when pia_alert_bridge_api_key is set (host_vars reads it from .env). Set +# the URL to null to disable delivery; events still go to the journal. +pia_alert_bridge_url: "http://alert-bridge.home.arpa/alerts" +pia_alert_source: "pia-gateway@minisforum" +pia_alert_bridge_api_key: "" + +# Deliberate credential removal. Disabling automation never deletes the +# on-host PIA credentials on its own; set this true (one run) to remove +# {{ pia_gateway_etc_dir }}/credentials.env, e.g. when decommissioning or +# after rotating the PIA password elsewhere. +pia_remove_host_credentials: false diff --git a/ansible/roles/pia-gateway/files/pia_gateway.py b/ansible/roles/pia-gateway/files/pia_gateway.py new file mode 100644 index 0000000..2b81a4e --- /dev/null +++ b/ansible/roles/pia-gateway/files/pia_gateway.py @@ -0,0 +1,2092 @@ +#!/usr/bin/env python3 +"""pia-gateway: PIA WireGuard egress gateway controller for minisforum. + +Installed by ansible/roles/pia-gateway. One program, several subcommands: + + check read-only status classification (journal + status/metrics files) + recover one bounded recovery pass (timer-driven, every 60s) + renew proactive same-key re-registration when the last verified + registration is older than the configured age (daily timer) + register explicit registration transaction (used by Ansible and operators) + rewrite-config + re-render pia-wg.conf from the last verified registration + (no PIA API call), activate and verify + status / pause / resume / retry-now / state-reset / notify + +Invariants this program must never break (see the role README): + * The host's main default route and the global FORWARD policy are never + touched. Only table `pia`, the `PIA-VLAN50` chain, and (in an + emergency) one extra FORWARD rule scoped to the VLAN 50 source. + * Every failure path leaves VLAN 50 closed. No path opens the gateway + except a verified WireGuard activation. + * No PIA credential, token, or private key is ever written to stdout, + stderr, the journal, argv, a notification, or the metrics file. + +Only the Python standard library is used so the same file runs on the +gateway (python3.13) and under the repo's local unittest suite. +""" + +from __future__ import annotations + +import argparse +import contextlib +import dataclasses +import datetime as _dt +import fcntl +import http.client +import io +import json +import os +import random +import re +import secrets +import socket +import ssl +import subprocess +import sys +import tempfile +import time +import urllib.parse +from typing import Any, Iterable, Optional + +STATE_SCHEMA_VERSION = 1 +EMERGENCY_COMMENT = "pia-gateway-emergency" +SERVERLIST_URL = "https://serverlist.piaservers.net/vpninfo/servers/v6" +TOKEN_URL = "https://www.privateinternetaccess.com/api/client/v2/token" +ADDKEY_PORT = 1337 + +CLASS_HEALTHY = "healthy" +CLASS_BOOT_GRACE = "boot_grace" +CLASS_PAUSED = "paused" +CLASS_TUNNEL_STALE = "tunnel_stale" +CLASS_TUNNEL_ABSENT = "tunnel_absent" +CLASS_REGISTRATION_FAILURE = "registration_or_endpoint_failure" +CLASS_SAFETY_DRIFT = "safety_drift" +CLASS_HOST_CONNECTIVITY = "host_connectivity_failure" +CLASS_UNKNOWN = "unknown" + +FAIL_AUTH = "auth_failure" +FAIL_HOST_CONNECTIVITY = CLASS_HOST_CONNECTIVITY +FAIL_REGISTRATION = CLASS_REGISTRATION_FAILURE +FAIL_SAFETY = CLASS_SAFETY_DRIFT + +EXIT_OK = 0 +EXIT_UNHEALTHY = 1 +EXIT_STATE_ERROR = 2 +EXIT_SAFETY = 3 +EXIT_LOCKED = 4 +EXIT_USAGE = 5 + + +class GatewayError(Exception): + """Base class; message must never contain secrets.""" + + +class StateCorruptError(GatewayError): + pass + + +class SafetyError(GatewayError): + pass + + +class LockBusyError(GatewayError): + pass + + +class ApiError(GatewayError): + def __init__(self, kind: str, detail: str): + super().__init__(f"{kind}: {detail}") + self.kind = kind + self.detail = detail + + +# -------------------------------------------------------------------------- +# Configuration and credentials +# -------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Config: + interface: str = "pia-wg" + config_dir: str = "/etc/wireguard" + state_dir: str = "/var/lib/pia-gateway" + credentials_file: str = "/etc/pia-gateway/credentials.env" + ca_file: str = "/etc/wireguard/pia-ca.crt" + lock_file: str = "/run/lock/pia-gateway.lock" + route_table: str = "pia" + rule_priority: int = 100 + vlan50_subnet: str = "10.10.50.0/24" + iptables_chain: str = "PIA-VLAN50" + lan_interface: str = "enp1s0" + mss_clamp_enabled: bool = True + region_candidates: list = dataclasses.field(default_factory=lambda: ["hk", "taiwan", "japan"]) + handshake_max_age_sec: int = 180 + failure_threshold: int = 3 + startup_grace_sec: int = 120 + restart_wait_sec: int = 90 + activation_wait_sec: int = 90 + recovery_max_duration_sec: int = 600 + backoff_schedule_sec: list = dataclasses.field(default_factory=lambda: [300, 900, 1800, 3600]) + backoff_jitter_fraction: float = 0.1 + max_candidates_per_cycle: int = 6 + max_rotation_candidates: int = 3 + renewal_after_days: int = 21 + http_connect_timeout_sec: int = 10 + http_total_timeout_sec: int = 30 + clock_skew_allowance_sec: int = 60 + backup_keep: int = 5 + lock_wait_sec: int = 600 + alert_bridge_url: Optional[str] = None + alert_source: str = "pia-gateway" + metrics_file: str = "/var/lib/pia-gateway/metrics.prom" + status_file: str = "/var/lib/pia-gateway/status.json" + killswitch_unit: str = "pia-killswitch.service" + + @classmethod + def load(cls, path: str) -> "Config": + with open(path, "r", encoding="utf-8") as f: + raw = json.load(f) + known = {f.name for f in dataclasses.fields(cls)} + unknown = set(raw) - known + if unknown: + raise GatewayError(f"config: unknown keys {sorted(unknown)}") + cfg = cls(**raw) + if not cfg.region_candidates: + raise GatewayError("config: region_candidates must be non-empty") + if cfg.failure_threshold < 1: + raise GatewayError("config: failure_threshold must be >= 1") + return cfg + + @property + def wg_conf(self) -> str: + return os.path.join(self.config_dir, f"{self.interface}.conf") + + @property + def wg_key(self) -> str: + return os.path.join(self.config_dir, f"{self.interface}.key") + + @property + def state_file(self) -> str: + return os.path.join(self.state_dir, "state.json") + + @property + def wg_unit(self) -> str: + return f"wg-quick@{self.interface}.service" + + +@dataclasses.dataclass +class Credentials: + pia_user: str + pia_password: str + alert_bridge_api_key: Optional[str] = None + + def __repr__(self) -> str: # never leak via repr/str + return "Credentials()" + + __str__ = __repr__ + + +def load_credentials(path: str, require_pia: bool = True) -> Credentials: + values: dict[str, str] = {} + try: + st = os.stat(path) + except FileNotFoundError: + if require_pia: + raise GatewayError(f"credentials file {path} is missing") + return Credentials("", "", None) + if st.st_mode & 0o077: + raise GatewayError(f"credentials file {path} must be mode 0600 (group/other bits set)") + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + values[k.strip()] = v.strip() + user = values.get("PIA_USER", "") + password = values.get("PIA_PASSWORD", "") + if require_pia and (not user or not password): + raise GatewayError(f"credentials file {path} lacks PIA_USER/PIA_PASSWORD") + return Credentials(user, password, values.get("ALERT_BRIDGE_API_KEY") or None) + + +# -------------------------------------------------------------------------- +# System boundary (commands, clock, files) — replaced by fakes in tests +# -------------------------------------------------------------------------- + + +@dataclasses.dataclass +class CmdResult: + rc: int + out: str + err: str + + +class System: + def run(self, argv: list[str], stdin: Optional[str] = None, timeout: int = 60) -> CmdResult: + try: + p = subprocess.run( + argv, + input=stdin, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except FileNotFoundError: + return CmdResult(127, "", f"{argv[0]}: not found") + except subprocess.TimeoutExpired: + return CmdResult(124, "", f"{argv[0]}: timeout after {timeout}s") + return CmdResult(p.returncode, p.stdout, p.stderr) + + def now(self) -> int: + return int(time.time()) + + def monotonic(self) -> float: + return time.monotonic() + + def sleep(self, sec: float) -> None: + time.sleep(sec) + + def boot_id(self) -> str: + try: + with open("/proc/sys/kernel/random/boot_id", "r", encoding="utf-8") as f: + return f.read().strip() + except OSError: + return "unknown" + + def read_text(self, path: str) -> str: + with open(path, "r", encoding="utf-8") as f: + return f.read() + + def exists(self, path: str) -> bool: + return os.path.exists(path) + + def listdir(self, path: str) -> list[str]: + try: + return sorted(os.listdir(path)) + except FileNotFoundError: + return [] + + def unlink(self, path: str) -> None: + with contextlib.suppress(FileNotFoundError): + os.unlink(path) + + def rename(self, src: str, dst: str) -> None: + os.replace(src, dst) + + def copy_file(self, src: str, dst: str, mode: int = 0o600) -> None: + data = self.read_text(src) + self.write_atomic(dst, data, mode) + + def mtime(self, path: str) -> float: + return os.stat(path).st_mtime + + def makedirs(self, path: str, mode: int = 0o700) -> None: + os.makedirs(path, mode=mode, exist_ok=True) + os.chmod(path, mode) + + def write_atomic(self, path: str, data: str, mode: int = 0o600) -> None: + directory = os.path.dirname(path) or "." + fd, tmp = tempfile.mkstemp(dir=directory, prefix=f".{os.path.basename(path)}.", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp, mode) + os.replace(tmp, path) + dfd = os.open(directory, os.O_RDONLY) + try: + os.fsync(dfd) + finally: + os.close(dfd) + except BaseException: + with contextlib.suppress(FileNotFoundError): + os.unlink(tmp) + raise + + def random_jitter(self) -> float: + return random.random() + + +# -------------------------------------------------------------------------- +# Logging (journal-friendly, secret-free by construction) +# -------------------------------------------------------------------------- + + +class Log: + def __init__(self, stream: io.TextIOBase = None, json_mode: bool = False): + self.stream = stream or sys.stdout + self.json_mode = json_mode + self.lines: list[str] = [] + + def emit(self, level: str, msg: str, **fields: Any) -> None: + if self.json_mode: + rec = {"level": level, "msg": msg} + rec.update(fields) + line = json.dumps(rec, sort_keys=True) + else: + extra = " ".join(f"{k}={_fmt(v)}" for k, v in fields.items()) + line = f"{level} {msg}" + (f" {extra}" if extra else "") + self.lines.append(line) + print(line, file=self.stream, flush=True) + + def info(self, msg: str, **f: Any) -> None: + self.emit("INFO", msg, **f) + + def warn(self, msg: str, **f: Any) -> None: + self.emit("WARN", msg, **f) + + def error(self, msg: str, **f: Any) -> None: + self.emit("ERROR", msg, **f) + + +def _fmt(v: Any) -> str: + if isinstance(v, (list, dict)): + return json.dumps(v, separators=(",", ":")) + return str(v) + + +# -------------------------------------------------------------------------- +# Persistent state +# -------------------------------------------------------------------------- + + +def default_state() -> dict: + return { + "schema_version": STATE_SCHEMA_VERSION, + "boot_id": None, + "boot_grace_until": 0, + "restart_grace_until": 0, + "consecutive_failures": 0, + "consecutive_unknown": 0, + "last_check": None, + "last_healthy_epoch": None, + "recovery": { + "in_progress": False, + "backoff_index": 0, + "next_retry_epoch": None, + "last_failure_class": None, + "last_failure_epoch": None, + "last_failure_detail": None, + "cycles": 0, + }, + "last_recovery": None, + "last_verified_registration": None, + "active_registration": None, + "renewal": {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None}, + "paused": None, + "emergency_close_active": False, + "pending_notifications": [], + } + + +class StateStore: + def __init__(self, cfg: Config, system: System): + self.cfg = cfg + self.sys = system + + def load(self) -> dict: + path = self.cfg.state_file + if not self.sys.exists(path): + return default_state() + try: + raw = json.loads(self.sys.read_text(path)) + except (ValueError, OSError) as e: + raise StateCorruptError(f"state file {path} is unreadable or not JSON ({e.__class__.__name__})") + if not isinstance(raw, dict) or raw.get("schema_version") != STATE_SCHEMA_VERSION: + raise StateCorruptError( + f"state file {path} has schema_version={raw.get('schema_version') if isinstance(raw, dict) else '?'}, " + f"expected {STATE_SCHEMA_VERSION}; run `pia-gateway state-reset --confirm` after review" + ) + merged = default_state() + for k, v in raw.items(): + if isinstance(merged.get(k), dict) and isinstance(v, dict): + merged[k].update(v) + else: + merged[k] = v + return merged + + def save(self, state: dict) -> None: + self.sys.makedirs(self.cfg.state_dir, 0o700) + self.sys.write_atomic(self.cfg.state_file, json.dumps(state, indent=1, sort_keys=True) + "\n", 0o600) + + +# -------------------------------------------------------------------------- +# Lock +# -------------------------------------------------------------------------- + + +class Lock: + """flock-based; released automatically by the kernel if the process dies.""" + + def __init__(self, path: str, system: System): + self.path = path + self.sys = system + self.fd: Optional[int] = None + + def acquire(self, wait_sec: float = 0) -> None: + directory = os.path.dirname(self.path) + if directory and not os.path.isdir(directory): + os.makedirs(directory, mode=0o755, exist_ok=True) + fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600) + start = self.sys.monotonic() + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + self.fd = fd + return + except BlockingIOError: + if self.sys.monotonic() - start >= wait_sec: + os.close(fd) + raise LockBusyError(f"lock {self.path} is held by another pia-gateway action") + self.sys.sleep(1) + + def release(self) -> None: + if self.fd is not None: + with contextlib.suppress(OSError): + fcntl.flock(self.fd, fcntl.LOCK_UN) + os.close(self.fd) + self.fd = None + + +# -------------------------------------------------------------------------- +# Status classification +# -------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Status: + timestamp: int + tunnel_class: str + cls: str + reasons: list[str] + safety_ok: bool + tunnel_ok: bool + interface_up: bool + peer_present: bool + handshake_epoch: Optional[int] + handshake_age_sec: Optional[int] + peer_endpoint: Optional[str] + expected_peer: Optional[str] + route_state: str # open | closed | missing | unknown + configured_peer_line: Optional[str] + checks: dict + + def to_dict(self) -> dict: + d = dataclasses.asdict(self) + d["class"] = d.pop("cls") + return d + + +def parse_expected_peer(conf_text: str) -> Optional[str]: + in_peer = False + for line in conf_text.splitlines(): + s = line.strip() + if s.startswith("["): + in_peer = s.lower() == "[peer]" + continue + if in_peer and s.startswith("PublicKey"): + _, _, v = s.partition("=") + return v.strip() + return None + + +def parse_configured_peer_line(conf_text: str) -> Optional[str]: + for line in conf_text.splitlines(): + if line.startswith("# PIA WireGuard server actually used"): + return line[2:] + return None + + +def _split_rule_tokens(rule: str) -> list[str]: + return rule.split() + + +def _is_emergency_rule(rule: str) -> bool: + return f"--comment {EMERGENCY_COMMENT}" in rule or f'--comment "{EMERGENCY_COMMENT}"' in rule + + +class Classifier: + def __init__(self, cfg: Config, system: System): + self.cfg = cfg + self.sys = system + + def _rc(self, argv: list[str]) -> CmdResult: + return self.sys.run(argv, timeout=20) + + def classify(self, state: dict) -> Status: + cfg = self.cfg + now = self.sys.now() + reasons: list[str] = [] + safety_ok = True + checks: dict[str, Any] = {} + unknown = False + + # -- interface + link = self._rc(["ip", "-o", "link", "show", "dev", cfg.interface]) + interface_up = False + if link.rc == 0 and link.out.strip(): + m = re.search(r"<([^>]*)>", link.out) + flags = m.group(1).split(",") if m else [] + interface_up = "UP" in flags + checks["interface"] = "up" if interface_up else "down" + if not interface_up: + reasons.append("interface is administratively down") + else: + checks["interface"] = "absent" + reasons.append("interface absent") + + # -- expected peer from config + expected_peer = None + configured_line = None + if self.sys.exists(cfg.wg_conf): + try: + conf_text = self.sys.read_text(cfg.wg_conf) + expected_peer = parse_expected_peer(conf_text) + configured_line = parse_configured_peer_line(conf_text) + except OSError: + reasons.append("config unreadable") + else: + reasons.append("config missing") + checks["expected_peer_present_in_config"] = expected_peer is not None + + # -- handshake + handshake_epoch: Optional[int] = None + handshake_age: Optional[int] = None + peer_present = False + endpoint = None + handshake_ok = False + if interface_up: + hs = self._rc(["wg", "show", cfg.interface, "latest-handshakes"]) + ep = self._rc(["wg", "show", cfg.interface, "endpoints"]) + if hs.rc != 0: + reasons.append("wg show failed") + else: + peers = {} + for line in hs.out.splitlines(): + parts = line.split("\t") + if len(parts) == 2 and parts[1].strip().lstrip("-").isdigit(): + peers[parts[0].strip()] = int(parts[1].strip()) + elif line.strip(): + reasons.append("malformed wg output") + if expected_peer is None: + reasons.append("no expected peer to compare against") + elif expected_peer not in peers: + reasons.append("expected peer not present on interface") + else: + peer_present = True + handshake_epoch = peers[expected_peer] + if ep.rc == 0: + for line in ep.out.splitlines(): + parts = line.split("\t") + if len(parts) == 2 and parts[0].strip() == expected_peer: + endpoint = parts[1].strip() + if endpoint in (None, "", "(none)"): + reasons.append("peer has no endpoint") + if handshake_epoch <= 0: + reasons.append("no handshake recorded") + elif handshake_epoch > now + cfg.clock_skew_allowance_sec: + reasons.append(f"handshake timestamp {handshake_epoch - now}s in the future (clock jump?)") + else: + handshake_age = max(0, now - handshake_epoch) + if handshake_age > cfg.handshake_max_age_sec: + reasons.append(f"handshake {handshake_age}s old exceeds {cfg.handshake_max_age_sec}s") + else: + handshake_ok = True + checks["handshake_age_sec"] = handshake_age + + # -- policy rule + rule = self._rc(["ip", "rule", "show"]) + rule_ok = False + if rule.rc == 0: + pat = re.compile(rf"^{cfg.rule_priority}:\s+from {re.escape(cfg.vlan50_subnet)} lookup {re.escape(cfg.route_table)}\s*$", re.M) + rule_ok = bool(pat.search(rule.out)) + else: + unknown = True + checks["policy_rule"] = rule_ok + if not rule_ok: + safety_ok = False + reasons.append("policy rule missing") + + # -- table default + route = self._rc(["ip", "route", "show", "table", cfg.route_table]) + route_state = "unknown" + if route.rc == 0: + open_default = any( + l.startswith("default") and f"dev {cfg.interface}" in l for l in route.out.splitlines() + ) + unreachable_default = any(l.strip().startswith("unreachable default") for l in route.out.splitlines()) + if open_default: + route_state = "open" + elif unreachable_default: + route_state = "closed" + else: + route_state = "missing" + else: + unknown = True + checks["route_table_default"] = route_state + if route_state in ("missing", "unknown"): + safety_ok = False + reasons.append(f"table {cfg.route_table} default route {route_state}") + + # -- FORWARD ordering + fwd = self._rc(["iptables", "-S", "FORWARD"]) + jump_first = False + emergency_present = False + if fwd.rc == 0: + rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")] + effective = [] + for r in rules: + if _is_emergency_rule(r): + emergency_present = True + continue + effective.append(r) + expected_jump = f"-A FORWARD -s {cfg.vlan50_subnet} -j {cfg.iptables_chain}" + jump_first = bool(effective) and effective[0].strip() == expected_jump + else: + unknown = True + checks["forward_jump_first"] = jump_first + checks["emergency_rule_present"] = emergency_present + if not jump_first: + safety_ok = False + reasons.append("FORWARD jump to kill-switch chain is not first") + + # -- chain contents + chain = self._rc(["iptables", "-S", cfg.iptables_chain]) + chain_ok = False + accepts_ok = False + if chain.rc == 0: + rules = [l.strip() for l in chain.out.splitlines() if l.startswith(f"-A {cfg.iptables_chain}")] + drop_idx = next((i for i, r in enumerate(rules) if r == f"-A {cfg.iptables_chain} -j DROP"), None) + if drop_idx is not None and drop_idx == len(rules) - 1: + chain_ok = True + else: + reasons.append("kill-switch chain lacks a terminal unconditional DROP") + before_drop = rules[:drop_idx] if drop_idx is not None else rules + out_accept = f"-s {cfg.vlan50_subnet} -o {cfg.interface} -j ACCEPT" + in_accept_prefix = f"-d {cfg.vlan50_subnet} -i {cfg.interface}" + has_out = any(r.endswith(out_accept) for r in before_drop) + has_in = any(in_accept_prefix in r and r.endswith("-j ACCEPT") for r in before_drop) + for r in before_drop: + if r.endswith("-j ACCEPT") and f"-o {cfg.interface}" not in r and f"-i {cfg.interface}" not in r: + chain_ok = False + reasons.append("kill-switch chain has an ACCEPT not scoped to the tunnel interface") + break + accepts_ok = has_out and has_in + else: + unknown = True + reasons.append("kill-switch chain missing") + checks["chain_terminal_drop"] = chain_ok + checks["chain_tunnel_accepts"] = accepts_ok + if not chain_ok: + safety_ok = False + + # -- host default route + main = self._rc(["ip", "-4", "route", "show", "default"]) + main_ok = False + if main.rc == 0: + main_ok = f"dev {cfg.lan_interface}" in main.out and f"dev {cfg.interface}" not in main.out + else: + unknown = True + checks["host_default_route_ok"] = main_ok + if not main_ok: + safety_ok = False + reasons.append("host default route is not via the LAN interface") + + tunnel_ok = interface_up and peer_present and handshake_ok and route_state == "open" and accepts_ok + if interface_up and peer_present and handshake_ok and route_state != "open": + reasons.append("handshake fresh but table route is not open") + if interface_up and peer_present and handshake_ok and route_state == "open" and not accepts_ok: + reasons.append("tunnel ACCEPT rules missing from kill-switch chain") + + # -- classify + if not safety_ok: + tunnel_class = CLASS_SAFETY_DRIFT + elif unknown: + tunnel_class = CLASS_UNKNOWN + elif tunnel_ok: + tunnel_class = CLASS_HEALTHY + elif not interface_up or not peer_present or (handshake_ok and not tunnel_ok): + tunnel_class = CLASS_TUNNEL_ABSENT + else: + tunnel_class = CLASS_TUNNEL_STALE + + cls = tunnel_class + rec = state.get("recovery", {}) + if tunnel_class in (CLASS_TUNNEL_STALE, CLASS_TUNNEL_ABSENT): + grace = max(state.get("boot_grace_until") or 0, state.get("restart_grace_until") or 0) + if now < grace: + cls = CLASS_BOOT_GRACE + elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_HOST_CONNECTIVITY,): + cls = CLASS_HOST_CONNECTIVITY + elif rec.get("next_retry_epoch") and rec.get("last_failure_class") in (FAIL_REGISTRATION, FAIL_AUTH): + cls = CLASS_REGISTRATION_FAILURE + if state.get("paused") and cls not in (CLASS_HEALTHY, CLASS_SAFETY_DRIFT): + cls = CLASS_PAUSED + + return Status( + timestamp=now, + tunnel_class=tunnel_class, + cls=cls, + reasons=reasons, + safety_ok=safety_ok, + tunnel_ok=tunnel_ok, + interface_up=interface_up, + peer_present=peer_present, + handshake_epoch=handshake_epoch, + handshake_age_sec=handshake_age, + peer_endpoint=endpoint, + expected_peer=expected_peer, + route_state=route_state, + configured_peer_line=configured_line, + checks=checks, + ) + + +# -------------------------------------------------------------------------- +# PIA API client (stdlib only) +# -------------------------------------------------------------------------- + + +class _PinnedHTTPSConnection(http.client.HTTPSConnection): + """Connect to a fixed IP while verifying TLS against the hostname (SNI).""" + + def __init__(self, host: str, ip: str, port: int, context: ssl.SSLContext, connect_timeout: float, total_timeout: float): + super().__init__(host, port, timeout=connect_timeout, context=context) + self._pinned_ip = ip + self._total_timeout = total_timeout + + def connect(self) -> None: # noqa: D401 + sock = socket.create_connection((self._pinned_ip, self.port), self.timeout) + sock.settimeout(self._total_timeout) + self.sock = self._context.wrap_socket(sock, server_hostname=self.host) + + +class HttpClient: + def __init__(self, cfg: Config): + self.cfg = cfg + + def _system_ctx(self) -> ssl.SSLContext: + return ssl.create_default_context() + + def _pia_ctx(self) -> ssl.SSLContext: + ctx = ssl.create_default_context(cafile=self.cfg.ca_file) + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + return ctx + + def _request(self, conn: http.client.HTTPSConnection, method: str, path: str, body: Optional[bytes], headers: dict) -> tuple[int, bytes]: + try: + conn.request(method, path, body=body, headers=headers) + resp = conn.getresponse() + data = resp.read(4_000_000) + return resp.status, data + except ssl.SSLCertVerificationError as e: + raise ApiError("tls", "certificate verification failed") from e + except ssl.SSLError as e: + raise ApiError("tls", e.__class__.__name__) from e + except socket.timeout as e: + raise ApiError("timeout", "request timed out") from e + except (OSError, http.client.HTTPException) as e: + raise ApiError("network", e.__class__.__name__) from e + finally: + with contextlib.suppress(Exception): + conn.close() + + def get(self, url: str) -> tuple[int, bytes]: + u = urllib.parse.urlsplit(url) + conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx()) + return self._request(conn, "GET", u.path or "/", None, {"User-Agent": "pia-gateway"}) + + def post_multipart(self, url: str, fields: dict[str, str]) -> tuple[int, bytes]: + boundary = "----pia-gateway-" + secrets.token_hex(12) + parts = [] + for k, v in fields.items(): + parts.append(f"--{boundary}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n") + body = ("".join(parts) + f"--{boundary}--\r\n").encode("utf-8") + u = urllib.parse.urlsplit(url) + conn = http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=self.cfg.http_connect_timeout_sec, context=self._system_ctx()) + headers = { + "User-Agent": "pia-gateway", + "Content-Type": f"multipart/form-data; boundary={boundary}", + "Content-Length": str(len(body)), + } + return self._request(conn, "POST", u.path, body, headers) + + def get_pinned(self, hostname: str, ip: str, port: int, path: str, query: dict[str, str]) -> tuple[int, bytes]: + conn = _PinnedHTTPSConnection( + hostname, ip, port, self._pia_ctx(), self.cfg.http_connect_timeout_sec, self.cfg.http_total_timeout_sec + ) + full = path + "?" + urllib.parse.urlencode(query) + return self._request(conn, "GET", full, None, {"User-Agent": "pia-gateway"}) + + +@dataclasses.dataclass +class Candidate: + region_id: str + region_name: str + cn: str + ip: str + + +@dataclasses.dataclass +class PeerInfo: + server_key: str + peer_ip: str + server_port: int + server_ip: str + region_id: str + region_name: str + cn: str + + +_B64KEY = re.compile(r"^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$") +_IPV4 = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$") + + +def valid_wg_key(k: str) -> bool: + return bool(_B64KEY.match(k or "")) + + +def valid_ipv4(s: str) -> bool: + if not _IPV4.match(s or ""): + return False + return all(0 <= int(o) <= 255 for o in s.split(".")) + + +class PiaApi: + def __init__(self, cfg: Config, http: HttpClient, log: Log): + self.cfg = cfg + self.http = http + self.log = log + + def fetch_serverlist(self) -> dict: + status, data = self.http.get(SERVERLIST_URL) + if status != 200: + raise ApiError("http", f"serverlist HTTP {status}") + first = data.split(b"\n", 1)[0] + try: + doc = json.loads(first.decode("utf-8")) + except (ValueError, UnicodeDecodeError) as e: + raise ApiError("malformed", "serverlist is not JSON") from e + if not isinstance(doc, dict) or not isinstance(doc.get("regions"), list): + raise ApiError("malformed", "serverlist lacks a regions list") + return doc + + def build_candidates(self, doc: dict, regions: Iterable[str]) -> list[Candidate]: + by_id = {r.get("id"): r for r in doc["regions"] if isinstance(r, dict)} + out: list[Candidate] = [] + missing, empty = [], [] + for rid in regions: + r = by_id.get(rid) + if r is None: + missing.append(rid) + continue + servers = (r.get("servers") or {}).get("wg") or [] + if not servers: + empty.append(rid) + continue + for s in servers: + cn, ip = s.get("cn"), s.get("ip") + if not cn or not valid_ipv4(ip or ""): + continue + out.append(Candidate(rid, r.get("name") or rid, cn, ip)) + if missing: + raise ApiError("malformed", f"requested regions not in serverlist: {missing}") + if empty: + # A configured fallback region with no WireGuard servers right now + # must not block recovery through the regions that do have them. + self.log.warn("regions without WireGuard servers skipped", regions=empty) + if not out: + raise ApiError("malformed", "no usable WireGuard servers in requested regions") + return out + + def get_token(self, creds: Credentials) -> str: + last_status = None + for attempt in range(3): + try: + status, data = self.http.post_multipart(TOKEN_URL, {"username": creds.pia_user, "password": creds.pia_password}) + except ApiError as e: + if e.kind in ("tls",): + raise + last_status = e.kind + if attempt < 2: + continue + raise + last_status = status + if status == 200: + try: + token = json.loads(data.decode("utf-8")).get("token") + except (ValueError, AttributeError, UnicodeDecodeError) as e: + raise ApiError("malformed", "token response is not JSON") from e + if not token or not isinstance(token, str): + raise ApiError("malformed", "token response has no token") + return token + if status in (401, 403): + raise ApiError("auth", f"token HTTP {status}") + if attempt < 2 and status >= 500: + continue + break + raise ApiError("http", f"token HTTP {last_status}") + + def add_key(self, cand: Candidate, token: str, pubkey: str) -> PeerInfo: + status, data = self.http.get_pinned(cand.cn, cand.ip, ADDKEY_PORT, "/addKey", {"pt": token, "pubkey": pubkey}) + if status != 200: + raise ApiError("http", f"addKey HTTP {status}") + if not data.strip(): + raise ApiError("malformed", "addKey empty body") + try: + doc = json.loads(data.decode("utf-8")) + except (ValueError, UnicodeDecodeError) as e: + raise ApiError("malformed", "addKey body is not JSON") from e + if not isinstance(doc, dict) or doc.get("status") != "OK": + raise ApiError("status", f"addKey status={doc.get('status') if isinstance(doc, dict) else '?'}") + server_key = doc.get("server_key") + peer_ip = doc.get("peer_ip") + port = doc.get("server_port") + if not valid_wg_key(server_key or ""): + raise ApiError("malformed", "addKey server_key invalid") + if not valid_ipv4(peer_ip or ""): + raise ApiError("malformed", "addKey peer_ip invalid") + try: + port = int(port) + except (TypeError, ValueError): + raise ApiError("malformed", "addKey server_port invalid") + if not (1 <= port <= 65535): + raise ApiError("malformed", "addKey server_port out of range") + return PeerInfo(server_key, peer_ip, port, cand.ip, cand.region_id, cand.region_name, cand.cn) + + +# -------------------------------------------------------------------------- +# WireGuard config rendering / validation +# -------------------------------------------------------------------------- + + +def render_wg_conf(cfg: Config, private_key: str, peer: PeerInfo) -> str: + i = cfg.interface + t = cfg.route_table + s = cfg.vlan50_subnet + c = cfg.iptables_chain + lines = [ + "# Managed by pia-gateway (ansible/roles/pia-gateway). Do not edit by hand.", + "# Table = off: routing for this interface lives only in table " + f"{t}, reached only by traffic sourced from {s}.", + "[Interface]", + f"PrivateKey = {private_key}", + f"Address = {peer.peer_ip}/32", + "Table = off", + "", + f"PostUp = ip route replace default dev %i table {t}", + f"PostUp = iptables -I {c} 1 -s {s} -o %i -j ACCEPT", + f"PostUp = iptables -I {c} 1 -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT", + f"PostUp = iptables -t nat -A POSTROUTING -s {s} -o %i -j MASQUERADE", + ] + if cfg.mss_clamp_enabled: + lines.append( + f"PostUp = iptables -t mangle -A FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu" + ) + lines += [ + "", + f"PreDown = ip route replace unreachable default table {t}", + f"PreDown = iptables -t nat -D POSTROUTING -s {s} -o %i -j MASQUERADE 2>/dev/null || true", + f"PreDown = iptables -D {c} -d {s} -i %i -m state --state ESTABLISHED,RELATED -j ACCEPT 2>/dev/null || true", + f"PreDown = iptables -D {c} -s {s} -o %i -j ACCEPT 2>/dev/null || true", + ] + if cfg.mss_clamp_enabled: + lines.append( + f"PreDown = iptables -t mangle -D FORWARD -s {s} -o %i -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu 2>/dev/null || true" + ) + lines += [ + "", + "[Peer]", + f"# PIA WireGuard server actually used — {peer.region_name} ({peer.region_id}), {peer.cn}", + f"PublicKey = {peer.server_key}", + f"Endpoint = {peer.server_ip}:{peer.server_port}", + "AllowedIPs = 0.0.0.0/0", + "PersistentKeepalive = 25", + "", + ] + return "\n".join(lines) + + +def validate_wg_conf(cfg: Config, text: str) -> list[str]: + """Return a list of problems (empty when the staged config is acceptable).""" + problems: list[str] = [] + section = None + kv: dict[str, list[str]] = {} + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + if s.startswith("["): + section = s.lower() + continue + k, _, v = s.partition("=") + kv.setdefault(f"{section}:{k.strip()}", []).append(v.strip()) + if kv.get("[interface]:Table") != ["off"]: + problems.append("Table must be off") + pk = kv.get("[interface]:PrivateKey", [""])[0] + if not valid_wg_key(pk): + problems.append("PrivateKey invalid") + addr = kv.get("[interface]:Address", [""])[0] + if not addr.endswith("/32") or not valid_ipv4(addr[:-3]): + problems.append("Address must be a /32 IPv4") + if kv.get("[peer]:AllowedIPs") != ["0.0.0.0/0"]: + problems.append("AllowedIPs must be 0.0.0.0/0") + if not valid_wg_key(kv.get("[peer]:PublicKey", [""])[0]): + problems.append("peer PublicKey invalid") + ep = kv.get("[peer]:Endpoint", [""])[0] + host, _, port = ep.rpartition(":") + if not valid_ipv4(host) or not port.isdigit(): + problems.append("Endpoint invalid") + postup = kv.get("[interface]:PostUp", []) + predown = kv.get("[interface]:PreDown", []) + if not any(f"table {cfg.route_table}" in p and "default dev %i" in p for p in postup): + problems.append("PostUp lacks table route open") + if not any(f"unreachable default table {cfg.route_table}" in p for p in predown): + problems.append("PreDown lacks unreachable route") + for hook in postup + predown: + if "ip route" in hook and "table" not in hook: + problems.append("hook touches routing without a table qualifier") + if f"-s {cfg.vlan50_subnet}" not in hook and "iptables" in hook and f"-d {cfg.vlan50_subnet}" not in hook: + problems.append("iptables hook not scoped to VLAN 50") + if "-P FORWARD" in hook: + problems.append("hook changes FORWARD policy") + return problems + + +# -------------------------------------------------------------------------- +# Notifier +# -------------------------------------------------------------------------- + + +class Notifier: + def __init__(self, cfg: Config, creds: Credentials, http: HttpClient, log: Log): + self.cfg = cfg + self.creds = creds + self.http = http + self.log = log + + def send(self, level: str, message: str) -> bool: + if not self.cfg.alert_bridge_url or not self.creds.alert_bridge_api_key: + self.log.info("notify skipped (no alert-bridge configured)", level=level, message=message) + return True + u = urllib.parse.urlsplit(self.cfg.alert_bridge_url) + body = json.dumps({"source": self.cfg.alert_source, "message": message, "level": level}).encode("utf-8") + try: + conn_cls = http.client.HTTPSConnection if u.scheme == "https" else http.client.HTTPConnection + conn = conn_cls(u.hostname, u.port or (443 if u.scheme == "https" else 80), timeout=5) + conn.request( + "POST", + u.path or "/", + body=body, + headers={ + "Authorization": f"Bearer {self.creds.alert_bridge_api_key}", + "Content-Type": "application/json", + "Content-Length": str(len(body)), + }, + ) + resp = conn.getresponse() + resp.read(4096) + conn.close() + ok = 200 <= resp.status < 300 + if not ok: + self.log.warn("notify failed", http_status=resp.status) + return ok + except (OSError, http.client.HTTPException) as e: + self.log.warn("notify failed", error=e.__class__.__name__) + return False + + +# -------------------------------------------------------------------------- +# Controller +# -------------------------------------------------------------------------- + + +@dataclasses.dataclass +class RegistrationOutcome: + ok: bool + stage: str + failure_class: Optional[str] = None + detail: Optional[str] = None + region: Optional[str] = None + server_cn: Optional[str] = None + any_api_ok: bool = False + all_api_failed_same_way: bool = False + verified: Optional[Status] = None + + +class Controller: + def __init__(self, cfg: Config, system: System, log: Log, http: Optional[HttpClient] = None, creds: Optional[Credentials] = None): + self.cfg = cfg + self.sys = system + self.log = log + self.http = http or HttpClient(cfg) + self._creds = creds + self.store = StateStore(cfg, system) + self.classifier = Classifier(cfg, system) + self.api = PiaApi(cfg, self.http, log) + + # -- helpers ---------------------------------------------------------- + + @property + def creds(self) -> Credentials: + if self._creds is None: + self._creds = load_credentials(self.cfg.credentials_file, require_pia=True) + return self._creds + + def creds_optional(self) -> Credentials: + if self._creds is None: + try: + self._creds = load_credentials(self.cfg.credentials_file, require_pia=False) + except GatewayError as e: + self.log.warn("credentials unavailable", error=str(e)) + self._creds = Credentials("", "", None) + return self._creds + + def notifier(self) -> Notifier: + return Notifier(self.cfg, self.creds_optional(), self.http, self.log) + + def notify(self, state: dict, level: str, message: str) -> None: + pending = state.setdefault("pending_notifications", []) + pending.append({"level": level, "message": message, "epoch": self.sys.now()}) + del pending[:-20] + self.flush_notifications(state) + + def flush_notifications(self, state: dict) -> None: + pending = state.get("pending_notifications") or [] + n = self.notifier() + remaining = [] + for item in pending: + if not n.send(item["level"], item["message"]): + remaining.append(item) + state["pending_notifications"] = remaining + + def lock(self, wait_sec: float = 0) -> Lock: + lk = Lock(self.cfg.lock_file, self.sys) + lk.acquire(wait_sec) + return lk + + def write_status_outputs(self, st: Status, state: dict) -> None: + cfg = self.cfg + rec = state.get("recovery", {}) + doc = { + "status": st.to_dict(), + "controller": { + "paused": bool(state.get("paused")), + "consecutive_failures": state.get("consecutive_failures", 0), + "recovery_in_progress": rec.get("in_progress", False), + "backoff_index": rec.get("backoff_index", 0), + "next_retry_epoch": rec.get("next_retry_epoch"), + "last_failure_class": rec.get("last_failure_class"), + "last_verified_registration": state.get("last_verified_registration"), + "last_recovery": state.get("last_recovery"), + "emergency_close_active": state.get("emergency_close_active", False), + }, + } + try: + self.sys.makedirs(cfg.state_dir, 0o700) + self.sys.write_atomic(cfg.status_file, json.dumps(doc, indent=1, sort_keys=True) + "\n", 0o644) + self.sys.write_atomic(cfg.metrics_file, self.render_metrics(st, state), 0o644) + except OSError as e: + self.log.warn("status/metrics write failed", error=e.__class__.__name__) + + def render_metrics(self, st: Status, state: dict) -> str: + rec = state.get("recovery", {}) + lvr = state.get("last_verified_registration") or {} + lr = state.get("last_recovery") or {} + m = [ + ("pia_gateway_check_timestamp_seconds", st.timestamp), + ("pia_gateway_healthy", 1 if st.cls == CLASS_HEALTHY else 0), + ("pia_gateway_tunnel_ok", 1 if st.tunnel_ok else 0), + ("pia_gateway_safety_ok", 1 if st.safety_ok else 0), + ("pia_gateway_handshake_age_seconds", st.handshake_age_sec if st.handshake_age_sec is not None else -1), + ("pia_gateway_consecutive_failures", state.get("consecutive_failures", 0)), + ("pia_gateway_paused", 1 if state.get("paused") else 0), + ("pia_gateway_recovery_in_backoff", 1 if rec.get("next_retry_epoch") else 0), + ("pia_gateway_recovery_backoff_index", rec.get("backoff_index", 0)), + ("pia_gateway_last_verified_registration_timestamp_seconds", lvr.get("epoch") or 0), + ("pia_gateway_last_recovery_timestamp_seconds", lr.get("epoch") or 0), + ("pia_gateway_emergency_close_active", 1 if state.get("emergency_close_active") else 0), + ] + out = [f"# pia-gateway class={st.cls} tunnel_class={st.tunnel_class}"] + for name, val in m: + out.append(f"# TYPE {name} gauge") + out.append(f"{name} {val}") + return "\n".join(out) + "\n" + + def refresh_boot_grace(self, state: dict) -> None: + bid = self.sys.boot_id() + if state.get("boot_id") != bid: + state["boot_id"] = bid + state["boot_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec + state["restart_grace_until"] = 0 + self.log.info("new boot detected; startup grace applied", grace_sec=self.cfg.startup_grace_sec) + self.guard_clock_jump(state) + + def guard_clock_jump(self, state: dict) -> None: + """Persisted wall-clock deadlines further out than they could ever have been + scheduled mean the clock went backwards; clear them rather than stall.""" + now = self.sys.now() + horizon = self.cfg.backoff_schedule_sec[-1] * 2 + 300 + for section in ("recovery", "renewal"): + nxt = state.get(section, {}).get("next_retry_epoch") + if nxt and nxt - now > horizon: + self.log.warn("clock jump detected; clearing stale retry deadline", section=section, was_in_sec=nxt - now) + state[section]["next_retry_epoch"] = None + for key in ("boot_grace_until", "restart_grace_until"): + if (state.get(key) or 0) - now > self.cfg.startup_grace_sec * 2 + 300: + state[key] = 0 + + def current_status(self, state: dict) -> Status: + return self.classifier.classify(state) + + def log_status(self, st: Status) -> None: + self.log.info( + "status", + **{"class": st.cls}, + tunnel_class=st.tunnel_class, + handshake_age_sec=st.handshake_age_sec, + route=st.route_state, + safety_ok=st.safety_ok, + tunnel_ok=st.tunnel_ok, + endpoint=st.peer_endpoint, + reasons=st.reasons, + ) + if st.configured_peer_line: + self.log.info("configured-peer", line=st.configured_peer_line) + + # -- emergency closing ------------------------------------------------ + + def emergency_rule_argv(self, op: str) -> list[str]: + return ["iptables", op, "FORWARD"] + (["1"] if op == "-I" else []) + [ + "-s", self.cfg.vlan50_subnet, "-m", "comment", "--comment", EMERGENCY_COMMENT, "-j", "DROP", + ] + + def emergency_close(self, state: dict, st: Status) -> bool: + """Insert an emergency DROP first in FORWARD and close table pia. Returns effectiveness.""" + cfg = self.cfg + fwd0 = self.sys.run(["iptables", "-S", "FORWARD"]) + rules0 = [l for l in fwd0.out.splitlines() if l.startswith("-A FORWARD")] if fwd0.rc == 0 else [] + already_first = bool(rules0) and _is_emergency_rule(rules0[0]) + if not already_first: + # `-C` (existence) is not enough here: something could have + # prepended a rule ahead of ours (e.g. a `ufw reload` outside + # this role's own handler ordering) without removing it, in + # which case it still exists but is no longer FIRST — and a + # rule that isn't first can be pre-empted by whatever now sits + # above it. Reposition explicitly: delete wherever it is (`-D` + # matches by rule spec, not position), then insert at 1. + if any(_is_emergency_rule(r) for r in rules0): + d = self.sys.run(self.emergency_rule_argv("-D")) + if d.rc != 0: + self.log.error("emergency DROP removal (repositioning) failed", rc=d.rc) + ins = self.sys.run(self.emergency_rule_argv("-I")) + if ins.rc != 0: + self.log.error("emergency DROP insert failed", rc=ins.rc) + r2 = self.sys.run(["ip", "route", "replace", "unreachable", "default", "table", cfg.route_table]) + if r2.rc != 0: + self.log.error("emergency unreachable route failed", rc=r2.rc) + fwd = self.sys.run(["iptables", "-S", "FORWARD"]) + rules = [l for l in fwd.out.splitlines() if l.startswith("-A FORWARD")] + effective = fwd.rc == 0 and bool(rules) and _is_emergency_rule(rules[0]) and r2.rc == 0 + state["emergency_close_active"] = True + if effective: + self.log.error("SAFETY DRIFT: emergency close applied", reasons=st.reasons) + self.notify(state, "error", f"pia-gateway SAFETY DRIFT: kill switch invariants failed ({'; '.join(st.reasons)}). " + "Emergency VLAN 50 DROP applied; automatic recovery halted until repaired.") + else: + self.log.error("SAFETY DRIFT: emergency close NOT verified effective", reasons=st.reasons) + self.notify(state, "error", "pia-gateway CRITICAL: safety drift and the emergency close could not be verified " + f"effective ({'; '.join(st.reasons)}). Manual intervention required NOW.") + return effective + + def emergency_clear_if_repaired(self, state: dict, st: Status) -> None: + if not state.get("emergency_close_active"): + return + if not st.safety_ok: + return + r = self.sys.run(self.emergency_rule_argv("-C")) + if r.rc == 0: + d = self.sys.run(self.emergency_rule_argv("-D")) + if d.rc != 0: + self.log.error("emergency DROP removal failed", rc=d.rc) + return + state["emergency_close_active"] = False + self.log.info("kill switch repaired; emergency DROP removed") + self.notify(state, "info", "pia-gateway: kill switch invariants restored; emergency DROP removed.") + + # -- key handling ----------------------------------------------------- + + def _read_private_key(self) -> str: + key = self.sys.read_text(self.cfg.wg_key).strip() + if not valid_wg_key(key): + raise GatewayError("existing private key file is not a valid WireGuard key") + return key + + def _generate_key(self) -> str: + r = self.sys.run(["wg", "genkey"]) + key = r.out.strip() + if r.rc != 0 or not valid_wg_key(key): + raise GatewayError("wg genkey failed") + return key + + def _pubkey(self, private_key: str) -> str: + r = self.sys.run(["wg", "pubkey"], stdin=private_key + "\n") + pub = r.out.strip() + if r.rc != 0 or not valid_wg_key(pub): + raise GatewayError("wg pubkey failed") + return pub + + def clean_abandoned_stages(self) -> int: + cfg = self.cfg + n = 0 + prefix_key = f"{cfg.interface}.key.new" + prefix_conf = f"{cfg.interface}.conf.stage" + for name in self.sys.listdir(cfg.config_dir): + if name.startswith(prefix_key) or name.startswith(prefix_conf): + self.sys.unlink(os.path.join(cfg.config_dir, name)) + n += 1 + if n: + self.log.info("removed abandoned staged files", count=n) + return n + + def prune_backups(self) -> None: + cfg = self.cfg + for base in (f"{cfg.interface}.conf.bak-", f"{cfg.interface}.key.bak-"): + names = [n for n in self.sys.listdir(cfg.config_dir) if n.startswith(base)] + names.sort() + for old in names[: max(0, len(names) - cfg.backup_keep)]: + self.sys.unlink(os.path.join(cfg.config_dir, old)) + + def _ts(self) -> str: + return _dt.datetime.fromtimestamp(self.sys.now(), _dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + # -- activation and verification --------------------------------------- + + def assert_killswitch_active(self) -> None: + r = self.sys.run(["systemctl", "is-active", self.cfg.killswitch_unit]) + if r.out.strip() != "active": + raise SafetyError(f"{self.cfg.killswitch_unit} is not active; refusing to activate the tunnel") + + def restart_interface(self, state: dict) -> bool: + state["restart_grace_until"] = self.sys.now() + self.cfg.startup_grace_sec + r = self.sys.run(["systemctl", "restart", self.cfg.wg_unit], timeout=120) + if r.rc != 0: + self.log.error("restart failed", unit=self.cfg.wg_unit, rc=r.rc) + if "already exists" in (r.err or ""): + self.log.warn("orphaned interface detected; deleting and retrying start") + self.sys.run(["ip", "link", "delete", "dev", self.cfg.interface]) + r = self.sys.run(["systemctl", "start", self.cfg.wg_unit], timeout=120) + return r.rc == 0 + return False + return True + + def wait_for_verified(self, state: dict, since_epoch: int, timeout_sec: int, expected_peer: Optional[str] = None) -> Optional[Status]: + """Poll until two consecutive healthy classifications with a handshake newer than since_epoch.""" + start = self.sys.monotonic() + healthy_streak = 0 + last: Optional[Status] = None + while True: + st = self.classifier.classify({**state, "restart_grace_until": 0, "boot_grace_until": 0, "paused": None}) + last = st + # Strictly greater, not >=: `wg show`/`since_epoch` are both + # whole seconds, so a handshake reported at the exact same + # second as `since_epoch` is indistinguishable from one that + # predates activation and just happens to share that second + # due to truncation. Treating it as fresh would let a stale, + # pre-activation handshake pass verification — exactly the + # "addKey said OK but nothing actually changed" failure mode + # this whole check exists to rule out. A handshake that is + # genuinely new but lands in the same second as `since_epoch` + # is simply not counted yet; the next 5s poll picks it up. + fresh = st.handshake_epoch is not None and st.handshake_epoch > since_epoch + peer_ok = expected_peer is None or st.expected_peer == expected_peer + if st.tunnel_class == CLASS_HEALTHY and fresh and peer_ok: + healthy_streak += 1 + if healthy_streak >= 2: + return st + else: + healthy_streak = 0 + if st.tunnel_class == CLASS_SAFETY_DRIFT: + return None + if self.sys.monotonic() - start >= timeout_sec: + return None + self.sys.sleep(5) + + def backup_current_config(self) -> Optional[str]: + cfg = self.cfg + if not self.sys.exists(cfg.wg_conf): + return None + dst = os.path.join(cfg.config_dir, f"{cfg.interface}.conf.bak-{self._ts()}-{os.getpid()}") + self.sys.copy_file(cfg.wg_conf, dst, 0o600) + return dst + + def activate_config(self, state: dict, staged_path: str, expected_peer: str, since_epoch: int) -> Optional[Status]: + cfg = self.cfg + self.assert_killswitch_active() + self.sys.rename(staged_path, cfg.wg_conf) + if not self.restart_interface(state): + return None + return self.wait_for_verified(state, since_epoch, cfg.activation_wait_sec, expected_peer) + + def stage_conf(self, text: str) -> str: + problems = validate_wg_conf(self.cfg, text) + if problems: + raise GatewayError("staged config rejected: " + "; ".join(problems)) + path = os.path.join(self.cfg.config_dir, f"{self.cfg.interface}.conf.stage-{self._ts()}-{os.getpid()}") + self.sys.write_atomic(path, text, 0o600) + return path + + def restore_previous_config(self, state: dict, backup_path: Optional[str]) -> None: + cfg = self.cfg + if not backup_path or not self.sys.exists(backup_path): + self.log.warn("no previous config to restore") + return + self.sys.copy_file(backup_path, cfg.wg_conf, 0o600) + self.log.warn("restored previous config (best effort)", backup=os.path.basename(backup_path)) + self.restart_interface(state) + + # -- registration transaction ---------------------------------------- + + def registration_transaction( + self, + state: dict, + mode: str, + regions: Optional[list[str]], + via: str, + max_candidates: Optional[int] = None, + deadline_mono: Optional[float] = None, + ) -> RegistrationOutcome: + """Same-key or new-key registration: API -> stage -> activate -> verify. + + Never opens the gateway unless verification passes. On total failure the + previous config is restored (best effort) and the outcome says so. + """ + cfg = self.cfg + self.clean_abandoned_stages() + if mode not in ("same-key", "new-key"): + raise GatewayError(f"unknown registration mode {mode}") + + # key material + staged_key_path: Optional[str] = None + if mode == "new-key": + private_key = self._generate_key() + staged_key_path = os.path.join(cfg.config_dir, f"{cfg.interface}.key.new.{self._ts()}.{os.getpid()}") + self.sys.write_atomic(staged_key_path, private_key + "\n", 0o600) + elif self.sys.exists(cfg.wg_key): + private_key = self._read_private_key() + else: + private_key = self._generate_key() + self.sys.write_atomic(cfg.wg_key, private_key + "\n", 0o600) + self.log.info("generated initial private key") + pubkey = self._pubkey(private_key) + + # API path + try: + doc = self.api.fetch_serverlist() + except ApiError as e: + fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION + return RegistrationOutcome(False, "serverlist", fc, str(e)) + try: + cands = self.api.build_candidates(doc, regions or cfg.region_candidates) + except ApiError as e: + return RegistrationOutcome(False, "serverlist", FAIL_REGISTRATION, str(e)) + if max_candidates: + cands = cands[:max_candidates] + try: + token = self.api.get_token(self.creds) + except ApiError as e: + if e.kind == "auth": + return RegistrationOutcome(False, "token", FAIL_AUTH, str(e)) + fc = FAIL_HOST_CONNECTIVITY if e.kind in ("network", "timeout") else FAIL_REGISTRATION + return RegistrationOutcome(False, "token", fc, str(e)) + + any_api_ok = False + api_fail_kinds: list[str] = [] + last_detail = None + original_backup = self.backup_current_config() + for cand in cands: + if deadline_mono is not None and self.sys.monotonic() >= deadline_mono: + last_detail = "recovery time budget exhausted" + break + try: + peer = self.api.add_key(cand, token, pubkey) + except ApiError as e: + api_fail_kinds.append(e.kind) + last_detail = f"{cand.region_id}/{cand.cn}: {e}" + self.log.warn("addKey failed", region=cand.region_id, server=cand.cn, kind=e.kind, detail=e.detail) + continue + any_api_ok = True + self.log.info("addKey accepted", region=cand.region_id, server=cand.cn) + text = render_wg_conf(cfg, private_key, peer) + try: + staged = self.stage_conf(text) + except GatewayError as e: + last_detail = str(e) + self.log.warn("staged config rejected", region=cand.region_id, server=cand.cn, detail=str(e)) + continue + since = self.sys.now() + try: + verified = self.activate_config(state, staged, peer.server_key, since) + except SafetyError as e: + self.sys.unlink(staged) + return RegistrationOutcome(False, "activate", FAIL_SAFETY, str(e)) + if verified is not None: + if staged_key_path: + if self.sys.exists(cfg.wg_key): + self.sys.copy_file(cfg.wg_key, os.path.join(cfg.config_dir, f"{cfg.interface}.key.bak-{self._ts()}"), 0o600) + self.sys.rename(staged_key_path, cfg.wg_key) + self.prune_backups() + epoch = self.sys.now() + reg = { + "epoch": epoch, + "region": peer.region_id, + "region_name": peer.region_name, + "server_cn": peer.cn, + "server_ip": peer.server_ip, + "server_port": peer.server_port, + "server_key": peer.server_key, + "peer_ip": peer.peer_ip, + "pubkey": pubkey, + "mode": mode, + "via": via, + } + state["last_verified_registration"] = {k: reg[k] for k in ("epoch", "region", "region_name", "server_cn", "server_ip", "pubkey", "mode", "via")} + state["active_registration"] = reg + state["renewal"] = {"backoff_index": 0, "next_retry_epoch": None, "last_failure_epoch": None} + self.log.info("registration verified", region=peer.region_id, server=peer.cn, mode=mode, via=via, handshake_age_sec=verified.handshake_age_sec) + return RegistrationOutcome(True, "register", None, None, peer.region_id, peer.cn, True, verified=verified) + last_detail = f"{cand.region_id}/{cand.cn}: activated but no verified handshake" + self.log.warn("activation not verified; trying next candidate", region=cand.region_id, server=cand.cn) + + # all candidates failed + if staged_key_path: + self.sys.unlink(staged_key_path) + self.restore_previous_config(state, original_backup) + uniform = bool(api_fail_kinds) and not any_api_ok and len(set(api_fail_kinds)) == 1 + fc = FAIL_REGISTRATION + if uniform and api_fail_kinds[0] in ("network", "timeout"): + fc = FAIL_HOST_CONNECTIVITY + return RegistrationOutcome(False, "register", fc, last_detail, any_api_ok=any_api_ok, all_api_failed_same_way=uniform) + + # -- recovery cycle --------------------------------------------------- + + def schedule_backoff(self, state: dict, section: str, failure_class: str, detail: Optional[str]) -> int: + cfg = self.cfg + sec = state[section] + idx = min(sec.get("backoff_index", 0), len(cfg.backoff_schedule_sec) - 1) + base = cfg.backoff_schedule_sec[idx] + jitter = int(base * cfg.backoff_jitter_fraction * (2 * self.sys.random_jitter() - 1)) + delay = max(60, min(cfg.backoff_schedule_sec[-1], base + jitter)) + sec["backoff_index"] = min(idx + 1, len(cfg.backoff_schedule_sec) - 1) + sec["next_retry_epoch"] = self.sys.now() + delay + sec["last_failure_epoch"] = self.sys.now() + if section == "recovery": + sec["last_failure_class"] = failure_class + sec["last_failure_detail"] = detail + return delay + + def recovery_cycle(self, state: dict, st: Status) -> bool: + cfg = self.cfg + rec = state["recovery"] + start_mono = self.sys.monotonic() + deadline = start_mono + cfg.recovery_max_duration_sec + pre_age = st.handshake_age_sec + evidence = { + "pre_fix_class": st.tunnel_class, + "pre_fix_handshake_age_sec": pre_age, + "pre_fix_endpoint": st.peer_endpoint, + "pre_fix_reasons": st.reasons, + } + first_cycle = not rec.get("in_progress") + rec["in_progress"] = True + rec["cycles"] = rec.get("cycles", 0) + 1 + self.log.warn("recovery started", **evidence) + if first_cycle: + self.notify(state, "warn", f"pia-gateway: tunnel {st.tunnel_class} (handshake age {pre_age}s); automatic recovery started.") + + def finish_ok(stage: str, region: Optional[str], server: Optional[str], verified: Status) -> bool: + elapsed = int(self.sys.monotonic() - start_mono) + state["last_recovery"] = { + "epoch": self.sys.now(), + "stage": stage, + "region": region, + "server_cn": server, + "elapsed_sec": elapsed, + **evidence, + } + state["recovery"] = {**default_state()["recovery"]} + state["consecutive_failures"] = 0 + state["last_healthy_epoch"] = self.sys.now() + self.log.info("recovered", stage=stage, region=region, server=server, elapsed_sec=elapsed, handshake_age_sec=verified.handshake_age_sec) + self.notify(state, "info", f"pia-gateway: recovered via {stage} in {elapsed}s" + (f" ({region}/{server})" if server else "") + ".") + return True + + def finish_fail(failure_class: str, detail: Optional[str]) -> bool: + delay = self.schedule_backoff(state, "recovery", failure_class, detail) + self.log.error("recovery failed", failure_class=failure_class, detail=detail, retry_in_sec=delay) + level = "error" if failure_class in (FAIL_AUTH, FAIL_SAFETY) else "warn" + self.notify(state, level, f"pia-gateway: recovery failed ({failure_class}: {detail}); gateway stays closed, retry in {delay // 60} min.") + return False + + # stage 1: restart + since = self.sys.now() + if not self.restart_interface(state): + self.log.warn("restart stage failed; escalating to registration") + else: + verified = self.wait_for_verified(state, since, cfg.restart_wait_sec) + if verified is not None: + return finish_ok("restart", None, None, verified) + self.log.warn("restart did not restore a fresh handshake; escalating to registration") + + if self.sys.monotonic() >= deadline: + return finish_fail(FAIL_REGISTRATION, "time budget exhausted after restart stage") + + try: + # stage 2: same-key registration + out = self.registration_transaction(state, "same-key", None, "recover", cfg.max_candidates_per_cycle, deadline) + if out.ok: + return finish_ok("reregister", out.region, out.server_cn, out.verified) + if out.failure_class in (FAIL_AUTH, FAIL_HOST_CONNECTIVITY, FAIL_SAFETY): + return finish_fail(out.failure_class, out.detail) + if self.sys.monotonic() >= deadline: + return finish_fail(out.failure_class or FAIL_REGISTRATION, "time budget exhausted after same-key stage") + + # stage 3: one staged fresh-key attempt, only when the API accepted + # the old key but no activated candidate handshook (evidence of + # server-side key rejection), never on uniform API failures. + if out.any_api_ok and not out.all_api_failed_same_way: + self.log.warn("same-key registration accepted but never handshook; attempting one staged key rotation") + out2 = self.registration_transaction(state, "new-key", None, "recover", cfg.max_rotation_candidates, deadline) + if out2.ok: + return finish_ok("rotate", out2.region, out2.server_cn, out2.verified) + return finish_fail(out2.failure_class or FAIL_REGISTRATION, out2.detail) + return finish_fail(out.failure_class or FAIL_REGISTRATION, out.detail) + except GatewayError as e: + # e.g. credentials file missing/unreadable, wg tooling failure — + # still enter backoff so the restart stage is not re-run every 60s. + return finish_fail(FAIL_SAFETY if isinstance(e, SafetyError) else FAIL_REGISTRATION, str(e)) + + # -- commands --------------------------------------------------------- + + def cmd_check(self, as_json: bool = False) -> int: + try: + state = self.store.load() + except StateCorruptError as e: + self.log.error("state corrupt", error=str(e)) + state = default_state() + st = self.current_status(state) + if as_json: + print(json.dumps(st.to_dict(), sort_keys=True)) + else: + self.log_status(st) + self.write_status_outputs(st, state) + return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY + + def cmd_recover(self, force: bool = False) -> int: + cfg = self.cfg + try: + lk = self.lock(0) + except LockBusyError as e: + self.log.info("skipping; another action holds the lock", detail=str(e)) + return EXIT_OK + state = None + try: + # The lock is acquired BEFORE state is read, not after — every + # mutating command follows this order now (see cmd_pause for + # the race it closes: a load taken before the lock can go + # stale while this process waits, so saving it back afterwards + # would silently clobber whatever the lock-holder just wrote). + try: + state = self.store.load() + except StateCorruptError as e: + self.log.error("state corrupt; recovery halted (fail closed)", error=str(e)) + tmp = default_state() + self.notify(tmp, "error", f"pia-gateway: state file corrupt ({e}); automatic recovery halted until `state-reset --confirm`.") + return EXIT_STATE_ERROR + self.refresh_boot_grace(state) + self.flush_notifications(state) + st = self.current_status(state) + self.log_status(st) + state["last_check"] = {"epoch": st.timestamp, "class": st.cls, "reasons": st.reasons, "handshake_age_sec": st.handshake_age_sec} + cycles_before = state["recovery"].get("cycles", 0) + emergency_before = state.get("emergency_close_active", False) + rc = self._decide(state, st, force) + acted = state["recovery"].get("cycles", 0) != cycles_before or state.get("emergency_close_active") != emergency_before + self.write_status_outputs(self.current_status(state) if acted else st, state) + return rc + finally: + if state is not None: + self.store.save(state) + lk.release() + + def _decide(self, state: dict, st: Status, force: bool) -> int: + cfg = self.cfg + rec = state["recovery"] + if st.tunnel_class == CLASS_SAFETY_DRIFT: + state["consecutive_failures"] = 0 + self.emergency_close(state, st) + return EXIT_SAFETY + self.emergency_clear_if_repaired(state, st) + + if st.tunnel_class == CLASS_UNKNOWN: + state["consecutive_unknown"] = state.get("consecutive_unknown", 0) + 1 + if state["consecutive_unknown"] == cfg.failure_threshold: + self.notify(state, "warn", f"pia-gateway: status unknown for {cfg.failure_threshold} checks ({'; '.join(st.reasons)}); no action taken.") + return EXIT_UNHEALTHY + state["consecutive_unknown"] = 0 + + if st.tunnel_class == CLASS_HEALTHY: + if rec.get("in_progress") or rec.get("next_retry_epoch"): + self.log.info("tunnel healthy again without a completed recovery cycle; clearing backoff") + self.notify(state, "info", "pia-gateway: tunnel healthy again; backoff cleared.") + state["recovery"] = {**default_state()["recovery"]} + state["consecutive_failures"] = 0 + state["last_healthy_epoch"] = st.timestamp + return EXIT_OK + + if state.get("paused"): + self.log.warn("paused; no recovery action", since=state["paused"].get("since"), reason=state["paused"].get("reason")) + return EXIT_UNHEALTHY + + if st.cls == CLASS_BOOT_GRACE and not force: + self.log.info("within startup/restart grace; not counting as a failure") + return EXIT_UNHEALTHY + + state["consecutive_failures"] = state.get("consecutive_failures", 0) + 1 + if state["consecutive_failures"] < cfg.failure_threshold and not force: + self.log.warn("unhealthy; below threshold", consecutive=state["consecutive_failures"], threshold=cfg.failure_threshold) + return EXIT_UNHEALTHY + if rec.get("next_retry_epoch") and st.timestamp < rec["next_retry_epoch"] and not force: + self.log.warn("in backoff; not retrying yet", retry_in_sec=rec["next_retry_epoch"] - st.timestamp, last_failure=rec.get("last_failure_class")) + return EXIT_UNHEALTHY + if force: + rec["next_retry_epoch"] = None + ok = self.recovery_cycle(state, st) + return EXIT_OK if ok else EXIT_UNHEALTHY + + def cmd_renew(self, force_due: bool = False) -> int: + cfg = self.cfg + try: + lk = self.lock(0) + except LockBusyError: + self.log.info("renewal skipped; another action holds the lock") + return EXIT_OK + state = None + try: + try: + state = self.store.load() + except StateCorruptError as e: + self.log.error("state corrupt; renewal halted", error=str(e)) + return EXIT_STATE_ERROR + if state.get("paused"): + self.log.info("paused; renewal skipped") + return EXIT_OK + st = self.current_status(state) + if st.tunnel_class != CLASS_HEALTHY: + self.log.info("tunnel not healthy; renewal defers to recovery", **{"class": st.tunnel_class}) + return EXIT_OK + ren = state["renewal"] + now = self.sys.now() + if ren.get("next_retry_epoch") and now < ren["next_retry_epoch"] and not force_due: + self.log.info("renewal in backoff", retry_in_sec=ren["next_retry_epoch"] - now) + return EXIT_OK + lvr = state.get("last_verified_registration") + due = force_due or lvr is None or (now - int(lvr.get("epoch", 0))) >= cfg.renewal_after_days * 86400 + if not due: + self.log.info("renewal not due", age_days=(now - int(lvr["epoch"])) // 86400, after_days=cfg.renewal_after_days) + return EXIT_OK + self.log.info("renewal due; re-registering with the existing key", baseline_missing=lvr is None) + out = self.registration_transaction(state, "same-key", None, "renew", cfg.max_candidates_per_cycle) + if out.ok: + self.log.info("renewal verified", region=out.region, server=out.server_cn) + return EXIT_OK + delay = self.schedule_backoff(state, "renewal", out.failure_class or FAIL_REGISTRATION, out.detail) + self.log.error("renewal failed; previous registration timestamp unchanged", failure_class=out.failure_class, detail=out.detail, retry_in_sec=delay) + self.notify(state, "warn", f"pia-gateway: scheduled renewal failed ({out.failure_class}: {out.detail}); retry in {delay // 60} min.") + return EXIT_UNHEALTHY + finally: + if state is not None: + self.store.save(state) + lk.release() + + def cmd_register(self, mode: str, region: Optional[str], via: str) -> int: + cfg = self.cfg + try: + lk = self.lock(cfg.lock_wait_sec) + except LockBusyError as e: + self.log.error("could not acquire lock", detail=str(e)) + return EXIT_LOCKED + state = None + try: + try: + state = self.store.load() + except StateCorruptError as e: + self.log.error("state corrupt; refusing to register", error=str(e)) + return EXIT_STATE_ERROR + self.refresh_boot_grace(state) + st = self.current_status(state) + if not st.safety_ok: + self.log.error("safety invariants failing; refusing to register", reasons=st.reasons) + return EXIT_SAFETY + regions = [region] if region else None + out = self.registration_transaction(state, mode, regions, via) + summary = {"ok": out.ok, "stage": out.stage, "failure_class": out.failure_class, "detail": out.detail, "region": out.region, "server_cn": out.server_cn} + print(json.dumps(summary, sort_keys=True)) + if out.ok: + state["consecutive_failures"] = 0 + state["recovery"] = {**default_state()["recovery"]} + return EXIT_OK + return EXIT_UNHEALTHY + finally: + if state is not None: + self.store.save(state) + lk.release() + + def cmd_rewrite_config(self) -> int: + cfg = self.cfg + try: + lk = self.lock(cfg.lock_wait_sec) + except LockBusyError as e: + self.log.error("could not acquire lock", detail=str(e)) + return EXIT_LOCKED + state = None + try: + try: + state = self.store.load() + except StateCorruptError as e: + self.log.error("state corrupt; refusing to rewrite", error=str(e)) + return EXIT_STATE_ERROR + reg = state.get("active_registration") + if not reg: + self.log.error("no verified registration recorded; run `pia-gateway register` instead") + return EXIT_USAGE + st = self.current_status(state) + if not st.safety_ok: + self.log.error("safety invariants failing; refusing to rewrite", reasons=st.reasons) + return EXIT_SAFETY + peer = PeerInfo(reg["server_key"], reg["peer_ip"], int(reg["server_port"]), reg["server_ip"], reg["region"], reg.get("region_name", reg["region"]), reg["server_cn"]) + private_key = self._read_private_key() + if self._pubkey(private_key) != reg.get("pubkey"): + self.log.error("on-disk key does not match the recorded registration; run register") + return EXIT_USAGE + staged = self.stage_conf(render_wg_conf(cfg, private_key, peer)) + backup = self.backup_current_config() + since = self.sys.now() + verified = self.activate_config(state, staged, peer.server_key, since) + if verified is None: + self.log.error("rewritten config did not verify; restoring previous") + self.restore_previous_config(state, backup) + return EXIT_UNHEALTHY + self.prune_backups() + self.log.info("config rewritten and verified") + return EXIT_OK + finally: + if state is not None: + self.store.save(state) + lk.release() + + def cmd_status(self) -> int: + try: + state = self.store.load() + except StateCorruptError as e: + print(f"STATE CORRUPT: {e}") + return EXIT_STATE_ERROR + st = self.current_status(state) + rec = state["recovery"] + now = self.sys.now() + lines = [ + f"class: {st.cls} (tunnel: {st.tunnel_class})", + f"reasons: {'; '.join(st.reasons) or '-'}", + f"handshake age: {st.handshake_age_sec if st.handshake_age_sec is not None else '-'}s (max {self.cfg.handshake_max_age_sec}s)", + f"endpoint: {st.peer_endpoint or '-'}", + f"route table {self.cfg.route_table}: {st.route_state}", + f"safety ok: {st.safety_ok}", + f"paused: {state.get('paused') or False}", + f"consecutive failures: {state.get('consecutive_failures', 0)}", + f"recovery in progress: {rec.get('in_progress')}", + f"backoff next retry: {_rel(rec.get('next_retry_epoch'), now)} (last failure: {rec.get('last_failure_class') or '-'})", + f"last verified reg.: {_rel((state.get('last_verified_registration') or {}).get('epoch'), now)} " + f"{_reg_summary(state.get('last_verified_registration'))}", + f"last recovery: {_rel((state.get('last_recovery') or {}).get('epoch'), now)} " + f"{(state.get('last_recovery') or {}).get('stage') or ''}", + f"emergency close active: {state.get('emergency_close_active', False)}", + f"last check: {_rel((state.get('last_check') or {}).get('epoch'), now)}", + ] + if st.configured_peer_line: + lines.append(f"configured peer: {st.configured_peer_line}") + print("\n".join(lines)) + return EXIT_OK if st.cls == CLASS_HEALTHY else EXIT_UNHEALTHY + + def cmd_pause(self, reason: str) -> int: + # Same lock as recover/register/renew/rewrite-config, taken BEFORE + # the state load: pause has to wait for any in-flight action to + # finish (and save) before it reads state itself, or a recovery + # pass that read state before pause ran could save its own + # (unpaused) copy back afterwards and silently erase the pause. + try: + lk = self.lock(self.cfg.lock_wait_sec) + except LockBusyError as e: + self.log.error("could not acquire lock", detail=str(e)) + return EXIT_LOCKED + try: + try: + state = self.store.load() + except StateCorruptError as e: + self.log.error("state corrupt; pause not recorded", error=str(e)) + return EXIT_STATE_ERROR + state["paused"] = {"since": self.sys.now(), "reason": reason or "operator"} + self.store.save(state) + self.log.info("paused: recovery and renewal suppressed; kill switch unchanged", reason=reason) + return EXIT_OK + finally: + lk.release() + + def cmd_resume(self) -> int: + try: + lk = self.lock(self.cfg.lock_wait_sec) + except LockBusyError as e: + self.log.error("could not acquire lock", detail=str(e)) + return EXIT_LOCKED + try: + try: + state = self.store.load() + except StateCorruptError as e: + self.log.error("state corrupt; resume not recorded", error=str(e)) + return EXIT_STATE_ERROR + state["paused"] = None + state["consecutive_failures"] = 0 + state["recovery"]["next_retry_epoch"] = None + self.store.save(state) + self.log.info("resumed") + return EXIT_OK + finally: + lk.release() + + def cmd_retry_now(self) -> int: + return self.cmd_recover(force=True) + + def cmd_state_reset(self, confirm: bool) -> int: + if not confirm: + self.log.error("refusing without --confirm") + return EXIT_USAGE + try: + lk = self.lock(self.cfg.lock_wait_sec) + except LockBusyError as e: + self.log.error("could not acquire lock", detail=str(e)) + return EXIT_LOCKED + try: + old = None + if self.sys.exists(self.cfg.state_file): + old = self.cfg.state_file + f".corrupt-{self._ts()}" + self.sys.rename(self.cfg.state_file, old) + self.store.save(default_state()) + self.log.warn("state reset", previous_moved_to=old) + return EXIT_OK + finally: + lk.release() + + def cmd_notify(self, level: str, message: str) -> int: + ok = self.notifier().send(level, message) + return EXIT_OK if ok else EXIT_UNHEALTHY + + +def _rel(epoch: Optional[int], now: int) -> str: + if not epoch: + return "never" + d = now - int(epoch) + if d < 0: + return f"in {-d}s" + if d < 3600: + return f"{d // 60}m ago" + if d < 86400: + return f"{d // 3600}h ago" + return f"{d // 86400}d ago" + + +def _reg_summary(reg: Optional[dict]) -> str: + if not reg: + return "" + return f"({reg.get('region')}/{reg.get('server_cn')}, {reg.get('mode')}, via {reg.get('via')})" + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="pia-gateway", description=__doc__.split("\n\n")[0]) + p.add_argument("--config", default="/etc/pia-gateway/config.json") + p.add_argument("--json-log", action="store_true", help="emit JSON log lines") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check") + c.add_argument("--json", action="store_true") + sub.add_parser("recover") + r = sub.add_parser("renew") + r.add_argument("--force-due", action="store_true") + g = sub.add_parser("register") + g.add_argument("--mode", choices=["same-key", "new-key"], default="same-key") + g.add_argument("--region", default=None) + g.add_argument("--via", default="operator") + sub.add_parser("rewrite-config") + sub.add_parser("status") + pa = sub.add_parser("pause") + pa.add_argument("--reason", default="operator") + sub.add_parser("resume") + sub.add_parser("retry-now") + sr = sub.add_parser("state-reset") + sr.add_argument("--confirm", action="store_true") + n = sub.add_parser("notify") + n.add_argument("--level", choices=["info", "warn", "error"], default="error") + n.add_argument("--message", required=True) + return p + + +def main(argv: Optional[list[str]] = None, system: Optional[System] = None, http: Optional[HttpClient] = None, stream=None) -> int: + args = build_parser().parse_args(argv) + log = Log(stream=stream, json_mode=args.json_log) + try: + cfg = Config.load(args.config) + except (OSError, ValueError, GatewayError) as e: + log.error("config load failed", error=str(e)) + return EXIT_USAGE + ctl = Controller(cfg, system or System(), log, http) + try: + if args.cmd == "check": + return ctl.cmd_check(as_json=args.json) + if args.cmd == "recover": + return ctl.cmd_recover() + if args.cmd == "renew": + return ctl.cmd_renew(force_due=args.force_due) + if args.cmd == "register": + return ctl.cmd_register(args.mode, args.region, args.via) + if args.cmd == "rewrite-config": + return ctl.cmd_rewrite_config() + if args.cmd == "status": + return ctl.cmd_status() + if args.cmd == "pause": + return ctl.cmd_pause(args.reason) + if args.cmd == "resume": + return ctl.cmd_resume() + if args.cmd == "retry-now": + return ctl.cmd_retry_now() + if args.cmd == "state-reset": + return ctl.cmd_state_reset(args.confirm) + if args.cmd == "notify": + return ctl.cmd_notify(args.level, args.message) + except GatewayError as e: + log.error("failed", error=str(e)) + return EXIT_SAFETY if isinstance(e, SafetyError) else EXIT_UNHEALTHY + except Exception as e: # noqa: BLE001 - never let a traceback (with source context) reach the journal + log.error("unexpected failure", error=e.__class__.__name__) + return EXIT_UNHEALTHY + return EXIT_USAGE + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ansible/roles/pia-gateway/tasks/controller.yaml b/ansible/roles/pia-gateway/tasks/controller.yaml new file mode 100644 index 0000000..74b3246 --- /dev/null +++ b/ansible/roles/pia-gateway/tasks/controller.yaml @@ -0,0 +1,98 @@ +--- +# Part of role: pia-gateway +# Included by: tasks/main.yaml (before registration — the shared +# registration backend is this program) +# Description: Installs the on-host controller (files/pia_gateway.py), its +# root-only runtime config and credentials, and the systemd units for +# the recovery / renewal timers. Timers are only ENABLED at the end of +# tasks/main.yaml, after the kill switch, interface and a verified +# registration exist — this file never starts them. + +- name: Create the controller config directory (root-only) + ansible.builtin.file: + path: "{{ pia_gateway_etc_dir }}" + state: directory + mode: "0700" + owner: root + group: root + +- name: Create the controller state directory (root-only) + ansible.builtin.file: + path: "{{ pia_gateway_state_dir }}" + state: directory + mode: "0700" + owner: root + group: root + +- name: Install the pia-gateway program + ansible.builtin.copy: + src: pia_gateway.py + dest: "{{ pia_gateway_program }}" + mode: "0700" + owner: root + group: root + +- name: Render the controller runtime config + ansible.builtin.template: + src: pia-gateway-config.json.j2 + dest: "{{ pia_gateway_etc_dir }}/config.json" + mode: "0600" + owner: root + group: root + register: pia_gateway_config_result + +- name: Validate the rendered config loads + ansible.builtin.command: + cmd: "{{ pia_gateway_program }} --config {{ pia_gateway_etc_dir }}/config.json status" + register: pia_gateway_config_check + changed_when: false + failed_when: pia_gateway_config_check.rc not in [0, 1, 2] + # `status`'s own source (cmd_status in files/pia_gateway.py) only ever + # returns one of three documented exits: 0 (healthy), 1 (unhealthy — + # the expected result on a fresh install, since the kill switch and + # pia-wg interface don't exist yet at this point in the run), or 2 + # (state file corrupt — not a config problem either). Allowlisting + # those three and failing on EVERYTHING else (5 = bad config JSON/ + # unknown key from main()'s own Config.load(), 3/4 = exits this + # command cannot legitimately return, 126/127/a crash) means a real + # config regression is never masked by matching only one "bad" code — + # the previous version of this task only failed on rc==5 and silently + # ignored any other unexpected exit. + +- name: Provision PIA credentials for unattended re-registration (root-only) + ansible.builtin.template: + src: pia-gateway-credentials.env.j2 + dest: "{{ pia_gateway_etc_dir }}/credentials.env" + mode: "0600" + owner: root + group: root + no_log: true + when: not (pia_remove_host_credentials | bool) + # Only PIA_USER/PIA_PASSWORD (and the alert-bridge key when set) — never + # the whole .env, the vault password, or any Kubernetes secret. This is + # the deliberate availability/security tradeoff documented in the README: + # a host that must re-register while the laptop is off needs the account + # secret locally. + +- name: Remove the on-host PIA credentials (explicit request only) + ansible.builtin.file: + path: "{{ pia_gateway_etc_dir }}/credentials.env" + state: absent + when: pia_remove_host_credentials | bool + +- name: Deploy controller systemd units + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + owner: root + group: root + loop: + - pia-gateway-recover.service + - pia-gateway-recover.timer + - pia-gateway-renew.service + - pia-gateway-renew.timer + - pia-gateway-onfailure.service + notify: Reload systemd daemon + # The reload runs at the first flush_handlers in main.yaml, before any + # of these units is enabled; the enable tasks also pass daemon_reload. diff --git a/ansible/roles/pia-gateway/tasks/healthcheck.yaml b/ansible/roles/pia-gateway/tasks/healthcheck.yaml index 8706c8a..e1a4401 100644 --- a/ansible/roles/pia-gateway/tasks/healthcheck.yaml +++ b/ansible/roles/pia-gateway/tasks/healthcheck.yaml @@ -1,17 +1,17 @@ --- # Part of role: pia-gateway # Included by: tasks/main.yaml -# Description: Installs the observability-only health check (script, -# oneshot service, timer). See templates/pia-gateway-healthcheck.sh.j2 -# for exactly what it checks and why it never remediates. +# Description: Installs the observability-only health check (oneshot +# service + timer). Since the auto-recovery work it runs +# `pia-gateway check` — the same status classification the recovery +# controller uses — instead of the earlier standalone bash script, so +# the two can never disagree about what "healthy" means. It still never +# remediates. -- name: Deploy the health check script - ansible.builtin.template: - src: pia-gateway-healthcheck.sh.j2 - dest: /usr/local/sbin/pia-gateway-healthcheck.sh - mode: "0700" - owner: root - group: root +- name: Remove the superseded bash health check script + ansible.builtin.file: + path: /usr/local/sbin/pia-gateway-healthcheck.sh + state: absent - name: Deploy the health check systemd service ansible.builtin.template: @@ -40,3 +40,4 @@ name: pia-gateway-healthcheck.timer enabled: true state: started + daemon_reload: true diff --git a/ansible/roles/pia-gateway/tasks/main.yaml b/ansible/roles/pia-gateway/tasks/main.yaml index b50f12a..386abcb 100644 --- a/ansible/roles/pia-gateway/tasks/main.yaml +++ b/ansible/roles/pia-gateway/tasks/main.yaml @@ -2,10 +2,23 @@ # Part of role: pia-gateway # Called by: ansible/playbooks/pia-gateway.yaml # Description: Orchestrates minisforum's PIA WireGuard egress gateway for -# VPN VLAN 50 — registration, boot-ordered kill switch + policy routing, -# the WireGuard interface itself, and health checks. Split into included -# files because each stage has a distinct idempotency/ordering concern; -# see the comment at the top of each included file. +# VPN VLAN 50 — kill switch + policy routing, the on-host recovery +# controller, registration, the WireGuard interface itself, and health +# checks. Split into included files because each stage has a distinct +# idempotency/ordering concern; see the comment at the top of each. +# +# Ordering (changed with the auto-recovery work, deliberately): +# 1. tooling, baseline assertions +# 2. controller program + config + credentials (NOT its timers) +# 3. quiesce the recovery/renewal timers for the rest of the run so a +# timer-driven restart can never race an Ansible handler restart +# 4. kill switch + routing + firewall, handlers flushed so the closed +# state is live BEFORE anything can activate a tunnel +# 5. registration (shared backend activates + verifies under the lock; +# the legacy backend only writes the config and notifies a restart) +# 6. interface unit enabled, health check installed, handlers flushed +# 7. timers enabled/disabled per pia_auto_recovery_enabled / +# pia_renewal_enabled - name: Install WireGuard tooling and iproute2 ansible.builtin.apt: @@ -13,24 +26,57 @@ - wireguard - wireguard-tools - iproute2 - - python3-requests + - python3 state: present update_cache: true + # python3-requests is no longer needed: the controller uses only the + # standard library. - name: Pin reverse-path filtering for {{ pia_wg_interface }} ansible.posix.sysctl: name: "net.ipv4.conf.{{ pia_wg_interface }}.rp_filter" value: "{{ pia_rp_filter_mode | string }}" sysctl_file: /etc/sysctl.d/60-pia-gateway.conf - reload: false # interface does not exist yet on a first run; applied for real after the WG interface comes up (handlers/main.yaml) + reload: false # interface does not exist yet on a first run; applied for real after the WG interface comes up register: pia_rp_filter_pinned - name: Confirm host default route and ip_forward are untouched ansible.builtin.include_tasks: assert-baseline.yaml -- name: Register minisforum with PIA and obtain WireGuard peer config - ansible.builtin.include_tasks: register.yaml - no_log: true # PIA credentials and the derived private key pass through this include +- name: Install the recovery controller, its config and credentials + ansible.builtin.include_tasks: controller.yaml + +- name: Check whether the recovery timer is installed from a previous run + ansible.builtin.stat: + path: /etc/systemd/system/pia-gateway-recover.timer + register: pia_recover_timer_stat + +- name: Quiesce the recovery and renewal timers for the duration of this run + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - pia-gateway-recover.timer + - pia-gateway-renew.timer + when: pia_recover_timer_stat.stat.exists + failed_when: false + # failed_when: false — on the very first install the units were just + # deployed and never started; "stopping" them is a harmless no-op that + # some systemd versions report as an error for never-loaded timers. + +- name: Wait for any in-flight recovery/renewal pass to finish + ansible.builtin.command: + cmd: systemctl is-active pia-gateway-recover.service pia-gateway-renew.service + register: pia_inflight + changed_when: false + failed_when: false + until: "'active' not in (pia_inflight.stdout_lines | default([]) | map('trim') | list)" + retries: 90 + delay: 10 + when: pia_recover_timer_stat.stat.exists + # A pass can take up to pia_recovery_max_duration_sec (+ restart waits); + # we never kill it mid-activation. Registration below also takes the + # program's own lock, so this is belt-and-braces for the handler restarts. - name: Configure boot-ordered kill switch and policy routing ansible.builtin.include_tasks: routing.yaml @@ -38,6 +84,18 @@ - name: Configure firewall-layer kill switch and Technitium access ansible.builtin.include_tasks: firewall.yaml +- name: Apply kill-switch handlers before anything can activate a tunnel + ansible.builtin.meta: flush_handlers + +- name: Register minisforum with PIA (shared on-host transaction) + ansible.builtin.include_tasks: register-shared.yaml + when: pia_registration_backend == "shared" + +- name: Register minisforum with PIA (legacy in-Ansible flow) + ansible.builtin.include_tasks: register-legacy.yaml + no_log: true # PIA credentials and the derived private key pass through this include + when: pia_registration_backend == "legacy" + - name: Write and enable the pia-wg WireGuard interface ansible.builtin.include_tasks: interface.yaml @@ -51,3 +109,54 @@ sysctl_file: /etc/sysctl.d/60-pia-gateway.conf reload: true when: pia_rp_filter_pinned is succeeded + +- name: Apply remaining handlers while the timers are still quiesced + ansible.builtin.meta: flush_handlers + +- name: Re-render pia-wg.conf when only the controller config changed (no PIA call) + ansible.builtin.command: + cmd: "{{ pia_gateway_program }} rewrite-config" + register: pia_rewrite_result + changed_when: pia_rewrite_result.rc == 0 + failed_when: pia_rewrite_result.rc not in [0, 5] + when: + - pia_registration_backend == "shared" + - pia_gateway_config_result is changed + - not (pia_registered_this_run | default(false) | bool) + # Picks up hook-content changes (e.g. pia_mss_clamp_enabled) without a + # forced re-registration — the gap noted in the 2026-08-25 incident. + # Exit 5 = "no verified registration recorded yet" (legacy-registered + # host); that is not an error here, the next registration will render + # the current hooks. + +- name: Enable the recovery timer + ansible.builtin.systemd: + name: pia-gateway-recover.timer + enabled: true + state: started + daemon_reload: true + when: pia_auto_recovery_enabled | bool + +- name: Disable the recovery timer (pia_auto_recovery_enabled=false) + ansible.builtin.systemd: + name: pia-gateway-recover.timer + enabled: false + state: stopped + daemon_reload: true + when: not (pia_auto_recovery_enabled | bool) + +- name: Enable the renewal timer + ansible.builtin.systemd: + name: pia-gateway-renew.timer + enabled: true + state: started + daemon_reload: true + when: (pia_auto_recovery_enabled | bool) and (pia_renewal_enabled | bool) + +- name: Disable the renewal timer + ansible.builtin.systemd: + name: pia-gateway-renew.timer + enabled: false + state: stopped + daemon_reload: true + when: not ((pia_auto_recovery_enabled | bool) and (pia_renewal_enabled | bool)) diff --git a/ansible/roles/pia-gateway/tasks/register.yaml b/ansible/roles/pia-gateway/tasks/register-legacy.yaml similarity index 94% rename from ansible/roles/pia-gateway/tasks/register.yaml rename to ansible/roles/pia-gateway/tasks/register-legacy.yaml index 432bdc4..164c61d 100644 --- a/ansible/roles/pia-gateway/tasks/register.yaml +++ b/ansible/roles/pia-gateway/tasks/register-legacy.yaml @@ -1,8 +1,26 @@ --- # Part of role: pia-gateway -# Included by: tasks/main.yaml (with no_log: true — this file handles the -# PIA account password, the derived WireGuard private key, and the -# short-lived PIA auth token) +# Included by: tasks/main.yaml ONLY when pia_registration_backend == +# "legacy" (with no_log: true — this file handles the PIA account +# password, the derived WireGuard private key, and the short-lived PIA +# auth token) +# +# LEGACY PATH — superseded by tasks/register-shared.yaml, which calls the +# on-host controller (files/pia_gateway.py) so Ansible and the unattended +# recovery/renewal timers share ONE registration implementation that also +# activates and verifies a handshake. This file is kept, unchanged, only as +# the documented fallback until the shared path has passed a live manual +# re-registration on minisforum (README "Rollout"). Delete it — and +# addkey-attempt.yaml, the `legacy` option, and pia-wg.conf.j2 — once that +# is done. Differences to be aware of while it still exists: +# - it does NOT verify a handshake (config write + handler restart only) +# - it does NOT record a verified-registration baseline in the +# controller state, so the renewal timer treats the host as "never +# verified" until the shared path runs once +# - its rotation stages a fixed `.key.new` path with `creates:` (an +# abandoned stage could be reused); the controller removes any +# `.key.new*` file before its own attempts. +# # Description: Registers minisforum as a PIA WireGuard peer and writes # {{ pia_wg_config_dir }}/{{ pia_wg_interface }}.conf. Skipped once that # file exists, unless pia_force_reregister or pia_force_key_rotation is diff --git a/ansible/roles/pia-gateway/tasks/register-shared.yaml b/ansible/roles/pia-gateway/tasks/register-shared.yaml new file mode 100644 index 0000000..49a93d6 --- /dev/null +++ b/ansible/roles/pia-gateway/tasks/register-shared.yaml @@ -0,0 +1,64 @@ +--- +# Part of role: pia-gateway +# Included by: tasks/main.yaml when pia_registration_backend == "shared" +# Description: Registers with PIA through the on-host controller +# (`pia-gateway register`), i.e. the same token/addKey/stage/activate/ +# verify transaction the unattended recovery and renewal timers use. +# Runs when there is no config yet, no VERIFIED registration baseline in +# the controller's state, or an explicit force flag. Reports success only +# after a fresh WireGuard handshake was observed on the activated config +# — never on `addKey` HTTP status alone. +# +# Requires the kill switch to be active (tasks/routing.yaml runs first in +# main.yaml); the program refuses to activate otherwise (exit 3) and the +# play fails loudly rather than opening anything. + +- name: Check whether a PIA WireGuard config already exists + ansible.builtin.stat: + path: "{{ pia_wg_config_dir }}/{{ pia_wg_interface }}.conf" + register: pia_wg_conf_stat + +- name: Check whether the controller has a verified registration baseline + ansible.builtin.stat: + path: "{{ pia_gateway_state_dir }}/state.json" + register: pia_state_stat + +- name: Read the controller state (no secrets are stored in it) + ansible.builtin.slurp: + src: "{{ pia_gateway_state_dir }}/state.json" + register: pia_state_raw + when: pia_state_stat.stat.exists + +- name: Decide whether registration is needed this run + ansible.builtin.set_fact: + pia_has_baseline: >- + {{ pia_state_stat.stat.exists + and ((pia_state_raw.content | b64decode | from_json).last_verified_registration is not none) }} + pia_registration_mode: "{{ 'new-key' if (pia_force_key_rotation | bool) else 'same-key' }}" + pia_region_args: "{{ ['--region', pia_region] if (pia_region is not none and (pia_region | string | trim | length) > 0) else [] }}" + +- name: Run the shared PIA registration transaction + ansible.builtin.command: + argv: "{{ [pia_gateway_program, 'register', '--via', 'ansible', '--mode', pia_registration_mode] + pia_region_args }}" + register: pia_register_result + changed_when: true + failed_when: pia_register_result.rc != 0 + when: >- + (pia_force_reregister | bool) + or (pia_force_key_rotation | bool) + or not pia_wg_conf_stat.stat.exists + or not (pia_has_baseline | bool) + # argv form: no shell, no secrets in arguments (credentials are read from + # the root-only file by the program itself), so this task is deliberately + # NOT no_log — its stdout is the secret-free JSON summary and the + # per-candidate diagnostics, which are exactly what an operator needs + # when a run fails. The lock is taken inside the program (waits up to + # pia_recovery_lock_wait_sec for an in-flight timer run to finish). + +- name: Record that registration happened this run + ansible.builtin.set_fact: + pia_registered_this_run: "{{ pia_register_result is not skipped }}" + +- name: Show the registration summary + ansible.builtin.debug: + msg: "{{ pia_register_result.stdout_lines | default(['skipped — config and verified baseline already present']) }}" diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-config.json.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-config.json.j2 new file mode 100644 index 0000000..7e769b4 --- /dev/null +++ b/ansible/roles/pia-gateway/templates/pia-gateway-config.json.j2 @@ -0,0 +1,36 @@ +{ + "interface": {{ pia_wg_interface | to_json }}, + "config_dir": {{ pia_wg_config_dir | to_json }}, + "state_dir": {{ pia_gateway_state_dir | to_json }}, + "credentials_file": {{ (pia_gateway_etc_dir + '/credentials.env') | to_json }}, + "ca_file": {{ (pia_wg_config_dir + '/pia-ca.crt') | to_json }}, + "lock_file": "/run/lock/pia-gateway.lock", + "route_table": {{ pia_route_table_name | to_json }}, + "rule_priority": {{ pia_rule_priority | int }}, + "vlan50_subnet": {{ pia_vlan50_subnet | to_json }}, + "iptables_chain": {{ pia_iptables_chain | to_json }}, + "lan_interface": {{ pia_lan_interface | to_json }}, + "mss_clamp_enabled": {{ pia_mss_clamp_enabled | bool | to_json }}, + "region_candidates": {{ pia_region_candidates | to_json }}, + "handshake_max_age_sec": {{ pia_healthcheck_handshake_max_age_sec | int }}, + "failure_threshold": {{ pia_recovery_failure_threshold | int }}, + "startup_grace_sec": {{ pia_recovery_startup_grace_sec | int }}, + "restart_wait_sec": {{ pia_recovery_restart_wait_sec | int }}, + "activation_wait_sec": {{ pia_recovery_activation_wait_sec | int }}, + "recovery_max_duration_sec": {{ pia_recovery_max_duration_sec | int }}, + "backoff_schedule_sec": {{ pia_recovery_backoff_schedule_sec | to_json }}, + "backoff_jitter_fraction": 0.1, + "max_candidates_per_cycle": {{ pia_recovery_max_candidates_per_cycle | int }}, + "max_rotation_candidates": {{ pia_recovery_max_rotation_candidates | int }}, + "renewal_after_days": {{ pia_renewal_after_days | int }}, + "http_connect_timeout_sec": {{ pia_recovery_http_connect_timeout_sec | int }}, + "http_total_timeout_sec": {{ pia_recovery_http_total_timeout_sec | int }}, + "clock_skew_allowance_sec": 60, + "backup_keep": {{ pia_recovery_backup_keep | int }}, + "lock_wait_sec": {{ pia_recovery_lock_wait_sec | int }}, + "alert_bridge_url": {{ pia_alert_bridge_url | to_json }}, + "alert_source": {{ pia_alert_source | to_json }}, + "metrics_file": {{ (pia_gateway_state_dir + '/metrics.prom') | to_json }}, + "status_file": {{ (pia_gateway_state_dir + '/status.json') | to_json }}, + "killswitch_unit": "pia-killswitch.service" +} diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-credentials.env.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-credentials.env.j2 new file mode 100644 index 0000000..da94c94 --- /dev/null +++ b/ansible/roles/pia-gateway/templates/pia-gateway-credentials.env.j2 @@ -0,0 +1,7 @@ +# Managed by Ansible (role: pia-gateway). Root-only (0600). Read at runtime by +# /usr/local/sbin/pia-gateway; never passed via argv or environment. +PIA_USER={{ pia_user }} +PIA_PASSWORD={{ pia_password }} +{% if pia_alert_bridge_api_key | default('') | length > 0 %} +ALERT_BRIDGE_API_KEY={{ pia_alert_bridge_api_key }} +{% endif %} diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.service.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.service.j2 index b9a848d..a4d7df3 100644 --- a/ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.service.j2 +++ b/ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.service.j2 @@ -1,7 +1,12 @@ # Managed by Ansible (role: pia-gateway). Do not edit by hand. +# Read-only: `pia-gateway check` classifies status with the same code the +# recovery controller uses, logs it, and refreshes the status/metrics files. +# It never restarts, registers, or touches routing/firewall state. [Unit] Description=PIA gateway health check (observability only, no remediation) [Service] Type=oneshot -ExecStart=/usr/local/sbin/pia-gateway-healthcheck.sh +ExecStart={{ pia_gateway_program }} check +SuccessExitStatus=1 +LimitCORE=0 diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.sh.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.sh.j2 deleted file mode 100644 index 802e9c3..0000000 --- a/ansible/roles/pia-gateway/templates/pia-gateway-healthcheck.sh.j2 +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Managed by Ansible (role: pia-gateway). Do not edit by hand. -# -# Observability only. This script NEVER modifies routing, firewall, or -# WireGuard state — a failed check must close the path (already true by -# construction, via the kill switch in tasks/routing.yaml/firewall.yaml), -# never "recover" it. Output goes to the journal -# (journalctl -u pia-gateway-healthcheck); wire alerting off of that -# separately (see plan.md Phase 9) rather than adding remediation here. -# -# The "route decision" check below confirms what the kernel FIB *would* -# do for a VLAN 50 source — it does not send a packet and is not proof of -# actual PIA egress or identity. That proof requires the packet-capture -# test matrix in plan.md Phase 5; do not treat a clean run of this script -# as substituting for it. -set -uo pipefail - -WG_IF="{{ pia_wg_interface }}" -TABLE="{{ pia_route_table_name }}" -SUBNET="{{ pia_vlan50_subnet }}" -PRIORITY="{{ pia_rule_priority }}" -CHAIN="{{ pia_iptables_chain }}" -MAX_HANDSHAKE_AGE={{ pia_healthcheck_handshake_max_age_sec }} -PROBE_SRC="${SUBNET%.*}.2" # a representative VLAN 50 address for the route-decision probe only; never actually used as a source - -fail=0 - -# Informational only (not pass/fail) — which region/server this host is -# actually registered against right now, per the comment -# tasks/register.yaml writes into the generated config. Since PIA -# WireGuard registration now falls back across multiple regions -# (pia_region_candidates in defaults/main.yaml), "which one is live" -# isn't implied by config alone — it's whichever one answered OK last -# time the playbook ran, not necessarily the first-configured one. -configured_peer=$(grep -m1 '^# PIA WireGuard server actually used' "{{ pia_wg_config_dir }}/{{ pia_wg_interface }}.conf" 2>/dev/null || true) -if [[ -n "${configured_peer}" ]]; then - echo "INFO configured-peer: ${configured_peer#\# }" -fi - -if ip link show "${WG_IF}" up &>/dev/null; then - echo "PASS interface: ${WG_IF} is up" -else - echo "FAIL interface: ${WG_IF} is not up" - fail=1 -fi - -if command -v wg &>/dev/null && wg show "${WG_IF}" latest-handshakes &>/dev/null; then - handshake_epoch=$(wg show "${WG_IF}" latest-handshakes 2>/dev/null | awk '{print $2}') - now=$(date +%s) - if [[ -n "${handshake_epoch}" && "${handshake_epoch}" -gt 0 ]]; then - age=$((now - handshake_epoch)) - if (( age <= MAX_HANDSHAKE_AGE )); then - echo "PASS handshake: ${age}s old (max ${MAX_HANDSHAKE_AGE}s)" - else - echo "FAIL handshake: ${age}s old, exceeds max ${MAX_HANDSHAKE_AGE}s" - fail=1 - fi - else - echo "FAIL handshake: no handshake recorded yet" - fail=1 - fi -else - echo "WARN handshake: unable to query wg show (interface absent or wg missing)" - fail=1 -fi - -if ip rule show | grep -qE "^${PRIORITY}:[[:space:]]*from ${SUBNET} lookup ${TABLE}\$"; then - echo "PASS rule: from ${SUBNET} lookup ${TABLE} present at priority ${PRIORITY}" -else - echo "FAIL rule: from ${SUBNET} lookup ${TABLE} missing" - fail=1 -fi - -default_route=$(ip route show table "${TABLE}" 2>/dev/null | grep '^default' || true) -if [[ "${default_route}" == *"dev ${WG_IF}"* ]]; then - echo "PASS route: table ${TABLE} default is via ${WG_IF} (open)" -elif [[ "${default_route}" == *unreachable* ]] || ip route show table "${TABLE}" 2>/dev/null | grep -q '^unreachable default'; then - echo "PASS route: table ${TABLE} default is unreachable (closed, kill switch engaged)" -else - echo "FAIL route: table ${TABLE} has no default route at all (neither open via ${WG_IF} nor a terminal unreachable) — investigate immediately" - fail=1 -fi - -# Not a check on FORWARD's own default policy — that's deliberately left -# alone (see tasks/firewall.yaml for why: minisforum runs Flannel, which -# needs its own broad FORWARD ACCEPTs). What actually matters is that the -# dedicated chain exists, is jumped into first for this source, and ends -# in a real DROP. -if iptables -C FORWARD -s "${SUBNET}" -j "${CHAIN}" 2>/dev/null; then - echo "PASS firewall: FORWARD jumps to ${CHAIN} for ${SUBNET}" -else - echo "FAIL firewall: FORWARD does not jump to ${CHAIN} for ${SUBNET}" - fail=1 -fi - -if iptables -S "${CHAIN}" 2>/dev/null | grep -qE '^-A '"${CHAIN}"' -j DROP$'; then - echo "PASS firewall: ${CHAIN} ends in an unconditional DROP" -else - echo "FAIL firewall: ${CHAIN} has no unconditional DROP catch-all" - fail=1 -fi - -route_decision=$(ip route get 1.1.1.1 from "${PROBE_SRC}" 2>&1 || true) -if [[ "${route_decision}" == *"dev ${WG_IF}"* ]]; then - echo "PASS route-decision (not proof of live egress — see header comment): ${SUBNET} sourced traffic resolves via ${WG_IF}" -else - echo "WARN route-decision (not proof of live egress — see header comment): ${route_decision}" -fi - -exit "${fail}" diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-onfailure.service.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-onfailure.service.j2 new file mode 100644 index 0000000..353afcc --- /dev/null +++ b/ansible/roles/pia-gateway/templates/pia-gateway-onfailure.service.j2 @@ -0,0 +1,12 @@ +# Managed by Ansible (role: pia-gateway). Do not edit by hand. +# Fired by OnFailure= of the recovery/renewal units when they crash or refuse +# to run (corrupt state). Delivery goes through alert-bridge if configured; +# a delivery failure is logged and never retried here (the next successful +# recover pass flushes its own pending queue). +[Unit] +Description=PIA gateway controller failure notifier + +[Service] +Type=oneshot +ExecStart={{ pia_gateway_program }} notify --level error --message "pia-gateway: a controller unit failed on minisforum (journalctl -u pia-gateway-recover -u pia-gateway-renew)" +LimitCORE=0 diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-recover.service.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-recover.service.j2 new file mode 100644 index 0000000..6beed45 --- /dev/null +++ b/ansible/roles/pia-gateway/templates/pia-gateway-recover.service.j2 @@ -0,0 +1,19 @@ +# Managed by Ansible (role: pia-gateway). Do not edit by hand. +# +# One bounded recovery pass (files/pia_gateway.py `recover`). Exit codes 1 +# (unhealthy, decision made) and 3 (safety drift handled with an emergency +# close + alert) are normal outcomes, not unit failures — only a crash or a +# corrupt state file (2) is, and that triggers the OnFailure notifier. +[Unit] +Description=PIA gateway automatic recovery pass +After=pia-killswitch.service network-online.target +Wants=network-online.target +OnFailure=pia-gateway-onfailure.service + +[Service] +Type=oneshot +ExecStart={{ pia_gateway_program }} recover +SuccessExitStatus=1 3 +TimeoutStartSec={{ (pia_recovery_max_duration_sec | int) + 300 }} +LimitCORE=0 +PrivateTmp=yes diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-recover.timer.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-recover.timer.j2 new file mode 100644 index 0000000..78c7be0 --- /dev/null +++ b/ansible/roles/pia-gateway/templates/pia-gateway-recover.timer.j2 @@ -0,0 +1,13 @@ +# Managed by Ansible (role: pia-gateway). Do not edit by hand. +# systemd never starts a second instance while pia-gateway-recover.service is +# still running, and the program also holds /run/lock/pia-gateway.lock. +[Unit] +Description=Run the PIA gateway recovery pass every {{ pia_healthcheck_interval_sec }}s + +[Timer] +OnBootSec=90 +OnUnitActiveSec={{ pia_healthcheck_interval_sec }} +AccuracySec=5s + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-renew.service.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-renew.service.j2 new file mode 100644 index 0000000..9664fb7 --- /dev/null +++ b/ansible/roles/pia-gateway/templates/pia-gateway-renew.service.j2 @@ -0,0 +1,14 @@ +# Managed by Ansible (role: pia-gateway). Do not edit by hand. +[Unit] +Description=PIA gateway proactive registration renewal +After=pia-killswitch.service network-online.target +Wants=network-online.target +OnFailure=pia-gateway-onfailure.service + +[Service] +Type=oneshot +ExecStart={{ pia_gateway_program }} renew +SuccessExitStatus=1 +TimeoutStartSec={{ (pia_recovery_max_duration_sec | int) + 300 }} +LimitCORE=0 +PrivateTmp=yes diff --git a/ansible/roles/pia-gateway/templates/pia-gateway-renew.timer.j2 b/ansible/roles/pia-gateway/templates/pia-gateway-renew.timer.j2 new file mode 100644 index 0000000..580b5cf --- /dev/null +++ b/ansible/roles/pia-gateway/templates/pia-gateway-renew.timer.j2 @@ -0,0 +1,11 @@ +# Managed by Ansible (role: pia-gateway). Do not edit by hand. +[Unit] +Description=Daily check whether the PIA registration is older than {{ pia_renewal_after_days }} days + +[Timer] +OnCalendar={{ pia_renewal_calendar }} +RandomizedDelaySec={{ pia_renewal_jitter_sec }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/pia-gateway/tests/test_pia_gateway.py b/ansible/roles/pia-gateway/tests/test_pia_gateway.py new file mode 100644 index 0000000..fab44ec --- /dev/null +++ b/ansible/roles/pia-gateway/tests/test_pia_gateway.py @@ -0,0 +1,1279 @@ +"""Offline tests for ansible/roles/pia-gateway/files/pia_gateway.py. + +Run from the repo root: python3 -m unittest discover -s ansible/roles/pia-gateway/tests -v + +Everything that touches the host (ip/wg/iptables/systemctl, clock, HTTP) is +faked; real files are used inside a temporary directory so atomic writes, +modes, staging and backups are exercised for real. +""" + +from __future__ import annotations + +import importlib.util +import io +import json +import os +import re +import shutil +import stat +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +HERE = Path(__file__).resolve().parent +SRC = HERE.parent / "files" / "pia_gateway.py" +spec = importlib.util.spec_from_file_location("pia_gateway", SRC) +pg = importlib.util.module_from_spec(spec) +sys.modules["pia_gateway"] = pg +spec.loader.exec_module(pg) + +SUBNET = "10.10.50.0/24" +CHAIN = "PIA-VLAN50" +IFACE = "pia-wg" + +# Sentinels that must never appear in any output. +SECRET_USER = "sentinel-user-9f1c" +SECRET_PASS = "sentinel-pass-77aa" +SECRET_TOKEN = "sentinel-token-13bd" +SECRET_ALERT_KEY = "sentinel-alertkey-55ee" + +# Valid-looking 32-byte base64 keys (44 chars, last char from the allowed set). +PRIV_A = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +PRIV_B = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA=" +PUB_A = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaA=" +PUB_B = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbA=" +SRV_HK1 = "hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1hk1A=" +SRV_HK2 = "hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2hk2A=" +SRV_TW1 = "tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1tw1A=" +SRV_OLD = "oldoldoldoldoldoldoldoldoldoldoldoldoldoldA=" + +PUBKEYS = {PRIV_A: PUB_A, PRIV_B: PUB_B} +SERVER_KEYS = {"hk1.example": SRV_HK1, "hk2.example": SRV_HK2, "tw1.example": SRV_TW1} + + +def serverlist_doc(): + return { + "regions": [ + {"id": "hk", "name": "Hong Kong", "servers": {"wg": [{"cn": "hk1.example", "ip": "10.99.0.1"}, {"cn": "hk2.example", "ip": "10.99.0.2"}]}}, + {"id": "taiwan", "name": "Taiwan", "servers": {"wg": [{"cn": "tw1.example", "ip": "10.99.1.1"}]}}, + {"id": "japan", "name": "JP Tokyo", "servers": {"wg": []}}, + ] + } + + +class FakeSystem(pg.System): + """Models just enough of the gateway host for the controller.""" + + def __init__(self, tmp: Path): + self.tmp = tmp + self.t = 1_700_000_000 + self.mono = 1000.0 + self.boot = "boot-1" + self.iface_exists = True + self.iface_up = True + self.peers: dict[str, int] = {} + self.endpoints: dict[str, str] = {} + self.rule_present = True + self.table_default = "open" # open | closed | missing + self.forward_rules = [f"-A FORWARD -s {SUBNET} -j {CHAIN}", "-A FORWARD -j ufw-before-forward"] + self.chain_rules = self.open_chain() + self.main_default = "default via 10.10.40.1 dev enp1s0 proto static" + self.killswitch_active = True + self.restart_rc = 0 + self.restart_count = 0 + self.handshake_on_restart = "fresh" # fresh | stale | callable(server_key)->epoch|None + self.commands: list[list[str]] = [] + self.stdins: list[str] = [] + self.genkey_queue = [PRIV_B] + self.jitter = 0.5 + + @staticmethod + def open_chain(): + return [ + f"-A {CHAIN} -d {SUBNET} -i {IFACE} -m state --state RELATED,ESTABLISHED -j ACCEPT", + f"-A {CHAIN} -s {SUBNET} -o {IFACE} -j ACCEPT", + f"-A {CHAIN} -m limit --limit 5/min --limit-burst 5 -j LOG --log-prefix \"PIA-KILLSWITCH-DROP: \"", + f"-A {CHAIN} -j DROP", + ] + + @staticmethod + def closed_chain(): + return [ + f"-A {CHAIN} -m limit --limit 5/min --limit-burst 5 -j LOG --log-prefix \"PIA-KILLSWITCH-DROP: \"", + f"-A {CHAIN} -j DROP", + ] + + # clock + def now(self): + return self.t + + def monotonic(self): + return self.mono + + def sleep(self, sec): + self.t += int(sec) + self.mono += sec + + def boot_id(self): + return self.boot + + def random_jitter(self): + return self.jitter + + # helpers + def set_peer(self, key, age, endpoint="10.99.0.1:1337"): + self.peers = {key: self.t - age if age is not None else 0} + self.endpoints = {key: endpoint} + + def conf_peer(self, conf_path): + if os.path.exists(conf_path): + return pg.parse_expected_peer(open(conf_path).read()) + return None + + def _apply_restart(self, conf_path): + self.restart_count += 1 + if self.restart_rc != 0: + return pg.CmdResult(self.restart_rc, "", "Job for wg-quick@pia-wg.service failed") + self.t += 1 # a real restart takes measurable wall-clock time; keeps + # any "fresh" handshake strictly after whatever `since_epoch` the + # caller captured just before invoking this restart (see the + # `>` — not `>=` — comparison in wait_for_verified). + key = self.conf_peer(conf_path) + self.iface_exists = True + self.iface_up = True + self.table_default = "open" + self.chain_rules = self.open_chain() + beh = self.handshake_on_restart + if callable(beh): + epoch = beh(key) + elif beh == "fresh": + epoch = self.t + else: + epoch = self.t - 100_000 + self.peers = {key: epoch if epoch is not None else 0} + self.endpoints = {key: "10.99.0.9:1337"} + return pg.CmdResult(0, "", "") + + # command dispatch + def run(self, argv, stdin=None, timeout=60): + self.commands.append(list(argv)) + if stdin is not None: + self.stdins.append(stdin) + a = argv + if a[:2] == ["wg", "genkey"]: + return pg.CmdResult(0, self.genkey_queue.pop(0) + "\n", "") + if a[:2] == ["wg", "pubkey"]: + return pg.CmdResult(0, PUBKEYS[stdin.strip()] + "\n", "") + if a[:2] == ["wg", "show"]: + if not self.iface_exists: + return pg.CmdResult(1, "", "Unable to access interface: No such device") + if a[3] == "latest-handshakes": + return pg.CmdResult(0, "".join(f"{k}\t{v}\n" for k, v in self.peers.items()), "") + if a[3] == "endpoints": + return pg.CmdResult(0, "".join(f"{k}\t{v}\n" for k, v in self.endpoints.items()), "") + if a[:4] == ["ip", "-o", "link", "show"]: + if not self.iface_exists: + return pg.CmdResult(1, "", 'Device "pia-wg" does not exist.') + flags = "POINTOPOINT,NOARP,UP,LOWER_UP" if self.iface_up else "POINTOPOINT,NOARP" + return pg.CmdResult(0, f"5: {IFACE}: <{flags}> mtu 1420 qdisc noqueue state UNKNOWN\n", "") + if a[:3] == ["ip", "rule", "show"]: + lines = ["0:\tfrom all lookup local"] + if self.rule_present: + lines.append(f"100:\tfrom {SUBNET} lookup pia") + lines += ["32766:\tfrom all lookup main", "32767:\tfrom all lookup default"] + return pg.CmdResult(0, "\n".join(lines) + "\n", "") + if a[:3] == ["ip", "route", "show"] and "table" in a: + if self.table_default == "open": + return pg.CmdResult(0, f"default dev {IFACE} scope link\n", "") + if self.table_default == "closed": + return pg.CmdResult(0, "unreachable default\n", "") + return pg.CmdResult(0, "", "") + if a[:4] == ["ip", "-4", "route", "show"]: + return pg.CmdResult(0, self.main_default + "\n", "") + if a[:3] == ["ip", "route", "replace"]: + if "unreachable" in a: + self.table_default = "closed" + return pg.CmdResult(0, "", "") + if a[:4] == ["ip", "link", "delete", "dev"]: + self.iface_exists = False + return pg.CmdResult(0, "", "") + if a[:2] == ["iptables", "-S"]: + if a[2] == "FORWARD": + return pg.CmdResult(0, "-P FORWARD ACCEPT\n" + "\n".join(self.forward_rules) + "\n", "") + if a[2] == CHAIN: + if self.chain_rules is None: + return pg.CmdResult(1, "", "iptables: No chain/target/match by that name.") + return pg.CmdResult(0, f"-N {CHAIN}\n" + "\n".join(self.chain_rules) + "\n", "") + if a[:2] == ["iptables", "-C"]: + rule = "-A " + " ".join(a[2:]) + return pg.CmdResult(0 if rule in self.forward_rules else 1, "", "") + if a[:2] == ["iptables", "-I"]: + rule = "-A FORWARD " + " ".join(a[4:]) + self.forward_rules.insert(0, rule) + return pg.CmdResult(0, "", "") + if a[:2] == ["iptables", "-D"]: + rule = "-A " + " ".join(a[2:]) + if rule in self.forward_rules: + self.forward_rules.remove(rule) + return pg.CmdResult(0, "", "") + return pg.CmdResult(1, "", "") + if a[:2] == ["systemctl", "is-active"]: + return pg.CmdResult(0 if self.killswitch_active else 3, "active\n" if self.killswitch_active else "inactive\n", "") + if a[:2] == ["systemctl", "restart"] or a[:2] == ["systemctl", "start"]: + return self._apply_restart(str(self.tmp / "wg" / f"{IFACE}.conf")) + raise AssertionError(f"unexpected command {argv}") + + +class FakeHttp: + def __init__(self): + self.serverlist = ("ok", serverlist_doc()) + self.token = ("ok", SECRET_TOKEN) + self.addkey = {} # cn -> ("ok", {...}) | ("err", kind) | ("http", status) | ("raw", bytes) + self.calls = [] + + def get(self, url): + self.calls.append(("get", url)) + kind, val = self.serverlist + if kind == "err": + raise pg.ApiError(val, "fake") + if kind == "http": + return val, b"" + if kind == "raw": + return 200, val + return 200, (json.dumps(val) + "\n-----BEGIN SIG-----\nxxx\n").encode() + + def post_multipart(self, url, fields): + self.calls.append(("post", url, dict(fields))) + kind, val = self.token + if kind == "err": + raise pg.ApiError(val, "fake") + if kind == "http": + return val, b"{}" + if kind == "raw": + return 200, val + return 200, json.dumps({"token": val}).encode() + + def get_pinned(self, hostname, ip, port, path, query): + self.calls.append(("addkey", hostname, ip, port, path, dict(query))) + kind, val = self.addkey.get(hostname, ("http", 502)) + if kind == "err": + raise pg.ApiError(val, "fake") + if kind == "http": + return val, b"" + if kind == "raw": + return 200, val + body = {"status": "OK", "server_key": SERVER_KEYS[hostname], "peer_ip": "10.5.5.5", "server_port": 1337, "server_ip": ip} + body.update(val or {}) + return 200, json.dumps(body).encode() + + +class Base(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="pia-gateway-test-")) + (self.tmp / "wg").mkdir(mode=0o700) + (self.tmp / "state").mkdir(mode=0o700) + (self.tmp / "etc").mkdir(mode=0o700) + creds = self.tmp / "etc" / "credentials.env" + creds.write_text(f"PIA_USER={SECRET_USER}\nPIA_PASSWORD={SECRET_PASS}\nALERT_BRIDGE_API_KEY={SECRET_ALERT_KEY}\n") + creds.chmod(0o600) + (self.tmp / "wg" / "pia-ca.crt").write_text("dummy") + self.cfg = pg.Config( + config_dir=str(self.tmp / "wg"), + state_dir=str(self.tmp / "state"), + credentials_file=str(creds), + ca_file=str(self.tmp / "wg" / "pia-ca.crt"), + lock_file=str(self.tmp / "lock"), + metrics_file=str(self.tmp / "state" / "metrics.prom"), + status_file=str(self.tmp / "state" / "status.json"), + alert_bridge_url="http://alert-bridge.test/alerts", + ) + self.sysm = FakeSystem(self.tmp) + self.http = FakeHttp() + self.out = io.StringIO() + self.log = pg.Log(stream=self.out) + self.notified: list[tuple[str, str]] = [] + + def fake_send(_self, level, message): + self.notified.append((level, message)) + return True + + self._send_patch = mock.patch.object(pg.Notifier, "send", fake_send) + self._send_patch.start() + self.write_conf(PRIV_A, SRV_OLD, "10.99.0.1") + (self.tmp / "wg" / "pia-wg.key").write_text(PRIV_A + "\n") + (self.tmp / "wg" / "pia-wg.key").chmod(0o600) + self.sysm.set_peer(SRV_OLD, 30) + self.ctl = pg.Controller(self.cfg, self.sysm, self.log, http=self.http) + # Simulate a controller that has already run once on this boot, so the + # startup grace does not absorb the first checks of every test. The + # fresh-boot behaviour has its own test. + seeded = pg.default_state() + seeded["boot_id"] = self.sysm.boot + self.ctl.store.save(seeded) + + def tearDown(self): + self._send_patch.stop() + shutil.rmtree(self.tmp, ignore_errors=True) + + def write_conf(self, priv, server_key, ip): + peer = pg.PeerInfo(server_key, "10.5.5.5", 1337, ip, "hk", "Hong Kong", "old.example") + p = self.tmp / "wg" / "pia-wg.conf" + p.write_text(pg.render_wg_conf(self.cfg, priv, peer)) + p.chmod(0o600) + + def state(self): + return json.loads((self.tmp / "state" / "state.json").read_text()) + + def run_recover_n(self, n, force=False): + rcs = [] + for _ in range(n): + rcs.append(self.ctl.cmd_recover(force=force)) + self.sysm.sleep(60) + return rcs + + def go_stale(self): + self.sysm.set_peer(SRV_OLD, 5000) + + def all_output(self): + s = self.out.getvalue() + for p in (self.tmp / "state" / "status.json", self.tmp / "state" / "metrics.prom", self.tmp / "state" / "state.json"): + if p.exists(): + s += p.read_text() + s += json.dumps(self.notified) + s += json.dumps(self.sysm.commands) + return s + + def assert_no_secrets(self): + blob = self.all_output() + for sec in (SECRET_USER, SECRET_PASS, SECRET_TOKEN, SECRET_ALERT_KEY, PRIV_A, PRIV_B): + self.assertNotIn(sec, blob, f"secret {sec[:12]}... leaked into output") + + +class TestClassification(Base): + def test_healthy(self): + st = self.ctl.current_status(pg.default_state()) + self.assertEqual(st.cls, pg.CLASS_HEALTHY) + self.assertTrue(st.safety_ok and st.tunnel_ok) + self.assertEqual(st.handshake_age_sec, 30) + + def test_stale_handshake(self): + self.go_stale() + st = self.ctl.current_status(pg.default_state()) + self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE) + self.assertIn("handshake 5000s old exceeds 180s", st.reasons) + + def test_zero_handshake(self): + self.sysm.set_peer(SRV_OLD, None) + st = self.ctl.current_status(pg.default_state()) + self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE) + self.assertIn("no handshake recorded", st.reasons) + + def test_future_handshake_is_not_healthy(self): + self.sysm.set_peer(SRV_OLD, -5000) + st = self.ctl.current_status(pg.default_state()) + self.assertEqual(st.cls, pg.CLASS_TUNNEL_STALE) + self.assertTrue(any("future" in r for r in st.reasons)) + + def test_small_clock_skew_tolerated(self): + self.sysm.set_peer(SRV_OLD, -10) + st = self.ctl.current_status(pg.default_state()) + self.assertEqual(st.cls, pg.CLASS_HEALTHY) + + def test_malformed_wg_output(self): + self.sysm.peers = {} + orig = self.sysm.run + + def run(argv, stdin=None, timeout=60): + if argv[:2] == ["wg", "show"] and argv[3] == "latest-handshakes": + self.sysm.commands.append(argv) + return pg.CmdResult(0, "garbage without tab\n", "") + return orig(argv, stdin, timeout) + + self.sysm.run = run + st = self.ctl.current_status(pg.default_state()) + self.assertNotEqual(st.cls, pg.CLASS_HEALTHY) + self.assertIn("malformed wg output", st.reasons) + + def test_wrong_peer_on_interface(self): + self.sysm.set_peer(SRV_HK1, 10) + st = self.ctl.current_status(pg.default_state()) + self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT) + + def test_interface_absent(self): + self.sysm.iface_exists = False + st = self.ctl.current_status(pg.default_state()) + self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT) + + def test_fresh_handshake_but_route_closed_is_not_ok(self): + self.sysm.table_default = "closed" + st = self.ctl.current_status(pg.default_state()) + self.assertTrue(st.safety_ok) + self.assertFalse(st.tunnel_ok) + self.assertEqual(st.cls, pg.CLASS_TUNNEL_ABSENT) + + def test_boot_grace_class(self): + self.go_stale() + state = pg.default_state() + state["boot_grace_until"] = self.sysm.t + 60 + self.assertEqual(self.ctl.current_status(state).cls, pg.CLASS_BOOT_GRACE) + + def test_paused_class(self): + self.go_stale() + state = pg.default_state() + state["paused"] = {"since": 1, "reason": "x"} + st = self.ctl.current_status(state) + self.assertEqual(st.cls, pg.CLASS_PAUSED) + self.assertEqual(st.tunnel_class, pg.CLASS_TUNNEL_STALE) + + +class TestSafetyDrift(Base): + def drift_cases(self): + yield "missing rule", lambda: setattr(self.sysm, "rule_present", False) + yield "missing route", lambda: setattr(self.sysm, "table_default", "missing") + yield "jump below accept", lambda: setattr(self.sysm, "forward_rules", ["-A FORWARD -j ACCEPT", f"-A FORWARD -s {SUBNET} -j {CHAIN}"]) + yield "no terminal drop", lambda: setattr(self.sysm, "chain_rules", self.sysm.open_chain()[:-1]) + yield "wrong iface accept", lambda: setattr(self.sysm, "chain_rules", [f"-A {CHAIN} -s {SUBNET} -o enp1s0 -j ACCEPT"] + self.sysm.open_chain()) + yield "chain missing", lambda: setattr(self.sysm, "chain_rules", None) + yield "host default via tunnel", lambda: setattr(self.sysm, "main_default", "default dev pia-wg scope link") + + def test_each_drift_triggers_emergency_close_and_no_restart(self): + for name, apply in self.drift_cases(): + with self.subTest(name): + self.setUp() + apply() + rc = self.ctl.cmd_recover() + self.assertEqual(rc, pg.EXIT_SAFETY, name) + self.assertEqual(self.sysm.restart_count, 0, name) + self.assertTrue(self.sysm.forward_rules[0].startswith(f"-A FORWARD -s {SUBNET} -m comment --comment pia-gateway-emergency -j DROP"), name) + self.assertEqual(self.sysm.table_default, "closed", name) + self.assertTrue(self.state()["emergency_close_active"], name) + self.assertTrue(any(lvl == "error" and "SAFETY DRIFT" in msg for lvl, msg in self.notified), name) + self.assert_no_secrets() + self.tearDown() + + def test_emergency_rule_removed_after_repair(self): + self.sysm.rule_present = False + self.ctl.cmd_recover() + self.assertTrue(self.state()["emergency_close_active"]) + # operator repairs the kill switch and the tunnel + self.sysm.rule_present = True + self.sysm.table_default = "open" + self.sysm.set_peer(SRV_OLD, 5) + rc = self.ctl.cmd_recover() + self.assertEqual(rc, pg.EXIT_OK) + self.assertFalse(self.state()["emergency_close_active"]) + self.assertFalse(any("emergency" in r for r in self.sysm.forward_rules)) + + def test_emergency_close_is_idempotent(self): + self.sysm.rule_present = False + self.ctl.cmd_recover() + self.ctl.cmd_recover() + self.assertEqual(sum("emergency" in r for r in self.sysm.forward_rules), 1) + + def test_emergency_close_repositions_when_pushed_below_a_preceding_accept(self): + self.sysm.rule_present = False + self.ctl.cmd_recover() + self.assertTrue(pg._is_emergency_rule(self.sysm.forward_rules[0])) + # Something outside this role's own handler ordering (e.g. a + # manual `ufw reload`) prepends a broader ACCEPT ahead of our + # rule without removing it — it still exists, but is no longer + # first, so it could be pre-empted by whatever now sits above it. + self.sysm.forward_rules.insert(0, "-A FORWARD -j ACCEPT") + self.assertFalse(pg._is_emergency_rule(self.sysm.forward_rules[0])) + rc = self.ctl.cmd_recover() + self.assertEqual(rc, pg.EXIT_SAFETY) + self.assertTrue(pg._is_emergency_rule(self.sysm.forward_rules[0])) + self.assertEqual(sum(1 for r in self.sysm.forward_rules if pg._is_emergency_rule(r)), 1) + self.assertIn("-A FORWARD -j ACCEPT", self.sysm.forward_rules) + + def test_register_refuses_on_drift(self): + self.sysm.rule_present = False + rc = self.ctl.cmd_register("same-key", None, "ansible") + self.assertEqual(rc, pg.EXIT_SAFETY) + self.assertEqual(self.http.calls, []) + + +class TestRecoveryStateMachine(Base): + def test_healthy_runs_are_idempotent_and_do_not_mutate(self): + self.run_recover_n(3) + mutating = [ + c for c in self.sysm.commands + if c[:2] in (["systemctl", "restart"], ["systemctl", "start"], ["wg", "genkey"], ["iptables", "-I"], ["iptables", "-D"], ["ip", "link"]) + or c[:3] == ["ip", "route", "replace"] + ] + self.assertEqual(mutating, []) + self.assertEqual(self.state()["consecutive_failures"], 0) + + def test_threshold_before_action(self): + self.go_stale() + self.run_recover_n(2) + self.assertEqual(self.sysm.restart_count, 0) + self.assertEqual(self.state()["consecutive_failures"], 2) + + def test_boot_grace_not_counted(self): + self.go_stale() + self.sysm.boot = "boot-2" + self.run_recover_n(2) + self.assertEqual(self.state()["consecutive_failures"], 0) + self.assertEqual(self.sysm.restart_count, 0) + self.sysm.sleep(120) + self.run_recover_n(3) + self.assertEqual(self.sysm.restart_count, 1) + + def test_restart_success_recovers(self): + self.go_stale() + rcs = self.run_recover_n(3) + self.assertEqual(rcs[-1], pg.EXIT_OK) + self.assertEqual(self.sysm.restart_count, 1) + s = self.state() + self.assertEqual(s["last_recovery"]["stage"], "restart") + self.assertEqual(s["last_recovery"]["pre_fix_handshake_age_sec"], 5000 + 120) + self.assertEqual(s["consecutive_failures"], 0) + self.assertFalse(s["recovery"]["in_progress"]) + self.assertEqual(self.http.calls, []) + levels = [l for l, _ in self.notified] + self.assertEqual(levels, ["warn", "info"]) + self.assert_no_secrets() + + def test_restart_failure_escalates_to_same_key_registration(self): + self.go_stale() + self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK1 else None) + self.http.addkey = {"hk1.example": ("ok", {})} + rcs = self.run_recover_n(3) + self.assertEqual(rcs[-1], pg.EXIT_OK) + s = self.state() + self.assertEqual(s["last_recovery"]["stage"], "reregister") + self.assertEqual(s["last_verified_registration"]["server_cn"], "hk1.example") + self.assertEqual(s["last_verified_registration"]["mode"], "same-key") + self.assertEqual(s["last_verified_registration"]["pubkey"], PUB_A) + conf = (self.tmp / "wg" / "pia-wg.conf").read_text() + self.assertIn(f"PublicKey = {SRV_HK1}", conf) + self.assertIn("Endpoint = 10.99.0.1:1337", conf) + self.assertIn("Table = off", conf) + self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.conf").stat().st_mode), 0o600) + addkeys = [c for c in self.http.calls if c[0] == "addkey"] + self.assertEqual(addkeys[0][5]["pubkey"], PUB_A) + self.assertEqual(addkeys[0][5]["pt"], SECRET_TOKEN) + self.assertTrue(any(n.startswith("pia-wg.conf.bak-") for n in os.listdir(self.tmp / "wg"))) + self.assert_no_secrets() + + def test_later_candidate_after_activation_failure(self): + self.go_stale() + self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK2 else None) + self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {})} + self.run_recover_n(3) + s = self.state() + self.assertEqual(s["last_verified_registration"]["server_cn"], "hk2.example") + order = [c[1] for c in self.http.calls if c[0] == "addkey"] + self.assertEqual(order, ["hk1.example", "hk2.example"]) + + def test_region_order_hk_then_taiwan(self): + self.go_stale() + self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_TW1 else None) + self.http.addkey = {"hk1.example": ("http", 500), "hk2.example": ("err", "timeout"), "tw1.example": ("ok", {})} + self.run_recover_n(3) + order = [c[1] for c in self.http.calls if c[0] == "addkey"] + self.assertEqual(order, ["hk1.example", "hk2.example", "tw1.example"]) + self.assertEqual(self.state()["last_verified_registration"]["region"], "taiwan") + + def test_rotation_after_same_key_accepted_but_no_handshake(self): + self.go_stale() + + def beh(key): + # Only a config carrying the NEW key handshakes. + conf = (self.tmp / "wg" / "pia-wg.conf").read_text() + return self.sysm.t if f"PrivateKey = {PRIV_B}" in conf else None + + self.sysm.handshake_on_restart = beh + self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})} + rcs = self.run_recover_n(3) + self.assertEqual(rcs[-1], pg.EXIT_OK) + s = self.state() + self.assertEqual(s["last_recovery"]["stage"], "rotate") + self.assertEqual(s["last_verified_registration"]["mode"], "new-key") + self.assertEqual(s["last_verified_registration"]["pubkey"], PUB_B) + self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_B) + names = os.listdir(self.tmp / "wg") + self.assertTrue(any(n.startswith("pia-wg.key.bak-") for n in names)) + self.assertFalse(any(n.startswith("pia-wg.key.new") for n in names)) + self.assertFalse(any(".conf.stage" in n for n in names)) + self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.key").stat().st_mode), 0o600) + self.assert_no_secrets() + + def test_no_rotation_on_uniform_api_failure(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.addkey = {"hk1.example": ("err", "timeout"), "hk2.example": ("err", "timeout"), "tw1.example": ("err", "timeout")} + self.run_recover_n(3) + s = self.state() + self.assertEqual(s["recovery"]["last_failure_class"], pg.FAIL_HOST_CONNECTIVITY) + self.assertIsNotNone(s["recovery"]["next_retry_epoch"]) + self.assertEqual(len(self.sysm.genkey_queue), 1) # no key generated + self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A) + + def test_all_candidates_fail_restores_previous_and_backs_off(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("http", 500), "tw1.example": ("ok", {})} + rcs = self.run_recover_n(3) + self.assertEqual(rcs[-1], pg.EXIT_UNHEALTHY) + s = self.state() + self.assertTrue(s["recovery"]["in_progress"]) + self.assertIsNotNone(s["recovery"]["next_retry_epoch"]) + conf = (self.tmp / "wg" / "pia-wg.conf").read_text() + self.assertIn(f"PublicKey = {SRV_OLD}", conf) + self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A) + self.assertFalse(any(n.startswith("pia-wg.key.new") for n in os.listdir(self.tmp / "wg"))) + # kill switch untouched: chain still ends in DROP, jump still first + self.assertEqual(self.sysm.forward_rules[0], f"-A FORWARD -s {SUBNET} -j {CHAIN}") + self.assertEqual(self.sysm.chain_rules[-1], f"-A {CHAIN} -j DROP") + self.assertTrue(any("recovery failed" in m for _, m in self.notified)) + self.assert_no_secrets() + + def test_auth_failure_no_rotation_error_alert(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.token = ("http", 401) + self.run_recover_n(3) + s = self.state() + self.assertEqual(s["recovery"]["last_failure_class"], pg.FAIL_AUTH) + self.assertFalse(any(c[0] == "addkey" for c in self.http.calls)) + self.assertTrue(any(l == "error" for l, _ in self.notified)) + self.assertEqual(len(self.sysm.genkey_queue), 1) + + def test_serverlist_unreachable_is_host_connectivity(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + self.run_recover_n(3) + self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_HOST_CONNECTIVITY) + self.assertFalse(any(c[0] in ("post", "addkey") for c in self.http.calls)) + + def test_malformed_serverlist(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("raw", b"not json\n") + self.run_recover_n(3) + self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_REGISTRATION) + + def test_malformed_and_invalid_addkey_responses_skip_to_next(self): + self.go_stale() + self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_TW1 else None) + self.http.addkey = { + "hk1.example": ("raw", b""), + "hk2.example": ("ok", {"peer_ip": "999.1.1.1"}), + "tw1.example": ("ok", {}), + } + self.run_recover_n(3) + self.assertEqual(self.state()["last_verified_registration"]["server_cn"], "tw1.example") + + def test_tls_failure_kind_recorded(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.addkey = {"hk1.example": ("err", "tls"), "hk2.example": ("err", "tls"), "tw1.example": ("err", "tls")} + self.run_recover_n(3) + self.assertEqual(self.state()["recovery"]["last_failure_class"], pg.FAIL_REGISTRATION) + self.assertIn("kind=tls", self.out.getvalue()) + + def test_backoff_sequence_bounded(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + delays = [] + self.run_recover_n(3) + for _ in range(5): + s = self.state() + delays.append(s["recovery"]["next_retry_epoch"] - s["recovery"]["last_failure_epoch"]) + self.sysm.t = s["recovery"]["next_retry_epoch"] + 1 + self.ctl.cmd_recover() + self.assertEqual(delays, [300, 900, 1800, 3600, 3600]) + + def test_backoff_prevents_hammering(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + self.run_recover_n(3) + before = self.sysm.restart_count + self.run_recover_n(4) + self.assertEqual(self.sysm.restart_count, before) + + def test_retry_now_bypasses_backoff(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + self.run_recover_n(3) + before = self.sysm.restart_count + self.sysm.handshake_on_restart = "fresh" + rc = self.ctl.cmd_retry_now() + self.assertEqual(rc, pg.EXIT_OK) + self.assertEqual(self.sysm.restart_count, before + 1) + + def test_credentials_missing_still_backs_off(self): + os.unlink(self.cfg.credentials_file) + self.ctl._creds = None + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.run_recover_n(3) + s = self.state() + self.assertIsNotNone(s["recovery"]["next_retry_epoch"]) + before = self.sysm.restart_count + self.run_recover_n(2) + self.assertEqual(self.sysm.restart_count, before) + + def test_time_budget_bounds_a_cycle(self): + self.cfg.recovery_max_duration_sec = 100 + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})} + self.run_recover_n(3) + addkeys = [c for c in self.http.calls if c[0] == "addkey"] + self.assertLess(len(addkeys), 3) + self.assertIn("time budget", self.out.getvalue()) + + def test_recovered_without_our_action_clears_backoff(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + self.run_recover_n(3) + self.sysm.set_peer(SRV_OLD, 5) + rc = self.ctl.cmd_recover() + self.assertEqual(rc, pg.EXIT_OK) + self.assertIsNone(self.state()["recovery"]["next_retry_epoch"]) + self.assertFalse(self.state()["recovery"]["in_progress"]) + + def test_backward_clock_jump_clears_stale_deadline(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + self.run_recover_n(3) + self.assertIsNotNone(self.state()["recovery"]["next_retry_epoch"]) + self.sysm.t -= 6 * 3600 # NTP steps the clock back six hours + self.sysm.handshake_on_restart = "fresh" + self.http.serverlist = ("ok", serverlist_doc()) + self.sysm.set_peer(SRV_OLD, 5000) + rc = self.ctl.cmd_recover() + self.assertIn("clock jump detected", self.out.getvalue()) + self.assertEqual(rc, pg.EXIT_OK) + self.assertEqual(self.state()["last_recovery"]["stage"], "restart") + + def test_normal_backoff_is_not_mistaken_for_clock_jump(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + self.run_recover_n(3) + self.ctl.cmd_recover() + self.assertNotIn("clock jump detected", self.out.getvalue()) + self.assertIsNotNone(self.state()["recovery"]["next_retry_epoch"]) + + def test_orphaned_interface_recovery_on_restart(self): + self.go_stale() + calls = {"n": 0} + orig = self.sysm._apply_restart + + def restart(conf_path): + calls["n"] += 1 + if calls["n"] == 1: + self.sysm.restart_count += 1 + return pg.CmdResult(1, "", "RTNETLINK answers: File exists\n`pia-wg' already exists") + return orig(conf_path) + + self.sysm._apply_restart = restart + self.run_recover_n(3) + self.assertIn(["ip", "link", "delete", "dev", IFACE], self.sysm.commands) + self.assertEqual(self.state()["last_recovery"]["stage"], "restart") + + +class TestHandshakeVerificationBoundary(Base): + """wait_for_verified's freshness check, at the exact one-second boundary. + + `wg show`'s handshake epoch and `since_epoch` are both whole seconds, + so a handshake reported in the SAME second as `since_epoch` is + indistinguishable from one that predates activation and merely shares + that truncated second. It must never count as proof of a NEW + handshake — that's the whole point of comparing against `since_epoch` + at all (an `addKey` returning `status: OK` was already established + elsewhere as necessary-but-not-sufficient; the same logic applies one + layer down, to the handshake timestamp itself). + """ + + def test_handshake_at_exactly_since_epoch_is_rejected(self): + since_epoch = self.sysm.t + self.sysm.set_peer(SRV_OLD, 0) # handshake_epoch == self.sysm.t == since_epoch, exactly + verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD) + self.assertIsNone(verified) + + def test_handshake_one_second_after_since_epoch_is_accepted(self): + since_epoch = self.sysm.t + self.sysm.sleep(1) + self.sysm.set_peer(SRV_OLD, 0) # handshake_epoch == self.sysm.t == since_epoch + 1 + verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD) + self.assertIsNotNone(verified) + + def test_handshake_one_second_before_since_epoch_is_rejected(self): + self.sysm.sleep(1) + since_epoch = self.sysm.t + self.sysm.set_peer(SRV_OLD, 1) # handshake_epoch == self.sysm.t - 1 == since_epoch - 1 + verified = self.ctl.wait_for_verified(pg.default_state(), since_epoch, timeout_sec=1, expected_peer=SRV_OLD) + self.assertIsNone(verified) + + +class TestPersistenceAndConcurrency(Base): + def test_state_survives_new_controller(self): + self.go_stale() + self.sysm.handshake_on_restart = "stale" + self.http.serverlist = ("err", "network") + self.run_recover_n(3) + s1 = self.state() + ctl2 = pg.Controller(self.cfg, self.sysm, self.log, http=self.http) + self.sysm.boot = "boot-2" + ctl2.cmd_recover() + s2 = self.state() + self.assertEqual(s1["recovery"]["next_retry_epoch"], s2["recovery"]["next_retry_epoch"]) + self.assertEqual(s2["boot_id"], "boot-2") + + def test_corrupt_state_fails_closed(self): + (self.tmp / "state" / "state.json").write_text("{not json") + self.go_stale() + rc = self.ctl.cmd_recover() + self.assertEqual(rc, pg.EXIT_STATE_ERROR) + self.assertEqual(self.sysm.restart_count, 0) + self.assertTrue(any("corrupt" in m for _, m in self.notified)) + self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_STATE_ERROR) + self.assertEqual(self.ctl.cmd_register("same-key", None, "operator"), pg.EXIT_STATE_ERROR) + self.assertEqual(self.ctl.cmd_rewrite_config(), pg.EXIT_STATE_ERROR) + self.assertEqual(self.ctl.cmd_pause("x"), pg.EXIT_STATE_ERROR) + self.assertEqual(self.ctl.cmd_resume(), pg.EXIT_STATE_ERROR) + + def test_wrong_schema_version_fails_closed(self): + s = pg.default_state() + s["schema_version"] = 99 + (self.tmp / "state" / "state.json").write_text(json.dumps(s)) + self.assertEqual(self.ctl.cmd_recover(), pg.EXIT_STATE_ERROR) + + def test_state_reset_requires_confirm_and_preserves_old(self): + (self.tmp / "state" / "state.json").write_text("{bad") + self.assertEqual(self.ctl.cmd_state_reset(confirm=False), pg.EXIT_USAGE) + self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_OK) + names = os.listdir(self.tmp / "state") + self.assertTrue(any(n.startswith("state.json.corrupt-") for n in names)) + self.assertEqual(self.state()["schema_version"], pg.STATE_SCHEMA_VERSION) + + def test_atomic_write_no_temp_left_and_mode(self): + p = self.tmp / "state" / "x.json" + self.sysm.write_atomic(str(p), "abc", 0o600) + self.assertEqual(p.read_text(), "abc") + self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o600) + self.assertEqual([n for n in os.listdir(self.tmp / "state") if n.endswith(".tmp")], []) + + def test_interrupted_write_keeps_original(self): + p = self.tmp / "state" / "state.json" + p.write_text("original") + with mock.patch.object(pg.os, "replace", side_effect=OSError("boom")): + with self.assertRaises(OSError): + self.sysm.write_atomic(str(p), "new", 0o600) + self.assertEqual(p.read_text(), "original") + self.assertEqual([n for n in os.listdir(self.tmp / "state") if n.endswith(".tmp")], []) + + def test_lock_contention_skips_without_acting(self): + self.go_stale() + self.run_recover_n(2) + holder = pg.Lock(self.cfg.lock_file, self.sysm) + holder.acquire(0) + try: + rc = self.ctl.cmd_recover() + self.assertEqual(rc, pg.EXIT_OK) + self.assertEqual(self.sysm.restart_count, 0) + self.assertIn("another action holds the lock", self.out.getvalue()) + self.assertEqual(self.ctl.cmd_register("same-key", None, "operator"), pg.EXIT_LOCKED) + finally: + holder.release() + rc = self.ctl.cmd_recover() + self.assertEqual(self.sysm.restart_count, 1) + + def test_lock_released_on_exception(self): + with mock.patch.object(self.ctl, "_decide", side_effect=RuntimeError("x")): + with self.assertRaises(RuntimeError): + self.ctl.cmd_recover() + lk = pg.Lock(self.cfg.lock_file, self.sysm) + lk.acquire(0) + lk.release() + + def test_pause_and_resume(self): + self.ctl.cmd_pause("maintenance") + self.go_stale() + self.run_recover_n(5) + self.assertEqual(self.sysm.restart_count, 0) + self.assertTrue(self.state()["paused"]) + self.assertEqual(self.ctl.cmd_renew(force_due=True), pg.EXIT_OK) + self.assertEqual(self.http.calls, []) + self.ctl.cmd_resume() + self.run_recover_n(3) + self.assertEqual(self.sysm.restart_count, 1) + + def test_pause_does_not_open_gateway(self): + self.sysm.table_default = "closed" + self.sysm.chain_rules = self.sysm.closed_chain() + self.ctl.cmd_pause("x") + self.ctl.cmd_recover() + self.assertEqual(self.sysm.table_default, "closed") + self.assertEqual(self.sysm.restart_count, 0) + + def test_pause_and_resume_serialize_with_an_inflight_recovery_pass(self): + # Simulate a recovery pass that is mid-flight: it already holds the + # SAME lock pause/resume now also take, and has not yet saved its + # (unpaused) state back to disk — this is exactly the window in + # which the old, unlocked pause/resume could silently lose the + # race and have their write clobbered by the pass finishing later. + self.cfg.lock_wait_sec = 3 + holder = pg.Lock(self.cfg.lock_file, self.sysm) + holder.acquire(0) + try: + self.assertEqual(self.ctl.cmd_pause("maintenance"), pg.EXIT_LOCKED) + self.assertEqual(self.ctl.cmd_resume(), pg.EXIT_LOCKED) + # blocked, not silently skipped-and-lost: state is untouched + self.assertIsNone(self.state().get("paused")) + finally: + # the in-flight pass "finishes": saves its own state (as + # cmd_recover's finally block does) and releases the lock. + self.ctl.store.save(pg.default_state()) + holder.release() + # now that the lock is free, pause proceeds against the CURRENT + # on-disk state rather than a copy read before it waited + self.assertEqual(self.ctl.cmd_pause("maintenance"), pg.EXIT_OK) + self.assertTrue(self.state()["paused"]) + # a recovery pass started afterwards must also acquire the lock + # and load state fresh before it can act, so it observes the + # pause and never clears or overwrites it + self.go_stale() + self.run_recover_n(5) + self.assertEqual(self.sysm.restart_count, 0) + self.assertTrue(self.state()["paused"]) + + def test_state_reset_serializes_with_lock(self): + holder = pg.Lock(self.cfg.lock_file, self.sysm) + holder.acquire(0) + try: + self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_LOCKED) + finally: + holder.release() + self.assertEqual(self.ctl.cmd_state_reset(confirm=True), pg.EXIT_OK) + + +class TestRenewal(Base): + def seed_registration(self, age_days): + s = pg.default_state() + s["last_verified_registration"] = {"epoch": self.sysm.t - age_days * 86400, "region": "hk", "server_cn": "old.example", "pubkey": PUB_A, "mode": "same-key", "via": "ansible"} + self.ctl.store.save(s) + + def test_not_due(self): + self.seed_registration(10) + self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK) + self.assertEqual(self.http.calls, []) + + def test_due_and_succeeds_advances_timestamp(self): + self.seed_registration(22) + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK) + s = self.state() + self.assertEqual(s["last_verified_registration"]["via"], "renew") + self.assertEqual(s["last_verified_registration"]["mode"], "same-key") + self.assertGreaterEqual(s["last_verified_registration"]["epoch"], self.sysm.t - 60) + self.assertEqual(len(self.sysm.genkey_queue), 1) + + def test_due_and_fails_keeps_timestamp_and_backs_off(self): + self.seed_registration(22) + old = self.state()["last_verified_registration"]["epoch"] + self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_OLD else None) + self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {}), "tw1.example": ("ok", {})} + self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_UNHEALTHY) + s = self.state() + self.assertEqual(s["last_verified_registration"]["epoch"], old) + self.assertIsNotNone(s["renewal"]["next_retry_epoch"]) + self.assertIn(f"PublicKey = {SRV_OLD}", (self.tmp / "wg" / "pia-wg.conf").read_text()) + self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK) # in backoff, no call + self.assertTrue(any("renewal failed" in m for _, m in self.notified)) + + def test_force_due(self): + self.seed_registration(1) + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + self.assertEqual(self.ctl.cmd_renew(force_due=True), pg.EXIT_OK) + self.assertEqual(self.state()["last_verified_registration"]["via"], "renew") + + def test_defers_to_recovery_when_unhealthy(self): + self.seed_registration(30) + self.go_stale() + self.assertEqual(self.ctl.cmd_renew(), pg.EXIT_OK) + self.assertEqual(self.http.calls, []) + + +class TestRegisterCommand(Base): + def test_fresh_install_generates_key_and_registers(self): + os.unlink(self.tmp / "wg" / "pia-wg.conf") + os.unlink(self.tmp / "wg" / "pia-wg.key") + self.sysm.iface_exists = False + self.sysm.table_default = "closed" + self.sysm.chain_rules = self.sysm.closed_chain() + self.sysm.genkey_queue = [PRIV_A] + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + rc = self.ctl.cmd_register("same-key", None, "ansible") + self.assertEqual(rc, pg.EXIT_OK) + self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A) + self.assertEqual(stat.S_IMODE((self.tmp / "wg" / "pia-wg.key").stat().st_mode), 0o600) + s = self.state() + self.assertEqual(s["last_verified_registration"]["via"], "ansible") + self.assertEqual(s["active_registration"]["server_key"], SRV_HK1) + self.assert_no_secrets() + + def test_register_refuses_when_killswitch_inactive(self): + self.sysm.killswitch_active = False + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + rc = self.ctl.cmd_register("same-key", None, "ansible") + self.assertEqual(rc, pg.EXIT_UNHEALTHY) + self.assertEqual(self.sysm.restart_count, 0) + self.assertIn(f"PublicKey = {SRV_OLD}", (self.tmp / "wg" / "pia-wg.conf").read_text()) + + def test_region_override(self): + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"tw1.example": ("ok", {})} + rc = self.ctl.cmd_register("same-key", "taiwan", "operator") + self.assertEqual(rc, pg.EXIT_OK) + self.assertEqual([c[1] for c in self.http.calls if c[0] == "addkey"], ["tw1.example"]) + + def test_new_key_mode_with_abandoned_stage_cleanup(self): + (self.tmp / "wg" / "pia-wg.key.new").write_text("abandoned\n") + (self.tmp / "wg" / "pia-wg.key.new.20260101T000000Z.1").write_text("abandoned2\n") + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + rc = self.ctl.cmd_register("new-key", None, "operator") + self.assertEqual(rc, pg.EXIT_OK) + names = os.listdir(self.tmp / "wg") + self.assertFalse(any(n.startswith("pia-wg.key.new") for n in names)) + self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_B) + self.assertIn(f"pubkey", json.dumps(self.state()["last_verified_registration"])) + self.assertEqual(self.state()["last_verified_registration"]["pubkey"], PUB_B) + + def test_same_key_does_not_rotate(self): + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + self.ctl.cmd_register("same-key", None, "operator") + self.assertEqual((self.tmp / "wg" / "pia-wg.key").read_text().strip(), PRIV_A) + self.assertEqual(len(self.sysm.genkey_queue), 1) + + def test_backups_bounded(self): + self.cfg.backup_keep = 2 + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + for i in range(5): + self.sysm.sleep(1) + self.ctl.cmd_register("same-key", None, "operator") + baks = [n for n in os.listdir(self.tmp / "wg") if n.startswith("pia-wg.conf.bak-")] + self.assertEqual(len(baks), 2) + + def test_rewrite_config_uses_recorded_registration(self): + self.sysm.handshake_on_restart = "fresh" + self.http.addkey = {"hk1.example": ("ok", {})} + self.ctl.cmd_register("same-key", None, "operator") + self.cfg.mss_clamp_enabled = False + calls_before = len(self.http.calls) + rc = self.ctl.cmd_rewrite_config() + self.assertEqual(rc, pg.EXIT_OK) + self.assertEqual(len(self.http.calls), calls_before) + conf = (self.tmp / "wg" / "pia-wg.conf").read_text() + self.assertNotIn("TCPMSS", conf) + self.assertIn(f"PublicKey = {SRV_HK1}", conf) + + +class TestConfigValidation(Base): + def render(self, **kw): + peer = pg.PeerInfo(SRV_HK1, "10.5.5.5", 1337, "10.99.0.1", "hk", "Hong Kong", "hk1.example") + return pg.render_wg_conf(self.cfg, PRIV_A, peer) + + def test_rendered_config_is_valid(self): + self.assertEqual(pg.validate_wg_conf(self.cfg, self.render()), []) + + def test_rejects_table_not_off(self): + text = self.render().replace("Table = off", "Table = auto") + self.assertIn("Table must be off", pg.validate_wg_conf(self.cfg, text)) + + def test_rejects_bad_keys_and_endpoint(self): + text = self.render().replace(f"PublicKey = {SRV_HK1}", "PublicKey = nope").replace("Endpoint = 10.99.0.1:1337", "Endpoint = bad") + problems = pg.validate_wg_conf(self.cfg, text) + self.assertIn("peer PublicKey invalid", problems) + self.assertIn("Endpoint invalid", problems) + + def test_rejects_unscoped_hook_and_policy_change(self): + text = self.render().replace( + f"PostUp = ip route replace default dev %i table pia", + "PostUp = ip route replace default dev %i\nPostUp = iptables -P FORWARD ACCEPT", + ) + problems = pg.validate_wg_conf(self.cfg, text) + self.assertTrue(any("without a table" in p for p in problems)) + self.assertIn("hook changes FORWARD policy", problems) + self.assertIn("PostUp lacks table route open", problems) + + def test_rejects_wide_allowed_ips_change(self): + text = self.render().replace("AllowedIPs = 0.0.0.0/0", "AllowedIPs = ::/0") + self.assertIn("AllowedIPs must be 0.0.0.0/0", pg.validate_wg_conf(self.cfg, text)) + + def test_expected_peer_parse(self): + self.assertEqual(pg.parse_expected_peer(self.render()), SRV_HK1) + self.assertIn("Hong Kong (hk), hk1.example", pg.parse_configured_peer_line(self.render())) + + +class TestSecretsAndModes(Base): + def test_credentials_mode_enforced(self): + os.chmod(self.cfg.credentials_file, 0o644) + with self.assertRaises(pg.GatewayError): + pg.load_credentials(self.cfg.credentials_file) + + def test_credentials_repr_redacted(self): + c = pg.load_credentials(self.cfg.credentials_file) + self.assertNotIn(SECRET_PASS, repr(c) + str(c)) + + def test_no_secret_in_argv_stdin_logs_or_files_across_full_cycle(self): + self.go_stale() + self.sysm.handshake_on_restart = lambda key: (self.sysm.t if key == SRV_HK2 else None) + self.http.addkey = {"hk1.example": ("ok", {}), "hk2.example": ("ok", {})} + self.run_recover_n(3) + self.ctl.cmd_check() + self.ctl.cmd_status() + self.assert_no_secrets() + for argv in self.sysm.commands: + for tok in argv: + self.assertNotIn(SECRET_TOKEN, tok) + self.assertNotIn(PRIV_A, tok) + # the private key goes to `wg pubkey` via stdin only (never argv) and nowhere else + for s in self.sysm.stdins: + self.assertIn(s.strip(), (PRIV_A, PRIV_B)) + + def test_notification_payloads_are_secret_free_and_failures_do_not_block(self): + self._send_patch.stop() + sent = [] + + def flaky(_self, level, message): + sent.append((level, message)) + return False + + with mock.patch.object(pg.Notifier, "send", flaky): + self.go_stale() + self.run_recover_n(3) + self._send_patch.start() + self.assertEqual(self.state()["last_recovery"]["stage"], "restart") + self.assertGreaterEqual(len(self.state()["pending_notifications"]), 1) + for _, m in sent: + for sec in (SECRET_PASS, SECRET_TOKEN, PRIV_A): + self.assertNotIn(sec, m) + + def test_pending_notifications_flushed_later_and_bounded(self): + s = pg.default_state() + s["pending_notifications"] = [{"level": "info", "message": f"m{i}", "epoch": 1} for i in range(30)] + self.ctl.store.save(s) + self.ctl.cmd_recover() + self.assertEqual(self.state()["pending_notifications"], []) + self.assertEqual(len([m for _, m in self.notified if m.startswith("m")]), 30) + + def test_state_and_status_file_modes(self): + self.ctl.cmd_recover() + self.assertEqual(stat.S_IMODE((self.tmp / "state" / "state.json").stat().st_mode), 0o600) + self.assertTrue((self.tmp / "state" / "metrics.prom").exists()) + m = (self.tmp / "state" / "metrics.prom").read_text() + self.assertIn("pia_gateway_healthy 1", m) + self.assertIn("pia_gateway_check_timestamp_seconds", m) + + def test_check_is_read_only(self): + self.go_stale() + before = (self.tmp / "state" / "state.json").read_text() + rc = self.ctl.cmd_check() + self.assertEqual(rc, pg.EXIT_UNHEALTHY) + self.assertEqual(self.sysm.restart_count, 0) + self.assertEqual((self.tmp / "state" / "state.json").read_text(), before) + self.assertIn("class=tunnel_stale", self.out.getvalue()) + + def test_main_never_prints_traceback(self): + cfgp = self.tmp / "config.json" + cfgp.write_text(json.dumps(dataclass_dict(self.cfg))) + sysm = self.sysm + + class Boom(FakeSystem): + def run(self, argv, stdin=None, timeout=60): + raise RuntimeError("kaboom " + SECRET_PASS) + + out = io.StringIO() + rc = pg.main(["--config", str(cfgp), "recover"], system=Boom(self.tmp), http=self.http, stream=out) + self.assertEqual(rc, pg.EXIT_UNHEALTHY) + self.assertNotIn("Traceback", out.getvalue()) + self.assertNotIn(SECRET_PASS, out.getvalue()) + self.assertIn("RuntimeError", out.getvalue()) + + def test_config_rejects_unknown_keys(self): + cfgp = self.tmp / "config.json" + d = dataclass_dict(self.cfg) + d["typo_key"] = 1 + cfgp.write_text(json.dumps(d)) + out = io.StringIO() + rc = pg.main(["--config", str(cfgp), "status"], system=self.sysm, http=self.http, stream=out) + self.assertEqual(rc, pg.EXIT_USAGE) + + +def dataclass_dict(cfg): + import dataclasses + + return dataclasses.asdict(cfg) + + +class TestHttpHelpers(unittest.TestCase): + def test_key_and_ip_validators(self): + self.assertTrue(pg.valid_wg_key(PUB_A)) + self.assertFalse(pg.valid_wg_key("short")) + self.assertFalse(pg.valid_wg_key(PUB_A[:-1] + "B")) # bad final char + self.assertTrue(pg.valid_ipv4("10.0.0.1")) + self.assertFalse(pg.valid_ipv4("300.0.0.1")) + self.assertFalse(pg.valid_ipv4("::1")) + + def test_candidates_order_and_missing_region(self): + api = pg.PiaApi(pg.Config(), None, pg.Log(stream=io.StringIO())) + cands = api.build_candidates(serverlist_doc(), ["hk", "taiwan"]) + self.assertEqual([c.cn for c in cands], ["hk1.example", "hk2.example", "tw1.example"]) + with self.assertRaises(pg.ApiError): + api.build_candidates(serverlist_doc(), ["hk", "nowhere"]) + with self.assertRaises(pg.ApiError): + api.build_candidates(serverlist_doc(), ["japan"]) # exists but no wg servers -> nothing usable + # an empty fallback region is skipped, not fatal + self.assertEqual([c.cn for c in api.build_candidates(serverlist_doc(), ["japan", "hk"])], ["hk1.example", "hk2.example"]) + + def test_token_retries_then_auth(self): + http = FakeHttp() + api = pg.PiaApi(pg.Config(), http, pg.Log(stream=io.StringIO())) + http.token = ("http", 401) + with self.assertRaises(pg.ApiError) as cm: + api.get_token(pg.Credentials("u", "p")) + self.assertEqual(cm.exception.kind, "auth") + http.token = ("err", "network") + with self.assertRaises(pg.ApiError): + api.get_token(pg.Credentials("u", "p")) + self.assertEqual(len([c for c in http.calls if c[0] == "post"]), 4) + + def test_multipart_body_contains_fields_once(self): + captured = {} + + class C(pg.HttpClient): + def _request(self, conn, method, path, body, headers): + captured["body"] = body + captured["headers"] = headers + return 200, b'{"token":"t"}' + + c = C(pg.Config()) + c.post_multipart(pg.TOKEN_URL, {"username": "u1", "password": "p1"}) + self.assertIn(b'name="username"\r\n\r\nu1\r\n', captured["body"]) + self.assertIn(b'name="password"\r\n\r\np1\r\n', captured["body"]) + self.assertTrue(captured["headers"]["Content-Type"].startswith("multipart/form-data; boundary=")) + + +if __name__ == "__main__": + unittest.main()