homelab/manifests/multus/vlan50-egress-guard-script.yaml

# Apply: kubectl apply -f manifests/multus/vlan50-egress-guard-script.yaml
# Description: Shared init-container script used by every VLAN 50
#   workload (canary, qBittorrent, JDownloader) to remove the automatic
#   eth0 default route, add the explicit eth0/net1 routes plan.md
#   requires, and install a netns-local iptables egress guard as an
#   independently-enforced backstop against Internet fallback through
#   eth0 — see plan.md "Pod layer" requirements and "Kubernetes routing
#   and egress requirements". Not relied on alone: this is defense in
#   depth alongside the explicit routing it also sets up, not a
#   substitute for it.
#
# Consumed by an initContainer with NET_ADMIN (and only that container —
# app containers must not carry NET_ADMIN). Runs once per pod netns
# creation; the resulting routes/iptables state persists for the pod's
# lifetime (container restarts within the same pod do not recreate the
# netns — a full pod reschedule does, and reruns this init container
# fresh, which is the intended behavior). This has NOT been exercised
# against a live pod; treat every line as needing the Phase 5 canary
# proof (packet capture + restart test) before trusting it in production.
#
# Required env on the init container: VLAN50_GATEWAY, TECHNITIUM_IP,
# POD_CIDR, SERVICE_CIDR, NODE_IP (nik-debian's own node IP —
# kubelet-originated probe traffic to the pod arrives via the primary
# eth0 gateway and its replies must go back the same way, not out net1),
# EXPECTED_VLAN50_IP (this workload's static VLAN 50 address, without a
# prefix — .100 for the canary, .10/.11/.12 for qBittorrent/JDownloader/
# the future browser-vpn-proxy — must match the /24 the NAD's static
# IPAM assigned via the pod's own k8s.v1.cni.cncf.io/networks annotation,
# checked below rather than trusted blindly).
# Optional: EXTRA_ETH0_CIDR (space-separated, for any additional
# narrowly-required cluster path — leave unset unless a specific need is
# identified and reviewed; do not widen this to a blanket 10.0.0.0/8 or
# similar).
#
# The init container needs NET_RAW in addition to NET_ADMIN (both, drop
# ALL otherwise) — arping (used below for the duplicate-address and
# gateway-reachability checks) builds raw ARP frames over an AF_PACKET
# socket, which the kernel gates on CAP_NET_RAW specifically; NET_ADMIN
# alone is not sufficient and arping fails immediately without it. This
# is a hard runtime dependency of this script now, not optional
# hardening — a pod wiring this ConfigMap in without also granting
# NET_RAW will fail closed at the arping step every time.
#
# Corrected from an earlier version after code review: that version
# deleted the eth0 default route and then routed pod/service CIDR
# directly `dev eth0` with no gateway — remote pod addresses (on other
# nodes) are not generally on-link, so those routes would have silently
# failed to actually reach anything once the implicit default route
# (which was the only thing making them reachable) was gone. This
# version captures the real gateway from the default route *before*
# deleting it, and uses that captured gateway explicitly for every eth0
# route added afterward. It also replaced a blind `iptables -F OUTPUT`
# (which would flush any pre-existing OUTPUT rules from other sources,
# not just ours) with a dedicated chain and a single jump, matching the
# same "own chain, don't touch what isn't ours" approach used on
# minisforum's host-level kill switch.
apiVersion: v1
kind: ConfigMap
metadata:
  name: vlan50-egress-guard-script
  namespace: downloads
data:
  guard.sh: |
    #!/bin/sh
    set -eu

    : "${VLAN50_GATEWAY:?required}"
    : "${TECHNITIUM_IP:?required}"
    : "${POD_CIDR:?required}"
    : "${SERVICE_CIDR:?required}"
    : "${NODE_IP:?required}"
    : "${EXPECTED_VLAN50_IP:?required}"

    # Safe to print in full: interface/address/route state only, never
    # touches the PIA token, WireGuard keys, or any other credential —
    # those live entirely on minisforum's side of the tunnel, not in this
    # pod's netns at all.
    print_diagnostics() {
      echo "[vlan50-egress-guard] diagnostics follow:" >&2
      ip -details link show >&2 || true
      ip -4 address show >&2 || true
      ip -4 route show table all >&2 || true
    }

    echo "[vlan50-egress-guard] waiting for net1"
    ready=0
    for _ in $(seq 1 30); do
      if ip link show net1 >/dev/null 2>&1; then
        ready=1
        break
      fi
      sleep 1
    done
    if [ "${ready}" -ne 1 ]; then
      echo "[vlan50-egress-guard] FATAL: net1 did not appear after 30s — refusing to start without the VLAN 50 attachment (Multus/NAD required, no eth0-only fallback)" >&2
      print_diagnostics
      exit 1
    fi

    echo "[vlan50-egress-guard] validating net1's address: expecting ${EXPECTED_VLAN50_IP}/24"
    addr_ok=0
    for _ in $(seq 1 15); do
      if ip -4 -o address show dev net1 | awk '{print $4}' | grep -qx "${EXPECTED_VLAN50_IP}/24"; then
        addr_ok=1
        break
      fi
      sleep 1
    done
    if [ "${addr_ok}" -ne 1 ]; then
      echo "[vlan50-egress-guard] FATAL: net1 does not carry the expected ${EXPECTED_VLAN50_IP}/24 after 15s (interface existing is not enough — the static IPAM result itself has to match what this workload is supposed to have). Actual net1 address(es):" >&2
      ip -4 -o address show dev net1 >&2 || true
      print_diagnostics
      exit 1
    fi
    echo "[vlan50-egress-guard] net1 has ${EXPECTED_VLAN50_IP}/24"

    # "Duplicate or tentative" for IPv4: the kernel's IFA_F_TENTATIVE flag
    # (what `ip addr show` reports as "tentative") is an IPv6 DAD concept
    # only — the static IPAM CNI plugin used here does no IPv4 DAD of its
    # own, so there is no tentative *state* to check for IPv4. What
    # actually matters — whether some OTHER host on VLAN 50 already holds
    # this address — is checked directly with an ARP probe instead.
    echo "[vlan50-egress-guard] checking ${EXPECTED_VLAN50_IP} is not already in use on VLAN 50 (ARP duplicate check)"
    if ! arping -D -c 2 -w 2 -I net1 "${EXPECTED_VLAN50_IP}" >/tmp/arping-dup.log 2>&1; then
      echo "[vlan50-egress-guard] FATAL: ${EXPECTED_VLAN50_IP} appears to already be in use on VLAN 50 — arping -D got a reply from another host, meaning this address is a duplicate. Refusing to proceed with a conflicting static IP. arping output:" >&2
      cat /tmp/arping-dup.log >&2 || true
      print_diagnostics
      exit 1
    fi
    echo "[vlan50-egress-guard] no duplicate detected for ${EXPECTED_VLAN50_IP}"

    # L2 (ARP) reachability, checked before this gateway is trusted with
    # the default route below — deliberately not an ICMP ping instead:
    # ARP resolution is a hard prerequisite for delivering any IP packet
    # over Ethernet at all, so it is a reliable, low-false-positive proof
    # the gateway is actually there; an ICMP probe would additionally
    # depend on the gateway choosing to answer echo requests, which many
    # routers disable for unrelated security reasons while still routing
    # traffic completely normally — that would risk failing this check
    # closed for a gateway that works fine.
    echo "[vlan50-egress-guard] checking VLAN 50 gateway ${VLAN50_GATEWAY} answers ARP on net1 before trusting it"
    if ! arping -c 3 -w 3 -I net1 "${VLAN50_GATEWAY}" >/tmp/arping-gw.log 2>&1; then
      echo "[vlan50-egress-guard] FATAL: VLAN 50 gateway ${VLAN50_GATEWAY} did not answer ARP on net1 — refusing to install it as the default route. arping output:" >&2
      cat /tmp/arping-gw.log >&2 || true
      print_diagnostics
      exit 1
    fi
    echo "[vlan50-egress-guard] gateway ${VLAN50_GATEWAY} is reachable"

    echo "[vlan50-egress-guard] capturing the original eth0 default gateway"
    ETH0_GATEWAY=$(ip route show default dev eth0 2>/dev/null | awk '/^default/ {print $3; exit}')
    if [ -z "${ETH0_GATEWAY}" ]; then
      echo "[vlan50-egress-guard] FATAL: could not identify eth0's original default gateway — refusing to proceed. Pod/service CIDR routes below need it explicitly (those destinations are not on-link on eth0), and continuing without it would either leave them unreachable or silently do nothing." >&2
      print_diagnostics
      exit 1
    fi
    echo "[vlan50-egress-guard] eth0 gateway: ${ETH0_GATEWAY}"

    echo "[vlan50-egress-guard] removing the automatic default route on eth0"
    ip route del default dev eth0 2>/dev/null || true
    ip -6 route del default 2>/dev/null || true

    echo "[vlan50-egress-guard] explicit eth0 routes via the captured gateway: pod CIDR, service CIDR, node IP${EXTRA_ETH0_CIDR:+, extra}"
    ip route replace "${POD_CIDR}" via "${ETH0_GATEWAY}" dev eth0
    ip route replace "${SERVICE_CIDR}" via "${ETH0_GATEWAY}" dev eth0
    ip route replace "${NODE_IP}/32" via "${ETH0_GATEWAY}" dev eth0
    for cidr in ${EXTRA_ETH0_CIDR:-}; do
      ip route replace "${cidr}" via "${ETH0_GATEWAY}" dev eth0
    done

    echo "[vlan50-egress-guard] net1 routes: Technitium /32, default"
    ip route replace "${TECHNITIUM_IP}/32" via "${VLAN50_GATEWAY}" dev net1
    ip route replace default via "${VLAN50_GATEWAY}" dev net1

    echo "[vlan50-egress-guard] installing IPv4 netns egress guard (dedicated chain, not a blind OUTPUT flush)"
    iptables -N VLAN50-GUARD 2>/dev/null || true
    iptables -F VLAN50-GUARD
    if ! iptables -C OUTPUT -j VLAN50-GUARD 2>/dev/null; then
      iptables -I OUTPUT 1 -j VLAN50-GUARD
    fi
    iptables -A VLAN50-GUARD -o lo -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -d "${POD_CIDR}" -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -d "${SERVICE_CIDR}" -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -d "${NODE_IP}/32" -j ACCEPT
    for cidr in ${EXTRA_ETH0_CIDR:-}; do
      iptables -A VLAN50-GUARD -o eth0 -d "${cidr}" -j ACCEPT
    done
    iptables -A VLAN50-GUARD -o net1 -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -m limit --limit 5/minute -j LOG --log-prefix "VLAN50-EGRESS-GUARD-DROP: "
    iptables -A VLAN50-GUARD -o eth0 -j DROP

    echo "[vlan50-egress-guard] blocking IPv6 entirely (both interfaces) — IPv4-only design"
    ip6tables -P OUTPUT DROP
    ip6tables -P FORWARD DROP
    ip6tables -A OUTPUT -o lo -j ACCEPT

    echo "[vlan50-egress-guard] final state:"
    ip route show
    iptables -S OUTPUT
    iptables -S VLAN50-GUARD