homelab/manifests/multus/vlan50-egress-guard-script.yaml

# Apply: kubectl apply -f manifests/multus/vlan50-egress-guard-script.yaml
# Description: Shared init-container script used by every VLAN 50
#   workload (canary, qBittorrent, JDownloader) to remove the automatic
#   eth0 default route, add the explicit eth0/net1 routes plan.md
#   requires, and install a netns-local iptables egress guard as an
#   independently-enforced backstop against Internet fallback through
#   eth0 — see plan.md "Pod layer" requirements and "Kubernetes routing
#   and egress requirements". Not relied on alone: this is defense in
#   depth alongside the explicit routing it also sets up, not a
#   substitute for it.
#
# Consumed by an initContainer with NET_ADMIN (and only that container —
# app containers must not carry NET_ADMIN). Runs once per pod netns
# creation; the resulting routes/iptables state persists for the pod's
# lifetime (container restarts within the same pod do not recreate the
# netns — a full pod reschedule does, and reruns this init container
# fresh, which is the intended behavior). This has NOT been exercised
# against a live pod; treat every line as needing the Phase 5 canary
# proof (packet capture + restart test) before trusting it in production.
#
# Required env on the init container: VLAN50_GATEWAY, TECHNITIUM_IP,
# POD_CIDR, SERVICE_CIDR, NODE_IP (nik-debian's own node IP —
# kubelet-originated probe traffic to the pod arrives via the primary
# eth0 gateway and its replies must go back the same way, not out net1).
# Optional: EXTRA_ETH0_CIDR (space-separated, for any additional
# narrowly-required cluster path — leave unset unless a specific need is
# identified and reviewed; do not widen this to a blanket 10.0.0.0/8 or
# similar).
#
# Corrected from an earlier version after code review: that version
# deleted the eth0 default route and then routed pod/service CIDR
# directly `dev eth0` with no gateway — remote pod addresses (on other
# nodes) are not generally on-link, so those routes would have silently
# failed to actually reach anything once the implicit default route
# (which was the only thing making them reachable) was gone. This
# version captures the real gateway from the default route *before*
# deleting it, and uses that captured gateway explicitly for every eth0
# route added afterward. It also replaced a blind `iptables -F OUTPUT`
# (which would flush any pre-existing OUTPUT rules from other sources,
# not just ours) with a dedicated chain and a single jump, matching the
# same "own chain, don't touch what isn't ours" approach used on
# minisforum's host-level kill switch.
apiVersion: v1
kind: ConfigMap
metadata:
  name: vlan50-egress-guard-script
  namespace: downloads
data:
  guard.sh: |
    #!/bin/sh
    set -eu

    : "${VLAN50_GATEWAY:?required}"
    : "${TECHNITIUM_IP:?required}"
    : "${POD_CIDR:?required}"
    : "${SERVICE_CIDR:?required}"
    : "${NODE_IP:?required}"

    echo "[vlan50-egress-guard] waiting for net1"
    ready=0
    for _ in $(seq 1 30); do
      if ip link show net1 >/dev/null 2>&1; then
        ready=1
        break
      fi
      sleep 1
    done
    if [ "${ready}" -ne 1 ]; then
      echo "[vlan50-egress-guard] FATAL: net1 did not appear after 30s — refusing to start without the VLAN 50 attachment (Multus/NAD required, no eth0-only fallback)" >&2
      exit 1
    fi

    echo "[vlan50-egress-guard] capturing the original eth0 default gateway"
    ETH0_GATEWAY=$(ip route show default dev eth0 2>/dev/null | awk '/^default/ {print $3; exit}')
    if [ -z "${ETH0_GATEWAY}" ]; then
      echo "[vlan50-egress-guard] FATAL: could not identify eth0's original default gateway — refusing to proceed. Pod/service CIDR routes below need it explicitly (those destinations are not on-link on eth0), and continuing without it would either leave them unreachable or silently do nothing." >&2
      exit 1
    fi
    echo "[vlan50-egress-guard] eth0 gateway: ${ETH0_GATEWAY}"

    echo "[vlan50-egress-guard] removing the automatic default route on eth0"
    ip route del default dev eth0 2>/dev/null || true
    ip -6 route del default 2>/dev/null || true

    echo "[vlan50-egress-guard] explicit eth0 routes via the captured gateway: pod CIDR, service CIDR, node IP${EXTRA_ETH0_CIDR:+, extra}"
    ip route replace "${POD_CIDR}" via "${ETH0_GATEWAY}" dev eth0
    ip route replace "${SERVICE_CIDR}" via "${ETH0_GATEWAY}" dev eth0
    ip route replace "${NODE_IP}/32" via "${ETH0_GATEWAY}" dev eth0
    for cidr in ${EXTRA_ETH0_CIDR:-}; do
      ip route replace "${cidr}" via "${ETH0_GATEWAY}" dev eth0
    done

    echo "[vlan50-egress-guard] net1 routes: Technitium /32, default"
    ip route replace "${TECHNITIUM_IP}/32" via "${VLAN50_GATEWAY}" dev net1
    ip route replace default via "${VLAN50_GATEWAY}" dev net1

    echo "[vlan50-egress-guard] installing IPv4 netns egress guard (dedicated chain, not a blind OUTPUT flush)"
    iptables -N VLAN50-GUARD 2>/dev/null || true
    iptables -F VLAN50-GUARD
    if ! iptables -C OUTPUT -j VLAN50-GUARD 2>/dev/null; then
      iptables -I OUTPUT 1 -j VLAN50-GUARD
    fi
    iptables -A VLAN50-GUARD -o lo -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -d "${POD_CIDR}" -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -d "${SERVICE_CIDR}" -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -d "${NODE_IP}/32" -j ACCEPT
    for cidr in ${EXTRA_ETH0_CIDR:-}; do
      iptables -A VLAN50-GUARD -o eth0 -d "${cidr}" -j ACCEPT
    done
    iptables -A VLAN50-GUARD -o net1 -j ACCEPT
    iptables -A VLAN50-GUARD -o eth0 -m limit --limit 5/minute -j LOG --log-prefix "VLAN50-EGRESS-GUARD-DROP: "
    iptables -A VLAN50-GUARD -o eth0 -j DROP

    echo "[vlan50-egress-guard] blocking IPv6 entirely (both interfaces) — IPv4-only design"
    ip6tables -P OUTPUT DROP
    ip6tables -P FORWARD DROP
    ip6tables -A OUTPUT -o lo -j ACCEPT

    echo "[vlan50-egress-guard] final state:"
    ip route show
    iptables -S OUTPUT
    iptables -S VLAN50-GUARD