diff --git a/manifests/multus/00-crd.yaml b/manifests/multus/00-crd.yaml new file mode 100644 index 0000000..61de3fb --- /dev/null +++ b/manifests/multus/00-crd.yaml @@ -0,0 +1,57 @@ +# Apply: kubectl apply -f manifests/multus/00-crd.yaml +# Description: NetworkAttachmentDefinition CRD. Cluster-scoped. +# argocd.argoproj.io/sync-wave -1 so it lands before Multus's own RBAC +# and DaemonSet, which reference this kind. +# +# Verbatim from k8snetworkplumbingwg/multus-cni's official +# deployments/multus-daemonset-thick.yml (commit 95a0932350f0060cd23209 +# 6876e24672a1de83fa, fetched 2026-08-24 — see 02-daemonset.yaml's header +# for the same pin) — not adapted at all; this resource has nothing +# k3s-specific about it. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + group: k8s.cni.cncf.io + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - nad + - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing + Working Group to express the intent for attaching pods to one or more logical or physical + networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this represen + tation of an object. Servers should convert recognized schemas to the + latest internal value, and may reject unrecognized values. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' + type: object + properties: + config: + description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' + type: string diff --git a/manifests/multus/01-rbac.yaml b/manifests/multus/01-rbac.yaml new file mode 100644 index 0000000..c89b50b --- /dev/null +++ b/manifests/multus/01-rbac.yaml @@ -0,0 +1,77 @@ +# Apply: kubectl apply -f manifests/multus/01-rbac.yaml +# Description: ServiceAccount and cluster-scoped RBAC for the Multus +# DaemonSet (02-daemonset.yaml). sync-wave 0 — after the CRD, before +# the DaemonSet that uses this ServiceAccount. +# +# Verbatim from k8snetworkplumbingwg/multus-cni's official +# deployments/multus-daemonset-thick.yml (commit 95a0932350f0060cd23209 +# 6876e24672a1de83fa, fetched 2026-08-24 — see 02-daemonset.yaml's header +# for the same pin), including the resource.k8s.io (Dynamic Resource +# Allocation) and events.k8s.io rules upstream grants — left in even +# though k3s v1.32 may not register that API group, since RBAC rules for +# a nonexistent group/resource are inert, not an error, and removing +# them would be inventing a divergence from upstream rather than fixing +# anything. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multus + namespace: kube-system + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: multus + annotations: + argocd.argoproj.io/sync-wave: "0" +rules: + - apiGroups: ["k8s.cni.cncf.io"] + resources: + - "*" + verbs: + - "*" + - apiGroups: + - "" + resources: + - pods + - pods/status + verbs: + - get + - list + - update + - watch + - apiGroups: + - "resource.k8s.io" + resources: + - resourceclaims + - resourceclaims/status + - resourceslices + verbs: + - get + - list + - apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: multus + annotations: + argocd.argoproj.io/sync-wave: "0" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus +subjects: + - kind: ServiceAccount + name: multus + namespace: kube-system diff --git a/manifests/multus/02-daemonset.yaml b/manifests/multus/02-daemonset.yaml new file mode 100644 index 0000000..52dc422 --- /dev/null +++ b/manifests/multus/02-daemonset.yaml @@ -0,0 +1,216 @@ +# Apply: kubectl apply -f manifests/multus/02-daemonset.yaml +# Description: Multus thick-plugin DaemonSet, restricted by nodeSelector +# to nik-debian only (the sole node with a VLAN 50 trunk — deliberately +# not cluster-wide, so minisforum and nik-gpu's CNI chain is never +# touched by this at all). sync-wave 0 — after RBAC. +# +# Rebuilt from k8snetworkplumbingwg/multus-cni's official +# deployments/multus-daemonset-thick.yml, commit +# 95a0932350f0060cd232096876e24672a1de83fa (master, fetched and read in +# full 2026-08-24 — https://raw.githubusercontent.com/ +# k8snetworkplumbingwg/multus-cni/master/deployments/ +# multus-daemonset-thick.yml). An earlier version of this file was NOT +# a faithful copy of that manifest — it used an invented entrypoint +# (/entrypoint/cni-installer.sh, which does not exist in the real image) +# and an incomplete set of mounts/config keys, reconstructed from memory +# rather than the real source. This version changes only what k3s and +# this topology actually require, everything else (entrypoints, daemon +# config keys, volume set, RBAC-adjacent mounts) is verbatim: +# +# 1. cni/cnibin volumes' hostPath: k3s's real CNI conf/bin dirs +# (/var/lib/rancher/k3s/agent/etc/cni/net.d, +# /var/lib/rancher/k3s/data/cni — confirmed LIVE on nik-debian +# 2026-08-23; there is no /opt/cni/bin or /etc/cni/net.d on this +# host, upstream's own defaults would silently no-op on k3s +# without this). The container-side mountPaths upstream defines +# (/host/etc/cni/net.d, /host/opt/cni/bin) are unchanged — only +# which host directory backs them moves. +# 2. All OTHER hostPath volumes (host-run, host-var-lib-cni-multus, +# host-var-lib-kubelet, host-run-k8s-cni-cncf-io, host-run-netns, +# hostroot, multus-conf-dir) are left at upstream's standard paths +# — k3s does not relocate /run, /var/lib/kubelet, /run/netns, or / +# itself, only the CNI-specific directories. This assumption (that +# kubelet's own root-dir is standard on k3s) is NOT independently +# verified against a live k3s node the way the CNI paths are — +# confirm before applying if this cluster ever changes +# --root-dir on the kubelet. +# 3. nodeSelector restricting it to nik-debian (upstream is +# cluster-wide by default). +# 4. Image pinned to the v4.3.0-thick digest (upstream's own quickstart +# file uses the "snapshot-thick" moving tag, which is explicitly a +# dev/nightly build, not something to run unpinned in production). +# Digest confirmed live via the GHCR OCI Distribution API +# 2026-08-24: v4.3.0-thick and stable-thick both resolve to +# sha256:2b9671447f3ea4e7e56730843dbf59445b9307246f393b61386b896 +# d56ae51c9, i.e. v4.3.0 is genuinely the current stable release, +# not just a same-named coincidence. +# +# Do not hand-edit the daemon-config.json keys, the container +# command/args, or the mount set below without re-diffing against the +# URL above — none of that was invented this time, it's copied. +apiVersion: v1 +kind: ConfigMap +metadata: + name: multus-daemon-config + namespace: kube-system + labels: + tier: node + app: multus + annotations: + argocd.argoproj.io/sync-wave: "0" +data: + daemon-config.json: | + { + "chrootDir": "/hostroot", + "cniVersion": "0.3.1", + "logLevel": "verbose", + "logToStderr": true, + "cniConfigDir": "/host/etc/cni/net.d", + "multusAutoconfigDir": "/host/etc/cni/net.d", + "multusConfigFile": "auto", + "socketDir": "/host/run/multus/" + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-multus-ds + namespace: kube-system + labels: + tier: node + app: multus + name: multus + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + selector: + matchLabels: + name: multus + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: multus + name: multus + spec: + # Not upstream — restricts this DaemonSet to the one node with a + # VLAN 50 trunk. minisforum and nik-gpu never run Multus at all. + nodeSelector: + kubernetes.io/hostname: nik-debian + hostNetwork: true + hostPID: true + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + serviceAccountName: multus + containers: + - name: kube-multus + # VERIFY the digest still resolves before applying — see + # header comment for how it was obtained and when. + image: "ghcr.io/k8snetworkplumbingwg/multus-cni@sha256:2b9671447f3ea4e7e56730843dbf59445b9307246f393b61386b896d56ae51c9" # v4.3.0-thick + command: ["/usr/src/multus-cni/bin/multus-daemon"] + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni + mountPath: /host/etc/cni/net.d + # multus-daemon expects that cnibin path must be identical between pod and container host. + # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be mount to '/opt/cni/bin' in multus-daemon, + # not to any other directory, like '/opt/bin' or '/usr/bin'. + - name: cnibin + mountPath: /opt/cni/bin + - name: host-run + mountPath: /host/run + - name: host-var-lib-cni-multus + mountPath: /var/lib/cni/multus + - name: host-var-lib-kubelet + mountPath: /var/lib/kubelet + mountPropagation: HostToContainer + - name: host-run-k8s-cni-cncf-io + mountPath: /run/k8s.cni.cncf.io + - name: host-run-netns + mountPath: /run/netns + mountPropagation: HostToContainer + - name: multus-daemon-config + mountPath: /etc/cni/net.d/multus.d + readOnly: true + - name: hostroot + mountPath: /hostroot + mountPropagation: HostToContainer + - mountPath: /etc/cni/multus/net.d + name: multus-conf-dir + env: + - name: MULTUS_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + initContainers: + - name: install-multus-binary + # Same image/digest as the main container — see header comment. + image: "ghcr.io/k8snetworkplumbingwg/multus-cni@sha256:2b9671447f3ea4e7e56730843dbf59445b9307246f393b61386b896d56ae51c9" # v4.3.0-thick + command: + - "/usr/src/multus-cni/bin/install_multus" + - "-d" + - "/host/opt/cni/bin" + - "-t" + - "thick" + resources: + requests: + cpu: "10m" + memory: "15Mi" + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + mountPropagation: Bidirectional + terminationGracePeriodSeconds: 30 + volumes: + - name: cni + hostPath: + # k3s CNI conf dir — NOT the standard /etc/cni/net.d. + path: /var/lib/rancher/k3s/agent/etc/cni/net.d + - name: cnibin + hostPath: + # k3s CNI bin dir — NOT the standard /opt/cni/bin. + path: /var/lib/rancher/k3s/data/cni + - name: hostroot + hostPath: + path: / + - name: multus-daemon-config + configMap: + name: multus-daemon-config + items: + - key: daemon-config.json + path: daemon-config.json + - name: host-run + hostPath: + path: /run + - name: host-var-lib-cni-multus + hostPath: + path: /var/lib/cni/multus + - name: host-var-lib-kubelet + hostPath: + path: /var/lib/kubelet + - name: host-run-k8s-cni-cncf-io + hostPath: + path: /run/k8s.cni.cncf.io + - name: host-run-netns + hostPath: + path: /run/netns/ + - name: multus-conf-dir + hostPath: + path: /etc/cni/multus/net.d diff --git a/manifests/multus/10-nad-vlan50.yaml b/manifests/multus/10-nad-vlan50.yaml new file mode 100644 index 0000000..8c281c5 --- /dev/null +++ b/manifests/multus/10-nad-vlan50.yaml @@ -0,0 +1,48 @@ +# Apply: kubectl apply -f manifests/multus/10-nad-vlan50.yaml +# Description: Namespace-scoped NetworkAttachmentDefinition for VLAN 50, +# macvlan bridge mode on nik-debian's enp1s0.50 (ansible/roles/ +# vlan50-parent). macvlan bridge chosen over ipvlan L2 per the +# home-network handoff confirming the managed switch accepts multiple +# source MACs on nik-debian's port — see plan.md Phase 4 / the +# coordination handoff in the session that authored this. sync-wave 1 — +# after Multus itself (00/01/02), before the canary (20) or any +# workload attachment. +# +# ipam.type is "static" deliberately, not a shared pool/allocator +# (Whereabouts etc.) — this cluster only ever attaches a handful of fixed +# workloads to VLAN 50 (qBittorrent .10, JDownloader .11, one reserved +# .12, one canary from 10.10.50.100-149), so each pod's own +# k8s.v1.cni.cncf.io/networks annotation supplies its ips/gateway +# directly; there is no dynamic allocation to arbitrate. Excluded from +# any future allocator by simply never being handed to one: .1 (Flint), +# .10/.11/.12 (static workloads), and .100-149 (canary/dynamic) are never +# referenced anywhere else in this repo. +# +# capabilities.ips: true is required by the Network Plumbing Working +# Group CNI spec for a plugin/IPAM combination to accept a runtime- +# supplied "ips" field at all (https://www.cni.dev/plugins/current/ipam/ +# static/) — without it at the top level of the NAD config, the per-pod +# k8s.v1.cni.cncf.io/networks annotation's "ips" array is not a +# recognized capability and the static IPAM plugin has no defined way to +# receive it. +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vlan50 + namespace: downloads +spec: + config: | + { + "cniVersion": "0.3.1", + "name": "vlan50", + "type": "macvlan", + "master": "enp1s0.50", + "mode": "bridge", + "mtu": 1500, + "capabilities": { + "ips": true + }, + "ipam": { + "type": "static" + } + } diff --git a/manifests/multus/20-canary.yaml b/manifests/multus/20-canary.yaml new file mode 100644 index 0000000..1691232 --- /dev/null +++ b/manifests/multus/20-canary.yaml @@ -0,0 +1,83 @@ +# Apply: kubectl apply -f manifests/multus/20-canary.yaml +# Delete when done: kubectl delete -f manifests/multus/20-canary.yaml +# Description: TEMPORARY canary pod for Phase 5 validation (plan.md) — +# proves eth0/net1 routing, kill-switch failure modes, and the netns +# egress guard before qBittorrent/JDownloader are touched. Not a +# long-lived resource; delete it once the Phase 5 validation matrix has +# run. sync-wave 2 — after Multus (0) and the NAD (1). +# +# *** DO NOT SYNC/APPLY until: *** +# - Phases 0-4 have actually been applied and verified live (this file +# is drafted alongside them, not proof they work). +# - The user has explicitly approved running the canary specifically — +# per this task's own gates, Argo CD sync of this file needs the same +# explicit go-ahead as any other live change, even though the +# multus.yaml Application it lives under is manual-sync-only anyway. +# +# Image: nicolaka/netshoot:v0.11 — a widely-used network-debugging image +# (iproute2 + iptables + curl/dig/tcpdump). Tag not independently +# verified against a live pull; confirm it still resolves before +# applying, same caveat as the Multus image tag in 02-daemonset.yaml. +apiVersion: v1 +kind: Pod +metadata: + name: vlan50-canary + namespace: downloads + labels: + app: vlan50-canary + annotations: + argocd.argoproj.io/sync-wave: "2" + k8s.v1.cni.cncf.io/networks: | + [{"name": "vlan50", "namespace": "downloads", "interface": "net1", "ips": ["10.10.50.100/24"]}] +spec: + nodeSelector: + kubernetes.io/hostname: nik-debian + restartPolicy: Never + # Same reasoning as qbittorrent.yaml/jdownloader.yaml: no cluster + # DNS/CoreDNS resolution needed, single Technitium resolver, routed via + # net1 so Technitium sees the real VLAN 50 source. + dnsPolicy: None + dnsConfig: + nameservers: + - "10.10.40.53" + initContainers: + - name: vlan50-egress-guard + image: nicolaka/netshoot:v0.11 + command: ["/bin/sh", "/scripts/guard.sh"] + env: + - name: VLAN50_GATEWAY + value: "10.10.50.1" + - name: TECHNITIUM_IP + value: "10.10.40.53" + - name: POD_CIDR + value: "10.42.0.0/16" + - name: SERVICE_CIDR + value: "10.43.0.0/16" + - name: NODE_IP + value: "10.10.40.20" + securityContext: + capabilities: + drop: ["ALL"] + add: ["NET_ADMIN"] + volumeMounts: + - name: guard-script + mountPath: /scripts + containers: + - name: netshoot + image: nicolaka/netshoot:v0.11 + command: ["sleep", "infinity"] + securityContext: + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: guard-script + configMap: + name: vlan50-egress-guard-script + defaultMode: 365 # octal 0555, r-xr-xr-x — yamllint here forbids octal literals diff --git a/manifests/multus/README.md b/manifests/multus/README.md new file mode 100644 index 0000000..870420b --- /dev/null +++ b/manifests/multus/README.md @@ -0,0 +1,108 @@ +# Multus + VLAN 50 NAD + canary + +Implements plan.md Phase 4/5. Reconciled by `argocd/apps/multus.yaml`, +which is deliberately **manual-sync only** — see that file's header +comment. Nothing here applies to the cluster just because it's merged to +`main`. + +## Files (sync-wave order) + +| File | Wave | What | +| --- | --- | --- | +| `00-crd.yaml` | -1 | `NetworkAttachmentDefinition` CRD | +| `01-rbac.yaml` | 0 | Multus ServiceAccount/ClusterRole/ClusterRoleBinding | +| `02-daemonset.yaml` | 0 | Multus thick-plugin DaemonSet — **nodeSelector restricted to `nik-debian` only**, k3s-specific CNI paths | +| `10-nad-vlan50.yaml` | 1 | VLAN 50 `NetworkAttachmentDefinition`, macvlan bridge on `enp1s0.50`, static IPAM | +| `vlan50-egress-guard-script.yaml` | (no wave annotation — apply alongside 10) | Shared init-container script consumed by the canary and, later, qBittorrent/JDownloader | +| `20-canary.yaml` | 2 | Temporary canary pod — **do not sync without separate explicit approval**, see its own header | +| `reserved/browser-vpn-proxy.yaml` | n/a | The `.12` third workload, prepared not deployed — see its own header for why it lives in a subdirectory `argocd/apps/multus.yaml` never scans at all | + +## Before applying anything here + +1. Ansible Phases 2/3 (`ansible/roles/pia-gateway`, `ansible/roles/ + vlan50-parent`) must already be live and verified — the NAD's `master: + enp1s0.50` and the whole VLAN 50 return path depend on both. +2. **Verify the Multus DaemonSet manifest** (`02-daemonset.yaml`) against + the current `k8snetworkplumbingwg/multus-cni` `deployments/multus- + daemonset-thick.yml` — see that file's own header comment for exactly + what's unverified (image tag, daemon flags) versus live-confirmed (the + k3s CNI paths). +3. Sync order matters even within this one manual-sync Application: 00 → + 01 → 02, confirm the Multus pod is actually Running on `nik-debian` + and `/var/lib/rancher/k3s/agent/etc/cni/net.d` now has a generated + `00-multus.conf`, *then* 10, *then* — only with separate approval — 20. + +## Why macvlan bridge, not ipvlan + +The managed switch accepts multiple source MAC addresses on nik-debian's +port (confirmed in the session that authored this plan — see the +coordination handoff in the final report of that session). macvlan bridge +is plan.md's stated preference whenever that's true, and is simpler than +ipvlan L2 (each attached pod gets its own real MAC, no L2 address-sharing +edge cases). + +## Why `ipam.type: static` and no shared allocator (Whereabouts etc.) + +Only ever a handful of fixed IPs on this network — qBittorrent (`.10`), +JDownloader (`.11`), one reserved-not-deployed (`.12`), and a single +canary (`.100`). A dynamic IPAM controller (CRD, webhook, its own failure +modes) is unjustified complexity for that; each pod's own +`k8s.v1.cni.cncf.io/networks` annotation just states its IP directly. + +## Rollback + +**Deleting the Kubernetes-side resources alone is not safe and can break +all new pod scheduling on nik-debian** — not just Multus-attached pods. +An earlier version of this doc claimed leaving the generated +`00-multus.conf` in place was harmless once Multus was gone; that's +wrong. Multus's "auto" config mode does **not** replace or rename the +original Flannel CNI conf file — it adds `00-multus.conf` alongside it, +which sorts first and wins by kubelet's own file-ordering convention. +`kubectl delete`ing the DaemonSet removes the running daemon and its pod, +but does **not** remove that generated conf file or the multus shim +binary the init container copied onto the host — both are host +filesystem side effects `kubectl delete` has no reach into. If +`00-multus.conf` is left behind after the daemon it depends on is gone, +containerd/kubelet will try to invoke a CNI plugin that's no longer +there for **every** new pod sandbox on that node, Multus-attached or not +— the node effectively stops being able to start any new pod until that +file is dealt with. + +Correct order — host-side cleanup on nik-debian *before* the Kubernetes +side: + +```bash +# 1. On nik-debian (needs root — not reachable via kubectl): +ssh -p 430 nik@10.10.40.20 +sudo ls /var/lib/rancher/k3s/agent/etc/cni/net.d/ +# Confirm both files are present: 00-multus.conf (generated) and the +# original Flannel conf (something like 10-flannel.conflist — the +# exact name wasn't captured during discovery; read whatever's there). +sudo cp /var/lib/rancher/k3s/agent/etc/cni/net.d/00-multus.conf \ + /root/00-multus.conf.removed-$(date +%Y%m%d%H%M%S) # capture for diagnosis first +sudo rm /var/lib/rancher/k3s/agent/etc/cni/net.d/00-multus.conf +# Optional cleanup, not required for correctness once the conf file +# above is gone — the multus shim binary itself, if you want it off +# the host too: +sudo rm -f /var/lib/rancher/k3s/data/cni/multus-shim # confirm the actual filename first; do not guess-delete + +# 2. Prove an ordinary pod can still be newly scheduled on nik-debian +# before touching anything else (from your workstation): +kubectl run vlan50-rollback-probe --image=busybox:1.36 --restart=Never \ + --overrides='{"spec":{"nodeSelector":{"kubernetes.io/hostname":"nik-debian"}}}' \ + -- sleep 60 +kubectl wait --for=condition=Ready pod/vlan50-rollback-probe --timeout=60s +kubectl delete pod vlan50-rollback-probe + +# 3. Only now, the Kubernetes-side resources, in this order: +kubectl delete -f manifests/multus/20-canary.yaml # if it was ever applied +kubectl delete -f manifests/multus/10-nad-vlan50.yaml # only after no pod still references it +kubectl delete -f manifests/multus/vlan50-egress-guard-script.yaml +kubectl delete -f manifests/multus/02-daemonset.yaml +kubectl delete -f manifests/multus/01-rbac.yaml +kubectl delete -f manifests/multus/00-crd.yaml # only after no NAD remains +``` + +None of this has been exercised against the live cluster — step 2's +probe pod is exactly how to prove it actually worked before considering +the rollback complete, not an optional extra. diff --git a/manifests/multus/reserved/browser-vpn-proxy.yaml b/manifests/multus/reserved/browser-vpn-proxy.yaml new file mode 100644 index 0000000..4eecc16 --- /dev/null +++ b/manifests/multus/reserved/browser-vpn-proxy.yaml @@ -0,0 +1,162 @@ +# NOT applied by any Argo CD Application. Deliberately kept in +# manifests/multus/reserved/ rather than manifests/multus/ directly: +# argocd/apps/multus.yaml's source has no directory.recurse: true, so it +# only ever scans files directly in manifests/multus/, never this +# subdirectory — even selective `argocd app sync multus --resource ...` +# has no way to target a resource Argo CD doesn't know exists. This is +# stronger isolation than the canary's own manual-sync-only Application +# gate, on purpose: plan.md Phase 8/12 requires this workload stay out of +# reconciliation entirely until qBittorrent AND JDownloader have +# independently passed their canary, validation, and soak gates — not +# just "not yet approved to sync", but "not a candidate for sync at all". +# +# When ready to actually build this (not yet — this file is prepared, +# not deployed): +# 1. Move it into manifests/multus/ (a sync-wave-numbered filename, +# e.g. 30-browser-vpn-proxy.yaml) once qBittorrent/JDownloader are +# both soaked. +# 2. Create the browser-vpn-proxy-credentials Secret first (see the +# companion secret script this needs — not written yet, follow the +# exact pattern of manifests/media/pia-secret.sh: add +# BROWSER_PROXY_USER/BROWSER_PROXY_PASSWORD to .env, a script that +# kubeseals a Secret with those as PROXY_USER/PROXY_PASSWORD). +# 3. Verify the NodePort below (30889) doesn't collide with anything +# live — manifests/network/gluetun-proxy.yaml already uses 30888 +# for the existing browser SOCKS5 proxy this is meant to eventually +# replace; confirm 30889 is actually free before applying. +# +# Design: +# - serjs/go-socks5-proxy (github.com/serjs/socks5-server) — a small, +# single-purpose Go SOCKS5 server, not a VPN client itself. It +# doesn't need to be, here: unlike the legacy manifests/network/ +# gluetun-proxy.yaml it's replacing (which runs its own OpenVPN +# client), PIA egress for this workload comes from the same +# netns-level routing as qBittorrent/JDownloader — the init +# container below sends this pod's default route out net1/PIA, the +# SOCKS5 server itself just needs to be a plain, correctly-behaving +# proxy. Pinned to v0.0.4's digest, confirmed live against Docker +# Hub's registry API 2026-08-24: +# sha256:6828ddb2a6a93dec85209b69cd3842bd80094f7dd52f35ee1dd0f9e9165e2188 +# - REQUIRE_AUTH=true + PROXY_USER/PROXY_PASSWORD (from the Secret +# above) AND ALLOWED_IPS restricted to the home LAN range — both, not +# either/or, matching plan.md's "authentication or LAN-restricted +# exposure" as a floor, not a ceiling. +# - Service is NodePort, not LoadBalancer/Ingress — reachable from the +# LAN at :30889, never exposed publicly (no port-forward +# on Flint/the router for it, same as the existing gluetun-proxy). +# +# Browser configuration — this is the part that actually matters for +# "DNS must be resolved through the proxy": configuring a SOCKS5 proxy +# address alone is NOT enough. By default most browsers resolve +# hostnames locally (via the LAN's normal DNS) and only send the already- +# resolved IP through the proxy — that leaks every site you visit to +# your normal LAN resolver and bypasses Technitium/PIA for DNS entirely, +# defeating the point. "socks5h" (vs plain "socks5") in a proxy URL is +# the conventional way tools signal "resolve hostnames on the proxy side, +# not locally" (e.g. curl --socks5-hostname, or a proxy URL scheme of +# socks5h://). In Firefox specifically: Settings -> Network Settings -> +# Manual proxy configuration -> SOCKS Host set to this Service -> SOCKS +# v5 selected -> the checkbox "Proxy DNS when using SOCKS v5" MUST be +# checked, or Firefox resolves locally despite the proxy being SOCKS5. +# Verify this actually holds (a DNS leak test site, or packet capture +# showing no DNS queries leaving the browser's own host) before trusting +# it — do not assume the checkbox alone is sufficient without checking. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: browser-vpn-proxy + namespace: downloads +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: browser-vpn-proxy + template: + metadata: + labels: + app: browser-vpn-proxy + annotations: + k8s.v1.cni.cncf.io/networks: | + [{"name": "vlan50", "namespace": "downloads", "interface": "net1", "ips": ["10.10.50.12/24"]}] + spec: + nodeSelector: + node-role: storage + kubernetes.io/hostname: nik-debian + dnsPolicy: None + dnsConfig: + nameservers: + - "10.10.40.53" + initContainers: + - name: vlan50-egress-guard + image: nicolaka/netshoot:v0.11 + command: ["/bin/sh", "/scripts/guard.sh"] + env: + - name: VLAN50_GATEWAY + value: "10.10.50.1" + - name: TECHNITIUM_IP + value: "10.10.40.53" + - name: POD_CIDR + value: "10.42.0.0/16" + - name: SERVICE_CIDR + value: "10.43.0.0/16" + - name: NODE_IP + value: "10.10.40.20" + securityContext: + capabilities: + drop: ["ALL"] + add: ["NET_ADMIN"] + volumeMounts: + - name: guard-script + mountPath: /scripts + containers: + - name: socks5 + image: "serjs/go-socks5-proxy@sha256:6828ddb2a6a93dec85209b69cd3842bd80094f7dd52f35ee1dd0f9e9165e2188" # v0.0.4 + ports: + - containerPort: 1080 + securityContext: + capabilities: + drop: ["ALL"] + env: + - name: REQUIRE_AUTH + value: "true" + - name: PROXY_USER + valueFrom: + secretKeyRef: + name: browser-vpn-proxy-credentials + key: PROXY_USER + - name: PROXY_PASSWORD + valueFrom: + secretKeyRef: + name: browser-vpn-proxy-credentials + key: PROXY_PASSWORD + - name: ALLOWED_IPS + value: "10.10.40.0/24" # LAN only — narrow this further if a specific client range is confirmed + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: guard-script + configMap: + name: vlan50-egress-guard-script + defaultMode: 365 # octal 0555, r-xr-xr-x +--- +apiVersion: v1 +kind: Service +metadata: + name: browser-vpn-proxy + namespace: downloads +spec: + selector: + app: browser-vpn-proxy + type: NodePort + ports: + - name: socks5 + port: 1080 + targetPort: 1080 + nodePort: 30889 # verify unused before applying — see header diff --git a/manifests/multus/vlan50-egress-guard-script.yaml b/manifests/multus/vlan50-egress-guard-script.yaml new file mode 100644 index 0000000..159406b --- /dev/null +++ b/manifests/multus/vlan50-egress-guard-script.yaml @@ -0,0 +1,122 @@ +# Apply: kubectl apply -f manifests/multus/vlan50-egress-guard-script.yaml +# Description: Shared init-container script used by every VLAN 50 +# workload (canary, qBittorrent, JDownloader) to remove the automatic +# eth0 default route, add the explicit eth0/net1 routes plan.md +# requires, and install a netns-local iptables egress guard as an +# independently-enforced backstop against Internet fallback through +# eth0 — see plan.md "Pod layer" requirements and "Kubernetes routing +# and egress requirements". Not relied on alone: this is defense in +# depth alongside the explicit routing it also sets up, not a +# substitute for it. +# +# Consumed by an initContainer with NET_ADMIN (and only that container — +# app containers must not carry NET_ADMIN). Runs once per pod netns +# creation; the resulting routes/iptables state persists for the pod's +# lifetime (container restarts within the same pod do not recreate the +# netns — a full pod reschedule does, and reruns this init container +# fresh, which is the intended behavior). This has NOT been exercised +# against a live pod; treat every line as needing the Phase 5 canary +# proof (packet capture + restart test) before trusting it in production. +# +# Required env on the init container: VLAN50_GATEWAY, TECHNITIUM_IP, +# POD_CIDR, SERVICE_CIDR, NODE_IP (nik-debian's own node IP — +# kubelet-originated probe traffic to the pod arrives via the primary +# eth0 gateway and its replies must go back the same way, not out net1). +# Optional: EXTRA_ETH0_CIDR (space-separated, for any additional +# narrowly-required cluster path — leave unset unless a specific need is +# identified and reviewed; do not widen this to a blanket 10.0.0.0/8 or +# similar). +# +# Corrected from an earlier version after code review: that version +# deleted the eth0 default route and then routed pod/service CIDR +# directly `dev eth0` with no gateway — remote pod addresses (on other +# nodes) are not generally on-link, so those routes would have silently +# failed to actually reach anything once the implicit default route +# (which was the only thing making them reachable) was gone. This +# version captures the real gateway from the default route *before* +# deleting it, and uses that captured gateway explicitly for every eth0 +# route added afterward. It also replaced a blind `iptables -F OUTPUT` +# (which would flush any pre-existing OUTPUT rules from other sources, +# not just ours) with a dedicated chain and a single jump, matching the +# same "own chain, don't touch what isn't ours" approach used on +# minisforum's host-level kill switch. +apiVersion: v1 +kind: ConfigMap +metadata: + name: vlan50-egress-guard-script + namespace: downloads +data: + guard.sh: | + #!/bin/sh + set -eu + + : "${VLAN50_GATEWAY:?required}" + : "${TECHNITIUM_IP:?required}" + : "${POD_CIDR:?required}" + : "${SERVICE_CIDR:?required}" + : "${NODE_IP:?required}" + + echo "[vlan50-egress-guard] waiting for net1" + ready=0 + for _ in $(seq 1 30); do + if ip link show net1 >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 + done + if [ "${ready}" -ne 1 ]; then + echo "[vlan50-egress-guard] FATAL: net1 did not appear after 30s — refusing to start without the VLAN 50 attachment (Multus/NAD required, no eth0-only fallback)" >&2 + exit 1 + fi + + echo "[vlan50-egress-guard] capturing the original eth0 default gateway" + ETH0_GATEWAY=$(ip route show default dev eth0 2>/dev/null | awk '/^default/ {print $3; exit}') + if [ -z "${ETH0_GATEWAY}" ]; then + echo "[vlan50-egress-guard] FATAL: could not identify eth0's original default gateway — refusing to proceed. Pod/service CIDR routes below need it explicitly (those destinations are not on-link on eth0), and continuing without it would either leave them unreachable or silently do nothing." >&2 + exit 1 + fi + echo "[vlan50-egress-guard] eth0 gateway: ${ETH0_GATEWAY}" + + echo "[vlan50-egress-guard] removing the automatic default route on eth0" + ip route del default dev eth0 2>/dev/null || true + ip -6 route del default 2>/dev/null || true + + echo "[vlan50-egress-guard] explicit eth0 routes via the captured gateway: pod CIDR, service CIDR, node IP${EXTRA_ETH0_CIDR:+, extra}" + ip route replace "${POD_CIDR}" via "${ETH0_GATEWAY}" dev eth0 + ip route replace "${SERVICE_CIDR}" via "${ETH0_GATEWAY}" dev eth0 + ip route replace "${NODE_IP}/32" via "${ETH0_GATEWAY}" dev eth0 + for cidr in ${EXTRA_ETH0_CIDR:-}; do + ip route replace "${cidr}" via "${ETH0_GATEWAY}" dev eth0 + done + + echo "[vlan50-egress-guard] net1 routes: Technitium /32, default" + ip route replace "${TECHNITIUM_IP}/32" via "${VLAN50_GATEWAY}" dev net1 + ip route replace default via "${VLAN50_GATEWAY}" dev net1 + + echo "[vlan50-egress-guard] installing IPv4 netns egress guard (dedicated chain, not a blind OUTPUT flush)" + iptables -N VLAN50-GUARD 2>/dev/null || true + iptables -F VLAN50-GUARD + if ! iptables -C OUTPUT -j VLAN50-GUARD 2>/dev/null; then + iptables -I OUTPUT 1 -j VLAN50-GUARD + fi + iptables -A VLAN50-GUARD -o lo -j ACCEPT + iptables -A VLAN50-GUARD -o eth0 -d "${POD_CIDR}" -j ACCEPT + iptables -A VLAN50-GUARD -o eth0 -d "${SERVICE_CIDR}" -j ACCEPT + iptables -A VLAN50-GUARD -o eth0 -d "${NODE_IP}/32" -j ACCEPT + for cidr in ${EXTRA_ETH0_CIDR:-}; do + iptables -A VLAN50-GUARD -o eth0 -d "${cidr}" -j ACCEPT + done + iptables -A VLAN50-GUARD -o net1 -j ACCEPT + iptables -A VLAN50-GUARD -o eth0 -m limit --limit 5/minute -j LOG --log-prefix "VLAN50-EGRESS-GUARD-DROP: " + iptables -A VLAN50-GUARD -o eth0 -j DROP + + echo "[vlan50-egress-guard] blocking IPv6 entirely (both interfaces) — IPv4-only design" + ip6tables -P OUTPUT DROP + ip6tables -P FORWARD DROP + ip6tables -A OUTPUT -o lo -j ACCEPT + + echo "[vlan50-egress-guard] final state:" + ip route show + iptables -S OUTPUT + iptables -S VLAN50-GUARD