feat: add GPU node setup with Docker and NVIDIA support, including configuration files and playbook updates

ansible/ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+inventory = inventory.yaml
+inject_facts_as_vars = False
+deprecation_warnings = False

ansible/group_vars/all/vault.yaml

@@ -0,0 +1,12 @@
+$ANSIBLE_VAULT;1.1;AES256
+31386562396163363266666232663537363730663161363165303536383934353136313133333437
+6236323335383962333464336235346532366133363234390a346633623166393834393430393338
+34373562656663623836326539363738326538653434336334363231633962366332663837336264
+3530316332666434360a666263353964386338366539303961616136626139663338393634333661
+37653538303831633837333738623466343637613966613533646663383535326262383766346565
+34636161353830646462366364373562653032636166663732646530366136303138306563366235
+32343761383435303633613137616666363638336366383364316630313537613363356436333966
+33623632373439653232373833376535646337623830616337306664393266633838333736353933
+63633636653635633835303735616533336239636564303931376535393765383137383336306364
+39613361383466383139663731353536346162343863336263336366306136376162663335623230
+623933336133643533363833323437396531

ansible/host_vars/gpu-node.yaml

@@ -0,0 +1,52 @@
+# Host vars for: gpu-node (GPU workstation — spot K3s agent)
+ansible_python_interpreter: /usr/bin/python3.12
+
+# ── common ─────────────────────────────────────────────────────────────────────
+timezone: Asia/Tokyo
+username: nik
+
+base_packages:
+  - curl
+  - git
+  - htop
+  - vim
+  - wget
+  - ca-certificates
+  - gnupg
+  - lsb-release
+  - build-essential
+
+ufw_allowed_ports:
+  - { port: "430",   proto: tcp, comment: "SSH" }
+  - { port: "11434", proto: tcp, comment: "Ollama API" }
+  - { port: "61208", proto: tcp, comment: "Glances web UI" }
+
+data_dirs: []
+
+# ── nvidia ─────────────────────────────────────────────────────────────────────
+nvidia_driver_version: "570"
+cuda_version: "12-8"
+
+# ── k3s-agent ──────────────────────────────────────────────────────────────────
+k3s_server_url: "https://192.168.7.77:6443"
+k3s_node_token: "{{ vault_k3s_node_token }}"
+
+# Check current cluster version with: k3s --version on minisforum
+k3s_version: "v1.32.4+k3s1"
+
+k3s_node_labels:
+  node-role: gpu
+  nik4nao.com/node-type: spot
+  nik4nao.com/gpu: "true"
+
+k3s_node_taints:
+  - "spot=true:NoSchedule"
+
+# ── ollama ─────────────────────────────────────────────────────────────────────
+ollama_port: 11434
+ollama_models:
+  - qwen3:4b
+ollama_models_dir: /usr/share/ollama/.ollama/models
+
+# ── glances ────────────────────────────────────────────────────────────────────
+# no extra vars — uses role defaults

ansible/playbooks/setup-gpu-node.yaml

@@ -6,6 +6,7 @@
 
   roles:
     - role: common
+    - role: docker
     - role: nvidia
     - role: k3s-agent
     - role: ollama

ansible/roles/docker/tasks/main.yaml

@@ -0,0 +1,43 @@
+---
+# Part of role: docker
+# Called by: ansible/playbooks/setup-gpu-node.yaml
+# Description: Installs Docker CE on Ubuntu, adds user to docker group.
+
+- name: Add Docker GPG key
+  ansible.builtin.shell:
+    cmd: >
+      curl -fsSL https://download.docker.com/linux/ubuntu/gpg |
+      gpg --dearmor -o /usr/share/keyrings/docker.gpg
+    creates: /usr/share/keyrings/docker.gpg
+
+- name: Add Docker apt repository
+  ansible.builtin.apt_repository:
+    repo: >
+      deb [arch=amd64 signed-by=/usr/share/keyrings/docker.gpg]
+      https://download.docker.com/linux/ubuntu
+      {{ ansible_facts['distribution_release'] }} stable
+    filename: docker
+    state: present
+
+- name: Install Docker CE
+  ansible.builtin.apt:
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+    state: present
+    update_cache: true
+
+- name: Ensure Docker service is running and enabled
+  ansible.builtin.systemd:
+    name: docker
+    state: started
+    enabled: true
+
+- name: Add user to docker group
+  ansible.builtin.user:
+    name: "{{ username }}"
+    groups: docker
+    append: true

ansible/roles/nvidia/handlers/main.yaml

@@ -0,0 +1,9 @@
+---
+# Part of role: nvidia
+# Called by: ansible/playbooks/setup-gpu-node.yaml
+# Description: Restarts Docker after nvidia-container-toolkit runtime configuration.
+- name: restart docker
+  ansible.builtin.systemd:
+    name: docker
+    state: restarted
+    daemon_reload: true

ansible/roles/nvidia/tasks/main.yaml

@@ -0,0 +1,78 @@
+---
+# Part of role: nvidia
+# Called by: ansible/playbooks/setup-gpu-node.yaml
+# Description: Installs NVIDIA drivers, CUDA toolkit, and nvidia-container-toolkit.
+#              Configures Docker and K3s containerd runtimes for GPU access.
+
+- name: Add NVIDIA CUDA apt keyring
+  ansible.builtin.shell:
+    cmd: >
+      curl -fsSL
+      https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
+      -o /tmp/cuda-keyring.deb && dpkg -i /tmp/cuda-keyring.deb
+    creates: /usr/share/keyrings/cuda-archive-keyring.gpg
+
+- name: Add NVIDIA container toolkit keyring
+  ansible.builtin.shell:
+    cmd: >
+      curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey |
+      gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+    creates: /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+
+- name: Add NVIDIA container toolkit repo
+  ansible.builtin.shell:
+    cmd: >
+      curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list |
+      sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' |
+      tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
+    creates: /etc/apt/sources.list.d/nvidia-container-toolkit.list
+
+- name: Update apt cache
+  ansible.builtin.apt:
+    update_cache: true
+
+- name: Install NVIDIA driver
+  ansible.builtin.apt:
+    name: "nvidia-driver-{{ nvidia_driver_version }}"
+    state: present
+  register: nvidia_driver_install
+
+- name: Install CUDA toolkit
+  ansible.builtin.apt:
+    name: "cuda-toolkit-{{ cuda_version }}"
+    state: present
+
+- name: Install nvidia-container-toolkit
+  ansible.builtin.apt:
+    name: nvidia-container-toolkit
+    state: present
+
+- name: Add CUDA to system PATH
+  ansible.builtin.copy:
+    dest: /etc/profile.d/cuda.sh
+    content: |
+      export PATH=/usr/local/cuda/bin:$PATH
+      export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+    mode: "0644"
+
+- name: Configure Docker runtime for NVIDIA
+  ansible.builtin.command:
+    cmd: nvidia-ctk runtime configure --runtime=docker
+  changed_when: true
+  notify: restart docker
+
+- name: Reboot if driver was just installed
+  ansible.builtin.reboot:
+    reboot_timeout: 300
+  when: nvidia_driver_install.changed
+
+- name: Verify NVIDIA driver loaded
+  ansible.builtin.command: nvidia-smi
+  changed_when: false
+  failed_when: false
+  register: nvidia_smi_check
+
+- name: Assert nvidia-smi succeeded
+  ansible.builtin.assert:
+    that: nvidia_smi_check.rc == 0
+    fail_msg: "nvidia-smi failed — driver may not have loaded correctly"

ansible/roles/nvidia/templates/k3s-containerd-config.toml.j2

@@ -0,0 +1,10 @@
+# Managed by Ansible — do not edit manually
+# Adds nvidia as an additional runtime — pods request it via runtimeClassName: nvidia
+# runc remains the default runtime for all other workloads
+version = 2
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
+  BinaryName = "/usr/bin/nvidia-container-runtime"

ansible/roles/ollama/handlers/main.yaml

@@ -6,7 +6,7 @@
 - name: restart ollama
   become: true
   command: launchctl kickstart -k system/com.ollama.ollama
-  when: ansible_system == 'Darwin'
+  when: ansible_facts['system'] == 'Darwin'
 
 - name: restart ollama linux
   ansible.builtin.systemd:
@@ -14,4 +14,4 @@
     state: restarted
     daemon_reload: true
   become: true
-  when: ansible_system == 'Linux'
+  when: ansible_facts['system'] == 'Linux'

ansible/roles/ollama/tasks/main.yaml

@@ -10,7 +10,7 @@
   community.general.homebrew:
     name: ollama
     state: present
-  when: ansible_system == 'Darwin'
+  when: ansible_facts['system'] == 'Darwin'
 
 - name: Deploy ollama launchd plist
   template:
@@ -21,7 +21,7 @@
     mode: "0644"
   become: true
   notify: restart ollama
-  when: ansible_system == 'Darwin'
+  when: ansible_facts['system'] == 'Darwin'
 
 - name: Load ollama launchd service
   become: true
@@ -29,14 +29,22 @@
   args:
     creates: /var/run/ollama.pid
   ignore_errors: true
-  when: ansible_system == 'Darwin'
+  when: ansible_facts['system'] == 'Darwin'
 
 # ── Linux ──────────────────────────────────────────────────────────────────────
 - name: Install ollama via install script
   ansible.builtin.shell:
     cmd: curl -fsSL https://ollama.com/install.sh | sh
-    creates: /usr/local/bin/ollama
+    creates: /etc/systemd/system/ollama.service
-  when: ansible_system == 'Linux'
+  when: ansible_facts['system'] == 'Linux'
+
+- name: Create ollama systemd override directory
+  ansible.builtin.file:
+    path: /etc/systemd/system/ollama.service.d
+    state: directory
+    mode: "0755"
+  become: true
+  when: ansible_facts['system'] == 'Linux'
 
 - name: Deploy ollama systemd override
   ansible.builtin.template:
@@ -47,7 +55,7 @@
     mode: "0644"
   become: true
   notify: restart ollama linux
-  when: ansible_system == 'Linux'
+  when: ansible_facts['system'] == 'Linux'
 
 - name: Enable and start ollama service
   ansible.builtin.systemd:
@@ -56,7 +64,7 @@
     enabled: true
     daemon_reload: true
   become: true
-  when: ansible_system == 'Linux'
+  when: ansible_facts['system'] == 'Linux'
 
 # ── shared ─────────────────────────────────────────────────────────────────────
 - name: Wait for ollama to be ready
@@ -76,7 +84,7 @@
 
 - name: Pull ollama models
   command: >
-    {{ '/opt/homebrew/bin/ollama' if ansible_system == 'Darwin' else '/usr/local/bin/ollama' }}
+    {{ '/opt/homebrew/bin/ollama' if ansible_facts['system'] == 'Darwin' else '/usr/local/bin/ollama' }}
     pull {{ item }}
   loop: "{{ ollama_models }}"
   when: item not in (ollama_tags.json.models | map(attribute='name') | list)

manifests/home-services/nvidia-device-plugin.yaml

@@ -0,0 +1,48 @@
+# Config for: NVIDIA GPU device plugin
+# Applied by: kubectl apply -f manifests/home-services/nvidia-device-plugin.yaml
+# Description: Exposes nvidia.com/gpu resource on gpu-node so K3s can schedule GPU workloads.
+---
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: nvidia
+handler: nvidia
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-device-plugin
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: nvidia-device-plugin
+  template:
+    metadata:
+      labels:
+        app: nvidia-device-plugin
+    spec:
+      runtimeClassName: nvidia
+      tolerations:
+        - key: spot
+          operator: Equal
+          value: "true"
+          effect: NoSchedule
+      nodeSelector:
+        nik4nao.com/gpu: "true"
+      containers:
+        - name: nvidia-device-plugin
+          image: nvcr.io/nvidia/k8s-device-plugin:v0.17.0
+          args:
+            - --device-discovery-strategy=nvml
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+          volumeMounts:
+            - name: device-plugin
+              mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+        - name: device-plugin
+          hostPath:
+            path: /var/lib/kubelet/device-plugins

router/uci-base.conf

@@ -0,0 +1,373 @@
+package attendedsysupgrade
+
+config server 'server'
+	option url 'https://sysupgrade.openwrt.org'
+
+config client 'client'
+	option upgrade_packages '1'
+	option auto_search '0'
+	option advanced_mode '0'
+	option login_check_for_upgrades '1'
+
+config owut 'owut'
+
+package dhcp
+
+config dnsmasq
+	option domainneeded '1'
+	option boguspriv '1'
+	option filterwin2k '0'
+	option localise_queries '1'
+	option rebind_protection '1'
+	option rebind_localhost '1'
+	option local '/lan/'
+	option domain 'lan'
+	option expandhosts '1'
+	option nonegcache '0'
+	option cachesize '1000'
+	option authoritative '1'
+	option readethers '1'
+	option leasefile '/tmp/dhcp.leases'
+	option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
+	option nonwildcard '1'
+	option localservice '1'
+	option ednspacket_max '1232'
+	option filter_aaaa '0'
+	option filter_a '0'
+
+config dhcp 'lan'
+	option interface 'lan'
+	option start '100'
+	option limit '150'
+	option leasetime '12h'
+	option dhcpv4 'server'
+	option dhcpv6 'server'
+	option ra 'server'
+	option ra_slaac '1'
+	list ra_flags 'managed-config'
+	list ra_flags 'other-config'
+
+config dhcp 'wan'
+	option interface 'wan'
+	option ignore '1'
+
+config odhcpd 'odhcpd'
+	option maindhcp '0'
+	option leasefile '/tmp/odhcpd.leases'
+	option leasetrigger '/usr/sbin/odhcpd-update'
+	option loglevel '4'
+	option piodir '/tmp/odhcpd-piodir'
+	option hostsdir '/tmp/hosts'
+
+package dropbear
+
+config dropbear 'main'
+	option PasswordAuth 'off'
+	option RootPasswordAuth 'off'
+	option Port '430'
+
+package firewall
+
+config defaults
+	option syn_flood '1'
+	option input 'REJECT'
+	option output 'ACCEPT'
+	option forward 'REJECT'
+
+config zone
+	option name 'lan'
+	list network 'lan'
+	option input 'ACCEPT'
+	option output 'ACCEPT'
+	option forward 'ACCEPT'
+
+config zone
+	option name 'wan'
+	list network 'wan'
+	list network 'wan6'
+	option input 'REJECT'
+	option output 'ACCEPT'
+	option forward 'DROP'
+	option masq '1'
+	option mtu_fix '1'
+
+config forwarding
+	option src 'lan'
+	option dest 'wan'
+
+config rule
+	option name 'Allow-DHCP-Renew'
+	option src 'wan'
+	option proto 'udp'
+	option dest_port '68'
+	option target 'ACCEPT'
+	option family 'ipv4'
+
+config rule
+	option name 'Allow-Ping'
+	option src 'wan'
+	option proto 'icmp'
+	option icmp_type 'echo-request'
+	option family 'ipv4'
+	option target 'ACCEPT'
+
+config rule
+	option name 'Allow-IGMP'
+	option src 'wan'
+	option proto 'igmp'
+	option family 'ipv4'
+	option target 'ACCEPT'
+
+config rule
+	option name 'Allow-DHCPv6'
+	option src 'wan'
+	option proto 'udp'
+	option dest_port '546'
+	option family 'ipv6'
+	option target 'ACCEPT'
+
+config rule
+	option name 'Allow-MLD'
+	option src 'wan'
+	option proto 'icmp'
+	option src_ip 'fe80::/10'
+	list icmp_type '130/0'
+	list icmp_type '131/0'
+	list icmp_type '132/0'
+	list icmp_type '143/0'
+	option family 'ipv6'
+	option target 'ACCEPT'
+
+config rule
+	option name 'Allow-ICMPv6-Input'
+	option src 'wan'
+	option proto 'icmp'
+	list icmp_type 'echo-request'
+	list icmp_type 'echo-reply'
+	list icmp_type 'destination-unreachable'
+	list icmp_type 'packet-too-big'
+	list icmp_type 'time-exceeded'
+	list icmp_type 'bad-header'
+	list icmp_type 'unknown-header-type'
+	list icmp_type 'router-solicitation'
+	list icmp_type 'neighbour-solicitation'
+	list icmp_type 'router-advertisement'
+	list icmp_type 'neighbour-advertisement'
+	option limit '1000/sec'
+	option family 'ipv6'
+	option target 'ACCEPT'
+
+config rule
+	option name 'Allow-ICMPv6-Forward'
+	option src 'wan'
+	option dest '*'
+	option proto 'icmp'
+	list icmp_type 'echo-request'
+	list icmp_type 'echo-reply'
+	list icmp_type 'destination-unreachable'
+	list icmp_type 'packet-too-big'
+	list icmp_type 'time-exceeded'
+	list icmp_type 'bad-header'
+	list icmp_type 'unknown-header-type'
+	option limit '1000/sec'
+	option family 'ipv6'
+	option target 'ACCEPT'
+
+config rule
+	option name 'Allow-IPSec-ESP'
+	option src 'wan'
+	option dest 'lan'
+	option proto 'esp'
+	option target 'ACCEPT'
+
+config rule
+	option name 'Allow-ISAKMP'
+	option src 'wan'
+	option dest 'lan'
+	option dest_port '500'
+	option proto 'udp'
+	option target 'ACCEPT'
+
+package luci
+
+config core 'main'
+	option lang 'auto'
+	option mediaurlbase '/luci-static/bootstrap'
+	option resourcebase '/luci-static/resources'
+	option ubuspath '/ubus/'
+
+config extern 'flash_keep'
+	option uci '/etc/config/'
+	option dropbear '/etc/dropbear/'
+	option openvpn '/etc/openvpn/'
+	option passwd '/etc/passwd'
+	option opkg '/etc/opkg.conf'
+	option firewall '/etc/firewall.user'
+	option uploads '/lib/uci/upload/'
+
+config internal 'languages'
+
+config internal 'sauth'
+	option sessionpath '/tmp/luci-sessions'
+	option sessiontime '3600'
+
+config internal 'ccache'
+	option enable '1'
+
+config internal 'themes'
+	option Bootstrap '/luci-static/bootstrap'
+	option BootstrapDark '/luci-static/bootstrap-dark'
+	option BootstrapLight '/luci-static/bootstrap-light'
+
+config internal 'apply'
+	option rollback '90'
+	option holdoff '4'
+	option timeout '5'
+	option display '1.5'
+
+config internal 'diag'
+	option dns 'openwrt.org'
+	option ping 'openwrt.org'
+	option route 'openwrt.org'
+
+package network
+
+config interface 'loopback'
+	option device 'lo'
+	option proto 'static'
+	list ipaddr '127.0.0.1/8'
+
+config globals 'globals'
+	option dhcp_default_duid '0004a365d32c3ee8471c918c7e1877ee52a3'
+	option ula_prefix 'fd4b:364b:726a::/48'
+
+config device
+	option name 'br-lan'
+	option type 'bridge'
+	list ports 'lan1'
+	list ports 'lan2'
+	list ports 'lan3'
+	list ports 'lan4'
+	list ports 'lan5'
+
+config interface 'lan'
+	option device 'br-lan'
+	option proto 'static'
+	option ipaddr '192.168.7.1'
+	option ip6assign '60'
+	option netmask '255.255.255.0'
+	option dns '192.168.7.77'
+
+config interface 'wan'
+	option device 'eth1'
+	option proto 'dhcp'
+	option peerdns '0'
+	option dns '192.168.7.77'
+
+config interface 'wan6'
+	option device 'eth1'
+	option proto 'dhcpv6'
+
+package rpcd
+
+config rpcd
+	option socket '/var/run/ubus/ubus.sock'
+	option timeout '30'
+
+config login
+	option username 'root'
+	option password '$p$root'
+	list read '*'
+	list write '*'
+
+package system
+
+config system
+	option hostname 'flint2'
+	option timezone 'GMT0'
+	option zonename 'UTC'
+	option ttylogin '0'
+	option log_size '128'
+	option urandom_seed '0'
+
+config timeserver 'ntp'
+	option enabled '1'
+	option enable_server '0'
+	list server '0.openwrt.pool.ntp.org'
+	list server '1.openwrt.pool.ntp.org'
+	list server '2.openwrt.pool.ntp.org'
+	list server '3.openwrt.pool.ntp.org'
+
+package ubihealthd
+
+package ubootenv
+
+config ubootenv
+	option dev '/dev/mmcblk0p1'
+	option offset '0x0'
+	option envsize '0x80000'
+
+package uhttpd
+
+config uhttpd 'main'
+	list listen_http '0.0.0.0:80'
+	list listen_http '[::]:80'
+	list listen_https '0.0.0.0:443'
+	list listen_https '[::]:443'
+	option redirect_https '0'
+	option home '/www'
+	option rfc1918_filter '1'
+	option max_requests '3'
+	option max_connections '100'
+	option cert '/etc/uhttpd.crt'
+	option key '/etc/uhttpd.key'
+	option cgi_prefix '/cgi-bin'
+	list lua_prefix '/cgi-bin/luci=/usr/lib/lua/luci/sgi/uhttpd.lua'
+	option script_timeout '60'
+	option network_timeout '30'
+	option http_keepalive '20'
+	option tcp_keepalive '1'
+	option ubus_prefix '/ubus'
+
+config cert 'defaults'
+	option days '397'
+	option key_type 'ec'
+	option bits '2048'
+	option ec_curve 'P-256'
+	option country 'ZZ'
+	option state 'Somewhere'
+	option location 'Unknown'
+	option commonname 'OpenWrt'
+
+package wireless
+
+config wifi-device 'radio0'
+	option type 'mac80211'
+	option path 'platform/soc/18000000.wifi'
+	option band '2g'
+	option channel '1'
+	option htmode 'HE20'
+
+config wifi-iface 'default_radio0'
+	option device 'radio0'
+	option network 'lan'
+	option mode 'ap'
+	option ssid 'OpenWrt'
+	option encryption 'none'
+	option disabled '1'
+
+config wifi-device 'radio1'
+	option type 'mac80211'
+	option path 'platform/soc/18000000.wifi+1'
+	option band '5g'
+	option channel '36'
+	option htmode 'HE80'
+
+config wifi-iface 'default_radio1'
+	option device 'radio1'
+	option network 'lan'
+	option mode 'ap'
+	option ssid 'OpenWrt'
+	option encryption 'none'
+	option disabled '1'
+