nik/homelab
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add GPU node setup with Docker and NVIDIA support, including configuration files and playbook updates

Browse Source
This commit is contained in:
Nik Afiq 2026-07-23 11:07:18 +09:00
parent f80848c57e
commit e7718ce356
13 changed files with 648 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ansible/ansible.cfg Normal file
View File

@ -0,0 +1,4 @@
[defaults]
inventory = inventory.yaml
inject_facts_as_vars = False
deprecation_warnings = False

12
ansible/group_vars/all/vault.yaml Normal file
View File

@ -0,0 +1,12 @@
$ANSIBLE_VAULT;1.1;AES256
31386562396163363266666232663537363730663161363165303536383934353136313133333437
6236323335383962333464336235346532366133363234390a346633623166393834393430393338
34373562656663623836326539363738326538653434336334363231633962366332663837336264
3530316332666434360a666263353964386338366539303961616136626139663338393634333661
37653538303831633837333738623466343637613966613533646663383535326262383766346565
34636161353830646462366364373562653032636166663732646530366136303138306563366235
32343761383435303633613137616666363638336366383364316630313537613363356436333966
33623632373439653232373833376535646337623830616337306664393266633838333736353933
63633636653635633835303735616533336239636564303931376535393765383137383336306364
39613361383466383139663731353536346162343863336263336366306136376162663335623230
623933336133643533363833323437396531

52
ansible/host_vars/gpu-node.yaml Normal file
View File

@ -0,0 +1,52 @@
# Host vars for: gpu-node (GPU workstation — spot K3s agent)
ansible_python_interpreter: /usr/bin/python3.12
# ── common ─────────────────────────────────────────────────────────────────────
timezone: Asia/Tokyo
username: nik
base_packages:
- curl
- git
- htop
- vim
- wget
- ca-certificates
- gnupg
- lsb-release
- build-essential
ufw_allowed_ports:
- { port: "430", proto: tcp, comment: "SSH" }
- { port: "11434", proto: tcp, comment: "Ollama API" }
- { port: "61208", proto: tcp, comment: "Glances web UI" }
data_dirs: []
# ── nvidia ─────────────────────────────────────────────────────────────────────
nvidia_driver_version: "570"
cuda_version: "12-8"
# ── k3s-agent ──────────────────────────────────────────────────────────────────
k3s_server_url: "https://192.168.7.77:6443"
k3s_node_token: "{{ vault_k3s_node_token }}"
# Check current cluster version with: k3s --version on minisforum
k3s_version: "v1.32.4+k3s1"
k3s_node_labels:
node-role: gpu
nik4nao.com/node-type: spot
nik4nao.com/gpu: "true"
k3s_node_taints:
- "spot=true:NoSchedule"
# ── ollama ─────────────────────────────────────────────────────────────────────
ollama_port: 11434
ollama_models:
- qwen3:4b
ollama_models_dir: /usr/share/ollama/.ollama/models
# ── glances ────────────────────────────────────────────────────────────────────
# no extra vars — uses role defaults

1
ansible/playbooks/setup-gpu-node.yaml
View File

@ -6,6 +6,7 @@
roles:
- role: common
- role: docker
- role: nvidia
- role: k3s-agent
- role: ollama

43
ansible/roles/docker/tasks/main.yaml Normal file
View File

@ -0,0 +1,43 @@
---
# Part of role: docker
# Called by: ansible/playbooks/setup-gpu-node.yaml
# Description: Installs Docker CE on Ubuntu, adds user to docker group.
- name: Add Docker GPG key
ansible.builtin.shell:
cmd: >
curl -fsSL https://download.docker.com/linux/ubuntu/gpg |
gpg --dearmor -o /usr/share/keyrings/docker.gpg
creates: /usr/share/keyrings/docker.gpg
- name: Add Docker apt repository
ansible.builtin.apt_repository:
repo: >
deb [arch=amd64 signed-by=/usr/share/keyrings/docker.gpg]
https://download.docker.com/linux/ubuntu
{{ ansible_facts['distribution_release'] }} stable
filename: docker
state: present
- name: Install Docker CE
ansible.builtin.apt:
name:
- docker-ce
- docker-ce-cli
- containerd.io
- docker-buildx-plugin
- docker-compose-plugin
state: present
update_cache: true
- name: Ensure Docker service is running and enabled
ansible.builtin.systemd:
name: docker
state: started
enabled: true
- name: Add user to docker group
ansible.builtin.user:
name: "{{ username }}"
groups: docker
append: true

9
ansible/roles/nvidia/handlers/main.yaml Normal file
View File

@ -0,0 +1,9 @@
---
# Part of role: nvidia
# Called by: ansible/playbooks/setup-gpu-node.yaml
# Description: Restarts Docker after nvidia-container-toolkit runtime configuration.
- name: restart docker
ansible.builtin.systemd:
name: docker
state: restarted
daemon_reload: true

78
ansible/roles/nvidia/tasks/main.yaml Normal file
View File

@ -0,0 +1,78 @@
---
# Part of role: nvidia
# Called by: ansible/playbooks/setup-gpu-node.yaml
# Description: Installs NVIDIA drivers, CUDA toolkit, and nvidia-container-toolkit.
# Configures Docker and K3s containerd runtimes for GPU access.
- name: Add NVIDIA CUDA apt keyring
ansible.builtin.shell:
cmd: >
curl -fsSL
https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
-o /tmp/cuda-keyring.deb && dpkg -i /tmp/cuda-keyring.deb
creates: /usr/share/keyrings/cuda-archive-keyring.gpg
- name: Add NVIDIA container toolkit keyring
ansible.builtin.shell:
cmd: >
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey |
gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
creates: /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
- name: Add NVIDIA container toolkit repo
ansible.builtin.shell:
cmd: >
curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list |
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' |
tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
creates: /etc/apt/sources.list.d/nvidia-container-toolkit.list
- name: Update apt cache
ansible.builtin.apt:
update_cache: true
- name: Install NVIDIA driver
ansible.builtin.apt:
name: "nvidia-driver-{{ nvidia_driver_version }}"
state: present
register: nvidia_driver_install
- name: Install CUDA toolkit
ansible.builtin.apt:
name: "cuda-toolkit-{{ cuda_version }}"
state: present
- name: Install nvidia-container-toolkit
ansible.builtin.apt:
name: nvidia-container-toolkit
state: present
- name: Add CUDA to system PATH
ansible.builtin.copy:
dest: /etc/profile.d/cuda.sh
content: |
export PATH=/usr/local/cuda/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
mode: "0644"
- name: Configure Docker runtime for NVIDIA
ansible.builtin.command:
cmd: nvidia-ctk runtime configure --runtime=docker
changed_when: true
notify: restart docker
- name: Reboot if driver was just installed
ansible.builtin.reboot:
reboot_timeout: 300
when: nvidia_driver_install.changed
- name: Verify NVIDIA driver loaded
ansible.builtin.command: nvidia-smi
changed_when: false
failed_when: false
register: nvidia_smi_check
- name: Assert nvidia-smi succeeded
ansible.builtin.assert:
that: nvidia_smi_check.rc == 0
fail_msg: "nvidia-smi failed — driver may not have loaded correctly"

10
ansible/roles/nvidia/templates/k3s-containerd-config.toml.j2 Normal file
View File

@ -0,0 +1,10 @@
# Managed by Ansible — do not edit manually
# Adds nvidia as an additional runtime — pods request it via runtimeClassName: nvidia
# runc remains the default runtime for all other workloads
version = 2
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
BinaryName = "/usr/bin/nvidia-container-runtime"

4
ansible/roles/ollama/handlers/main.yaml
View File

@ -6,7 +6,7 @@
- name: restart ollama
become: true
command: launchctl kickstart -k system/com.ollama.ollama
when: ansible_system == 'Darwin'
when: ansible_facts['system'] == 'Darwin'
- name: restart ollama linux
ansible.builtin.systemd:
@ -14,4 +14,4 @@
state: restarted
daemon_reload: true
become: true
when: ansible_system == 'Linux'
when: ansible_facts['system'] == 'Linux'

24
ansible/roles/ollama/tasks/main.yaml
View File

@ -10,7 +10,7 @@
community.general.homebrew:
name: ollama
state: present
when: ansible_system == 'Darwin'
when: ansible_facts['system'] == 'Darwin'
- name: Deploy ollama launchd plist
template:
@ -21,7 +21,7 @@
mode: "0644"
become: true
notify: restart ollama
when: ansible_system == 'Darwin'
when: ansible_facts['system'] == 'Darwin'
- name: Load ollama launchd service
become: true
@ -29,14 +29,22 @@
args:
creates: /var/run/ollama.pid
ignore_errors: true
when: ansible_system == 'Darwin'
when: ansible_facts['system'] == 'Darwin'
# ── Linux ──────────────────────────────────────────────────────────────────────
- name: Install ollama via install script
ansible.builtin.shell:
cmd: curl -fsSL https://ollama.com/install.sh | sh
creates: /usr/local/bin/ollama
when: ansible_system == 'Linux'
creates: /etc/systemd/system/ollama.service
when: ansible_facts['system'] == 'Linux'
- name: Create ollama systemd override directory
ansible.builtin.file:
path: /etc/systemd/system/ollama.service.d
state: directory
mode: "0755"
become: true
when: ansible_facts['system'] == 'Linux'
- name: Deploy ollama systemd override
ansible.builtin.template:
@ -47,7 +55,7 @@
mode: "0644"
become: true
notify: restart ollama linux
when: ansible_system == 'Linux'
when: ansible_facts['system'] == 'Linux'
- name: Enable and start ollama service
ansible.builtin.systemd:
@ -56,7 +64,7 @@
enabled: true
daemon_reload: true
become: true
when: ansible_system == 'Linux'
when: ansible_facts['system'] == 'Linux'
# ── shared ─────────────────────────────────────────────────────────────────────
- name: Wait for ollama to be ready
@ -76,7 +84,7 @@
- name: Pull ollama models
command: >
{{ '/opt/homebrew/bin/ollama' if ansible_system == 'Darwin' else '/usr/local/bin/ollama' }}
{{ '/opt/homebrew/bin/ollama' if ansible_facts['system'] == 'Darwin' else '/usr/local/bin/ollama' }}
pull {{ item }}
loop: "{{ ollama_models }}"
when: item not in (ollama_tags.json.models | map(attribute='name') | list)

48
manifests/home-services/nvidia-device-plugin.yaml Normal file
View File

@ -0,0 +1,48 @@
# Config for: NVIDIA GPU device plugin
# Applied by: kubectl apply -f manifests/home-services/nvidia-device-plugin.yaml
# Description: Exposes nvidia.com/gpu resource on gpu-node so K3s can schedule GPU workloads.
---
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
name: nvidia
handler: nvidia
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nvidia-device-plugin
namespace: kube-system
spec:
selector:
matchLabels:
app: nvidia-device-plugin
template:
metadata:
labels:
app: nvidia-device-plugin
spec:
runtimeClassName: nvidia
tolerations:
- key: spot
operator: Equal
value: "true"
effect: NoSchedule
nodeSelector:
nik4nao.com/gpu: "true"
containers:
- name: nvidia-device-plugin
image: nvcr.io/nvidia/k8s-device-plugin:v0.17.0
args:
- --device-discovery-strategy=nvml
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
volumeMounts:
- name: device-plugin
mountPath: /var/lib/kubelet/device-plugins
volumes:
- name: device-plugin
hostPath:
path: /var/lib/kubelet/device-plugins

BIN
router/backup-base.tar.gz Normal file
View File

Binary file not shown.

373
router/uci-base.conf Normal file
View File

@ -0,0 +1,373 @@
package attendedsysupgrade
config server 'server'
option url 'https://sysupgrade.openwrt.org'
config client 'client'
option upgrade_packages '1'
option auto_search '0'
option advanced_mode '0'
option login_check_for_upgrades '1'
config owut 'owut'
package dhcp
config dnsmasq
option domainneeded '1'
option boguspriv '1'
option filterwin2k '0'
option localise_queries '1'
option rebind_protection '1'
option rebind_localhost '1'
option local '/lan/'
option domain 'lan'
option expandhosts '1'
option nonegcache '0'
option cachesize '1000'
option authoritative '1'
option readethers '1'
option leasefile '/tmp/dhcp.leases'
option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
option nonwildcard '1'
option localservice '1'
option ednspacket_max '1232'
option filter_aaaa '0'
option filter_a '0'
config dhcp 'lan'
option interface 'lan'
option start '100'
option limit '150'
option leasetime '12h'
option dhcpv4 'server'
option dhcpv6 'server'
option ra 'server'
option ra_slaac '1'
list ra_flags 'managed-config'
list ra_flags 'other-config'
config dhcp 'wan'
option interface 'wan'
option ignore '1'
config odhcpd 'odhcpd'
option maindhcp '0'
option leasefile '/tmp/odhcpd.leases'
option leasetrigger '/usr/sbin/odhcpd-update'
option loglevel '4'
option piodir '/tmp/odhcpd-piodir'
option hostsdir '/tmp/hosts'
package dropbear
config dropbear 'main'
option PasswordAuth 'off'
option RootPasswordAuth 'off'
option Port '430'
package firewall
config defaults
option syn_flood '1'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
config zone
option name 'lan'
list network 'lan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
config zone
option name 'wan'
list network 'wan'
list network 'wan6'
option input 'REJECT'
option output 'ACCEPT'
option forward 'DROP'
option masq '1'
option mtu_fix '1'
config forwarding
option src 'lan'
option dest 'wan'
config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'
option family 'ipv4'
config rule
option name 'Allow-Ping'
option src 'wan'
option proto 'icmp'
option icmp_type 'echo-request'
option family 'ipv4'
option target 'ACCEPT'
config rule
option name 'Allow-IGMP'
option src 'wan'
option proto 'igmp'
option family 'ipv4'
option target 'ACCEPT'
config rule
option name 'Allow-DHCPv6'
option src 'wan'
option proto 'udp'
option dest_port '546'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-MLD'
option src 'wan'
option proto 'icmp'
option src_ip 'fe80::/10'
list icmp_type '130/0'
list icmp_type '131/0'
list icmp_type '132/0'
list icmp_type '143/0'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-ICMPv6-Input'
option src 'wan'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
list icmp_type 'router-solicitation'
list icmp_type 'neighbour-solicitation'
list icmp_type 'router-advertisement'
list icmp_type 'neighbour-advertisement'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-ICMPv6-Forward'
option src 'wan'
option dest '*'
option proto 'icmp'
list icmp_type 'echo-request'
list icmp_type 'echo-reply'
list icmp_type 'destination-unreachable'
list icmp_type 'packet-too-big'
list icmp_type 'time-exceeded'
list icmp_type 'bad-header'
list icmp_type 'unknown-header-type'
option limit '1000/sec'
option family 'ipv6'
option target 'ACCEPT'
config rule
option name 'Allow-IPSec-ESP'
option src 'wan'
option dest 'lan'
option proto 'esp'
option target 'ACCEPT'
config rule
option name 'Allow-ISAKMP'
option src 'wan'
option dest 'lan'
option dest_port '500'
option proto 'udp'
option target 'ACCEPT'
package luci
config core 'main'
option lang 'auto'
option mediaurlbase '/luci-static/bootstrap'
option resourcebase '/luci-static/resources'
option ubuspath '/ubus/'
config extern 'flash_keep'
option uci '/etc/config/'
option dropbear '/etc/dropbear/'
option openvpn '/etc/openvpn/'
option passwd '/etc/passwd'
option opkg '/etc/opkg.conf'
option firewall '/etc/firewall.user'
option uploads '/lib/uci/upload/'
config internal 'languages'
config internal 'sauth'
option sessionpath '/tmp/luci-sessions'
option sessiontime '3600'
config internal 'ccache'
option enable '1'
config internal 'themes'
option Bootstrap '/luci-static/bootstrap'
option BootstrapDark '/luci-static/bootstrap-dark'
option BootstrapLight '/luci-static/bootstrap-light'
config internal 'apply'
option rollback '90'
option holdoff '4'
option timeout '5'
option display '1.5'
config internal 'diag'
option dns 'openwrt.org'
option ping 'openwrt.org'
option route 'openwrt.org'
package network
config interface 'loopback'
option device 'lo'
option proto 'static'
list ipaddr '127.0.0.1/8'
config globals 'globals'
option dhcp_default_duid '0004a365d32c3ee8471c918c7e1877ee52a3'
option ula_prefix 'fd4b:364b:726a::/48'
config device
option name 'br-lan'
option type 'bridge'
list ports 'lan1'
list ports 'lan2'
list ports 'lan3'
list ports 'lan4'
list ports 'lan5'
config interface 'lan'
option device 'br-lan'
option proto 'static'
option ipaddr '192.168.7.1'
option ip6assign '60'
option netmask '255.255.255.0'
option dns '192.168.7.77'
config interface 'wan'
option device 'eth1'
option proto 'dhcp'
option peerdns '0'
option dns '192.168.7.77'
config interface 'wan6'
option device 'eth1'
option proto 'dhcpv6'
package rpcd
config rpcd
option socket '/var/run/ubus/ubus.sock'
option timeout '30'
config login
option username 'root'
option password '$p$root'
list read '*'
list write '*'
package system
config system
option hostname 'flint2'
option timezone 'GMT0'
option zonename 'UTC'
option ttylogin '0'
option log_size '128'
option urandom_seed '0'
config timeserver 'ntp'
option enabled '1'
option enable_server '0'
list server '0.openwrt.pool.ntp.org'
list server '1.openwrt.pool.ntp.org'
list server '2.openwrt.pool.ntp.org'
list server '3.openwrt.pool.ntp.org'
package ubihealthd
package ubootenv
config ubootenv
option dev '/dev/mmcblk0p1'
option offset '0x0'
option envsize '0x80000'
package uhttpd
config uhttpd 'main'
list listen_http '0.0.0.0:80'
list listen_http '[::]:80'
list listen_https '0.0.0.0:443'
list listen_https '[::]:443'
option redirect_https '0'
option home '/www'
option rfc1918_filter '1'
option max_requests '3'
option max_connections '100'
option cert '/etc/uhttpd.crt'
option key '/etc/uhttpd.key'
option cgi_prefix '/cgi-bin'
list lua_prefix '/cgi-bin/luci=/usr/lib/lua/luci/sgi/uhttpd.lua'
option script_timeout '60'
option network_timeout '30'
option http_keepalive '20'
option tcp_keepalive '1'
option ubus_prefix '/ubus'
config cert 'defaults'
option days '397'
option key_type 'ec'
option bits '2048'
option ec_curve 'P-256'
option country 'ZZ'
option state 'Somewhere'
option location 'Unknown'
option commonname 'OpenWrt'
package wireless
config wifi-device 'radio0'
option type 'mac80211'
option path 'platform/soc/18000000.wifi'
option band '2g'
option channel '1'
option htmode 'HE20'
config wifi-iface 'default_radio0'
option device 'radio0'
option network 'lan'
option mode 'ap'
option ssid 'OpenWrt'
option encryption 'none'
option disabled '1'
config wifi-device 'radio1'
option type 'mac80211'
option path 'platform/soc/18000000.wifi+1'
option band '5g'
option channel '36'
option htmode 'HE80'
config wifi-iface 'default_radio1'
option device 'radio1'
option network 'lan'
option mode 'ap'
option ssid 'OpenWrt'
option encryption 'none'
option disabled '1'