diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..7d59817 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +inventory = inventory.yaml +inject_facts_as_vars = False +deprecation_warnings = False \ No newline at end of file diff --git a/ansible/group_vars/all/vault.yaml b/ansible/group_vars/all/vault.yaml new file mode 100644 index 0000000..fa5b478 --- /dev/null +++ b/ansible/group_vars/all/vault.yaml @@ -0,0 +1,12 @@ +$ANSIBLE_VAULT;1.1;AES256 +31386562396163363266666232663537363730663161363165303536383934353136313133333437 +6236323335383962333464336235346532366133363234390a346633623166393834393430393338 +34373562656663623836326539363738326538653434336334363231633962366332663837336264 +3530316332666434360a666263353964386338366539303961616136626139663338393634333661 +37653538303831633837333738623466343637613966613533646663383535326262383766346565 +34636161353830646462366364373562653032636166663732646530366136303138306563366235 +32343761383435303633613137616666363638336366383364316630313537613363356436333966 +33623632373439653232373833376535646337623830616337306664393266633838333736353933 +63633636653635633835303735616533336239636564303931376535393765383137383336306364 +39613361383466383139663731353536346162343863336263336366306136376162663335623230 +623933336133643533363833323437396531 diff --git a/ansible/host_vars/gpu-node.yaml b/ansible/host_vars/gpu-node.yaml new file mode 100644 index 0000000..bec4193 --- /dev/null +++ b/ansible/host_vars/gpu-node.yaml @@ -0,0 +1,52 @@ +# Host vars for: gpu-node (GPU workstation — spot K3s agent) +ansible_python_interpreter: /usr/bin/python3.12 + +# ── common ───────────────────────────────────────────────────────────────────── +timezone: Asia/Tokyo +username: nik + +base_packages: + - curl + - git + - htop + - vim + - wget + - ca-certificates + - gnupg + - lsb-release + - build-essential + +ufw_allowed_ports: + - { port: "430", proto: tcp, comment: "SSH" } + - { port: "11434", proto: tcp, comment: "Ollama API" } + - { port: "61208", proto: tcp, comment: "Glances web UI" } + +data_dirs: [] + +# ── nvidia ───────────────────────────────────────────────────────────────────── +nvidia_driver_version: "570" +cuda_version: "12-8" + +# ── k3s-agent ────────────────────────────────────────────────────────────────── +k3s_server_url: "https://192.168.7.77:6443" +k3s_node_token: "{{ vault_k3s_node_token }}" + +# Check current cluster version with: k3s --version on minisforum +k3s_version: "v1.32.4+k3s1" + +k3s_node_labels: + node-role: gpu + nik4nao.com/node-type: spot + nik4nao.com/gpu: "true" + +k3s_node_taints: + - "spot=true:NoSchedule" + +# ── ollama ───────────────────────────────────────────────────────────────────── +ollama_port: 11434 +ollama_models: + - qwen3:4b +ollama_models_dir: /usr/share/ollama/.ollama/models + +# ── glances ──────────────────────────────────────────────────────────────────── +# no extra vars — uses role defaults \ No newline at end of file diff --git a/ansible/playbooks/setup-gpu-node.yaml b/ansible/playbooks/setup-gpu-node.yaml index 7f96eb6..ae092f1 100644 --- a/ansible/playbooks/setup-gpu-node.yaml +++ b/ansible/playbooks/setup-gpu-node.yaml @@ -6,6 +6,7 @@ roles: - role: common + - role: docker - role: nvidia - role: k3s-agent - role: ollama diff --git a/ansible/roles/docker/tasks/main.yaml b/ansible/roles/docker/tasks/main.yaml new file mode 100644 index 0000000..26dc3eb --- /dev/null +++ b/ansible/roles/docker/tasks/main.yaml @@ -0,0 +1,43 @@ +--- +# Part of role: docker +# Called by: ansible/playbooks/setup-gpu-node.yaml +# Description: Installs Docker CE on Ubuntu, adds user to docker group. + +- name: Add Docker GPG key + ansible.builtin.shell: + cmd: > + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | + gpg --dearmor -o /usr/share/keyrings/docker.gpg + creates: /usr/share/keyrings/docker.gpg + +- name: Add Docker apt repository + ansible.builtin.apt_repository: + repo: > + deb [arch=amd64 signed-by=/usr/share/keyrings/docker.gpg] + https://download.docker.com/linux/ubuntu + {{ ansible_facts['distribution_release'] }} stable + filename: docker + state: present + +- name: Install Docker CE + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: true + +- name: Ensure Docker service is running and enabled + ansible.builtin.systemd: + name: docker + state: started + enabled: true + +- name: Add user to docker group + ansible.builtin.user: + name: "{{ username }}" + groups: docker + append: true \ No newline at end of file diff --git a/ansible/roles/nvidia/handlers/main.yaml b/ansible/roles/nvidia/handlers/main.yaml new file mode 100644 index 0000000..0bbf08a --- /dev/null +++ b/ansible/roles/nvidia/handlers/main.yaml @@ -0,0 +1,9 @@ +--- +# Part of role: nvidia +# Called by: ansible/playbooks/setup-gpu-node.yaml +# Description: Restarts Docker after nvidia-container-toolkit runtime configuration. +- name: restart docker + ansible.builtin.systemd: + name: docker + state: restarted + daemon_reload: true \ No newline at end of file diff --git a/ansible/roles/nvidia/tasks/main.yaml b/ansible/roles/nvidia/tasks/main.yaml new file mode 100644 index 0000000..0e26d04 --- /dev/null +++ b/ansible/roles/nvidia/tasks/main.yaml @@ -0,0 +1,78 @@ +--- +# Part of role: nvidia +# Called by: ansible/playbooks/setup-gpu-node.yaml +# Description: Installs NVIDIA drivers, CUDA toolkit, and nvidia-container-toolkit. +# Configures Docker and K3s containerd runtimes for GPU access. + +- name: Add NVIDIA CUDA apt keyring + ansible.builtin.shell: + cmd: > + curl -fsSL + https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb + -o /tmp/cuda-keyring.deb && dpkg -i /tmp/cuda-keyring.deb + creates: /usr/share/keyrings/cuda-archive-keyring.gpg + +- name: Add NVIDIA container toolkit keyring + ansible.builtin.shell: + cmd: > + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | + gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + creates: /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + +- name: Add NVIDIA container toolkit repo + ansible.builtin.shell: + cmd: > + curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | + tee /etc/apt/sources.list.d/nvidia-container-toolkit.list + creates: /etc/apt/sources.list.d/nvidia-container-toolkit.list + +- name: Update apt cache + ansible.builtin.apt: + update_cache: true + +- name: Install NVIDIA driver + ansible.builtin.apt: + name: "nvidia-driver-{{ nvidia_driver_version }}" + state: present + register: nvidia_driver_install + +- name: Install CUDA toolkit + ansible.builtin.apt: + name: "cuda-toolkit-{{ cuda_version }}" + state: present + +- name: Install nvidia-container-toolkit + ansible.builtin.apt: + name: nvidia-container-toolkit + state: present + +- name: Add CUDA to system PATH + ansible.builtin.copy: + dest: /etc/profile.d/cuda.sh + content: | + export PATH=/usr/local/cuda/bin:$PATH + export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH + mode: "0644" + +- name: Configure Docker runtime for NVIDIA + ansible.builtin.command: + cmd: nvidia-ctk runtime configure --runtime=docker + changed_when: true + notify: restart docker + +- name: Reboot if driver was just installed + ansible.builtin.reboot: + reboot_timeout: 300 + when: nvidia_driver_install.changed + +- name: Verify NVIDIA driver loaded + ansible.builtin.command: nvidia-smi + changed_when: false + failed_when: false + register: nvidia_smi_check + +- name: Assert nvidia-smi succeeded + ansible.builtin.assert: + that: nvidia_smi_check.rc == 0 + fail_msg: "nvidia-smi failed — driver may not have loaded correctly" \ No newline at end of file diff --git a/ansible/roles/nvidia/templates/k3s-containerd-config.toml.j2 b/ansible/roles/nvidia/templates/k3s-containerd-config.toml.j2 new file mode 100644 index 0000000..b36cc43 --- /dev/null +++ b/ansible/roles/nvidia/templates/k3s-containerd-config.toml.j2 @@ -0,0 +1,10 @@ +# Managed by Ansible — do not edit manually +# Adds nvidia as an additional runtime — pods request it via runtimeClassName: nvidia +# runc remains the default runtime for all other workloads +version = 2 + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" \ No newline at end of file diff --git a/ansible/roles/ollama/handlers/main.yaml b/ansible/roles/ollama/handlers/main.yaml index 195b14f..fd72776 100644 --- a/ansible/roles/ollama/handlers/main.yaml +++ b/ansible/roles/ollama/handlers/main.yaml @@ -6,7 +6,7 @@ - name: restart ollama become: true command: launchctl kickstart -k system/com.ollama.ollama - when: ansible_system == 'Darwin' + when: ansible_facts['system'] == 'Darwin' - name: restart ollama linux ansible.builtin.systemd: @@ -14,4 +14,4 @@ state: restarted daemon_reload: true become: true - when: ansible_system == 'Linux' \ No newline at end of file + when: ansible_facts['system'] == 'Linux' \ No newline at end of file diff --git a/ansible/roles/ollama/tasks/main.yaml b/ansible/roles/ollama/tasks/main.yaml index 1c924c2..8eaf907 100644 --- a/ansible/roles/ollama/tasks/main.yaml +++ b/ansible/roles/ollama/tasks/main.yaml @@ -10,7 +10,7 @@ community.general.homebrew: name: ollama state: present - when: ansible_system == 'Darwin' + when: ansible_facts['system'] == 'Darwin' - name: Deploy ollama launchd plist template: @@ -21,7 +21,7 @@ mode: "0644" become: true notify: restart ollama - when: ansible_system == 'Darwin' + when: ansible_facts['system'] == 'Darwin' - name: Load ollama launchd service become: true @@ -29,14 +29,22 @@ args: creates: /var/run/ollama.pid ignore_errors: true - when: ansible_system == 'Darwin' + when: ansible_facts['system'] == 'Darwin' # ── Linux ────────────────────────────────────────────────────────────────────── - name: Install ollama via install script ansible.builtin.shell: cmd: curl -fsSL https://ollama.com/install.sh | sh - creates: /usr/local/bin/ollama - when: ansible_system == 'Linux' + creates: /etc/systemd/system/ollama.service + when: ansible_facts['system'] == 'Linux' + +- name: Create ollama systemd override directory + ansible.builtin.file: + path: /etc/systemd/system/ollama.service.d + state: directory + mode: "0755" + become: true + when: ansible_facts['system'] == 'Linux' - name: Deploy ollama systemd override ansible.builtin.template: @@ -47,7 +55,7 @@ mode: "0644" become: true notify: restart ollama linux - when: ansible_system == 'Linux' + when: ansible_facts['system'] == 'Linux' - name: Enable and start ollama service ansible.builtin.systemd: @@ -56,7 +64,7 @@ enabled: true daemon_reload: true become: true - when: ansible_system == 'Linux' + when: ansible_facts['system'] == 'Linux' # ── shared ───────────────────────────────────────────────────────────────────── - name: Wait for ollama to be ready @@ -76,7 +84,7 @@ - name: Pull ollama models command: > - {{ '/opt/homebrew/bin/ollama' if ansible_system == 'Darwin' else '/usr/local/bin/ollama' }} + {{ '/opt/homebrew/bin/ollama' if ansible_facts['system'] == 'Darwin' else '/usr/local/bin/ollama' }} pull {{ item }} loop: "{{ ollama_models }}" when: item not in (ollama_tags.json.models | map(attribute='name') | list) diff --git a/manifests/home-services/nvidia-device-plugin.yaml b/manifests/home-services/nvidia-device-plugin.yaml new file mode 100644 index 0000000..df2873b --- /dev/null +++ b/manifests/home-services/nvidia-device-plugin.yaml @@ -0,0 +1,48 @@ +# Config for: NVIDIA GPU device plugin +# Applied by: kubectl apply -f manifests/home-services/nvidia-device-plugin.yaml +# Description: Exposes nvidia.com/gpu resource on gpu-node so K3s can schedule GPU workloads. +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin + namespace: kube-system +spec: + selector: + matchLabels: + app: nvidia-device-plugin + template: + metadata: + labels: + app: nvidia-device-plugin + spec: + runtimeClassName: nvidia + tolerations: + - key: spot + operator: Equal + value: "true" + effect: NoSchedule + nodeSelector: + nik4nao.com/gpu: "true" + containers: + - name: nvidia-device-plugin + image: nvcr.io/nvidia/k8s-device-plugin:v0.17.0 + args: + - --device-discovery-strategy=nvml + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins \ No newline at end of file diff --git a/router/backup-base.tar.gz b/router/backup-base.tar.gz new file mode 100644 index 0000000..cad43f9 Binary files /dev/null and b/router/backup-base.tar.gz differ diff --git a/router/uci-base.conf b/router/uci-base.conf new file mode 100644 index 0000000..130f45c --- /dev/null +++ b/router/uci-base.conf @@ -0,0 +1,373 @@ +package attendedsysupgrade + +config server 'server' + option url 'https://sysupgrade.openwrt.org' + +config client 'client' + option upgrade_packages '1' + option auto_search '0' + option advanced_mode '0' + option login_check_for_upgrades '1' + +config owut 'owut' + +package dhcp + +config dnsmasq + option domainneeded '1' + option boguspriv '1' + option filterwin2k '0' + option localise_queries '1' + option rebind_protection '1' + option rebind_localhost '1' + option local '/lan/' + option domain 'lan' + option expandhosts '1' + option nonegcache '0' + option cachesize '1000' + option authoritative '1' + option readethers '1' + option leasefile '/tmp/dhcp.leases' + option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto' + option nonwildcard '1' + option localservice '1' + option ednspacket_max '1232' + option filter_aaaa '0' + option filter_a '0' + +config dhcp 'lan' + option interface 'lan' + option start '100' + option limit '150' + option leasetime '12h' + option dhcpv4 'server' + option dhcpv6 'server' + option ra 'server' + option ra_slaac '1' + list ra_flags 'managed-config' + list ra_flags 'other-config' + +config dhcp 'wan' + option interface 'wan' + option ignore '1' + +config odhcpd 'odhcpd' + option maindhcp '0' + option leasefile '/tmp/odhcpd.leases' + option leasetrigger '/usr/sbin/odhcpd-update' + option loglevel '4' + option piodir '/tmp/odhcpd-piodir' + option hostsdir '/tmp/hosts' + +package dropbear + +config dropbear 'main' + option PasswordAuth 'off' + option RootPasswordAuth 'off' + option Port '430' + +package firewall + +config defaults + option syn_flood '1' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + +config zone + option name 'lan' + list network 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + +config zone + option name 'wan' + list network 'wan' + list network 'wan6' + option input 'REJECT' + option output 'ACCEPT' + option forward 'DROP' + option masq '1' + option mtu_fix '1' + +config forwarding + option src 'lan' + option dest 'wan' + +config rule + option name 'Allow-DHCP-Renew' + option src 'wan' + option proto 'udp' + option dest_port '68' + option target 'ACCEPT' + option family 'ipv4' + +config rule + option name 'Allow-Ping' + option src 'wan' + option proto 'icmp' + option icmp_type 'echo-request' + option family 'ipv4' + option target 'ACCEPT' + +config rule + option name 'Allow-IGMP' + option src 'wan' + option proto 'igmp' + option family 'ipv4' + option target 'ACCEPT' + +config rule + option name 'Allow-DHCPv6' + option src 'wan' + option proto 'udp' + option dest_port '546' + option family 'ipv6' + option target 'ACCEPT' + +config rule + option name 'Allow-MLD' + option src 'wan' + option proto 'icmp' + option src_ip 'fe80::/10' + list icmp_type '130/0' + list icmp_type '131/0' + list icmp_type '132/0' + list icmp_type '143/0' + option family 'ipv6' + option target 'ACCEPT' + +config rule + option name 'Allow-ICMPv6-Input' + option src 'wan' + option proto 'icmp' + list icmp_type 'echo-request' + list icmp_type 'echo-reply' + list icmp_type 'destination-unreachable' + list icmp_type 'packet-too-big' + list icmp_type 'time-exceeded' + list icmp_type 'bad-header' + list icmp_type 'unknown-header-type' + list icmp_type 'router-solicitation' + list icmp_type 'neighbour-solicitation' + list icmp_type 'router-advertisement' + list icmp_type 'neighbour-advertisement' + option limit '1000/sec' + option family 'ipv6' + option target 'ACCEPT' + +config rule + option name 'Allow-ICMPv6-Forward' + option src 'wan' + option dest '*' + option proto 'icmp' + list icmp_type 'echo-request' + list icmp_type 'echo-reply' + list icmp_type 'destination-unreachable' + list icmp_type 'packet-too-big' + list icmp_type 'time-exceeded' + list icmp_type 'bad-header' + list icmp_type 'unknown-header-type' + option limit '1000/sec' + option family 'ipv6' + option target 'ACCEPT' + +config rule + option name 'Allow-IPSec-ESP' + option src 'wan' + option dest 'lan' + option proto 'esp' + option target 'ACCEPT' + +config rule + option name 'Allow-ISAKMP' + option src 'wan' + option dest 'lan' + option dest_port '500' + option proto 'udp' + option target 'ACCEPT' + +package luci + +config core 'main' + option lang 'auto' + option mediaurlbase '/luci-static/bootstrap' + option resourcebase '/luci-static/resources' + option ubuspath '/ubus/' + +config extern 'flash_keep' + option uci '/etc/config/' + option dropbear '/etc/dropbear/' + option openvpn '/etc/openvpn/' + option passwd '/etc/passwd' + option opkg '/etc/opkg.conf' + option firewall '/etc/firewall.user' + option uploads '/lib/uci/upload/' + +config internal 'languages' + +config internal 'sauth' + option sessionpath '/tmp/luci-sessions' + option sessiontime '3600' + +config internal 'ccache' + option enable '1' + +config internal 'themes' + option Bootstrap '/luci-static/bootstrap' + option BootstrapDark '/luci-static/bootstrap-dark' + option BootstrapLight '/luci-static/bootstrap-light' + +config internal 'apply' + option rollback '90' + option holdoff '4' + option timeout '5' + option display '1.5' + +config internal 'diag' + option dns 'openwrt.org' + option ping 'openwrt.org' + option route 'openwrt.org' + +package network + +config interface 'loopback' + option device 'lo' + option proto 'static' + list ipaddr '127.0.0.1/8' + +config globals 'globals' + option dhcp_default_duid '0004a365d32c3ee8471c918c7e1877ee52a3' + option ula_prefix 'fd4b:364b:726a::/48' + +config device + option name 'br-lan' + option type 'bridge' + list ports 'lan1' + list ports 'lan2' + list ports 'lan3' + list ports 'lan4' + list ports 'lan5' + +config interface 'lan' + option device 'br-lan' + option proto 'static' + option ipaddr '192.168.7.1' + option ip6assign '60' + option netmask '255.255.255.0' + option dns '192.168.7.77' + +config interface 'wan' + option device 'eth1' + option proto 'dhcp' + option peerdns '0' + option dns '192.168.7.77' + +config interface 'wan6' + option device 'eth1' + option proto 'dhcpv6' + +package rpcd + +config rpcd + option socket '/var/run/ubus/ubus.sock' + option timeout '30' + +config login + option username 'root' + option password '$p$root' + list read '*' + list write '*' + +package system + +config system + option hostname 'flint2' + option timezone 'GMT0' + option zonename 'UTC' + option ttylogin '0' + option log_size '128' + option urandom_seed '0' + +config timeserver 'ntp' + option enabled '1' + option enable_server '0' + list server '0.openwrt.pool.ntp.org' + list server '1.openwrt.pool.ntp.org' + list server '2.openwrt.pool.ntp.org' + list server '3.openwrt.pool.ntp.org' + +package ubihealthd + +package ubootenv + +config ubootenv + option dev '/dev/mmcblk0p1' + option offset '0x0' + option envsize '0x80000' + +package uhttpd + +config uhttpd 'main' + list listen_http '0.0.0.0:80' + list listen_http '[::]:80' + list listen_https '0.0.0.0:443' + list listen_https '[::]:443' + option redirect_https '0' + option home '/www' + option rfc1918_filter '1' + option max_requests '3' + option max_connections '100' + option cert '/etc/uhttpd.crt' + option key '/etc/uhttpd.key' + option cgi_prefix '/cgi-bin' + list lua_prefix '/cgi-bin/luci=/usr/lib/lua/luci/sgi/uhttpd.lua' + option script_timeout '60' + option network_timeout '30' + option http_keepalive '20' + option tcp_keepalive '1' + option ubus_prefix '/ubus' + +config cert 'defaults' + option days '397' + option key_type 'ec' + option bits '2048' + option ec_curve 'P-256' + option country 'ZZ' + option state 'Somewhere' + option location 'Unknown' + option commonname 'OpenWrt' + +package wireless + +config wifi-device 'radio0' + option type 'mac80211' + option path 'platform/soc/18000000.wifi' + option band '2g' + option channel '1' + option htmode 'HE20' + +config wifi-iface 'default_radio0' + option device 'radio0' + option network 'lan' + option mode 'ap' + option ssid 'OpenWrt' + option encryption 'none' + option disabled '1' + +config wifi-device 'radio1' + option type 'mac80211' + option path 'platform/soc/18000000.wifi+1' + option band '5g' + option channel '36' + option htmode 'HE80' + +config wifi-iface 'default_radio1' + option device 'radio1' + option network 'lan' + option mode 'ap' + option ssid 'OpenWrt' + option encryption 'none' + option disabled '1' +