diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 06989fa..d099402 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -30,6 +30,7 @@ jobs: discord-bot: ${{ steps.filter.outputs.discord-bot }} tts-gateway: ${{ steps.filter.outputs.tts-gateway }} tts-sidecar: ${{ steps.filter.outputs.tts-sidecar }} + tts-model: ${{ steps.filter.outputs.tts-model }} steps: - uses: actions/checkout@v6 @@ -64,8 +65,13 @@ jobs: - 'go.work.sum' - 'tts-gateway/**' - '!tts-gateway/sidecar/**' + - '!tts-gateway/model/**' tts-sidecar: - 'tts-gateway/sidecar/**' + # Not a Go module - just the committed checkpoint/hparams + a Dockerfile that + # copies them in, so it doesn't need the gen/go.work paths the Go images do. + tts-model: + - 'tts-gateway/model/**' test: runs-on: ubuntu-latest @@ -241,3 +247,30 @@ jobs: tags: | ${{ env.IMAGE_PREFIX }}/tts-sidecar:${{ github.sha }} ${{ env.IMAGE_PREFIX }}/tts-sidecar:latest + + build-tts-model: + needs: [test, changes] + if: github.ref == 'refs/heads/main' && needs.changes.outputs.tts-model == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: docker/setup-buildx-action@v4 + + - uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + # Just a busybox base + COPY of the committed checkpoint/hparams - no build steps + # worth caching, and this only runs when tts-gateway/model/** actually changed anyway. + - uses: docker/build-push-action@v7 + with: + context: tts-gateway/model + file: tts-gateway/model/Dockerfile + push: true + platforms: linux/amd64 + tags: | + ${{ env.IMAGE_PREFIX }}/tts-model:${{ github.sha }} + ${{ env.IMAGE_PREFIX }}/tts-model:latest diff --git a/CLAUDE.md b/CLAUDE.md index f41e9c1..defebe9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -175,12 +175,11 @@ rather than trusting this description to stay accurate): default `RollingUpdate` — nik-gpu only has one allocatable GPU, so a rolling update would deadlock waiting for a GPU still held by the pod it's replacing. - Model artifact distribution (checkpoint + hparams) is an `emptyDir` volume populated at pod - start by an `initContainers` entry (`model-init`) that copies from a small, versioned image - (`gitea.nik4nao.com/nik/tts-model:`, built from `tts-gateway/model/Dockerfile`) — not a - raw `hostPath` into the node's disk anymore. That model image still can't be built by CI (the - checkpoint isn't committed to git, ~455MB, and only ever existed on nik-gpu's local disk) — it's - built and pushed manually, on nik-gpu, whenever the checkpoint/config change. See - `tts-gateway/README.md` for the exact commands. + start by an `initContainers` entry (`model-init`) that copies from `gitea.nik4nao.com/nik/tts-model:latest` + — not a raw `hostPath` into the node's disk anymore. That image is built from + `tts-gateway/model/Dockerfile`, whose checkpoint/hparams are committed directly to git (a + deliberate exception to not committing large binaries — see that directory) and built/pushed + by CI's `build-tts-model` job like the other four images, path-filtered on `tts-gateway/model/**`. - mTLS (`TLS_DIR`) is currently commented out on `tts-gateway`, for plaintext `grpcurl` testing from outside the cluster during initial rollout — a live TODO, not a permanent decision, unlike every other service here which assumes mTLS-or-trusted-network as its only access boundary (see @@ -188,6 +187,6 @@ rather than trusting this description to stay accurate): ## CI -`.gitea/workflows/ci.yaml` always runs `go vet`/`go test` for all five modules (`gen`, `ai-gateway`, `ha-gateway`, `discord-bot`, `tts-gateway`) on every push/PR. On pushes to `main`, a `changes` job (`dorny/paths-filter`) determines which of the five images actually need rebuilding based on which paths changed, so an edit scoped to one service doesn't rebuild (and re-push) all of them — `gen/`, `go.work`, and `go.work.sum` count as shared and mark every Go-based image as changed, since a dependency bump there can affect all of them. Each `build-*` job is gated on that output and, when it runs, uses `docker/build-push-action`'s registry-based cache (`cache-from`/`cache-to: type=registry,ref=.../:buildcache`) so unchanged Docker layers (e.g. `go mod download`, `apt-get`/`pip install`) don't get redone on every run — the *first* run after adding this has nothing to pull from and builds fully fresh, subsequent ones should be much faster. `tts-sidecar` (the Python/CUDA inference sidecar under `tts-gateway/sidecar/`) is a large ~13GB image; it builds fine on a generic runner since only *running* it needs a GPU, not building it. +`.gitea/workflows/ci.yaml` always runs `go vet`/`go test` for all five Go modules (`gen`, `ai-gateway`, `ha-gateway`, `discord-bot`, `tts-gateway`) on every push/PR. On pushes to `main`, a `changes` job (`dorny/paths-filter`) determines which of the six images (`ai-gateway`, `ha-gateway`, `discord-bot`, `tts-gateway`, `tts-sidecar`, `tts-model`) actually need rebuilding based on which paths changed, so an edit scoped to one service doesn't rebuild (and re-push) all of them — `gen/`, `go.work`, and `go.work.sum` count as shared and mark every Go-based image as changed, since a dependency bump there can affect all of them (`tts-model` isn't a Go module, so it's untouched by that rule — only `tts-gateway/model/**` triggers it). Each `build-*` job is gated on that output and, when it runs, uses `docker/build-push-action`'s registry-based cache (`cache-from`/`cache-to: type=registry,ref=.../:buildcache`) so unchanged Docker layers (e.g. `go mod download`, `apt-get`/`pip install`) don't get redone on every run — the *first* run after adding this has nothing to pull from and builds fully fresh, subsequent ones should be much faster (`build-tts-model` skips this, since a busybox+`COPY` image has no build steps worth caching). `tts-sidecar` (the Python/CUDA inference sidecar under `tts-gateway/sidecar/`) is a large ~13GB image; it builds fine on a generic runner since only *running* it needs a GPU, not building it. `tts-model`'s own build context (`tts-gateway/model/`) is ~455MB (the committed checkpoint) — a deliberate, one-time exception to this repo otherwise keeping large binaries out of git. Note that `ai-gateway/Dockerfile`, `ha-gateway/Dockerfile`, `discord-bot/Dockerfile`, and `tts-gateway/Dockerfile` each `COPY` every other service directory (not just their own) because `go.work` lists all five Go modules as workspace members — Go's workspace-mode module resolution needs every listed directory present in the build context, even ones a given service doesn't otherwise depend on. Adding a new module to `go.work` means adding a matching `COPY` line (both the manifest-only and full-source copies, see below) to the other Dockerfiles too, or their builds break. Each of those four Dockerfiles also copies every module's `go.mod`/`go.sum` first and runs `go mod download` *before* copying full source, so that layer's cache survives source-only edits instead of being invalidated by every commit. diff --git a/TTS_GATEWAY_PLAN.md b/TTS_GATEWAY_PLAN.md index e4d0420..41ae37b 100644 --- a/TTS_GATEWAY_PLAN.md +++ b/TTS_GATEWAY_PLAN.md @@ -172,9 +172,12 @@ WAV writer, `ffmpeg` transcode adapter, full `Synthesize` RPC wiring, speaker-no **Phase 5 — Containerize & deploy to nik-gpu** — **done.** Deployed and confirmed working in production; see `handoff.md` for status. Model artifact distribution is resolved as a small -versioned image (`tts-gateway/model/Dockerfile`) copied into a shared volume by a k8s -`initContainer`, built manually on nik-gpu (the checkpoint never existed anywhere CI can reach) — -see `tts-gateway/README.md`'s "Model artifact distribution (production)" section. +image (`tts-gateway/model/Dockerfile`) copied into a shared volume by a k8s `initContainer` — +the checkpoint/hparams are committed directly into `tts-gateway/model/` (a deliberate one-time +exception to not committing large binaries) and CI builds/pushes that image like every other one +here (`build-tts-model` in `.gitea/workflows/ci.yaml`). An initial attempt at building it manually +on nik-gpu instead hit a registry-auth wall there, which is what prompted committing the +checkpoint and automating the build instead — see `tts-gateway/README.md`. **Phase 6 — Client integration** — **done.** `discord-bot` registers a `/speak` command calling this gateway via the same secondary-adapter + gRPC-client pattern used for its diff --git a/handoff.md b/handoff.md index c6930d5..79e2ebf 100644 --- a/handoff.md +++ b/handoff.md @@ -6,11 +6,15 @@ inference, real playable audio). This file is a punch list for picking the remai fresh session — it doesn't re-explain things that are already documented elsewhere; it points to where. -**Update (later same day):** items 1 and 3 below are now done — model artifact distribution is a -versioned image + k8s `initContainer` (see `tts-gateway/README.md`), and `discord-bot` has a -working `/speak` command (see `TTS_GATEWAY_PLAN.md` Phase 6). Only item 2 (mTLS) remains, left -untouched per an explicit decision to verify these two first. The two "what's left" entries below -are kept as-written for their historical reasoning/decision trail rather than rewritten in place. +**Update (later same day):** items 1 and 3 below are now done. Model artifact distribution ended +up fully automated, not just "worked around" — after an initial attempt at a manually-built image +hit a nik-gpu registry-auth wall, the checkpoint/hparams were committed directly into +`tts-gateway/model/` (a deliberate one-time exception to not committing large binaries) and CI now +builds+pushes the `tts-model` image itself (`build-tts-model` in `.gitea/workflows/ci.yaml`), same +as every other image here — no manual nik-gpu step at all anymore. `discord-bot` has a working +`/speak` command (see `TTS_GATEWAY_PLAN.md` Phase 6). Only item 2 (mTLS) remains, left untouched +per an explicit decision to verify these two first. The two "what's left" entries below are kept +as-written for their historical reasoning/decision trail rather than rewritten in place. ## Where things stand diff --git a/tts-gateway/README.md b/tts-gateway/README.md index 63f0609..26960e4 100644 --- a/tts-gateway/README.md +++ b/tts-gateway/README.md @@ -66,34 +66,17 @@ The server also registers gRPC health checks and reflection. | `CHECKPOINT_PATH` | `/models/G_790000.pth` | Path to the model checkpoint | | `PORT` | `50054` | HTTP listen port | -Neither the checkpoint nor the hparams file is committed to git - both must -be supplied at runtime (see below). In production, they're baked into a -small versioned image (`tts-gateway/model/Dockerfile`) that a k8s -`initContainer` copies into a shared volume before `tts-sidecar` starts - -see "Model artifact distribution (production)" below. For local/manual -testing on nik-gpu, a plain bind mount (as shown below) is simpler. - -### Model artifact distribution (production) - -CI can never build the model image - the checkpoint (~455MB) and hparams -aren't committed to git, and nik-gpu's local disk at `/data/tts-gateway` is -the only place they exist. Build and push it manually, directly on nik-gpu -(not via `docker --context nik-gpu` from elsewhere - the build context is -gathered client-side, so it would look for `/data/tts-gateway` on the wrong -machine), whenever the checkpoint or config change: - -```bash -ssh nik-gpu -docker build -f ~/repo/home-service/tts-gateway/model/Dockerfile \ - -t gitea.nik4nao.com/nik/tts-model:g790000-v1 /data/tts-gateway -docker push gitea.nik4nao.com/nik/tts-model:g790000-v1 -``` - -Bump the tag (`v2`, `v3`, ...) rather than overwriting one - non-`:latest` -tags default to `imagePullPolicy: IfNotPresent`, so a node that already -pulled a tag won't re-pull an overwritten one. The k8s Deployment -(`~/repo/homelab/manifests/home-services/tts-gateway.yaml`) references the -tag explicitly in its `model-init` init container. +The checkpoint and hparams file live directly in `tts-gateway/model/` (a +deliberate, one-time exception to not committing large binaries - this asset +never changes and there's no other artifact store in play). CI builds and +pushes `gitea.nik4nao.com/nik/tts-model` from there exactly like the other +four images (`build-tts-model` in `.gitea/workflows/ci.yaml`), path-filtered +so it only rebuilds when `tts-gateway/model/**` actually changes. In +production, a k8s `initContainer` copies that image's files into a shared +volume before `tts-sidecar` starts - see +`~/repo/homelab/manifests/home-services/tts-gateway.yaml`'s `model-init` +container. To update the checkpoint: replace `tts-gateway/model/G_790000.pth` +and/or `uma.json`, commit, push to `main` - CI handles the rest. ## Running And Testing On nik-gpu @@ -213,8 +196,8 @@ sidecar/ # Python/libtorch inference servic - Concurrency is unbounded on the Go side, but the sidecar's model forward pass is inherently single-threaded per GPU; heavy concurrent load will just queue at the sidecar. -- Model artifact distribution to production is a manually-built-and-pushed - versioned image (see above), not something CI can automate - the - checkpoint only ever existed on nik-gpu's local disk. +- The model checkpoint is committed directly in `tts-gateway/model/` rather + than fetched from an external artifact store - a deliberate exception + given how rarely it changes, not a pattern to repeat for other assets. - `discord-bot`'s `/speak` command calls this service and plays the result in a Discord voice channel (`discord-bot/internal/adapters/primary/discord/voice.go`). diff --git a/tts-gateway/model/Dockerfile b/tts-gateway/model/Dockerfile index 7167621..917dced 100644 --- a/tts-gateway/model/Dockerfile +++ b/tts-gateway/model/Dockerfile @@ -1,7 +1,6 @@ -# One-time (or only-when-the-checkpoint-changes) manually built and pushed by a human on -# nik-gpu - CI can never build this: the checkpoint and hparams aren't committed to git -# (~455MB, and nik-gpu's local disk at /data/tts-gateway is the only place they exist). -# Build CONTEXT must be that directory, not a checkout of this repo - see -# tts-gateway/README.md for the exact commands. +# Built and pushed by CI (see .gitea/workflows/ci.yaml's build-tts-model job), same as the +# other four images - the checkpoint/hparams are committed directly in this directory (a +# deliberate exception to "don't commit large binaries", made because this asset never +# changes and there's no other artifact store in play). Build context is this directory. FROM busybox:1.36 COPY G_790000.pth uma.json /models/ diff --git a/tts-gateway/model/G_790000.pth b/tts-gateway/model/G_790000.pth new file mode 100644 index 0000000..4ad869b Binary files /dev/null and b/tts-gateway/model/G_790000.pth differ diff --git a/tts-gateway/model/uma.json b/tts-gateway/model/uma.json new file mode 100644 index 0000000..58fad7d --- /dev/null +++ b/tts-gateway/model/uma.json @@ -0,0 +1,53 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 1000, + "seed": 1234, + "epochs": 10000, + "learning_rate": 2e-4, + "betas": [0.8, 0.99], + "eps": 1e-9, + "batch_size": 12, + "fp16_run": true, + "lr_decay": 0.999875, + "segment_size": 8192, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0 + }, + "data": { + "training_files":"filelists/uma_text_train.txt.cleaned", + "validation_files":"filelists/uma_text_val.txt.cleaned", + "text_cleaners":["japanese_cleaners"], + "max_wav_value": 32768.0, + "sampling_rate": 22050, + "filter_length": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": null, + "add_blank": true, + "n_speakers": 92, + "cleaned_text": true + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + "upsample_rates": [8,8,2,2], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [16,16,4,4], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256 + } +} \ No newline at end of file