From f30f35b44172a4bc8511ed74a6f60294adcda7f7 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Sun, 5 Jul 2026 04:38:52 +0900 Subject: [PATCH] Add automation scripts for Blue Archive mailbox and cafe routines --- CLAUDE.md | 59 +++++++++++++++++++++++++++++++ ba_dailies.sh | 69 +++++++++++++++++++++++++++++++++++-- scripts/detect_and_click.py | 69 +++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 CLAUDE.md create mode 100644 scripts/detect_and_click.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..40dacfb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,59 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +Automation for daily chores in the mobile game Blue Archive: claiming mailbox rewards and farming/collecting cafe affection income. It drives the actual game client via simulated mouse clicks and keypresses (`xdotool`) against fixed screen coordinates, plus one template-matching detector for a dynamic (moving) UI element. + +## Two-machine architecture + +This repo is developed on macOS (`nik-macbookair`) but the code only runs on a separate Linux box, `nik-gpu`, where the Blue Archive client and its X display actually live (`DISPLAY=:0`, GDM `XAUTHORITY` under `/run/user/1000`). There is no local way to execute or test these scripts — they must be deployed to `nik-gpu` to run for real. + +- `ba_dailies.sh` and `scripts/detect_and_click.py` are pushed to `nik-gpu` with `scp`/`rsync` (e.g. `scp ba_dailies.sh nik-gpu:~/ba_dailies.sh`, `scp scripts/detect_and_click.py nik-gpu:~/ba_scripts/detect_and_click.py`) and invoked there over `ssh`. +- `assets/cafe_sparkle.png` (the template image) is likewise copied to `nik-gpu:~/ba_assets/cafe_sparkle.png`. +- `detect_and_click.py` runs a Python venv on `nik-gpu` at `~/.venvs/ba-auto-daily/bin/python3` (needs `opencv-python`/`numpy`); `ba_dailies.sh` invokes it via that fixed path (`VENV_PYTHON` in the script). +- Screenshot -> detect -> click for the cafe sparkle happens entirely on `nik-gpu` in one local pipeline (see the docstring in `scripts/detect_and_click.py`) rather than round-tripping images over SSH, because the sparkle target moves fast enough that a multi-hop pipeline would miss the click. +- `screenshots/` holds reference captures (taken on `nik-gpu`, pulled back for inspection) used to work out coordinates and template thresholds when coordinates drift after a game UI update — they are not test fixtures consumed by any script. + +Because there's no local execution path, "testing a change" means syntax-checking locally and then deploying to `nik-gpu` and running it against the live game there: +```bash +bash -n ba_dailies.sh +python3 -m py_compile scripts/detect_and_click.py +``` + +## Dependencies on `nik-gpu` + +These are host-level prerequisites, not managed by this repo — confirm they're present before assuming a failure is a code bug: +- `xdotool` (window focus, clicks, keypresses) +- `scrot` (screenshot capture for the detector) +- A venv at `~/.venvs/ba-auto-daily/` with `opencv-python` and `numpy` installed, Python binary at `~/.venvs/ba-auto-daily/bin/python3` + +Quick check: +```bash +ssh nik-gpu "which xdotool scrot && ~/.venvs/ba-auto-daily/bin/python3 -c 'import cv2, numpy; print(cv2.__version__)'" +``` +## Working conventions + +Use `./scratchpad` (create if missing) in the project root for temporary/intermediate files — e.g. cropped calibration images from `screenshots/cafe/sparkle/`, one-off debug output. Never write to `/tmp` or `/private/tmp`. + +## `ba_dailies.sh` + +Entry point, run on `nik-gpu` as `./ba_dailies.sh [mailbox|cafe]`: +- No argument: focuses the game window, runs mailbox claim, then the full cafe routine. +- `mailbox`: focus + claim mailbox only. +- `cafe`: focus + cafe routine only (both rooms + income claim). + +All interaction points (icon/button coordinates, max click-attempts per cafe room) are top-of-file constants — when the in-game UI shifts or the window resolution changes, update the coordinates there rather than inline in the functions. `focus_game` finds the window via `xdotool search --name "BlueArchive"` and hard-fails if the game isn't running. + +The cafe routine (`do_cafe`) alternates between two cafe rooms; for each room it repeatedly calls into `detect_and_click.py` to find and click affection sparkles (up to `CAFE_MAX_CLICKS_PER_ROOM` times) before moving on, then claims cafe income at the end. + +**Known gap — needs verification:** the manual cafe flow this automates includes a few conditional steps not obviously covered above: closing a popup that only appears if a student is "rotated," zooming out/centering the view before detection starts, and closing a rank-up popup that can appear after a successful click. Confirm these are actually handled somewhere in `do_cafe` (or decide they're unnecessary in practice) rather than assuming coverage from this description alone. + +## `scripts/detect_and_click.py` + +Standalone script (runs on `nik-gpu`, called once per sparkle-click attempt from the shell loop): screenshots the game window with `scrot`, template-matches `cafe_sparkle.png` using a color-masked `cv2.matchTemplate` (masks to yellow/white sparkle pixels so it doesn't match on background art), clicks the best match (offset-corrected — the template's anchor point isn't the click point), and reports its result on stdout/exit code (`MATCH x y score` / exit 0, or `NO_MATCH` / exit 1) so the caller shell loop can decide whether to keep clicking. + +`THRESHOLD = 0.97` and `OFFSET_X`/`OFFSET_Y` are the values most likely to need retuning if detection starts missing or mis-clicking — use the `screenshots/cafe/sparkle/` reference captures to recalibrate. When cropping or annotating these captures for calibration, use `./scratchpad` in the project root for the intermediate files, not `/tmp`. + +**Performance note — needs verification:** this script is invoked as a fresh process per click attempt, and Python + OpenCV cold-start has real overhead (commonly 200–500ms). If the sparkle target moves fast, confirm this hasn't caused missed detections in practice against the live game. If it has, consider a persistent process the shell loop talks to (pipe/socket) instead of a per-attempt cold start. \ No newline at end of file diff --git a/ba_dailies.sh b/ba_dailies.sh index 79ceb72..073858c 100644 --- a/ba_dailies.sh +++ b/ba_dailies.sh @@ -6,6 +6,13 @@ WIN_NAME="BlueArchive" MAILBOX_ICON="1726 60" CLAIM_ALL="1691 1128" +CAFE_ICON="165 1100" +CAFE_ROOM_SWITCH="190 160" +CAFE_INCOME="1780 1105" +VENV_PYTHON="$HOME/.venvs/ba-auto-daily/bin/python3" +DETECT_SPARKLE="$HOME/ba_scripts/detect_and_click.py" +CAFE_MAX_CLICKS_PER_ROOM=15 + get_window() { xdotool search --name "$WIN_NAME" | head -1 } @@ -53,6 +60,62 @@ do_mailbox() { echo "[mailbox] Done." } -focus_game -do_mailbox -echo "All done." +do_cafe_room() { + local i + for ((i = 0; i < CAFE_MAX_CLICKS_PER_ROOM; i++)); do + if ! "$VENV_PYTHON" "$DETECT_SPARKLE" | grep -q "^MATCH"; then + break + fi + sleep 1 + press_enter + done +} + +do_cafe() { + echo "[cafe] Opening cafe..." + click "$CAFE_ICON" + sleep 3 + press_enter + echo "[cafe] Room 1: farming affection..." + do_cafe_room + echo "[cafe] Switching to room 2..." + click "$CAFE_ROOM_SWITCH" + sleep 3 + press_enter + echo "[cafe] Room 2: farming affection..." + do_cafe_room + echo "[cafe] Claiming cafe income..." + click "$CAFE_INCOME" + sleep 2 + press_enter + sleep 2 + press_enter + sleep 2 + echo "[cafe] Exiting cafe..." + press_esc + sleep 1.5 + press_esc + sleep 1.5 + echo "[cafe] Done." +} + +case "$1" in + mailbox) + focus_game + do_mailbox + ;; + cafe) + focus_game + do_cafe + ;; + "") + focus_game + do_mailbox + do_cafe + echo "All done." + ;; + *) + echo "Unknown phase: $1" + exit 1 + ;; +esac diff --git a/scripts/detect_and_click.py b/scripts/detect_and_click.py new file mode 100644 index 0000000..cd5726b --- /dev/null +++ b/scripts/detect_and_click.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Screenshot the game window, find a cafe affection sparkle, click it. + +Runs entirely on nik-gpu (screenshot -> detect -> click all local) so the +whole cycle finishes in well under a second - roaming students move fast +enough that a multi-hop SSH/scp pipeline misses the click. + +Prints "MATCH x y score" and exits 0 if a sparkle was found and clicked, +or prints "NO_MATCH" and exits 1 otherwise. +""" +import subprocess +import sys + +import cv2 +import numpy as np + +TEMPLATE_PATH = "/home/nik/ba_assets/cafe_sparkle.png" +SHOT_PATH = "/tmp/ba_live.png" +OFFSET_X = 75 +OFFSET_Y = 47 +THRESHOLD = 0.97 + +ENV = {"DISPLAY": ":0", "XAUTHORITY": "/run/user/1000/gdm/Xauthority"} + + +def screenshot(): + subprocess.run( + ["scrot", "-a", "0,0,1920,1200", "-o", SHOT_PATH], + env=ENV, check=True, + ) + + +def find_sparkles(): + template = cv2.imread(TEMPLATE_PATH) + b, g, r = cv2.split(template.astype(np.int16)) + yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200)) + mask_plane = (yellow_white.astype(np.uint8)) * 255 + mask = cv2.merge([mask_plane, mask_plane, mask_plane]) + th, tw = template.shape[:2] + + img = cv2.imread(SHOT_PATH) + result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask) + locs = np.where(result >= THRESHOLD) + points = sorted(zip(*locs[::-1]), key=lambda p: -result[p[1], p[0]]) + + merged = [] + for x, y in points: + if all(abs(x - mx) > tw // 2 or abs(y - my) > th // 2 for mx, my, _ in merged): + merged.append((x, y, result[y, x])) + return [(x + tw // 2, y + th // 2, score) for x, y, score in merged] + + +def click(x, y): + subprocess.run( + ["xdotool", "mousemove", str(x), str(y), "click", "1"], + env=ENV, check=True, + ) + + +if __name__ == "__main__": + screenshot() + sparkles = find_sparkles() + if not sparkles: + print("NO_MATCH") + sys.exit(1) + x, y, score = sparkles[0] + click(x + OFFSET_X, y + OFFSET_Y) + print(f"MATCH {x} {y} {score:.4f}") + sys.exit(0)