From 3a4534f205c26efd3703c86cc9d2aa6a26dcdf05 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Sun, 5 Jul 2026 16:31:26 +0900 Subject: [PATCH] Migrate cafe and mailbox tasks to Python: replace legacy Bash scripts, implement state verification, and integrate sparkle detection. --- CLAUDE.md | 17 ++-- README.md | 17 ++-- ba_auto/config.py | 10 ++- ba_auto/detector.py | 36 ++++++++- ba_auto/navigation.py | 21 ++++- ba_auto/reference_notes/mapping.md | 2 +- ba_auto/tasks/cafe.py | 93 +++++++++++++++++++++- ba_auto/tasks/mailbox.py | 12 +-- plan.md | 70 +++++++---------- scripts/ba_dailies_legacy.sh | 122 ----------------------------- scripts/detect_and_click.py | 69 ---------------- setup.sh | 14 ++-- 12 files changed, 204 insertions(+), 279 deletions(-) delete mode 100644 scripts/ba_dailies_legacy.sh delete mode 100644 scripts/detect_and_click.py diff --git a/CLAUDE.md b/CLAUDE.md index c7012da..f4ced61 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -306,7 +306,7 @@ This keeps the project close to the reference architecture while replacing only Image matching and color matching should live in `ba_auto/detector.py` or in clearly named helper classes/functions. -Existing logic from `scripts/detect_and_click.py` should be migrated into reusable Python functions. +The cafe sparkle-matching logic formerly in `scripts/detect_and_click.py` (now retired) has been migrated here as `find_cafe_sparkle()`, called in-process rather than as a per-click subprocess. The detector should support: @@ -421,16 +421,17 @@ When debugging image matching, write debug images to `./scratchpad`. ## Existing features -Current project state (Phase 1 skeleton in place, mailbox migrated): +Current project state (mailbox and cafe both migrated; no Bash feature logic remains): - `ba_dailies.sh` is a thin launcher that execs `ba_daily.py` - `ba_daily.py` dispatches `mailbox`/`cafe`/default flow to `ba_auto/tasks/` -- `ba_auto/tasks/mailbox.py` is real Python: it clicks with `ba_auto/driver.py` primitives and verifies state with `driver.color_at` (ported from `module/mail.py`'s `rgb_in_range` pattern) before pressing further keys — no legacy bridge -- `ba_auto/tasks/cafe.py` still bridges to `scripts/ba_dailies_legacy.sh` (the pre-migration click sequence), marked with a TODO — the real logic has not moved into Python yet -- cafe's bridge still shells out to `scripts/detect_and_click.py` for OpenCV sparkle detection, unchanged -- `ba_auto/driver.py` has the primitives (`run_command`, `focus_game`, `click`, `keypress`, `screenshot`, `wait`, `color_at`) wired into `mailbox.py`; not yet used by `cafe.py` -- `ba_auto/detector.py` and `ba_auto/navigation.py` are placeholders -- Live testing found the old fixed mailbox-icon coordinate was marginal and could miss, cascading into an unverified Escape press that triggers Blue Archive's own "exit the game?" confirmation — see `plan.md` Phase 5 for details. This is the concrete reason `mailbox.py` verifies state before acting rather than trusting fixed coordinates blindly. +- `ba_auto/tasks/mailbox.py` and `ba_auto/tasks/cafe.py` are both real Python: they click with `ba_auto/driver.py` primitives and verify state with `driver.color_at`/`ba_auto/navigation.py` (ported from `module/mail.py` and `module/cafe_reward.py`'s `rgb_in_range`/`co_detect` pattern) before pressing further keys — no legacy bridge remains +- `ba_auto/detector.py` has `find_cafe_sparkle()`, the sparkle template-match ported in-process from the now-deleted `scripts/detect_and_click.py` +- `scripts/ba_dailies_legacy.sh` and `scripts/detect_and_click.py` have been deleted — nothing references them anymore +- `ba_auto/driver.py` primitives (`run_command`, `focus_game`, `click`, `keypress`, `screenshot`, `wait`, `color_at`) are wired into both task modules +- `ba_auto/navigation.py` has two shared state probes used by both tasks: `is_on_subscreen` (any mailbox/cafe/shop-style panel vs. the home screen) and `is_modal_open` (a dimmed dialog overlay) +- Live testing found both the mailbox-icon and cafe-icon fixed coordinates were flaky (missed the first click, worked on retry) and that neither task verified anything before proceeding, so a missed click cascaded into blind actions and could reach an unverified Escape press on the home screen — which triggers Blue Archive's own "exit the game?" confirmation. See `plan.md` Phases 5–6 for the full writeup. This is the concrete reason both tasks verify state before acting rather than trusting fixed coordinates blindly. +- Not yet verified for cafe: rank-up popups mid-pat-loop, and whether camera zoom/pan can drift over a long unattended run (the reference project zooms out before detecting; ours does not, and testing didn't reproduce a failure from skipping it — see `plan.md` Phase 6 "Not verified" list) Migration goal: diff --git a/README.md b/README.md index 6591955..b08131e 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Current task status: | Command | Implementation | |---|---| | `mailbox` | Real Python (`ba_auto/tasks/mailbox.py`). Verifies the mailbox panel actually opened (via a pixel-color probe, `driver.color_at`) before clicking "claim all" or pressing any further keys. Retries the open-click up to 3 times before giving up safely. | -| `cafe` | Still a bridge: `ba_auto/tasks/cafe.py` shells out to `scripts/ba_dailies_legacy.sh cafe`, the original unverified fixed-coordinate click sequence, unchanged. Not yet hardened — see "Known issue" below. | +| `cafe` | Real Python (`ba_auto/tasks/cafe.py`). Verifies each room/dialog transition the same way as `mailbox` before acting; sparkle detection runs in-process via `ba_auto/detector.py` instead of shelling out per click. See "Fixed: the exit-game dialog bug" below for what this replaced. | ## Prerequisites on nik-gpu @@ -55,7 +55,7 @@ rsync -av --delete \ ssh nik-gpu "cd ~/repo/ba-auto-daily && ./setup.sh" ``` -`setup.sh` copies the synced files into the fixed runtime paths (`~/ba_dailies.sh`, `~/ba_daily.py`, `~/ba_auto/`, `~/ba_scripts/`, `~/ba_assets/`) and (re)installs the venv. Re-run both commands any time you change code — there is no auto-deploy. +`setup.sh` copies the synced files into the fixed runtime paths (`~/ba_dailies.sh`, `~/ba_daily.py`, `~/ba_auto/`, `~/ba_assets/`) and (re)installs the venv. Re-run both commands any time you change code — there is no auto-deploy. ## Running it @@ -73,7 +73,7 @@ You can also run these remotely without a separate `ssh` login step: ssh nik-gpu "~/ba_dailies.sh mailbox" ``` -Exit code `0` means the script ran to completion — it does **not** by itself guarantee the in-game action succeeded (`mailbox` checks and logs this explicitly; `cafe` currently does not). +Exit code `0` means the script ran to completion without a Python exception — it does **not** by itself guarantee the in-game action succeeded. Both `mailbox` and `cafe` log what they actually detected and did (or why they safely aborted), so check the log output, not just the exit code. ## How to watch/verify it @@ -86,11 +86,13 @@ Exit code `0` means the script ran to completion — it does **not** by itself g ``` - **Read the log output.** Each task prints what it's doing, e.g. `mailbox` prints `panel not detected after click (attempt N/3)` if a click misses, and `nothing to claim` vs `claiming all` depending on what it found. -## Known issue: don't trust `cafe` unattended yet +## Fixed: the exit-game dialog bug -Live-testing `mailbox` surfaced a real bug in the *original*, unverified click sequence (see `plan.md` Phase 5 for the full writeup): a slightly-off icon coordinate caused a missed click, and the fixed sequence blindly kept going — clicking, pressing Enter, pressing Escape — with no idea whether any of it landed. The trailing Escape ended up hitting the bare home screen, which triggers Blue Archive's own **"Exit the game?"** confirmation dialog. +Live-testing both `mailbox` and `cafe` surfaced the same real bug in the *original*, unverified click sequences (see `plan.md` Phases 5–6 for the full writeup): a slightly-off icon coordinate occasionally caused a missed click, and the fixed sequence blindly kept going — clicking, pressing Enter, pressing Escape — with no idea whether any of it landed. The trailing Escape ended up hitting the bare home screen, which triggers Blue Archive's own **"Exit the game?"** confirmation dialog. `cafe`'s sequence is much longer than `mailbox`'s (open → pat loop ×15 → switch room → pat loop ×15 → claim income → exit), so a missed click there had more room to cascade. -`mailbox.py` now guards against this (verifies state before acting, retries, aborts safely instead of guessing). **`cafe` still uses the old unverified bridge and has the same failure mode.** If you run `~/ba_dailies.sh cafe` and something looks off, check the screen before pressing anything — if you see an unexpected confirmation dialog, press **Escape/Cancel**, never Enter/OK, until you've confirmed what it's asking. +Both `mailbox.py` and `cafe.py` now guard against this: they verify state via pixel-color probes before acting, retry a bounded number of times on a missed click, and abort safely (no further keypresses) instead of guessing. This was verified against the live game, including an actual income claim (gold and AP increased as expected) and a real sparkle detect-and-click. + +That said, this is one round of live testing, not exhaustive coverage — see `plan.md` Phase 6 "Not verified" for open risks (rank-up popups mid-loop, long-run camera zoom/pan drift). If you ever see an unexpected confirmation dialog while running either command, press **Escape/Cancel**, never Enter/OK, until you've confirmed what it's asking. ## Local checks (nik-macbookair) @@ -98,7 +100,6 @@ The game can't run here, so this only catches syntax errors, not behavior: ```bash bash -n ba_dailies.sh -bash -n scripts/ba_dailies_legacy.sh python3 -m py_compile ba_daily.py ba_auto/*.py ba_auto/tasks/*.py ``` @@ -107,5 +108,5 @@ Real verification only happens by actually running against the live game on `nik ## More detail - `CLAUDE.md` — architecture rules and conventions for this repo. -- `plan.md` — feature-by-feature migration status and backlog, including the mailbox bug writeup. +- `plan.md` — feature-by-feature migration status and backlog, including the mailbox/cafe exit-game-dialog bug writeup. - `ba_auto/reference_notes/mapping.md` — maps each feature to its `baas-reference` source. diff --git a/ba_auto/config.py b/ba_auto/config.py index ab1421b..699c7d0 100644 --- a/ba_auto/config.py +++ b/ba_auto/config.py @@ -6,11 +6,15 @@ XAUTHORITY = "/run/user/1000/gdm/Xauthority" ENV = {**os.environ, "DISPLAY": DISPLAY, "XAUTHORITY": XAUTHORITY} WINDOW_NAME = "BlueArchive" - -# TODO: remove once cafe is ported off the legacy Bash bridge (mailbox no longer uses this). -LEGACY_SCRIPT = os.path.expanduser("~/ba_scripts/ba_dailies_legacy.sh") +ASSET_DIR = os.path.expanduser("~/ba_assets") # Refined from (1726, 60): that coordinate sat on the edge of the icon's # hitbox and intermittently missed during live testing. MAILBOX_ICON = (1732, 50) CLAIM_ALL = (1691, 1128) + +CAFE_ICON = (165, 1100) +CAFE_ROOM_SWITCH = (190, 160) +CAFE_INCOME = (1780, 1105) +CAFE_MAX_CLICKS_PER_ROOM = 15 +CAFE_SPARKLE_TEMPLATE = os.path.join(ASSET_DIR, "cafe_sparkle.png") diff --git a/ba_auto/detector.py b/ba_auto/detector.py index 942351e..dba3ba0 100644 --- a/ba_auto/detector.py +++ b/ba_auto/detector.py @@ -1,3 +1,35 @@ -"""Image/color matching helpers (OpenCV-based); not wired up yet.""" +"""Image/color matching helpers (OpenCV-based). Ported from scripts/detect_and_click.py.""" +import cv2 +import numpy as np -# TODO: migrate scripts/detect_and_click.py sparkle-matching logic here. +from ba_auto import config, driver + +SPARKLE_SHOT_PATH = "/tmp/ba_live.png" +SPARKLE_CLICK_OFFSET = (75, 47) +SPARKLE_THRESHOLD = 0.97 + + +def find_cafe_sparkle(): + driver.screenshot(SPARKLE_SHOT_PATH) + template = cv2.imread(config.CAFE_SPARKLE_TEMPLATE) + b, g, r = cv2.split(template.astype(np.int16)) + yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200)) + mask_plane = (yellow_white.astype(np.uint8)) * 255 + mask = cv2.merge([mask_plane, mask_plane, mask_plane]) + th, tw = template.shape[:2] + + img = cv2.imread(SPARKLE_SHOT_PATH) + result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask) + locs = np.where(result >= SPARKLE_THRESHOLD) + points = sorted(zip(*locs[::-1]), key=lambda p: -result[p[1], p[0]]) + + merged = [] + for x, y in points: + if all(abs(x - mx) > tw // 2 or abs(y - my) > th // 2 for mx, my, _ in merged): + merged.append((x, y, result[y, x])) + if not merged: + return None + + x, y, score = merged[0] + ox, oy = SPARKLE_CLICK_OFFSET + return (x + tw // 2 + ox, y + th // 2 + oy, score) diff --git a/ba_auto/navigation.py b/ba_auto/navigation.py index f7a98c3..406b486 100644 --- a/ba_auto/navigation.py +++ b/ba_auto/navigation.py @@ -1,3 +1,20 @@ -"""Shared navigation helpers (home, menu, popups, back/escape); not wired up yet.""" +"""Shared navigation helpers (home, menu, popups, back/escape).""" -# TODO: extract common navigation flows here as tasks are ported off Bash. +# The mailbox/cafe/shop-style header bar renders a plain light background +# here; the home screen shows character art instead. +SUBSCREEN_HEADER_PROBE = (500, 10) +SUBSCREEN_HEADER_MIN_CHANNEL = 200 + +# Any modal dialog dims the screen behind it to roughly this darkness. +MODAL_DIM_PROBE = (960, 200) +MODAL_DIM_MAX_CHANNEL = 150 + + +def is_on_subscreen(driver): + r, g, b = driver.color_at(*SUBSCREEN_HEADER_PROBE) + return r > SUBSCREEN_HEADER_MIN_CHANNEL and g > SUBSCREEN_HEADER_MIN_CHANNEL and b > SUBSCREEN_HEADER_MIN_CHANNEL + + +def is_modal_open(driver): + r, g, b = driver.color_at(*MODAL_DIM_PROBE) + return r < MODAL_DIM_MAX_CHANNEL and g < MODAL_DIM_MAX_CHANNEL and b < MODAL_DIM_MAX_CHANNEL diff --git a/ba_auto/reference_notes/mapping.md b/ba_auto/reference_notes/mapping.md index f978ebb..303252d 100644 --- a/ba_auto/reference_notes/mapping.md +++ b/ba_auto/reference_notes/mapping.md @@ -5,7 +5,7 @@ Maps each local feature to the corresponding `~/repo/baas-reference/module/...` | Local feature | Reference file | Reference functions/classes | Local file | Backend replacements | Status | |---|---|---|---|---|---| | Mailbox | `module/mail.py` | `to_mail`, `implement` | `ba_auto/tasks/mailbox.py` | tap/click via xdotool, screenshot via scrot, `color.rgb_in_range` → `driver.color_at` pixel-probe check | Migrated: real Python, state-verified via color probe (no legacy bridge) | -| Cafe | Need to confirm in reference | Need to inspect | `ba_auto/tasks/cafe.py` | template matching via OpenCV, click via xdotool | Bridged to `scripts/ba_dailies_legacy.sh` + `scripts/detect_and_click.py`; Python port pending (plan.md Phase 6) | +| Cafe | `module/cafe_reward.py` | `to_cafe`, `interaction_for_cafe_solve_method3`, `collect` | `ba_auto/tasks/cafe.py` | `picture.co_detect`/`color.rgb_in_range` → `driver.color_at` pixel-probe checks; sparkle template match ported in-process into `ba_auto/detector.py` (`find_cafe_sparkle`) | Migrated: real Python, state-verified via color probes (no legacy bridge) | | Stamina/AP | `module/collect_daily_free_power.py`, `module/collect_daily_task_power.py` | Need to inspect | `ba_auto/tasks/stamina.py` | color checks/clicks via local driver | Not started | | Group/Club AP | `module/group.py` | Need to inspect | `ba_auto/tasks/group.py` | fixed click + state check via local driver | Not started | | Bounty | `module/rewarded_task.py` | Need to inspect | `ba_auto/tasks/bounty.py` | sweep/color/OCR adaptation | Not started | diff --git a/ba_auto/tasks/cafe.py b/ba_auto/tasks/cafe.py index 734929e..6209bd9 100644 --- a/ba_auto/tasks/cafe.py +++ b/ba_auto/tasks/cafe.py @@ -1,6 +1,93 @@ -"""Cafe daily task.""" +"""Cafe daily task. Ported from baas-reference module/cafe_reward.py's state-probe pattern.""" + +from ba_auto import detector, navigation + +ROOM_OPEN_RETRIES = 3 + +# "受取" (claim) renders as flat grey when there is nothing to collect yet. +CLAIM_PROBE = (960, 850) +CLAIM_DISABLED_RGB = (218, 218, 218) +CLAIM_DISABLED_TOLERANCE = 15 + + +def _claim_disabled(driver): + r, g, b = driver.color_at(*CLAIM_PROBE) + tr, tg, tb = CLAIM_DISABLED_RGB + return ( + abs(r - tr) <= CLAIM_DISABLED_TOLERANCE + and abs(g - tg) <= CLAIM_DISABLED_TOLERANCE + and abs(b - tb) <= CLAIM_DISABLED_TOLERANCE + ) + + +def _enter_room(driver, coords): + for attempt in range(1, ROOM_OPEN_RETRIES + 1): + driver.click(*coords) + driver.wait(3) + if navigation.is_on_subscreen(driver): + # dismiss the "visited student list" notice shown on room entry + driver.keypress("Return") + driver.wait(1) + return True + print(f"[cafe] room not detected after click (attempt {attempt}/{ROOM_OPEN_RETRIES})") + return False + + +def _pat_room(driver, config): + for _ in range(config.CAFE_MAX_CLICKS_PER_ROOM): + match = detector.find_cafe_sparkle() + if match is None: + break + x, y, _score = match + driver.click(x, y) + driver.wait(1) + driver.keypress("Return") + + +def _claim_income(driver, config): + driver.click(*config.CAFE_INCOME) + driver.wait(2) + if not navigation.is_modal_open(driver): + print("[cafe] income panel not detected, skipping claim") + return + + if _claim_disabled(driver): + print("[cafe] nothing to claim") + else: + print("[cafe] claiming income") + driver.keypress("Return") + driver.wait(2) + driver.keypress("Return") + driver.wait(2) + + if navigation.is_modal_open(driver): + driver.keypress("Escape") + driver.wait(1.5) def run(driver, config): - # TODO: migrate old Bash cafe logic into this Python task. - driver.run_command([config.LEGACY_SCRIPT, "cafe"]) + driver.focus_game() + + if not _enter_room(driver, config.CAFE_ICON): + print("[cafe] could not confirm cafe is open, aborting without pressing further keys") + return + + print("[cafe] room 1: farming affection") + _pat_room(driver, config) + + if not _enter_room(driver, config.CAFE_ROOM_SWITCH): + print("[cafe] could not confirm room switch, stopping before income claim") + if navigation.is_on_subscreen(driver): + driver.keypress("Escape") + driver.wait(1.5) + return + + print("[cafe] room 2: farming affection") + _pat_room(driver, config) + + _claim_income(driver, config) + + if navigation.is_on_subscreen(driver): + driver.keypress("Escape") + driver.wait(1.5) + print("[cafe] Done.") diff --git a/ba_auto/tasks/mailbox.py b/ba_auto/tasks/mailbox.py index f3b2fe9..6badc90 100644 --- a/ba_auto/tasks/mailbox.py +++ b/ba_auto/tasks/mailbox.py @@ -1,9 +1,6 @@ """Mailbox daily task. Ported from baas-reference module/mail.py's color-probe pattern.""" -# (500, 10) sits on the mailbox panel's plain header background; the home -# screen shows character art there instead, so this tells open vs not-open. -HEADER_PROBE = (500, 10) -HEADER_OPEN_MIN_CHANNEL = 200 +from ba_auto import navigation # "Claim all" renders as flat grey (153,153,153) when there is nothing to claim. CLAIM_ALL_PROBE = (1710, 1128) @@ -13,11 +10,6 @@ CLAIM_ALL_DISABLED_TOLERANCE = 12 OPEN_RETRIES = 3 -def _is_open(driver): - r, g, b = driver.color_at(*HEADER_PROBE) - return r > HEADER_OPEN_MIN_CHANNEL and g > HEADER_OPEN_MIN_CHANNEL and b > HEADER_OPEN_MIN_CHANNEL - - def _claim_all_disabled(driver): r, g, b = driver.color_at(*CLAIM_ALL_PROBE) tr, tg, tb = CLAIM_ALL_DISABLED_RGB @@ -35,7 +27,7 @@ def run(driver, config): for attempt in range(1, OPEN_RETRIES + 1): driver.click(*config.MAILBOX_ICON) driver.wait(1.5) - if _is_open(driver): + if navigation.is_on_subscreen(driver): opened = True break print(f"[mailbox] panel not detected after click (attempt {attempt}/{OPEN_RETRIES})") diff --git a/plan.md b/plan.md index 6e57b21..4b17e18 100644 --- a/plan.md +++ b/plan.md @@ -76,13 +76,12 @@ This is the intended direction. It does not have to be completed all at once. | `~/repo/ba-auto-daily/ba_dailies.sh` | Thin launcher only. It should call the Python entry point. Do not add new feature logic here. | | `~/repo/ba-auto-daily/ba_daily.py` | Main Python CLI entry point. Dispatches tasks such as mailbox, cafe, stamina, group, etc. | | `~/repo/ba-auto-daily/ba_auto/driver.py` | Local PC/Steam/Proton control backend. Wraps xdotool, scrot, waits, clicks, swipes, keypresses, screenshots, and window focus. | -| `~/repo/ba-auto-daily/ba_auto/detector.py` | OpenCV/template/color matching helpers. Existing `scripts/detect_and_click.py` logic should be migrated here. | -| `~/repo/ba-auto-daily/ba_auto/navigation.py` | Shared navigation helpers: home, menu, close popup, back, open feature screens. | +| `~/repo/ba-auto-daily/ba_auto/detector.py` | OpenCV/template/color matching helpers. Currently has `find_cafe_sparkle()`, ported in-process from the retired `scripts/detect_and_click.py`. | +| `~/repo/ba-auto-daily/ba_auto/navigation.py` | Shared navigation/state-probe helpers: `is_on_subscreen`, `is_modal_open`, used by both `mailbox.py` and `cafe.py`. | | `~/repo/ba-auto-daily/ba_auto/tasks/` | Feature implementations. Each task should adapt the relevant `baas-reference/module/...` logic where possible. | | `~/repo/ba-auto-daily/ba_auto/reference_notes/mapping.md` | Reference mapping table: local feature → reference module → local implementation → driver gaps. | | `~/repo/ba-auto-daily/assets/` | Locally captured template images, such as cafe sparkle. Do not blindly copy assets from the reference repo. | | `~/repo/ba-auto-daily/screenshots/` | Human reference screenshots, mostly Moonlight/game captures, used for calibration and debugging. | -| `~/repo/ba-auto-daily/scripts/` | Transitional scripts. Long-term reusable Python logic should move into `ba_auto/`. | | `~/repo/ba-auto-daily/setup.sh` | Bootstrap/deploy helper for `nik-gpu`. Should install/check dependencies and copy runtime files. | | `~/repo/baas-reference/` | Read-only GPL-3.0 reference clone. Study and adapt. Never edit. | @@ -98,11 +97,7 @@ Preferred runtime layout: | `nik-gpu:~/ba_assets/` | Runtime assets/templates. | | `nik-gpu:~/.venvs/ba-auto-daily/` | Python virtual environment. | -Current older layout may include: - -| Path | What it is | -|---|---| -| `nik-gpu:~/ba_scripts/detect_and_click.py` | Old standalone detector helper. Should eventually be replaced by `ba_auto/detector.py`. | +`nik-gpu:~/ba_scripts/` may still contain `detect_and_click.py` and `ba_dailies_legacy.sh` left over from before both mailbox and cafe were migrated off them. Neither is deployed or referenced by anything anymore (`setup.sh` stopped copying them once Phase 6 landed) — safe to delete manually on `nik-gpu`, just not automated here. ## Implementation strategy @@ -140,7 +135,7 @@ Initial seed: | Local feature | Reference file | Reference functions/classes | Local file | Backend replacements | Status | |---|---|---|---|---|---| | Mailbox | Need to confirm in reference | Need to inspect | `ba_auto/tasks/mailbox.py` | tap/click via xdotool, screenshot via scrot | Existing Bash behavior; migrate to Python | -| Cafe | Need to confirm in reference | Need to inspect | `ba_auto/tasks/cafe.py` | template matching via OpenCV, click via xdotool | Existing Bash + helper behavior; migrate to Python | +| Cafe | `module/cafe_reward.py` | `to_cafe`, `interaction_for_cafe_solve_method3`, `collect` | `ba_auto/tasks/cafe.py` | `picture.co_detect`/`color.rgb_in_range` → `driver.color_at` pixel-probe checks; sparkle template match ported in-process into `ba_auto/detector.py` | Migrated: real Python, state-verified via color probes, no legacy bridge | | Stamina/AP | `module/collect_daily_free_power.py`, `module/collect_daily_task_power.py` | Need to inspect | `ba_auto/tasks/stamina.py` | color checks/clicks via local driver | Not started | | Group/Club AP | `module/group.py` | Need to inspect | `ba_auto/tasks/group.py` | fixed click + state check via local driver | Not started | | Bounty | `module/rewarded_task.py` | Need to inspect | `ba_auto/tasks/bounty.py` | sweep/color/OCR adaptation | Not started | @@ -157,8 +152,8 @@ Do not implement a feature without filling at least the relevant row. | Feature | Current status | Target status | |---|---|---| | Mailbox claim | Migrated: `ba_auto/tasks/mailbox.py` uses `driver.color_at` to verify the panel opened before acting (found live-testing bug: a marginal icon coordinate could miss and cascade into pressing Escape on the home screen, which triggers Blue Archive's own exit-game confirmation) | Done | -| Cafe pats + income | Bridged: `ba_auto/tasks/cafe.py` calls `scripts/ba_dailies_legacy.sh cafe` (which still shells out to `scripts/detect_and_click.py`) | Migrate to `ba_auto/tasks/cafe.py` and `ba_auto/detector.py` (Phase 6) | -| Shared driver | `ba_auto/driver.py` built (`run_command`, `focus_game`, `click`, `keypress`, `screenshot`, `wait`); not yet wired into tasks | Wire into mailbox/cafe as they migrate off the Bash bridge | +| Cafe pats + income | Migrated: `ba_auto/tasks/cafe.py` verifies each room/dialog opened via `driver.color_at` before acting; sparkle detection now runs in-process via `ba_auto/detector.py` instead of a per-click subprocess | Done | +| Shared driver | `ba_auto/driver.py` built (`run_command`, `focus_game`, `click`, `keypress`, `screenshot`, `wait`, `color_at`); wired into both `mailbox.py` and `cafe.py` | Extend with new primitives as future tasks need them | | Python CLI | Built: `ba_daily.py` dispatches `mailbox`/`cafe`/default flow | Extend as new tasks are added | | Reference mapping | Built: `ba_auto/reference_notes/mapping.md` | Fill in reference file/function columns per feature | | Everything else | Not started | Implement reference-first in Python | @@ -210,7 +205,7 @@ Keep compatibility with: ### Phase 3: Driver extraction -**Status: Primitives added to `ba_auto/driver.py`, not yet wired into task modules.** +**Status: Done.** Primitives (including `color_at`, added during the mailbox/cafe hardening work) are wired into both `ba_auto/tasks/mailbox.py` and `ba_auto/tasks/cafe.py`. Move shell interactions into `ba_auto/driver.py`. @@ -229,11 +224,9 @@ wait_until(...) ### Phase 4: Detector extraction -**Status: Not started — `ba_auto/detector.py` is currently a placeholder.** +**Status: Done (scoped).** `scripts/detect_and_click.py`'s sparkle-matching logic (masked template match against `assets/cafe_sparkle.png`) was ported into `ba_auto/detector.py` as `find_cafe_sparkle()`, called in-process from `ba_auto/tasks/cafe.py` — this removed the old per-click Python cold start (a fresh `cv2`/`numpy` import per subprocess call) that CLAUDE.md's driver-layer guidance specifically warns against. `scripts/detect_and_click.py` had no remaining callers once this landed, so it was deleted rather than kept as a compatibility wrapper. The more generic primitives listed below (`load_template`, `match_template`, etc.) have not been built — only the one concrete sparkle-matching function needed so far exists; generalize when a second detector use case actually needs it. -Move `scripts/detect_and_click.py` logic into `ba_auto/detector.py`. - -Detector primitives should include: +Detector primitives, generalize later if needed: ``` load_template(...) @@ -244,8 +237,6 @@ color_mask(...) debug_write_match(...) ``` -The old script may remain as a compatibility wrapper temporarily, but the reusable logic should live under `ba_auto/`. - ### Phase 5: Mailbox migration **Status: Done.** Live testing surfaced a real bug: the old `MAILBOX_ICON` coordinate `(1726, 60)` sat on the edge of the icon's hitbox and intermittently missed, and the fixed click sequence had no way to notice — it cascaded into pressing Escape on the bare home screen, which triggers Blue Archive's own "exit the game?" confirmation (dismissed safely with Cancel during testing; no game state was lost). The Python port in `ba_auto/tasks/mailbox.py` fixes the coordinate and, following `module/mail.py`'s `rgb_in_range` pattern, verifies the panel actually opened (and whether "claim all" is disabled) via `driver.color_at` before pressing any further keys, with a bounded retry and a safe abort if the panel never appears. @@ -260,27 +251,30 @@ The CLI should call it through Python. ### Phase 6: Cafe migration -**Status: Not started — `ba_auto/tasks/cafe.py` currently bridges to `scripts/ba_dailies_legacy.sh`.** +**Status: Done.** Same root cause as the mailbox bug (Phase 5), confirmed by step-by-step live replay with screenshots: `CAFE_ICON` clicks are flaky (missed on the first attempt, worked on retry at the identical coordinate — this is xdotool/Proton click-registration flakiness, not a coordinate-precision problem), and the old sequence had zero verification across its ~12 steps (open → dismiss notice → pat loop ×15 → switch room → dismiss notice → pat loop ×15 → claim income ×2 Enter → ×2 Escape). A missed click anywhere cascades into blind actions on whatever screen is actually showing, which — same as mailbox — very likely ends with an unverified Escape hitting the home screen and triggering Blue Archive's own exit-game confirmation. -Move cafe logic from Bash to: +`ba_auto/tasks/cafe.py` now verifies state at every transition using `driver.color_at`, following `module/cafe_reward.py`'s `picture.co_detect`/`rgb_in_range` pattern: -``` -ba_auto/tasks/cafe.py -``` +- opening the cafe icon and the room-switch button both retry (bounded) and confirm the panel actually opened via the same subscreen-header probe as mailbox, now shared in `ba_auto/navigation.is_on_subscreen` +- the "visited student list" notice that appears on every room entry is dismissed with Enter; this is harmless as a no-op if no popup is actually present (verified live), so no separate presence check was needed there +- the income dialog's own dimmed-overlay backdrop is checked (`navigation.is_modal_open`) before pressing Enter to claim, and the "receive" button's disabled-grey color is checked before attempting to claim at all (mirrors `collect()`'s `rgb_in_range` gate in the reference) +- the closing Escape(s) only fire when a subscreen/modal is confirmed still open, never blindly -During migration, verify: +Verified live (two full runs against the real game, plus a manual step-by-step replay of every transition): -- both rooms still work -- sparkle detection still works -- cafe income claim still works -- rank-up popups are handled or explicitly documented as not handled -- student rotation popups are handled or explicitly documented as not handled -- view centering/zoom state is robust -- repeated detection does not suffer from Python cold-start delay +- both rooms open and pat correctly +- sparkle detection still works and now runs in-process (see Phase 4) instead of shelling out per click +- cafe income claim works (confirmed gold +81,251 / AP +61 on an actual claim) and correctly no-ops when there's nothing to collect +- the reference's `zoom_out` step (camera zoom before sparkle detection — CLAUDE.md's "view centering/zoom" gap) was **not** ported: detection matched at 0.99 confidence without it in live testing, so it wasn't reproducibly broken here. Left as a documented open risk below rather than added speculatively. + +Not verified / open risks: + +- rank-up popups: not observed during testing (no student ranked up while testing), still unhandled if one appears mid-loop +- whether zoom/pan state could drift over a long unattended run and eventually break sparkle detection (see above — no evidence of this yet, but the reference project treats it as necessary) ### Phase 7: setup.sh update -**Status: Done — `setup.sh` now deploys `ba_daily.py`, `ba_auto/`, and `scripts/ba_dailies_legacy.sh`.** +**Status: Done — `setup.sh` deploys `ba_daily.py` and `ba_auto/`.** `scripts/ba_dailies_legacy.sh` and `scripts/detect_and_click.py` were deleted once mailbox and cafe both migrated off them (Phases 5–6); `setup.sh` no longer references either. Update `setup.sh` so it deploys: @@ -355,11 +349,9 @@ This should become a reusable primitive, not arena-specific code. ### 1. Migration to Python-first -This is the highest priority. +**Status: Done.** See Phases 1–7 above for the detailed history, including the mailbox and cafe exit-game-dialog bug and its fix. -Do this before adding new features. - -Goal: +Goal (all done): - Bash launcher only - Python CLI @@ -369,12 +361,6 @@ Goal: - cafe migrated - reference mapping started -Reference: - -- Current local implementation -- Existing `ba_dailies.sh` -- Existing `scripts/detect_and_click.py` - ### 2. Stamina/AP sweep Claim: diff --git a/scripts/ba_dailies_legacy.sh b/scripts/ba_dailies_legacy.sh deleted file mode 100644 index 92cc0ac..0000000 --- a/scripts/ba_dailies_legacy.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# Legacy pre-migration click sequences; bridged from ba_auto/tasks/*.py until ported to Python (see CLAUDE.md). -export DISPLAY=:0 -export XAUTHORITY=/run/user/1000/gdm/Xauthority - -WIN_NAME="BlueArchive" -MAILBOX_ICON="1726 60" -CLAIM_ALL="1691 1128" - -CAFE_ICON="165 1100" -CAFE_ROOM_SWITCH="190 160" -CAFE_INCOME="1780 1105" -VENV_PYTHON="$HOME/.venvs/ba-auto-daily/bin/python3" -DETECT_SPARKLE="$HOME/ba_scripts/detect_and_click.py" -CAFE_MAX_CLICKS_PER_ROOM=15 - -get_window() { - xdotool search --name "$WIN_NAME" | head -1 -} - -click() { - local coords="$1" - xdotool mousemove $coords click 1 - sleep 0.5 -} - -press_enter() { - xdotool key Return - sleep 0.5 -} - -press_esc() { - xdotool key Escape - sleep 0.5 -} - -focus_game() { - local win - win=$(get_window) - if [ -z "$win" ]; then - echo "ERROR: Blue Archive window not found. Is the game running?" - exit 1 - fi - xdotool windowactivate "$win" - sleep 0.5 -} - -do_mailbox() { - echo "[mailbox] Opening mailbox..." - click "$MAILBOX_ICON" - sleep 1.5 - echo "[mailbox] Claiming all..." - click "$CLAIM_ALL" - sleep 1 - echo "[mailbox] Confirming claim..." - press_enter - sleep 1 - echo "[mailbox] Closing mailbox..." - press_esc - sleep 1 - echo "[mailbox] Done." -} - -do_cafe_room() { - local i - for ((i = 0; i < CAFE_MAX_CLICKS_PER_ROOM; i++)); do - if ! "$VENV_PYTHON" "$DETECT_SPARKLE" | grep -q "^MATCH"; then - break - fi - sleep 1 - press_enter - done -} - -do_cafe() { - echo "[cafe] Opening cafe..." - click "$CAFE_ICON" - sleep 3 - press_enter - echo "[cafe] Room 1: farming affection..." - do_cafe_room - echo "[cafe] Switching to room 2..." - click "$CAFE_ROOM_SWITCH" - sleep 3 - press_enter - echo "[cafe] Room 2: farming affection..." - do_cafe_room - echo "[cafe] Claiming cafe income..." - click "$CAFE_INCOME" - sleep 2 - press_enter - sleep 2 - press_enter - sleep 2 - echo "[cafe] Exiting cafe..." - press_esc - sleep 1.5 - press_esc - sleep 1.5 - echo "[cafe] Done." -} - -case "$1" in - mailbox) - focus_game - do_mailbox - ;; - cafe) - focus_game - do_cafe - ;; - "") - focus_game - do_mailbox - do_cafe - echo "All done." - ;; - *) - echo "Unknown phase: $1" - exit 1 - ;; -esac diff --git a/scripts/detect_and_click.py b/scripts/detect_and_click.py deleted file mode 100644 index cd5726b..0000000 --- a/scripts/detect_and_click.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 -"""Screenshot the game window, find a cafe affection sparkle, click it. - -Runs entirely on nik-gpu (screenshot -> detect -> click all local) so the -whole cycle finishes in well under a second - roaming students move fast -enough that a multi-hop SSH/scp pipeline misses the click. - -Prints "MATCH x y score" and exits 0 if a sparkle was found and clicked, -or prints "NO_MATCH" and exits 1 otherwise. -""" -import subprocess -import sys - -import cv2 -import numpy as np - -TEMPLATE_PATH = "/home/nik/ba_assets/cafe_sparkle.png" -SHOT_PATH = "/tmp/ba_live.png" -OFFSET_X = 75 -OFFSET_Y = 47 -THRESHOLD = 0.97 - -ENV = {"DISPLAY": ":0", "XAUTHORITY": "/run/user/1000/gdm/Xauthority"} - - -def screenshot(): - subprocess.run( - ["scrot", "-a", "0,0,1920,1200", "-o", SHOT_PATH], - env=ENV, check=True, - ) - - -def find_sparkles(): - template = cv2.imread(TEMPLATE_PATH) - b, g, r = cv2.split(template.astype(np.int16)) - yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200)) - mask_plane = (yellow_white.astype(np.uint8)) * 255 - mask = cv2.merge([mask_plane, mask_plane, mask_plane]) - th, tw = template.shape[:2] - - img = cv2.imread(SHOT_PATH) - result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask) - locs = np.where(result >= THRESHOLD) - points = sorted(zip(*locs[::-1]), key=lambda p: -result[p[1], p[0]]) - - merged = [] - for x, y in points: - if all(abs(x - mx) > tw // 2 or abs(y - my) > th // 2 for mx, my, _ in merged): - merged.append((x, y, result[y, x])) - return [(x + tw // 2, y + th // 2, score) for x, y, score in merged] - - -def click(x, y): - subprocess.run( - ["xdotool", "mousemove", str(x), str(y), "click", "1"], - env=ENV, check=True, - ) - - -if __name__ == "__main__": - screenshot() - sparkles = find_sparkles() - if not sparkles: - print("NO_MATCH") - sys.exit(1) - x, y, score = sparkles[0] - click(x + OFFSET_X, y + OFFSET_Y) - print(f"MATCH {x} {y} {score:.4f}") - sys.exit(0) diff --git a/setup.sh b/setup.sh index b1c5570..f2b749c 100755 --- a/setup.sh +++ b/setup.sh @@ -5,13 +5,12 @@ # not from macOS, since it installs into nik-gpu-local paths. # # It does NOT touch the game or take any screenshots; it only installs the -# venv and copies files to the fixed paths ba_dailies.sh/ba_daily.py/ -# detect_and_click.py expect (outside the repo, per the two-machine deploy -# convention in CLAUDE.md). +# venv and copies files to the fixed paths ba_dailies.sh/ba_daily.py/ba_auto +# expect (outside the repo, per the two-machine deploy convention in +# CLAUDE.md). set -e VENV_DIR="$HOME/.venvs/ba-auto-daily" -SCRIPTS_DIR="$HOME/ba_scripts" ASSETS_DIR="$HOME/ba_assets" echo "== Checking host tools ==" @@ -36,11 +35,8 @@ fi "$VENV_DIR/bin/pip" install --quiet opencv-python-headless numpy "$VENV_DIR/bin/python3" -c "import cv2, numpy; print('opencv', cv2.__version__, '/ numpy', numpy.__version__)" -echo "== Deploying scripts + assets to fixed paths ==" -mkdir -p "$SCRIPTS_DIR" "$ASSETS_DIR" -cp scripts/detect_and_click.py "$SCRIPTS_DIR/detect_and_click.py" -cp scripts/ba_dailies_legacy.sh "$SCRIPTS_DIR/ba_dailies_legacy.sh" -chmod +x "$SCRIPTS_DIR/ba_dailies_legacy.sh" +echo "== Deploying assets to fixed paths ==" +mkdir -p "$ASSETS_DIR" cp assets/cafe_sparkle.png "$ASSETS_DIR/cafe_sparkle.png" echo "== Deploying Python-first entry point =="