From a04ef5d0c5526c302d959278d6a5ea3f3ae53e24 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Sun, 5 Jul 2026 17:57:26 +0900 Subject: [PATCH] Refactor scratchpad directory references and update Phase 6 follow-up report - Changed references from './scratchpad' to '.scratchpad/' in graph.json and plan.md for consistency. - Expanded Phase 6 follow-up section in plan.md to clarify changes made to the pat detection logic: - Updated `find_cafe_sparkle()` to utilize multiple template scales for improved detection. - Modified `_pat_room` to allow polling for maximum clicks instead of breaking on the first miss. - Added mouse movement after each pat to prevent cursor occlusion of sparkles. - Verified that room entry and modal state checks function correctly, but end-to-end pat success remains untested due to lack of available interactions. --- .gitignore | 1 + CLAUDE.md | 10 ++++-- README.md | 2 +- ba_auto/config.py | 7 ++++ ba_auto/detector.py | 51 +++++++++++++++++++----------- ba_auto/driver.py | 7 +++- ba_auto/reference_notes/mapping.md | 2 +- ba_auto/tasks/cafe.py | 16 ++++++++-- graphify-out/graph.html | 2 +- graphify-out/graph.json | 4 +-- plan.md | 21 ++++++++++-- 11 files changed, 92 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index 4da9b0f..179cdea 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ graphify-out/cost.json graphify-out/cache/ __pycache__/ *.pyc +.scratchpad/ diff --git a/CLAUDE.md b/CLAUDE.md index f4ced61..a927e99 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -195,6 +195,10 @@ Future dependency: OCR engine, likely Tesseract or PaddleOCR. Do not introduce OCR casually. Add it only when implementing a feature that actually needs OCR. +## Working conventions + +Use `.scratchpad/` (create if missing) in the project root for temporary/intermediate files — e.g. cropped calibration images from `screenshots/cafe/sparkle/`, one-off debug output. Never write to `/tmp` or `/private/tmp`. + ## Bash policy `ba_dailies.sh` should be a thin launcher only. @@ -315,7 +319,7 @@ The detector should support: - threshold tuning - masked matching - click-offset handling -- debug image output to `./scratchpad` +- debug image output to `.scratchpad/` Avoid one Python cold start per click attempt where possible. Prefer long-running Python task logic that can take repeated screenshots and click repeatedly from one process. @@ -381,7 +385,7 @@ Use this format: ## Working conventions -Use `./scratchpad` for temporary or intermediate files. +Use `.scratchpad/` for temporary or intermediate files. Examples: @@ -417,7 +421,7 @@ Example: ssh nik-gpu "~/ba_dailies.sh cafe" ``` -When debugging image matching, write debug images to `./scratchpad`. +When debugging image matching, write debug images to `.scratchpad/`. ## Existing features diff --git a/README.md b/README.md index b08131e..824ac90 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Current task status: | Command | Implementation | |---|---| | `mailbox` | Real Python (`ba_auto/tasks/mailbox.py`). Verifies the mailbox panel actually opened (via a pixel-color probe, `driver.color_at`) before clicking "claim all" or pressing any further keys. Retries the open-click up to 3 times before giving up safely. | -| `cafe` | Real Python (`ba_auto/tasks/cafe.py`). Verifies each room/dialog transition the same way as `mailbox` before acting; sparkle detection runs in-process via `ba_auto/detector.py` instead of shelling out per click. See "Fixed: the exit-game dialog bug" below for what this replaced. | +| `cafe` | Real Python (`ba_auto/tasks/cafe.py`). Verifies each room/dialog transition the same way as `mailbox` before acting; sparkle detection runs in-process via `ba_auto/detector.py` instead of shelling out per click. See "Fixed: the exit-game dialog bug" below for what this replaced, and `plan.md` Phase 6 follow-up for the multi-scale detection + persistent-polling changes made after a "farming affection doesn't happen" report. | ## Prerequisites on nik-gpu diff --git a/ba_auto/config.py b/ba_auto/config.py index 699c7d0..1fc4e36 100644 --- a/ba_auto/config.py +++ b/ba_auto/config.py @@ -8,6 +8,10 @@ ENV = {**os.environ, "DISPLAY": DISPLAY, "XAUTHORITY": XAUTHORITY} WINDOW_NAME = "BlueArchive" ASSET_DIR = os.path.expanduser("~/ba_assets") +# Runtime working files (probe/detection screenshots); never /tmp, per CLAUDE.md. +SCRATCHPAD_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scratchpad") +os.makedirs(SCRATCHPAD_DIR, exist_ok=True) + # Refined from (1726, 60): that coordinate sat on the edge of the icon's # hitbox and intermittently missed during live testing. MAILBOX_ICON = (1732, 50) @@ -16,5 +20,8 @@ CLAIM_ALL = (1691, 1128) CAFE_ICON = (165, 1100) CAFE_ROOM_SWITCH = (190, 160) CAFE_INCOME = (1780, 1105) +# Max sparkle-detection attempts per room (hits and misses both count -- +# sparkles are on a per-student cooldown, so most checks legitimately find +# nothing and the loop keeps polling rather than giving up after one miss). CAFE_MAX_CLICKS_PER_ROOM = 15 CAFE_SPARKLE_TEMPLATE = os.path.join(ASSET_DIR, "cafe_sparkle.png") diff --git a/ba_auto/detector.py b/ba_auto/detector.py index dba3ba0..2fa2f2b 100644 --- a/ba_auto/detector.py +++ b/ba_auto/detector.py @@ -1,35 +1,50 @@ """Image/color matching helpers (OpenCV-based). Ported from scripts/detect_and_click.py.""" +import os + import cv2 import numpy as np from ba_auto import config, driver -SPARKLE_SHOT_PATH = "/tmp/ba_live.png" +SPARKLE_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "cafe_live.png") SPARKLE_CLICK_OFFSET = (75, 47) SPARKLE_THRESHOLD = 0.97 +# The cafe camera's zoom level isn't reset before farming, and the sparkle +# icon's on-screen size scales with it (see screenshots/cafe/sparkle/ +# 01_sparke_zoomed_centered.png vs 03_sparkle_zoomed_out.png) -- a single +# fixed-scale template match misses whenever the camera isn't at the exact +# zoom the template was captured at. Try a spread of scales instead. +SPARKLE_SCALES = (0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2) + + +def _masked_template(template): + b, g, r = cv2.split(template.astype(np.int16)) + yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200)) + mask_plane = (yellow_white.astype(np.uint8)) * 255 + return cv2.merge([mask_plane, mask_plane, mask_plane]) + def find_cafe_sparkle(): driver.screenshot(SPARKLE_SHOT_PATH) - template = cv2.imread(config.CAFE_SPARKLE_TEMPLATE) - b, g, r = cv2.split(template.astype(np.int16)) - yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200)) - mask_plane = (yellow_white.astype(np.uint8)) * 255 - mask = cv2.merge([mask_plane, mask_plane, mask_plane]) - th, tw = template.shape[:2] - + template_full = cv2.imread(config.CAFE_SPARKLE_TEMPLATE) img = cv2.imread(SPARKLE_SHOT_PATH) - result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask) - locs = np.where(result >= SPARKLE_THRESHOLD) - points = sorted(zip(*locs[::-1]), key=lambda p: -result[p[1], p[0]]) + th0, tw0 = template_full.shape[:2] - merged = [] - for x, y in points: - if all(abs(x - mx) > tw // 2 or abs(y - my) > th // 2 for mx, my, _ in merged): - merged.append((x, y, result[y, x])) - if not merged: + best = None + for scale in SPARKLE_SCALES: + tw, th = max(1, round(tw0 * scale)), max(1, round(th0 * scale)) + template = cv2.resize(template_full, (tw, th)) + mask = _masked_template(template) + + result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask) + _, max_val, _, max_loc = cv2.minMaxLoc(result) + if max_val >= SPARKLE_THRESHOLD and (best is None or max_val > best[0]): + best = (max_val, max_loc[0], max_loc[1], tw, th, scale) + + if best is None: return None - x, y, score = merged[0] + score, x, y, tw, th, scale = best ox, oy = SPARKLE_CLICK_OFFSET - return (x + tw // 2 + ox, y + th // 2 + oy, score) + return (x + tw // 2 + round(ox * scale), y + th // 2 + round(oy * scale), score) diff --git a/ba_auto/driver.py b/ba_auto/driver.py index 9afae8a..90f0ba2 100644 --- a/ba_auto/driver.py +++ b/ba_auto/driver.py @@ -1,4 +1,5 @@ """Local PC/Steam/Proton control backend (xdotool/scrot wrappers).""" +import os import subprocess import time @@ -6,7 +7,7 @@ import cv2 from ba_auto import config -PROBE_SHOT_PATH = "/tmp/ba_auto_probe.png" +PROBE_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "probe.png") def run_command(args, **kwargs): @@ -32,6 +33,10 @@ def click(x, y): wait(0.5) +def move_mouse(x, y): + run_command(["xdotool", "mousemove", str(x), str(y)]) + + def keypress(key): run_command(["xdotool", "key", key]) wait(0.5) diff --git a/ba_auto/reference_notes/mapping.md b/ba_auto/reference_notes/mapping.md index 303252d..5117ad2 100644 --- a/ba_auto/reference_notes/mapping.md +++ b/ba_auto/reference_notes/mapping.md @@ -5,7 +5,7 @@ Maps each local feature to the corresponding `~/repo/baas-reference/module/...` | Local feature | Reference file | Reference functions/classes | Local file | Backend replacements | Status | |---|---|---|---|---|---| | Mailbox | `module/mail.py` | `to_mail`, `implement` | `ba_auto/tasks/mailbox.py` | tap/click via xdotool, screenshot via scrot, `color.rgb_in_range` → `driver.color_at` pixel-probe check | Migrated: real Python, state-verified via color probe (no legacy bridge) | -| Cafe | `module/cafe_reward.py` | `to_cafe`, `interaction_for_cafe_solve_method3`, `collect` | `ba_auto/tasks/cafe.py` | `picture.co_detect`/`color.rgb_in_range` → `driver.color_at` pixel-probe checks; sparkle template match ported in-process into `ba_auto/detector.py` (`find_cafe_sparkle`) | Migrated: real Python, state-verified via color probes (no legacy bridge) | +| Cafe | `module/cafe_reward.py` | `to_cafe`, `interaction_for_cafe_solve_method3`, `collect` | `ba_auto/tasks/cafe.py` | `picture.co_detect`/`color.rgb_in_range` → `driver.color_at` pixel-probe checks; sparkle template match ported in-process into `ba_auto/detector.py` (`find_cafe_sparkle`, now multi-scale) | Migrated: real Python, state-verified via color probes (no legacy bridge). Pat loop now polls for the full attempt budget instead of stopping on the first miss (see `plan.md` Phase 6 follow-up) — not yet confirmed against a live sparkle since none was available during testing | | Stamina/AP | `module/collect_daily_free_power.py`, `module/collect_daily_task_power.py` | Need to inspect | `ba_auto/tasks/stamina.py` | color checks/clicks via local driver | Not started | | Group/Club AP | `module/group.py` | Need to inspect | `ba_auto/tasks/group.py` | fixed click + state check via local driver | Not started | | Bounty | `module/rewarded_task.py` | Need to inspect | `ba_auto/tasks/bounty.py` | sweep/color/OCR adaptation | Not started | diff --git a/ba_auto/tasks/cafe.py b/ba_auto/tasks/cafe.py index 6209bd9..11cda52 100644 --- a/ba_auto/tasks/cafe.py +++ b/ba_auto/tasks/cafe.py @@ -34,14 +34,26 @@ def _enter_room(driver, coords): def _pat_room(driver, config): + # Sparkles appear on a per-student cooldown, so most single checks find + # nothing -- the old Bash loop (and an earlier version of this one) gave + # up on the very first miss, which meant it essentially never farmed. + # Keep polling for the full budget instead of bailing early. + patted = 0 for _ in range(config.CAFE_MAX_CLICKS_PER_ROOM): match = detector.find_cafe_sparkle() if match is None: - break - x, y, _score = match + driver.wait(1) + continue + x, y, score = match driver.click(x, y) driver.wait(1) driver.keypress("Return") + # park the cursor away from the sparkle area so it can't occlude the + # next detection screenshot (see screenshots/cafe/sparkle/02_*_cursor_on_head.png) + driver.move_mouse(10, 1190) + patted += 1 + print(f"[cafe] patted sparkle at ({x}, {y}), score={score:.3f}") + print(f"[cafe] patted {patted} sparkle(s)" if patted else "[cafe] no sparkle found") def _claim_income(driver, config): diff --git a/graphify-out/graph.html b/graphify-out/graph.html index 9b10a42..cbb5e92 100644 --- a/graphify-out/graph.html +++ b/graphify-out/graph.html @@ -66,7 +66,7 @@
159 nodes · 176 edges · 23 communities