feat: batch cafe sparkle patting, add diagnostic capture, verify shop purchases via balance delta
Cafe: find_cafe_sparkles (replacing find_cafe_sparkle) matches every sparkle in one screenshot instead of the single best one; _pat_current_view now pats a whole batch per frame and stops after two consecutive empty rounds instead of a fixed 15-click budget. Cuts an empty room's farming time from ~100s to ~14s in live testing. _dismiss_rank_up_if_shown also saves a diagnostic screenshot on its own final failure, since the real "screen not confirmed" cron failures (100% reproducible in logs, 0% reproducible live across several debugging sessions) need hard evidence from the next real occurrence. shop_utils.confirm_purchase now requires the currency balance to have actually dropped by the expected spend before reporting success, instead of trusting a single post-click overlay read -- closes a real false positive caught live where a cron run logged a successful purchase that never actually happened. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
6d37e889eb
commit
6a800f731b
@ -77,10 +77,17 @@ MISSION_DAILY_GEM_CLAIM_BUTTON = (1400, 1095)
|
||||
CAFE_ICON = (165, 1100)
|
||||
CAFE_ROOM_SWITCH = (190, 160)
|
||||
CAFE_INCOME = (1780, 1105)
|
||||
# Max sparkle-detection attempts per room (hits and misses both count --
|
||||
# sparkles are on a per-student cooldown, so most checks legitimately find
|
||||
# nothing and the loop keeps polling rather than giving up after one miss).
|
||||
CAFE_MAX_CLICKS_PER_ROOM = 15
|
||||
# _pat_current_view (2026-09-19 redesign, per explicit user direction):
|
||||
# screenshot once, pat every sparkle found in that one frame, screenshot
|
||||
# again, repeat -- stopping once CAFE_EMPTY_ROUNDS_TO_STOP consecutive
|
||||
# rounds each find zero sparkles, rather than the old fixed budget of
|
||||
# individual click attempts (which re-screenshotted and re-matched once per
|
||||
# pat even when several sparkles were already visible in the same frame).
|
||||
# CAFE_MAX_SPARKLE_ROUNDS is a hard safety cap on top of that -- the real
|
||||
# stopping condition is the consecutive-empty-rounds count, this just
|
||||
# bounds worst case if that somehow never triggers.
|
||||
CAFE_MAX_SPARKLE_ROUNDS = 10
|
||||
CAFE_EMPTY_ROUNDS_TO_STOP = 2
|
||||
CAFE_SPARKLE_TEMPLATE = os.path.join(ASSET_DIR, "cafe_sparkle.png")
|
||||
# A pat that crosses an affection-rank threshold shows a full-screen "絆ラン
|
||||
# クアップ!" (Bond Rank Up!) cutscene with no cafe header visible at all.
|
||||
@ -1409,7 +1416,7 @@ BOUNTY_SWEEP_CONFIRM_TEXT_RECT = (605, 505, 1320, 615)
|
||||
# visual differences (see cafe.py's CLAIM_DISABLED_RGB, stamina.py's
|
||||
# MISSION_CLAIM_PROBE) rather than building new template assets, since every
|
||||
# state below reduces to a clean, high-contrast flat color rather than a
|
||||
# complex shape needing find_cafe_sparkle-style matching.
|
||||
# complex shape needing find_cafe_sparkles-style matching.
|
||||
#
|
||||
# RECALIBRATED 2026-07-29 after a real Blue Archive client UI update changed
|
||||
# this screen's whole layout. Re-verified live on nik-gpu the same day (see
|
||||
|
||||
@ -18,7 +18,7 @@ SPARKLE_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "cafe_live.png")
|
||||
# Real-usage bug (2026-07-16, reported live with a screenshot): when two
|
||||
# students stand close together, (75, 47) overshoots past the intended
|
||||
# target's own head and lands on a DIFFERENT, closer student's body instead
|
||||
# -- the click doesn't register as a pat there, so find_cafe_sparkle() just
|
||||
# -- the click doesn't register as a pat there, so find_cafe_sparkles() just
|
||||
# matches the same still-showing sparkle again next iteration, repeatedly
|
||||
# clicking nearly the same point for the rest of the room's budget (confirmed
|
||||
# live: 13 of 16 "pats" in one run clustered within a ~10px box, all
|
||||
@ -60,32 +60,75 @@ def _masked_template(template):
|
||||
return cv2.merge([mask_plane, mask_plane, mask_plane])
|
||||
|
||||
|
||||
def find_cafe_sparkle(image=None):
|
||||
"""`image` optionally supplies an already-decoded frame instead of
|
||||
taking a fresh screenshot -- for cafe.py's per-iteration capture, shared
|
||||
with that same iteration's navigation.is_header_bar_visible rank-up
|
||||
check (see cafe.py's _pat_current_view)."""
|
||||
# How close two sparkle detections' click points need to be (in px, both
|
||||
# axes) to be treated as the same real sparkle rather than two distinct
|
||||
# ones -- the multi-scale sweep in find_cafe_sparkles commonly matches one
|
||||
# real sparkle at several neighboring scales with slightly different
|
||||
# computed centers, and those need collapsing into a single click.
|
||||
# Conservative starting guess, not yet live-calibrated against two genuinely
|
||||
# distinct sparkles sitting close together: comfortably smaller than the
|
||||
# template's own on-screen size across the sampled scales (70x93 at 1.0x,
|
||||
# so ~42x56 to ~84x112 across SPARKLE_SCALES), on the assumption that two
|
||||
# different students' sparkles won't normally render closer together than
|
||||
# that. Tighten if a live run is ever seen merging two real sparkles into
|
||||
# one click, or loosen if the same sparkle still produces duplicate clicks.
|
||||
SPARKLE_DEDUPE_DISTANCE = 50
|
||||
|
||||
|
||||
def find_cafe_sparkles(image=None):
|
||||
"""Every distinct sparkle currently visible in one frame, as
|
||||
`(x, y, score)` tuples sorted by descending score -- not just the
|
||||
single best match `find_cafe_sparkle` used to return. `image`
|
||||
optionally supplies an already-decoded frame instead of taking a fresh
|
||||
screenshot -- for cafe.py's per-round capture, shared with that same
|
||||
round's navigation.is_header_bar_visible rank-up check (see cafe.py's
|
||||
_pat_current_view).
|
||||
|
||||
Finding every sparkle in one screenshot, instead of just the best one,
|
||||
lets the pat loop click through all of them from a single capture
|
||||
before re-screenshotting, rather than paying for a fresh scrot +
|
||||
multi-scale template match per individual pat when several sparkles are
|
||||
already visible at once. Replaces the original single-match
|
||||
find_cafe_sparkle (2026-09-19, per explicit user direction) as part of
|
||||
the same change that made _pat_current_view stop after two consecutive
|
||||
empty screenshots instead of a fixed click-attempt budget -- see that
|
||||
function's own docstring.
|
||||
"""
|
||||
template_full = cv2.imread(config.CAFE_SPARKLE_TEMPLATE)
|
||||
img = image if image is not None else driver.read_screenshot(SPARKLE_SHOT_PATH)
|
||||
th0, tw0 = template_full.shape[:2]
|
||||
ox, oy = SPARKLE_CLICK_OFFSET
|
||||
|
||||
best = None
|
||||
candidates = []
|
||||
for scale in SPARKLE_SCALES:
|
||||
tw, th = max(1, round(tw0 * scale)), max(1, round(th0 * scale))
|
||||
template = cv2.resize(template_full, (tw, th))
|
||||
mask = _masked_template(template)
|
||||
|
||||
result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask)
|
||||
_, max_val, _, max_loc = cv2.minMaxLoc(result)
|
||||
if max_val >= SPARKLE_THRESHOLD and (best is None or max_val > best[0]):
|
||||
best = (max_val, max_loc[0], max_loc[1], tw, th, scale)
|
||||
ys, xs = np.where(result >= SPARKLE_THRESHOLD)
|
||||
for x, y in zip(xs, ys):
|
||||
cx = int(x) + tw // 2 + round(ox * scale)
|
||||
cy = int(y) + th // 2 + round(oy * scale)
|
||||
candidates.append((float(result[y, x]), cx, cy))
|
||||
|
||||
if best is None:
|
||||
return None
|
||||
return _dedupe_sparkles(candidates)
|
||||
|
||||
score, x, y, tw, th, scale = best
|
||||
ox, oy = SPARKLE_CLICK_OFFSET
|
||||
return (x + tw // 2 + round(ox * scale), y + th // 2 + round(oy * scale), score)
|
||||
|
||||
def _dedupe_sparkles(candidates):
|
||||
"""Collapse candidates within SPARKLE_DEDUPE_DISTANCE of each other
|
||||
(the same real sparkle matched at several nearby scales/positions) into
|
||||
a single detection, keeping the highest-scoring one each time."""
|
||||
candidates = sorted(candidates, key=lambda c: c[0], reverse=True)
|
||||
kept = []
|
||||
for score, x, y in candidates:
|
||||
if any(
|
||||
abs(x - kx) < SPARKLE_DEDUPE_DISTANCE and abs(y - ky) < SPARKLE_DEDUPE_DISTANCE
|
||||
for _, kx, ky in kept
|
||||
):
|
||||
continue
|
||||
kept.append((score, x, y))
|
||||
return [(x, y, score) for score, x, y in kept]
|
||||
|
||||
|
||||
def find_template(template_path, region=None, threshold=0.85):
|
||||
@ -96,7 +139,7 @@ def find_template(template_path, region=None, threshold=0.85):
|
||||
-- used wherever the reference identifies a screen/state by comparing a
|
||||
fixed on-screen crop to a known template image, rather than by OCR (OCR
|
||||
reads stay on read_text/read_int; see CLAUDE.md's OCR policy for why
|
||||
these two stay separate detection paths). Unlike find_cafe_sparkle, this
|
||||
these two stay separate detection paths). Unlike find_cafe_sparkles, this
|
||||
project's UI chrome (nav icons, modal titles, result banners) renders at
|
||||
a fixed resolution with no camera zoom involved, so no multi-scale search
|
||||
or color masking is needed here -- a single plain match is enough. Add a
|
||||
|
||||
@ -90,8 +90,8 @@ def is_header_bar_visible(driver, image=None):
|
||||
# from the same frame instead of drifting across ~8 sequential captures).
|
||||
#
|
||||
# `image` optionally supplies an already-decoded frame (e.g. cafe.py's
|
||||
# per-iteration capture, shared with that same iteration's
|
||||
# find_cafe_sparkle call) instead of taking a fresh screenshot here.
|
||||
# per-round capture, shared with that same round's
|
||||
# find_cafe_sparkles call) instead of taking a fresh screenshot here.
|
||||
points = [(x, HEADER_ROW_Y) for x in HEADER_ROW_X_OFFSETS]
|
||||
for r, g, b in driver.colors_at(points, image=image):
|
||||
if not (r > SUBSCREEN_HEADER_MIN_CHANNEL and g > SUBSCREEN_HEADER_MIN_CHANNEL and b > SUBSCREEN_HEADER_MIN_CHANNEL):
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -93,6 +93,9 @@ _pat_current_view poll iteration instead of only right after a pat -- see
|
||||
that function's own comment.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from ba_auto import detector, navigation
|
||||
|
||||
ROOM_OPEN_RETRIES = 3
|
||||
@ -113,6 +116,28 @@ def _claim_disabled(driver):
|
||||
)
|
||||
|
||||
|
||||
def _save_diagnostic_screenshot(driver, config, label):
|
||||
# Added 2026-09-19, per explicit user request, after this task's own
|
||||
# "screen not confirmed (rank-up cutscene stuck?)" warning turned out to
|
||||
# fire on ~90% of real scheduled runs in the logs but could not be
|
||||
# reproduced on demand in several separate live debugging sessions
|
||||
# (fresh game relaunch, full login->cafe sequence, the cooldown-dialog-
|
||||
# dismiss path all included) -- every attempt to force it live
|
||||
# succeeded cleanly instead. Since the exact interrupting state
|
||||
# couldn't be captured live, the next REAL occurrence needs to leave
|
||||
# hard evidence of what the screen actually looked like on its own,
|
||||
# rather than relying on log text alone to diagnose after the fact.
|
||||
# Written to scratchpad/ per this project's own temporary-file
|
||||
# convention -- not auto-cleaned, review/delete via clean_scratchpad.sh
|
||||
# once a real occurrence has been captured and inspected.
|
||||
path = os.path.join(config.SCRATCHPAD_DIR, f"cafe_diag_{label}_{int(time.time())}.png")
|
||||
try:
|
||||
driver.read_screenshot(path)
|
||||
print(f"[cafe] saved diagnostic screenshot: {path}")
|
||||
except RuntimeError as exc:
|
||||
print(f"[cafe] could not save diagnostic screenshot: {exc}")
|
||||
|
||||
|
||||
def _enter_room(driver, coords):
|
||||
for attempt in range(1, ROOM_OPEN_RETRIES + 1):
|
||||
driver.click(*coords)
|
||||
@ -157,14 +182,32 @@ def _dismiss_rank_up_if_shown(driver, config, image=None):
|
||||
first_check_image = None
|
||||
driver.keypress("Return")
|
||||
driver.wait(1.5)
|
||||
return navigation.is_header_bar_visible(driver)
|
||||
confirmed = navigation.is_header_bar_visible(driver)
|
||||
if not confirmed:
|
||||
_save_diagnostic_screenshot(driver, config, "not_confirmed")
|
||||
return confirmed
|
||||
|
||||
|
||||
def _pat_current_view(driver, config):
|
||||
# Sparkles appear on a per-student cooldown, so most single checks find
|
||||
# nothing -- the old Bash loop (and an earlier version of this one) gave
|
||||
# up on the very first miss, which meant it essentially never farmed.
|
||||
# Keep polling for the full budget instead of bailing early.
|
||||
# Sparkles appear on a per-student cooldown. Original design: one
|
||||
# screenshot + one sparkle-match attempt per click, giving up after a
|
||||
# fixed budget of individual click attempts -- correct, but wasteful
|
||||
# whenever several sparkles were visible in the same frame, since each
|
||||
# still paid for its own fresh scrot + multi-scale template match.
|
||||
# Redesigned 2026-09-19, per explicit user direction: one screenshot,
|
||||
# find and pat EVERY sparkle visible in it (detector.find_cafe_sparkles,
|
||||
# not just the single best match), then take a new screenshot and
|
||||
# repeat -- stopping once config.CAFE_EMPTY_ROUNDS_TO_STOP consecutive
|
||||
# rounds each find nothing, rather than polling a fixed click-attempt
|
||||
# budget regardless of how many were found per frame.
|
||||
#
|
||||
# Patting through a whole batch from one screenshot without
|
||||
# re-capturing between individual pats is safe here specifically
|
||||
# because cafe sparkles sit on static room furniture/students, not a
|
||||
# reflowing list -- unlike Common Shop's item grid (see shop_utils.py's
|
||||
# own 2026-09-17 incident, where reusing stale positions after an
|
||||
# action that could reflow the grid was NOT safe). Patting one sparkle
|
||||
# doesn't move where any other already-detected sparkle is sitting.
|
||||
#
|
||||
# Real-usage bug (2026-07-16): the rank-up check used to run only once,
|
||||
# immediately after a pat, with no wait beforehand -- but the rank-up
|
||||
@ -172,7 +215,7 @@ def _pat_current_view(driver, config):
|
||||
# animation delay, so that single check could catch the tail end of the
|
||||
# still-normal room view (header genuinely still visible in that instant)
|
||||
# and conclude "clear" a beat before the actual cutscene appeared. Once
|
||||
# that happened, nothing ever checked again: find_cafe_sparkle() can't
|
||||
# that happened, nothing ever checked again: find_cafe_sparkles() can't
|
||||
# tell "stuck on an undetected cutscene" apart from the ordinary
|
||||
# "nothing to pat right now" case, so the loop just polled dead air for
|
||||
# the rest of its budget, the following camera pan dragged over a static
|
||||
@ -180,41 +223,40 @@ def _pat_current_view(driver, config):
|
||||
# return_to_home's generic cleanup failed to recover afterward -- this
|
||||
# exact cutscene's background happens to read under is_on_subscreen/
|
||||
# is_modal_open's thresholds too, so every "are we home" check downstream
|
||||
# falsely agreed nothing was wrong. Fixed by checking on EVERY poll
|
||||
# iteration, not just right after a pat, so a delayed appearance is
|
||||
# caught (and dismissed, via _dismiss_rank_up_if_shown's own Enter-press
|
||||
# loop) on the next iteration, roughly a second later, instead of never.
|
||||
#
|
||||
# One capture per iteration shared between the rank-up check and the
|
||||
# sparkle match (previously two separate scrot calls back-to-back with
|
||||
# nothing in between): the header-bar check only samples fixed chrome
|
||||
# pixels unrelated to the room scene, and the sparkle match already
|
||||
# reports "wherever it is in this one frame" regardless of whether that
|
||||
# frame is shared -- the room animating (a student walking around)
|
||||
# doesn't change that, since neither read depends on the other lagging
|
||||
# or leading it. See plan.md's "Performance improvement plan".
|
||||
# falsely agreed nothing was wrong. Fixed by checking before every
|
||||
# individual pat, not just once per round, so a delayed appearance is
|
||||
# caught mid-batch instead of only between rounds.
|
||||
patted = 0
|
||||
for _ in range(config.CAFE_MAX_CLICKS_PER_ROOM):
|
||||
empty_rounds = 0
|
||||
for _ in range(config.CAFE_MAX_SPARKLE_ROUNDS):
|
||||
image = detector.capture_screen()
|
||||
if not _dismiss_rank_up_if_shown(driver, config, image=image):
|
||||
print("[cafe] warning: cafe screen not confirmed (rank-up cutscene stuck?) -- stopping this view's pat loop rather than clicking blindly")
|
||||
return patted
|
||||
|
||||
matches = detector.find_cafe_sparkles(image=image)
|
||||
if not matches:
|
||||
empty_rounds += 1
|
||||
if empty_rounds >= config.CAFE_EMPTY_ROUNDS_TO_STOP:
|
||||
break
|
||||
match = detector.find_cafe_sparkle(image=image)
|
||||
if match is None:
|
||||
driver.wait(1)
|
||||
continue
|
||||
x, y, score = match
|
||||
empty_rounds = 0
|
||||
|
||||
for x, y, score in matches:
|
||||
driver.click(x, y)
|
||||
driver.wait(1)
|
||||
driver.keypress("Return")
|
||||
# park the cursor away from the sparkle area so it can't occlude the
|
||||
# next detection screenshot (see screenshots/cafe/sparkle/02_*_cursor_on_head.png)
|
||||
# park the cursor away from the sparkle area so it can't occlude
|
||||
# the next detection screenshot (see
|
||||
# screenshots/cafe/sparkle/02_*_cursor_on_head.png)
|
||||
driver.move_mouse(10, 1190)
|
||||
# No image= here -- the click/Return/mouse-move above are real state
|
||||
# changes since this iteration's capture, so this check must be fresh.
|
||||
# No image= here -- the click/Return/mouse-move above are real
|
||||
# state changes since this batch's own capture, so this check
|
||||
# must be fresh.
|
||||
if not _dismiss_rank_up_if_shown(driver, config):
|
||||
print("[cafe] warning: cafe screen not confirmed after a pat (rank-up cutscene stuck?) -- stopping this view's pat loop rather than clicking blindly")
|
||||
break
|
||||
return patted
|
||||
patted += 1
|
||||
print(f"[cafe] patted sparkle at ({x}, {y}), score={score:.3f}")
|
||||
return patted
|
||||
|
||||
@ -257,24 +257,58 @@ def select_targets_by_name(driver, config, targets):
|
||||
return selected, skipped
|
||||
|
||||
|
||||
def confirm_purchase(driver, config):
|
||||
def confirm_purchase(driver, config, *, balance_rect, balance_before, expected_spend):
|
||||
"""Click the bulk Buy button, then click through the purchase-confirm
|
||||
dialog and the reward-acquired banner. Both dim
|
||||
config.SHOP_OVERLAY_PROBE away from pure white; press Enter until it
|
||||
reads idle again, bounded by config.SHOP_PURCHASE_MAX_ENTER_PRESSES --
|
||||
this project doesn't press keys past a bound against an unrecognized
|
||||
state (see navigation.wait_for_state). Returns True once idle, False if
|
||||
it never clears within the bound.
|
||||
state (see navigation.wait_for_state).
|
||||
|
||||
Real live false positive (2026-09-19): the overlay can already read
|
||||
"idle" on the very first check -- there's nothing in a single
|
||||
overlay-darkness read that distinguishes "confirmed and closed" from
|
||||
"the Buy click missed and the confirm dialog never opened at all".
|
||||
Confirmed live: a real cron run logged "bought 4 item(s)" for a real
|
||||
credit amount, but those exact 4 items were still sitting unbought at
|
||||
their base price hours later -- the overlay had simply never dimmed at
|
||||
any point this function actually checked it. Fixed by also requiring
|
||||
the currency balance to have actually dropped by `expected_spend`
|
||||
once the overlay reports idle -- re-read with the same
|
||||
stray-misread-digit retry tolerance SHOP_PRICE_OCR_RETRIES already uses
|
||||
elsewhere in this module, so one bad OCR frame on the balance itself
|
||||
can't sink an otherwise-real purchase.
|
||||
|
||||
Returns True only once the overlay is idle AND the balance dropped by
|
||||
exactly `expected_spend`; False otherwise -- whether the overlay never
|
||||
cleared within the retry bound, or it cleared but the balance never
|
||||
moved (or moved by some other amount), meaning nothing was really
|
||||
bought despite looking "idle".
|
||||
"""
|
||||
driver.click(*config.SHOP_BUY_BUTTON)
|
||||
driver.wait(1.5)
|
||||
min_ch = config.SHOP_OVERLAY_IDLE_MIN_CHANNEL
|
||||
idle = False
|
||||
for _ in range(config.SHOP_PURCHASE_MAX_ENTER_PRESSES):
|
||||
r, g, b = driver.color_at(*config.SHOP_OVERLAY_PROBE)
|
||||
if r > min_ch and g > min_ch and b > min_ch:
|
||||
return True
|
||||
idle = True
|
||||
break
|
||||
driver.keypress("Return")
|
||||
driver.wait(1.5)
|
||||
if not idle:
|
||||
return False
|
||||
|
||||
balance_after = None
|
||||
for attempt in range(1, config.SHOP_PRICE_OCR_RETRIES + 1):
|
||||
balance_after = detector.read_int(balance_rect)
|
||||
if balance_after is not None and balance_before - balance_after == expected_spend:
|
||||
return True
|
||||
if attempt < config.SHOP_PRICE_OCR_RETRIES:
|
||||
driver.wait(0.3)
|
||||
|
||||
actual_spend = None if balance_after is None else balance_before - balance_after
|
||||
print(f"[shop] purchase overlay cleared but balance changed by {actual_spend}, expected {expected_spend} -- treating as not confirmed")
|
||||
return False
|
||||
|
||||
|
||||
@ -316,11 +350,11 @@ def run_shop_tab(driver, config, *, tab_button, targets, balance_rect, currency_
|
||||
print("[shop] nothing selected (all targets skipped), cancelling")
|
||||
return "nothing_selected"
|
||||
|
||||
if not confirm_purchase(driver, config):
|
||||
print("[shop] purchase confirmation did not resolve to an idle screen within the retry bound -- stopping, check the game manually")
|
||||
spent = sum(price for _, price in selected)
|
||||
if not confirm_purchase(driver, config, balance_rect=balance_rect, balance_before=balance, expected_spend=spent):
|
||||
print("[shop] purchase confirmation did not resolve to a verified balance change within the retry bound -- stopping, check the game manually")
|
||||
return "unrecognized_state"
|
||||
|
||||
spent = sum(price for _, price in selected)
|
||||
msg = f"[shop] bought {len(selected)} item(s) for {spent} {currency_label}"
|
||||
if skipped:
|
||||
msg += f", skipped {len(skipped)}"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user