Nik Afiq
511010b6d6
- Added `find_template` and `template_visible` functions for template matching in screenshots. - Introduced `read_int_white_on_dark` for OCR of bright text on dark backgrounds. - Ported arena functionality from reference, including ticket management, opponent selection, and reward collection. - Implemented safety checks for modal visibility and result confirmation to prevent unintended ticket spends. - Live-tested the arena task, confirming functionality across multiple tickets with real fight outcomes. - Updated mapping documentation to reflect new arena task implementation and its unique navigation requirements.
232 lines
10 KiB
Python
232 lines
10 KiB
Python
"""Image/color matching helpers (OpenCV-based). Ported from scripts/detect_and_click.py."""
|
|
import os
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import pytesseract
|
|
|
|
from ba_auto import config, driver
|
|
|
|
OCR_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "ocr_live.png")
|
|
OCR_UPSCALE = 3
|
|
|
|
SPARKLE_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "cafe_live.png")
|
|
SPARKLE_CLICK_OFFSET = (75, 47)
|
|
SPARKLE_THRESHOLD = 0.97
|
|
|
|
# The cafe camera's zoom level isn't reset before farming, and the sparkle
|
|
# icon's on-screen size scales with it (see screenshots/cafe/sparkle/
|
|
# 01_sparke_zoomed_centered.png vs 03_sparkle_zoomed_out.png) -- a single
|
|
# fixed-scale template match misses whenever the camera isn't at the exact
|
|
# zoom the template was captured at. Try a spread of scales instead.
|
|
SPARKLE_SCALES = (0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2)
|
|
|
|
|
|
def _masked_template(template):
|
|
b, g, r = cv2.split(template.astype(np.int16))
|
|
yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200))
|
|
mask_plane = (yellow_white.astype(np.uint8)) * 255
|
|
return cv2.merge([mask_plane, mask_plane, mask_plane])
|
|
|
|
|
|
def find_cafe_sparkle():
|
|
driver.screenshot(SPARKLE_SHOT_PATH)
|
|
template_full = cv2.imread(config.CAFE_SPARKLE_TEMPLATE)
|
|
img = cv2.imread(SPARKLE_SHOT_PATH)
|
|
th0, tw0 = template_full.shape[:2]
|
|
|
|
best = None
|
|
for scale in SPARKLE_SCALES:
|
|
tw, th = max(1, round(tw0 * scale)), max(1, round(th0 * scale))
|
|
template = cv2.resize(template_full, (tw, th))
|
|
mask = _masked_template(template)
|
|
|
|
result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask)
|
|
_, max_val, _, max_loc = cv2.minMaxLoc(result)
|
|
if max_val >= SPARKLE_THRESHOLD and (best is None or max_val > best[0]):
|
|
best = (max_val, max_loc[0], max_loc[1], tw, th, scale)
|
|
|
|
if best is None:
|
|
return None
|
|
|
|
score, x, y, tw, th, scale = best
|
|
ox, oy = SPARKLE_CLICK_OFFSET
|
|
return (x + tw // 2 + round(ox * scale), y + th // 2 + round(oy * scale), score)
|
|
|
|
|
|
def find_template(template_path, region=None, threshold=0.85):
|
|
"""Generic named-template match against a fresh screenshot, optionally
|
|
restricted to a screen sub-region `(x1, y1, x2, y2)`.
|
|
|
|
The local equivalent of the reference's core/picture.py::match_img_feature
|
|
-- used wherever the reference identifies a screen/state by comparing a
|
|
fixed on-screen crop to a known template image, rather than by OCR (OCR
|
|
reads stay on read_text/read_int; see CLAUDE.md's OCR policy for why
|
|
these two stay separate detection paths). Unlike find_cafe_sparkle, this
|
|
project's UI chrome (nav icons, modal titles, result banners) renders at
|
|
a fixed resolution with no camera zoom involved, so no multi-scale search
|
|
or color masking is needed here -- a single plain match is enough. Add a
|
|
scale sweep or mask back in for a specific template if live testing ever
|
|
finds one that needs it.
|
|
|
|
Returns the match's top-left `(x, y)` in full-screenshot coordinates, or
|
|
None if nothing scores >= threshold.
|
|
"""
|
|
driver.screenshot(OCR_SHOT_PATH)
|
|
img = cv2.imread(OCR_SHOT_PATH)
|
|
if region is not None:
|
|
x1, y1, x2, y2 = region
|
|
img = img[y1:y2, x1:x2]
|
|
else:
|
|
x1, y1 = 0, 0
|
|
|
|
template = cv2.imread(template_path)
|
|
if template is None:
|
|
raise FileNotFoundError(f"Template image not found: {template_path}")
|
|
|
|
result = cv2.matchTemplate(img, template, cv2.TM_CCOEFF_NORMED)
|
|
_, max_val, _, max_loc = cv2.minMaxLoc(result)
|
|
if max_val < threshold:
|
|
return None
|
|
return (x1 + max_loc[0], y1 + max_loc[1])
|
|
|
|
|
|
def template_visible(template_path, region=None, threshold=0.85):
|
|
"""Boolean presence check built on find_template -- the common case for
|
|
screen-state detection (e.g. arena's title-banner checks via
|
|
navigation.wait_for_state's `ends`/`reactions` check_fn contract), where
|
|
only whether a known screen is showing matters, not exactly where."""
|
|
return find_template(template_path, region=region, threshold=threshold) is not None
|
|
|
|
|
|
def _color_mask(region, rgb_min, rgb_max):
|
|
x1, y1, x2, y2 = region
|
|
driver.screenshot(OCR_SHOT_PATH)
|
|
img = cv2.imread(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
b, g, r = crop[:, :, 0].astype(np.int16), crop[:, :, 1].astype(np.int16), crop[:, :, 2].astype(np.int16)
|
|
(r_lo, g_lo, b_lo), (r_hi, g_hi, b_hi) = rgb_min, rgb_max
|
|
return (r >= r_lo) & (r <= r_hi) & (g >= g_lo) & (g <= g_hi) & (b >= b_lo) & (b <= b_hi)
|
|
|
|
|
|
def region_contains_color(region, rgb_min, rgb_max):
|
|
"""Whether any pixel within `region` (x1, y1, x2, y2) falls in the given
|
|
RGB range. Useful for presence checks on small, non-convex glyphs (e.g.
|
|
an arrow chevron) where a single fixed-point probe can land in the
|
|
glyph's own concave gap -- confirmed live: a centroid-based single point
|
|
for story_sweep's region-arrow chevron fell squarely in the notch
|
|
between its two strokes, reading as "absent" even while the arrow was
|
|
clearly rendered a few pixels away. See plan.md Phase 10.
|
|
"""
|
|
return bool(_color_mask(region, rgb_min, rgb_max).any())
|
|
|
|
|
|
def find_color_centroid(region, rgb_min, rgb_max):
|
|
"""Centroid `(x, y)` of every pixel within `region` (x1, y1, x2, y2)
|
|
falling in the given RGB range, or None if none match. Useful for
|
|
clicking a known-colored button whose exact position varies between
|
|
otherwise-similar dialogs -- e.g. story_sweep's sweep-result screen
|
|
shows the same cyan confirm-button color for its "SKIP" and final "OK"
|
|
states, ~120px apart vertically; finding it by color avoids hardcoding
|
|
both positions. See plan.md Phase 10.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
mask = _color_mask(region, rgb_min, rgb_max)
|
|
ys, xs = np.nonzero(mask)
|
|
if len(xs) == 0:
|
|
return None
|
|
return (x1 + int(xs.mean()), y1 + int(ys.mean()))
|
|
|
|
|
|
def _ocr_crop(region):
|
|
x1, y1, x2, y2 = region
|
|
driver.screenshot(OCR_SHOT_PATH)
|
|
img = cv2.imread(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
|
# This UI's text is consistently dark-on-light -- a hard black/white
|
|
# threshold measurably fixed real misreads during live calibration
|
|
# (e.g. a stage label's digit misread as a stray extra digit) that
|
|
# persisted across every psm mode until the anti-aliased grey edges were
|
|
# removed. Confirmed live against scratchpad/scroll_up20.png row labels.
|
|
_, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
|
|
return cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
|
|
|
|
def read_text(region, whitelist=None, psm=7, lang="eng"):
|
|
"""OCR a pixel rectangle `(x1, y1, x2, y2)` from a fresh screenshot.
|
|
|
|
Local equivalent of the reference's core/ocr/ocr.py Baas_ocr client
|
|
(get_region_res) -- without its socket/shared-memory server, which exists
|
|
there to make OCR fast across thousands of automation steps. This project
|
|
only needs occasional single-crop reads (a region number, a stage label),
|
|
so a plain in-process pytesseract call is enough; see CLAUDE.md's "OCR
|
|
policy" and Handoff.md. `lang="eng"` is sufficient for the digit/dash
|
|
labels this project reads (region numbers, "30-1"/"30-A" stage labels) --
|
|
no Japanese trained data is needed for those specific reads.
|
|
"""
|
|
crop = _ocr_crop(region)
|
|
tess_config = f"--psm {psm}"
|
|
if whitelist:
|
|
tess_config += f" -c tessedit_char_whitelist={whitelist}"
|
|
return pytesseract.image_to_string(crop, lang=lang, config=tess_config).strip()
|
|
|
|
|
|
def read_int_white_on_dark(region, psm=7):
|
|
"""OCR a digit crop that renders bright text directly on a dark card
|
|
background (e.g. arena's "Lv.90" self/opponent-level labels) -- the
|
|
opposite polarity of this project's usual dark-text-on-light-UI
|
|
assumption baked into _ocr_crop's plain THRESH_BINARY. Confirmed live:
|
|
read_int returns None against these labels, since a plain threshold
|
|
there produces white glyphs on a black field and tesseract doesn't
|
|
segment that the way it does the normal case. An inverted threshold
|
|
fixes it outright -- no color-relationship masking needed here, unlike
|
|
read_int_on_heart_badge's badge-outline contamination problem.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
driver.screenshot(OCR_SHOT_PATH)
|
|
img = cv2.imread(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
|
_, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY_INV)
|
|
upscaled = cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
|
|
text = pytesseract.image_to_string(upscaled, lang="eng", config=tess_config).strip()
|
|
digits = "".join(ch for ch in text if ch.isdigit())
|
|
return int(digits) if digits else None
|
|
|
|
|
|
def read_int(region, psm=7):
|
|
"""Local equivalent of the reference's recognize_int."""
|
|
digits = "".join(ch for ch in read_text(region, whitelist="0123456789", psm=psm) if ch.isdigit())
|
|
return int(digits) if digits else None
|
|
|
|
|
|
def read_int_on_heart_badge(region, psm=7):
|
|
"""OCR a small dark-navy digit rendered on lesson.py's pink/magenta
|
|
heart-shaped affection badge.
|
|
|
|
read_int's plain grayscale threshold (tuned for this UI's normal
|
|
dark-text-on-light-card look) misreads these: the heart's own outline
|
|
stroke is a darker, saturated magenta whose grayscale value happens to
|
|
fall on the same side of the threshold as the digit glyph, so it
|
|
survives as stray black marks tesseract sometimes fuses into extra
|
|
digits -- confirmed live, "13" -> "113", "19" -> "119". The digit color
|
|
is reliably R < G (navy/blue-grey) while every pink/magenta badge tone
|
|
sampled (fill and outline, light and dark) is R > G, so masking on that
|
|
channel relationship instead of raw brightness cleanly drops the badge
|
|
shape and keeps just the glyph.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
driver.screenshot(OCR_SHOT_PATH)
|
|
img = cv2.imread(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
b, g, r = crop[:, :, 0].astype(np.int16), crop[:, :, 1].astype(np.int16), crop[:, :, 2].astype(np.int16)
|
|
ink = (r < g) & (np.maximum(np.maximum(r, g), b) < 170)
|
|
binary = np.where(ink, 0, 255).astype(np.uint8)
|
|
upscaled = cv2.resize(binary, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
|
|
text = pytesseract.image_to_string(upscaled, lang="eng", config=tess_config).strip()
|
|
digits = "".join(ch for ch in text if ch.isdigit())
|
|
return int(digits) if digits else None
|