ba-auto-daily/ba_auto/detector.py

"""Image/color matching helpers (OpenCV-based). Ported from scripts/detect_and_click.py."""
import os

import cv2
import numpy as np
import pytesseract

from ba_auto import config, driver

OCR_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "ocr_live.png")
OCR_UPSCALE = 3

SPARKLE_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "cafe_live.png")
# Ported byte-for-byte from the original scripts/detect_and_click.py without
# recalibration (see plan.md's Phase 6 writeup) -- (75, 47) was never
# actually re-verified against this project's own real gameplay.
#
# Real-usage bug (2026-07-16, reported live with a screenshot): when two
# students stand close together, (75, 47) overshoots past the intended
# target's own head and lands on a DIFFERENT, closer student's body instead
# -- the click doesn't register as a pat there, so find_cafe_sparkle() just
# matches the same still-showing sparkle again next iteration, repeatedly
# clicking nearly the same point for the rest of the room's budget (confirmed
# live: 13 of 16 "pats" in one run clustered within a ~10px box, all
# ineffective).
#
# First recalibration attempt, (0, 0) (raw template-match center, no
# offset), was also wrong -- reported live ("it is clicking the actual
# sparkle instead of the student head"): a coincidental ambient thought-
# bubble animation at the raw-center test position was misread as a hover
# confirmation, when the raw sparkle position is not actually within the
# character's clickable area.
#
# Properly recalibrated live the same day via numbered candidate-point
# overlays (screenshots with several labeled offset options drawn on a real
# live sparkle, user picked the one actually landing on the head each time)
# against two independent real students in different poses (a sitting
# arcade-cabinet pose and a lying-down couch pose) -- both converged on
# roughly the same small offset, confirmed by the user directly on-screen
# rather than inferred from a single ambiguous signal like the (0,0) attempt
# was. (75, 47) was oversized (the close-together overshoot bug) and (0, 0)
# was undersized (clicks the sparkle glyph itself, not the character) --
# (50, 15) was the empirically-confirmed middle ground, then nudged 1px
# further right by the user after watching it land live.
SPARKLE_CLICK_OFFSET = (51, 15)
SPARKLE_THRESHOLD = 0.97

# The cafe camera's zoom level isn't reset before farming, and the sparkle
# icon's on-screen size scales with it (see screenshots/cafe/sparkle/
# 01_sparke_zoomed_centered.png vs 03_sparkle_zoomed_out.png) -- a single
# fixed-scale template match misses whenever the camera isn't at the exact
# zoom the template was captured at. Try a spread of scales instead.
SPARKLE_SCALES = (0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2)


def _masked_template(template):
    b, g, r = cv2.split(template.astype(np.int16))
    yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200))
    mask_plane = (yellow_white.astype(np.uint8)) * 255
    return cv2.merge([mask_plane, mask_plane, mask_plane])


def find_cafe_sparkle():
    driver.screenshot(SPARKLE_SHOT_PATH)
    template_full = cv2.imread(config.CAFE_SPARKLE_TEMPLATE)
    img = cv2.imread(SPARKLE_SHOT_PATH)
    th0, tw0 = template_full.shape[:2]

    best = None
    for scale in SPARKLE_SCALES:
        tw, th = max(1, round(tw0 * scale)), max(1, round(th0 * scale))
        template = cv2.resize(template_full, (tw, th))
        mask = _masked_template(template)

        result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask)
        _, max_val, _, max_loc = cv2.minMaxLoc(result)
        if max_val >= SPARKLE_THRESHOLD and (best is None or max_val > best[0]):
            best = (max_val, max_loc[0], max_loc[1], tw, th, scale)

    if best is None:
        return None

    score, x, y, tw, th, scale = best
    ox, oy = SPARKLE_CLICK_OFFSET
    return (x + tw // 2 + round(ox * scale), y + th // 2 + round(oy * scale), score)


def find_template(template_path, region=None, threshold=0.85):
    """Generic named-template match against a fresh screenshot, optionally
    restricted to a screen sub-region `(x1, y1, x2, y2)`.

    The local equivalent of the reference's core/picture.py::match_img_feature
    -- used wherever the reference identifies a screen/state by comparing a
    fixed on-screen crop to a known template image, rather than by OCR (OCR
    reads stay on read_text/read_int; see CLAUDE.md's OCR policy for why
    these two stay separate detection paths). Unlike find_cafe_sparkle, this
    project's UI chrome (nav icons, modal titles, result banners) renders at
    a fixed resolution with no camera zoom involved, so no multi-scale search
    or color masking is needed here -- a single plain match is enough. Add a
    scale sweep or mask back in for a specific template if live testing ever
    finds one that needs it.

    Returns the match's top-left `(x, y)` in full-screenshot coordinates, or
    None if nothing scores >= threshold.
    """
    driver.screenshot(OCR_SHOT_PATH)
    img = cv2.imread(OCR_SHOT_PATH)
    if region is not None:
        x1, y1, x2, y2 = region
        img = img[y1:y2, x1:x2]
    else:
        x1, y1 = 0, 0

    template = cv2.imread(template_path)
    if template is None:
        raise FileNotFoundError(f"Template image not found: {template_path}")

    result = cv2.matchTemplate(img, template, cv2.TM_CCOEFF_NORMED)
    _, max_val, _, max_loc = cv2.minMaxLoc(result)
    if max_val < threshold:
        return None
    return (x1 + max_loc[0], y1 + max_loc[1])


def template_visible(template_path, region=None, threshold=0.85):
    """Boolean presence check built on find_template -- the common case for
    screen-state detection (e.g. arena's title-banner checks via
    navigation.wait_for_state's `ends`/`reactions` check_fn contract), where
    only whether a known screen is showing matters, not exactly where."""
    return find_template(template_path, region=region, threshold=threshold) is not None


def _color_mask(region, rgb_min, rgb_max):
    x1, y1, x2, y2 = region
    driver.screenshot(OCR_SHOT_PATH)
    img = cv2.imread(OCR_SHOT_PATH)
    crop = img[y1:y2, x1:x2]
    b, g, r = crop[:, :, 0].astype(np.int16), crop[:, :, 1].astype(np.int16), crop[:, :, 2].astype(np.int16)
    (r_lo, g_lo, b_lo), (r_hi, g_hi, b_hi) = rgb_min, rgb_max
    return (r >= r_lo) & (r <= r_hi) & (g >= g_lo) & (g <= g_hi) & (b >= b_lo) & (b <= b_hi)


def region_contains_color(region, rgb_min, rgb_max):
    """Whether any pixel within `region` (x1, y1, x2, y2) falls in the given
    RGB range. Useful for presence checks on small, non-convex glyphs (e.g.
    an arrow chevron) where a single fixed-point probe can land in the
    glyph's own concave gap -- confirmed live: a centroid-based single point
    for story_sweep's region-arrow chevron fell squarely in the notch
    between its two strokes, reading as "absent" even while the arrow was
    clearly rendered a few pixels away. See plan.md Phase 10.
    """
    return bool(_color_mask(region, rgb_min, rgb_max).any())


def find_color_centroid(region, rgb_min, rgb_max, min_pixels=1):
    """Centroid `(x, y)` of every pixel within `region` (x1, y1, x2, y2)
    falling in the given RGB range, or None if fewer than `min_pixels`
    match. Useful for clicking a known-colored button whose exact position
    varies between otherwise-similar dialogs -- e.g. story_sweep's
    sweep-result screen shows the same cyan confirm-button color for its
    "SKIP" and final "OK" states, ~120px apart vertically; finding it by
    color avoids hardcoding both positions. See plan.md Phase 10.

    `min_pixels` defaults to 1 (any match counts, the original behavior) --
    raise it when the region can also contain sparse, incidental matches of
    the same color from unrelated content (e.g. bounty.py's result-button
    region briefly overlapping a reward-icon's own artwork, confirmed live
    to produce a few hundred stray matching pixels vs a real button's tens
    of thousands -- see config.BOUNTY_RESULT_BUTTON_MIN_PIXELS).
    """
    x1, y1, x2, y2 = region
    mask = _color_mask(region, rgb_min, rgb_max)
    ys, xs = np.nonzero(mask)
    if len(xs) < min_pixels:
        return None
    return (x1 + int(xs.mean()), y1 + int(ys.mean()))


def _ocr_crop(region):
    x1, y1, x2, y2 = region
    driver.screenshot(OCR_SHOT_PATH)
    img = cv2.imread(OCR_SHOT_PATH)
    crop = img[y1:y2, x1:x2]
    gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
    # This UI's text is consistently dark-on-light -- a hard black/white
    # threshold measurably fixed real misreads during live calibration
    # (e.g. a stage label's digit misread as a stray extra digit) that
    # persisted across every psm mode until the anti-aliased grey edges were
    # removed. Confirmed live against scratchpad/scroll_up20.png row labels.
    _, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
    return cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)


def read_text(region, whitelist=None, psm=7, lang="eng"):
    """OCR a pixel rectangle `(x1, y1, x2, y2)` from a fresh screenshot.

    Local equivalent of the reference's core/ocr/ocr.py Baas_ocr client
    (get_region_res) -- without its socket/shared-memory server, which exists
    there to make OCR fast across thousands of automation steps. This project
    only needs occasional single-crop reads (a region number, a stage label),
    so a plain in-process pytesseract call is enough; see CLAUDE.md's "OCR
    policy" and Handoff.md. `lang="eng"` is sufficient for the digit/dash
    labels this project reads (region numbers, "30-1"/"30-A" stage labels) --
    no Japanese trained data is needed for those specific reads.
    """
    crop = _ocr_crop(region)
    tess_config = f"--psm {psm}"
    if whitelist:
        tess_config += f" -c tessedit_char_whitelist={whitelist}"
    return pytesseract.image_to_string(crop, lang=lang, config=tess_config).strip()


def read_int_white_on_dark(region, psm=7):
    """OCR a digit crop that renders bright text directly on a dark card
    background (e.g. arena's "Lv.90" self/opponent-level labels) -- the
    opposite polarity of this project's usual dark-text-on-light-UI
    assumption baked into _ocr_crop's plain THRESH_BINARY. Confirmed live:
    read_int returns None against these labels, since a plain threshold
    there produces white glyphs on a black field and tesseract doesn't
    segment that the way it does the normal case. An inverted threshold
    fixes it outright -- no color-relationship masking needed here, unlike
    read_int_on_heart_badge's badge-outline contamination problem.
    """
    x1, y1, x2, y2 = region
    driver.screenshot(OCR_SHOT_PATH)
    img = cv2.imread(OCR_SHOT_PATH)
    crop = img[y1:y2, x1:x2]
    gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
    _, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY_INV)
    upscaled = cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
    tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
    text = pytesseract.image_to_string(upscaled, lang="eng", config=tess_config).strip()
    digits = "".join(ch for ch in text if ch.isdigit())
    return int(digits) if digits else None


def read_int(region, psm=7):
    """Local equivalent of the reference's recognize_int."""
    digits = "".join(ch for ch in read_text(region, whitelist="0123456789", psm=psm) if ch.isdigit())
    return int(digits) if digits else None


def read_int_bordered(region, psm=7, border=20):
    """read_int, but with a white margin added around the upscaled crop
    before OCR.

    event_sweep.py's stage-row numbers "08"/"09" (a leading-zero digit
    tight against the following one) were misread by plain read_int on
    every psm mode tried -- "08" consistently as "2", "09" as empty or "9"
    with the leading zero silently dropped -- even though the crop and its
    thresholded/upscaled version both look completely clean to the eye.
    "10"/"11"/"12" at the same crop dimensions read fine. Confirmed live
    (scratchpad/probe_ocr_fix_08_09.py) this is specifically a tesseract
    segmentation problem with a text blob that fills the crop edge-to-edge,
    with no surrounding whitespace context to anchor character boundaries
    -- adding a plain white cv2.copyMakeBorder margin (no other pipeline
    change) fixed both "08" and "09" to their exact correct digits at every
    psm mode tried, and left the already-working "10" unaffected. Kept as
    its own function rather than changed in read_int/_ocr_crop, since only
    this one tight edge-to-edge crop shape has ever been confirmed to need
    it -- every other OCR read in this project already has enough natural
    margin.
    """
    x1, y1, x2, y2 = region
    driver.screenshot(OCR_SHOT_PATH)
    img = cv2.imread(OCR_SHOT_PATH)
    crop = img[y1:y2, x1:x2]
    gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
    _, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
    upscaled = cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
    bordered = cv2.copyMakeBorder(upscaled, border, border, border, border, cv2.BORDER_CONSTANT, value=255)
    tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
    text = pytesseract.image_to_string(bordered, lang="eng", config=tess_config).strip()
    digits = "".join(ch for ch in text if ch.isdigit())
    return int(digits) if digits else None


def read_int_on_heart_badge(region, psm=7):
    """OCR a small dark-navy digit rendered on lesson.py's pink/magenta
    heart-shaped affection badge.

    read_int's plain grayscale threshold (tuned for this UI's normal
    dark-text-on-light-card look) misreads these: the heart's own outline
    stroke is a darker, saturated magenta whose grayscale value happens to
    fall on the same side of the threshold as the digit glyph, so it
    survives as stray black marks tesseract sometimes fuses into extra
    digits -- confirmed live, "13" -> "113", "19" -> "119". The digit color
    is reliably R < G (navy/blue-grey) while every pink/magenta badge tone
    sampled (fill and outline, light and dark) is R > G, so masking on that
    channel relationship instead of raw brightness cleanly drops the badge
    shape and keeps just the glyph.
    """
    x1, y1, x2, y2 = region
    driver.screenshot(OCR_SHOT_PATH)
    img = cv2.imread(OCR_SHOT_PATH)
    crop = img[y1:y2, x1:x2]
    b, g, r = crop[:, :, 0].astype(np.int16), crop[:, :, 1].astype(np.int16), crop[:, :, 2].astype(np.int16)
    ink = (r < g) & (np.maximum(np.maximum(r, g), b) < 170)
    binary = np.where(ink, 0, 255).astype(np.uint8)
    upscaled = cv2.resize(binary, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
    tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
    text = pytesseract.image_to_string(upscaled, lang="eng", config=tess_config).strip()
    digits = "".join(ch for ch in text if ch.isdigit())
    return int(digits) if digits else None