- Introduced a fallback mechanism using difflib for improved name matching when OCR results contain internal character corruption. - Implemented a two-stage fallback for name matching: stripping whitespace and using similarity scoring if no exact or substring matches are found. - Modified `select_targets_by_name` to handle incremental purchases, committing batches before reopening the shop tab to avoid under-purchases due to UI state resets. - Added retry logic for handling OCR instability, allowing multiple attempts to read item names before resorting to reopening the shop tab. - Updated the return values of `select_targets_by_name` to reflect purchased items and skipped targets, improving clarity on transaction outcomes.
471 lines
24 KiB
Python
471 lines
24 KiB
Python
"""Image/color matching helpers (OpenCV-based). Ported from scripts/detect_and_click.py."""
|
|
import os
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import pytesseract
|
|
|
|
from ba_auto import config, driver
|
|
|
|
OCR_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "ocr_live.png")
|
|
OCR_UPSCALE = 3
|
|
|
|
SPARKLE_SHOT_PATH = os.path.join(config.SCRATCHPAD_DIR, "cafe_live.png")
|
|
# Ported byte-for-byte from the original scripts/detect_and_click.py without
|
|
# recalibration (see plan.md's Phase 6 writeup) -- (75, 47) was never
|
|
# actually re-verified against this project's own real gameplay.
|
|
#
|
|
# Real-usage bug (2026-07-16, reported live with a screenshot): when two
|
|
# students stand close together, (75, 47) overshoots past the intended
|
|
# target's own head and lands on a DIFFERENT, closer student's body instead
|
|
# -- the click doesn't register as a pat there, so find_cafe_sparkles() just
|
|
# matches the same still-showing sparkle again next iteration, repeatedly
|
|
# clicking nearly the same point for the rest of the room's budget (confirmed
|
|
# live: 13 of 16 "pats" in one run clustered within a ~10px box, all
|
|
# ineffective).
|
|
#
|
|
# First recalibration attempt, (0, 0) (raw template-match center, no
|
|
# offset), was also wrong -- reported live ("it is clicking the actual
|
|
# sparkle instead of the student head"): a coincidental ambient thought-
|
|
# bubble animation at the raw-center test position was misread as a hover
|
|
# confirmation, when the raw sparkle position is not actually within the
|
|
# character's clickable area.
|
|
#
|
|
# Properly recalibrated live the same day via numbered candidate-point
|
|
# overlays (screenshots with several labeled offset options drawn on a real
|
|
# live sparkle, user picked the one actually landing on the head each time)
|
|
# against two independent real students in different poses (a sitting
|
|
# arcade-cabinet pose and a lying-down couch pose) -- both converged on
|
|
# roughly the same small offset, confirmed by the user directly on-screen
|
|
# rather than inferred from a single ambiguous signal like the (0,0) attempt
|
|
# was. (75, 47) was oversized (the close-together overshoot bug) and (0, 0)
|
|
# was undersized (clicks the sparkle glyph itself, not the character) --
|
|
# (50, 15) was the empirically-confirmed middle ground, then nudged 1px
|
|
# further right by the user after watching it land live.
|
|
SPARKLE_CLICK_OFFSET = (51, 15)
|
|
SPARKLE_THRESHOLD = 0.97
|
|
|
|
# The cafe camera's zoom level isn't reset before farming, and the sparkle
|
|
# icon's on-screen size scales with it (see screenshots/cafe/sparkle/
|
|
# 01_sparke_zoomed_centered.png vs 03_sparkle_zoomed_out.png) -- a single
|
|
# fixed-scale template match misses whenever the camera isn't at the exact
|
|
# zoom the template was captured at. Try a spread of scales instead.
|
|
SPARKLE_SCALES = (0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2)
|
|
|
|
|
|
def _masked_template(template):
|
|
b, g, r = cv2.split(template.astype(np.int16))
|
|
yellow_white = ((r > 180) & (g > 140) & (r - b > 60)) | ((r > 200) & (g > 200) & (b > 200))
|
|
mask_plane = (yellow_white.astype(np.uint8)) * 255
|
|
return cv2.merge([mask_plane, mask_plane, mask_plane])
|
|
|
|
|
|
# How close two sparkle detections' click points need to be (in px, both
|
|
# axes) to be treated as the same real sparkle rather than two distinct
|
|
# ones -- the multi-scale sweep in find_cafe_sparkles commonly matches one
|
|
# real sparkle at several neighboring scales with slightly different
|
|
# computed centers, and those need collapsing into a single click.
|
|
# Conservative starting guess, not yet live-calibrated against two genuinely
|
|
# distinct sparkles sitting close together: comfortably smaller than the
|
|
# template's own on-screen size across the sampled scales (70x93 at 1.0x,
|
|
# so ~42x56 to ~84x112 across SPARKLE_SCALES), on the assumption that two
|
|
# different students' sparkles won't normally render closer together than
|
|
# that. Tighten if a live run is ever seen merging two real sparkles into
|
|
# one click, or loosen if the same sparkle still produces duplicate clicks.
|
|
SPARKLE_DEDUPE_DISTANCE = 50
|
|
|
|
|
|
def find_cafe_sparkles(image=None):
|
|
"""Every distinct sparkle currently visible in one frame, as
|
|
`(x, y, score)` tuples sorted by descending score -- not just the
|
|
single best match `find_cafe_sparkle` used to return. `image`
|
|
optionally supplies an already-decoded frame instead of taking a fresh
|
|
screenshot -- for cafe.py's per-round capture, shared with that same
|
|
round's navigation.is_header_bar_visible rank-up check (see cafe.py's
|
|
_pat_current_view).
|
|
|
|
Finding every sparkle in one screenshot, instead of just the best one,
|
|
lets the pat loop click through all of them from a single capture
|
|
before re-screenshotting, rather than paying for a fresh scrot +
|
|
multi-scale template match per individual pat when several sparkles are
|
|
already visible at once. Replaces the original single-match
|
|
find_cafe_sparkle (2026-09-19, per explicit user direction) as part of
|
|
the same change that made _pat_current_view stop after two consecutive
|
|
empty screenshots instead of a fixed click-attempt budget -- see that
|
|
function's own docstring.
|
|
"""
|
|
template_full = cv2.imread(config.CAFE_SPARKLE_TEMPLATE)
|
|
img = image if image is not None else driver.read_screenshot(SPARKLE_SHOT_PATH)
|
|
th0, tw0 = template_full.shape[:2]
|
|
ox, oy = SPARKLE_CLICK_OFFSET
|
|
|
|
candidates = []
|
|
for scale in SPARKLE_SCALES:
|
|
tw, th = max(1, round(tw0 * scale)), max(1, round(th0 * scale))
|
|
template = cv2.resize(template_full, (tw, th))
|
|
mask = _masked_template(template)
|
|
|
|
result = cv2.matchTemplate(img, template, cv2.TM_CCORR_NORMED, mask=mask)
|
|
ys, xs = np.where(result >= SPARKLE_THRESHOLD)
|
|
for x, y in zip(xs, ys):
|
|
cx = int(x) + tw // 2 + round(ox * scale)
|
|
cy = int(y) + th // 2 + round(oy * scale)
|
|
candidates.append((float(result[y, x]), cx, cy))
|
|
|
|
return _dedupe_sparkles(candidates)
|
|
|
|
|
|
def _dedupe_sparkles(candidates):
|
|
"""Collapse candidates within SPARKLE_DEDUPE_DISTANCE of each other
|
|
(the same real sparkle matched at several nearby scales/positions) into
|
|
a single detection, keeping the highest-scoring one each time."""
|
|
candidates = sorted(candidates, key=lambda c: c[0], reverse=True)
|
|
kept = []
|
|
for score, x, y in candidates:
|
|
if any(
|
|
abs(x - kx) < SPARKLE_DEDUPE_DISTANCE and abs(y - ky) < SPARKLE_DEDUPE_DISTANCE
|
|
for _, kx, ky in kept
|
|
):
|
|
continue
|
|
kept.append((score, x, y))
|
|
return [(x, y, score) for score, x, y in kept]
|
|
|
|
|
|
def find_template(template_path, region=None, threshold=0.85):
|
|
"""Generic named-template match against a fresh screenshot, optionally
|
|
restricted to a screen sub-region `(x1, y1, x2, y2)`.
|
|
|
|
The local equivalent of the reference's core/picture.py::match_img_feature
|
|
-- used wherever the reference identifies a screen/state by comparing a
|
|
fixed on-screen crop to a known template image, rather than by OCR (OCR
|
|
reads stay on read_text/read_int; see CLAUDE.md's OCR policy for why
|
|
these two stay separate detection paths). Unlike find_cafe_sparkles, this
|
|
project's UI chrome (nav icons, modal titles, result banners) renders at
|
|
a fixed resolution with no camera zoom involved, so no multi-scale search
|
|
or color masking is needed here -- a single plain match is enough. Add a
|
|
scale sweep or mask back in for a specific template if live testing ever
|
|
finds one that needs it.
|
|
|
|
Returns the match's top-left `(x, y)` in full-screenshot coordinates, or
|
|
None if nothing scores >= threshold.
|
|
"""
|
|
img = driver.read_screenshot(OCR_SHOT_PATH)
|
|
if region is not None:
|
|
x1, y1, x2, y2 = region
|
|
img = img[y1:y2, x1:x2]
|
|
else:
|
|
x1, y1 = 0, 0
|
|
|
|
template = cv2.imread(template_path)
|
|
if template is None:
|
|
raise FileNotFoundError(f"Template image not found: {template_path}")
|
|
|
|
result = cv2.matchTemplate(img, template, cv2.TM_CCOEFF_NORMED)
|
|
_, max_val, _, max_loc = cv2.minMaxLoc(result)
|
|
if max_val < threshold:
|
|
return None
|
|
return (x1 + max_loc[0], y1 + max_loc[1])
|
|
|
|
|
|
def template_visible(template_path, region=None, threshold=0.85):
|
|
"""Boolean presence check built on find_template -- the common case for
|
|
screen-state detection (e.g. arena's title-banner checks via
|
|
navigation.wait_for_state's `ends`/`reactions` check_fn contract), where
|
|
only whether a known screen is showing matters, not exactly where."""
|
|
return find_template(template_path, region=region, threshold=threshold) is not None
|
|
|
|
|
|
def capture_screen():
|
|
"""One fresh, decoded screenshot -- for callers that need to read several
|
|
regions off the same frame instead of paying for a separate scrot
|
|
capture per region_contains_color()/read_int_on_heart_badge() call (see
|
|
lesson.py's _scan_open_grid_cells, which reads up to 27 cells per region
|
|
scan and, unbatched, was firing a fresh capture for nearly every one)."""
|
|
return driver.read_screenshot(OCR_SHOT_PATH)
|
|
|
|
|
|
def _color_mask(region, rgb_min, rgb_max, image=None):
|
|
x1, y1, x2, y2 = region
|
|
img = image if image is not None else driver.read_screenshot(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
b, g, r = crop[:, :, 0].astype(np.int16), crop[:, :, 1].astype(np.int16), crop[:, :, 2].astype(np.int16)
|
|
(r_lo, g_lo, b_lo), (r_hi, g_hi, b_hi) = rgb_min, rgb_max
|
|
return (r >= r_lo) & (r <= r_hi) & (g >= g_lo) & (g <= g_hi) & (b >= b_lo) & (b <= b_hi)
|
|
|
|
|
|
def region_contains_color(region, rgb_min, rgb_max, image=None):
|
|
"""Whether any pixel within `region` (x1, y1, x2, y2) falls in the given
|
|
RGB range. Useful for presence checks on small, non-convex glyphs (e.g.
|
|
an arrow chevron) where a single fixed-point probe can land in the
|
|
glyph's own concave gap -- confirmed live: a centroid-based single point
|
|
for story_sweep's region-arrow chevron fell squarely in the notch
|
|
between its two strokes, reading as "absent" even while the arrow was
|
|
clearly rendered a few pixels away. See plan.md Phase 10.
|
|
|
|
`image` optionally supplies an already-decoded frame (from
|
|
capture_screen()) instead of taking a fresh screenshot -- for batched
|
|
reads of several regions that are known not to change between them.
|
|
"""
|
|
return bool(_color_mask(region, rgb_min, rgb_max, image=image).any())
|
|
|
|
|
|
def find_color_centroid(region, rgb_min, rgb_max, min_pixels=1):
|
|
"""Centroid `(x, y)` of every pixel within `region` (x1, y1, x2, y2)
|
|
falling in the given RGB range, or None if fewer than `min_pixels`
|
|
match. Useful for clicking a known-colored button whose exact position
|
|
varies between otherwise-similar dialogs -- e.g. story_sweep's
|
|
sweep-result screen shows the same cyan confirm-button color for its
|
|
"SKIP" and final "OK" states, ~120px apart vertically; finding it by
|
|
color avoids hardcoding both positions. See plan.md Phase 10.
|
|
|
|
`min_pixels` defaults to 1 (any match counts, the original behavior) --
|
|
raise it when the region can also contain sparse, incidental matches of
|
|
the same color from unrelated content (e.g. bounty.py's result-button
|
|
region briefly overlapping a reward-icon's own artwork, confirmed live
|
|
to produce a few hundred stray matching pixels vs a real button's tens
|
|
of thousands -- see config.BOUNTY_RESULT_BUTTON_MIN_PIXELS).
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
mask = _color_mask(region, rgb_min, rgb_max)
|
|
ys, xs = np.nonzero(mask)
|
|
if len(xs) < min_pixels:
|
|
return None
|
|
return (x1 + int(xs.mean()), y1 + int(ys.mean()))
|
|
|
|
|
|
def _ocr_crop(region):
|
|
x1, y1, x2, y2 = region
|
|
img = driver.read_screenshot(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
|
# This UI's text is consistently dark-on-light -- a hard black/white
|
|
# threshold measurably fixed real misreads during live calibration
|
|
# (e.g. a stage label's digit misread as a stray extra digit) that
|
|
# persisted across every psm mode until the anti-aliased grey edges were
|
|
# removed. Confirmed live against scratchpad/scroll_up20.png row labels.
|
|
_, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
|
|
return cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
|
|
|
|
def read_text(region, whitelist=None, psm=7, lang="eng"):
|
|
"""OCR a pixel rectangle `(x1, y1, x2, y2)` from a fresh screenshot.
|
|
|
|
Local equivalent of the reference's core/ocr/ocr.py Baas_ocr client
|
|
(get_region_res) -- without its socket/shared-memory server, which exists
|
|
there to make OCR fast across thousands of automation steps. This project
|
|
only needs occasional single-crop reads (a region number, a stage label),
|
|
so a plain in-process pytesseract call is enough; see CLAUDE.md's "OCR
|
|
policy" and Handoff.md. `lang="eng"` is sufficient for the digit/dash
|
|
labels this project reads (region numbers, "30-1"/"30-A" stage labels) --
|
|
no Japanese trained data is needed for those specific reads.
|
|
"""
|
|
crop = _ocr_crop(region)
|
|
tess_config = f"--psm {psm}"
|
|
if whitelist:
|
|
tess_config += f" -c tessedit_char_whitelist={whitelist}"
|
|
return pytesseract.image_to_string(crop, lang=lang, config=tess_config).strip()
|
|
|
|
|
|
def read_int_white_on_dark(region, psm=7):
|
|
"""OCR a digit crop that renders bright text directly on a dark card
|
|
background (e.g. arena's "Lv.90" self/opponent-level labels) -- the
|
|
opposite polarity of this project's usual dark-text-on-light-UI
|
|
assumption baked into _ocr_crop's plain THRESH_BINARY. Confirmed live:
|
|
read_int returns None against these labels, since a plain threshold
|
|
there produces white glyphs on a black field and tesseract doesn't
|
|
segment that the way it does the normal case. An inverted threshold
|
|
fixes it outright -- no color-relationship masking needed here, unlike
|
|
read_int_on_heart_badge's badge-outline contamination problem.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
img = driver.read_screenshot(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
|
_, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY_INV)
|
|
upscaled = cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
|
|
text = pytesseract.image_to_string(upscaled, lang="eng", config=tess_config).strip()
|
|
digits = "".join(ch for ch in text if ch.isdigit())
|
|
return int(digits) if digits else None
|
|
|
|
|
|
def read_text_white_on_dark(region, whitelist=None, psm=7, lang="eng", threshold=150):
|
|
"""General-text equivalent of read_int_white_on_dark, for light-on-dark
|
|
UI outside this project's own game client -- e.g. a genuine Steam
|
|
client dialog (confirmed live 2026-09-20: three consecutive real cron
|
|
failures traced back to Steam itself being signed out, sitting on its
|
|
own account sign-in screen instead of ever launching the game --
|
|
login.py's `_steam_needs_sign_in` uses this to recognize that specific
|
|
state and give a diagnostic message that says so, instead of a generic
|
|
"game window never appeared").
|
|
|
|
`threshold` defaults to the same 150 read_int_white_on_dark already
|
|
uses, but is adjustable here since not every light-on-dark UI is
|
|
equally bright -- Steam's own "SIGN IN WITH ACCOUNT NAME" label topped
|
|
out at grayscale 126 in a live capture, well under 150, and needed a
|
|
lower threshold (confirmed live: 70-100 all read it cleanly) to
|
|
binarize into anything at all rather than a blank crop.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
img = driver.read_screenshot(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
|
_, thresh = cv2.threshold(gray, threshold, 255, cv2.THRESH_BINARY_INV)
|
|
upscaled = cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
tess_config = f"--psm {psm}"
|
|
if whitelist:
|
|
tess_config += f" -c tessedit_char_whitelist={whitelist}"
|
|
return pytesseract.image_to_string(upscaled, lang=lang, config=tess_config).strip()
|
|
|
|
|
|
def read_int(region, psm=7):
|
|
"""Local equivalent of the reference's recognize_int.
|
|
|
|
Whitelist includes ',' even though the return value is comma-stripped
|
|
digits only -- a real live misread (2026-09-24) found a digit-only
|
|
whitelist forces tesseract to misclassify a genuine thousands-separator
|
|
comma as a phantom digit instead of punctuation it's simply not allowed
|
|
to output, corrupting the whole read (confirmed live: Common Shop's
|
|
'中級レポート' at 125,000 read as 1250060/195900/4925000 depending on
|
|
the exact frame, never the correct value, across a digit-only whitelist
|
|
and multiple psm modes) -- allowing ',' through lets tesseract correctly
|
|
segment it as punctuation, and the existing isdigit() filter here already
|
|
strips it either way, so this is a strict improvement with no behavior
|
|
change for any comma-free read (region numbers, ticket counts, etc.).
|
|
This same whitelist also covers config.CREDIT_BALANCE_OCR_RECT reads
|
|
(routinely multi-comma, e.g. "412,271,037"), used by
|
|
shop_utils.confirm_purchase's balance-delta verification -- a likely
|
|
contributor to that function's own intermittent "purchase confirmation
|
|
did not resolve" false negatives, though not independently confirmed.
|
|
"""
|
|
digits = "".join(ch for ch in read_text(region, whitelist="0123456789,", psm=psm) if ch.isdigit())
|
|
return int(digits) if digits else None
|
|
|
|
|
|
def read_int_bordered(region, psm=7, border=20):
|
|
"""read_int, but with a white margin added around the upscaled crop
|
|
before OCR.
|
|
|
|
event_sweep.py's stage-row numbers "08"/"09" (a leading-zero digit
|
|
tight against the following one) were misread by plain read_int on
|
|
every psm mode tried -- "08" consistently as "2", "09" as empty or "9"
|
|
with the leading zero silently dropped -- even though the crop and its
|
|
thresholded/upscaled version both look completely clean to the eye.
|
|
"10"/"11"/"12" at the same crop dimensions read fine. Confirmed live
|
|
(scratchpad/probe_ocr_fix_08_09.py) this is specifically a tesseract
|
|
segmentation problem with a text blob that fills the crop edge-to-edge,
|
|
with no surrounding whitespace context to anchor character boundaries
|
|
-- adding a plain white cv2.copyMakeBorder margin (no other pipeline
|
|
change) fixed both "08" and "09" to their exact correct digits at every
|
|
psm mode tried, and left the already-working "10" unaffected. Kept as
|
|
its own function rather than changed in read_int/_ocr_crop, since only
|
|
this one tight edge-to-edge crop shape has ever been confirmed to need
|
|
it -- every other OCR read in this project already has enough natural
|
|
margin.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
img = driver.read_screenshot(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
|
_, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
|
|
upscaled = cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
bordered = cv2.copyMakeBorder(upscaled, border, border, border, border, cv2.BORDER_CONSTANT, value=255)
|
|
tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
|
|
text = pytesseract.image_to_string(bordered, lang="eng", config=tess_config).strip()
|
|
digits = "".join(ch for ch in text if ch.isdigit())
|
|
return int(digits) if digits else None
|
|
|
|
|
|
def read_lines(region, lang="jpn", psm=6, image=None):
|
|
"""OCR `region` and return `(x, y, text)` for every distinct text line
|
|
found in it -- `(x, y)` is that line's on-screen center (for clicking
|
|
it) and `text` is the line's full content, re-joined left-to-right from
|
|
tesseract's own per-token boxes (CJK text gets segmented into several
|
|
word/glyph tokens even on one visual line).
|
|
|
|
Unlike read_text/read_int (a single known-fixed crop, content only, no
|
|
position), this scans a region that may contain several stacked lines
|
|
at once and reports WHERE each one actually rendered -- for a caller
|
|
that needs to dynamically locate and click a label/button whose exact
|
|
position isn't a reliable fixed coordinate, e.g. a sidebar list that
|
|
reflows when the game adds a new entry above an existing one. Built for
|
|
gem_shop.py after a real live failure where a fixed sidebar coordinate,
|
|
aimed at 一般商品 (general products), silently started landing on
|
|
定額商品 instead once the game added a new entry above it -- see
|
|
gem_shop.py's module docstring and config.py's "RECALIBRATED AGAIN
|
|
2026-09-17" note for the full incident.
|
|
|
|
Only confirmed reliable for vertically-stacked lines rendered in this
|
|
UI's usual dark-text-on-light styling (same assumption `_ocr_crop`
|
|
already makes elsewhere) -- confirmed live NOT to segment a horizontal
|
|
row of side-by-side labels (e.g. gem shop's own
|
|
デイリー/ウィークリー/マンスリー sub-tabs) reliably in one call; OCR
|
|
each column as its own small region instead for that shape. Also not
|
|
reliable against a selected/highlighted item's own inverted (light-
|
|
text-on-dark-pill) styling -- confirmed live to misread rather than
|
|
just fail quietly, so prefer checking for a different, still-unselected
|
|
marker over re-reading a label the caller itself just selected.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
img = image if image is not None else driver.read_screenshot(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
gray = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
|
|
_, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
|
|
upscaled = cv2.resize(thresh, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
|
|
tess_config = f"--psm {psm}"
|
|
data = pytesseract.image_to_data(upscaled, lang=lang, config=tess_config, output_type=pytesseract.Output.DICT)
|
|
|
|
lines = {}
|
|
for i, word in enumerate(data["text"]):
|
|
word = word.strip()
|
|
if not word:
|
|
continue
|
|
key = (data["block_num"][i], data["par_num"][i], data["line_num"][i])
|
|
lines.setdefault(key, []).append((data["left"][i], data["top"][i], data["width"][i], data["height"][i], word))
|
|
|
|
results = []
|
|
for tokens in lines.values():
|
|
tokens.sort(key=lambda t: t[0])
|
|
joined = "".join(t[4] for t in tokens)
|
|
lefts = [t[0] for t in tokens]
|
|
rights = [t[0] + t[2] for t in tokens]
|
|
tops = [t[1] for t in tokens]
|
|
bottoms = [t[1] + t[3] for t in tokens]
|
|
cx = (min(lefts) + max(rights)) / 2 / OCR_UPSCALE
|
|
cy = (min(tops) + max(bottoms)) / 2 / OCR_UPSCALE
|
|
results.append((x1 + round(cx), y1 + round(cy), joined))
|
|
return results
|
|
|
|
|
|
def read_int_on_heart_badge(region, psm=7, image=None):
|
|
"""OCR a small dark-navy digit rendered on lesson.py's pink/magenta
|
|
heart-shaped affection badge.
|
|
|
|
read_int's plain grayscale threshold (tuned for this UI's normal
|
|
dark-text-on-light-card look) misreads these: the heart's own outline
|
|
stroke is a darker, saturated magenta whose grayscale value happens to
|
|
fall on the same side of the threshold as the digit glyph, so it
|
|
survives as stray black marks tesseract sometimes fuses into extra
|
|
digits -- confirmed live, "13" -> "113", "19" -> "119". The digit color
|
|
is reliably R < G (navy/blue-grey) while every pink/magenta badge tone
|
|
sampled (fill and outline, light and dark) is R > G, so masking on that
|
|
channel relationship instead of raw brightness cleanly drops the badge
|
|
shape and keeps just the glyph.
|
|
|
|
`image` optionally supplies an already-decoded frame (from
|
|
capture_screen()) instead of taking a fresh screenshot -- see
|
|
region_contains_color's own `image` param for why.
|
|
"""
|
|
x1, y1, x2, y2 = region
|
|
img = image if image is not None else driver.read_screenshot(OCR_SHOT_PATH)
|
|
crop = img[y1:y2, x1:x2]
|
|
b, g, r = crop[:, :, 0].astype(np.int16), crop[:, :, 1].astype(np.int16), crop[:, :, 2].astype(np.int16)
|
|
ink = (r < g) & (np.maximum(np.maximum(r, g), b) < 170)
|
|
binary = np.where(ink, 0, 255).astype(np.uint8)
|
|
upscaled = cv2.resize(binary, None, fx=OCR_UPSCALE, fy=OCR_UPSCALE, interpolation=cv2.INTER_CUBIC)
|
|
tess_config = f"--psm {psm} -c tessedit_char_whitelist=0123456789"
|
|
text = pytesseract.image_to_string(upscaled, lang="eng", config=tess_config).strip()
|
|
digits = "".join(ch for ch in text if ch.isdigit())
|
|
return int(digits) if digits else None
|