#!/usr/bin/env python3 """The controls on the box's desktop, from its accessibility tree (AT-SPI), as JSON: {"elements": [{"role", "name", "value", "x", "y", "w", "h", "active", "flags", "window"}]} in screen pixels. Web pages are in it too (Chromium runs with renderer accessibility on). AnyBot numbers them on the bot's picture; a click by number lands on the centre. Adapted from XBot's box/box-a11y (which skips anything not "showing"; Chromium can say that of its whole window). Usage: a11y [--limit N] [--budget SECONDS] Exit 2 when AT-SPI itself is unreachable. """ import json import os import sys import time import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) # Clicked or typed into (they get a number). ACTIVE = { "push button", "toggle button", "radio button", "check box", "link", "menu item", "check menu item", "radio menu item", "tab", "page tab", "list item", "tree item", "combo box", "spin button", "slider", "entry", "password text", "menu", } # Listed when they carry words (headings, labels, cells...). NAMED = {"heading", "label", "static", "text", "image", "table cell", "column header", "row header", "icon", "tool tip", "alert", "dialog", "paragraph"} # The desktop's own parts are visible in the picture; the list is for the windows worked in. SKIP_APPS = {"fluxbox", "a11y", "python3", "xterm-wrapper"} def main(): limit, budget = 400, 3.0 args = sys.argv[1:] for i, a in enumerate(args): if a == "--limit": limit = int(args[i + 1]) if a == "--budget": budget = float(args[i + 1]) if not os.environ.get("DBUS_SESSION_BUS_ADDRESS"): try: for line in open("/tmp/anybot-dbus.env", encoding="utf-8"): k, _, v = line.strip().partition("=") if k == "DBUS_SESSION_BUS_ADDRESS" and v: os.environ[k] = v except OSError: pass os.environ.setdefault("DISPLAY", ":1") try: import gi gi.require_version("Atspi", "2.0") from gi.repository import Atspi Atspi.init() desktop = Atspi.get_desktop(0) apps = desktop.get_child_count() except Exception as exc: # noqa: BLE001 print(f"AT-SPI unreachable: {exc}", file=sys.stderr) return 2 sw, sh = 0, 0 try: geo = os.popen("xdotool getdisplaygeometry 2>/dev/null").read().split() sw, sh = int(geo[0]), int(geo[1]) except Exception: # noqa: BLE001 pass deadline = time.monotonic() + budget out = [] def state(node): try: return node.get_state_set() except Exception: # noqa: BLE001 return None def extents(node): try: r = node.get_component_iface().get_extents(Atspi.CoordType.SCREEN) return int(r.x), int(r.y), int(r.width), int(r.height) except Exception: # noqa: BLE001 return None def onscreen(b): x, y, w, h = b if w <= 0 or h <= 0 or x + w <= 0 or y + h <= 0: return False return not (sw and x >= sw) and not (sh and y >= sh) def walk(node, depth, window): if time.monotonic() > deadline or len(out) >= limit or depth > 45: return try: role = node.get_role_name() except Exception: # noqa: BLE001 return st = state(node) showing = bool(st and st.contains(Atspi.StateType.SHOWING)) visible = bool(st and st.contains(Atspi.StateType.VISIBLE)) # A page that isn't showing is a tab in the background: none of it is on the screen. if role in ("document web", "document frame") and not showing: return # Otherwise "showing" isn't trusted: Chromium can call its whole window not showing (it # took itself for minimized under this window manager) while its page is on the screen. # Visible with a place on the screen is what counts; containers are walked through. box = extents(node) if depth > 1 and (not visible or box is None or not onscreen(box)): return try: name = (node.get_name() or "").strip() except Exception: # noqa: BLE001 name = "" if role in ("frame", "dialog", "window") and name: window = name editable = bool(st and st.contains(Atspi.StateType.EDITABLE)) value = "" if (role in ("entry", "password text", "text", "spin button", "slider") or editable) and role != "password text": try: t = node.get_text_iface() if t is not None: value = (t.get_text(0, min(t.get_character_count(), 200)) or "").strip() except Exception: # noqa: BLE001 pass active = role in ACTIVE or (editable and role in ("text", "paragraph", "section", "document web")) if depth > 1 and box is not None and (active or (role in NAMED and (name or value))): flags = [] if st is not None: for s, word in ((Atspi.StateType.FOCUSED, "focused"), (Atspi.StateType.CHECKED, "checked"), (Atspi.StateType.SELECTED, "selected")): if st.contains(s): flags.append(word) if not st.contains(Atspi.StateType.ENABLED): flags.append("disabled") x, y, w, h = box out.append({"role": role, "name": name[:120], "value": value[:200], "x": x, "y": y, "w": w, "h": h, "active": active, "flags": flags, "window": window}) try: n = node.get_child_count() except Exception: # noqa: BLE001 return for i in range(min(n, 2000)): try: c = node.get_child_at_index(i) except Exception: # noqa: BLE001 continue if c is not None: walk(c, depth + 1, window) for i in range(apps): try: app = desktop.get_child_at_index(i) name = app.get_name() or "" except Exception: # noqa: BLE001 continue if name in SKIP_APPS: continue walk(app, 0, name) json.dump({"elements": out, "truncated": len(out) >= limit or time.monotonic() > deadline}, sys.stdout, ensure_ascii=False) return 0 if __name__ == "__main__": sys.exit(main())