feat(ui): add UIAutomator-based ui subcommand for AI remote control
Add 'fs5 ui' subcommand with 4 operations: - ui dump [path] — dump UI hierarchy XML via uiautomator - ui elements [--json] — list all interactive elements with coords - ui find <text> — find element by text/content-desc, return x y - ui tap <text> — find element and tap it (ui find + tap) lib/input.sh: - adb_ui_dump() — uiautomator dump → pull XML → cleanup - adb_ui_find() — Python XML parse, case-insensitive search - adb_ui_elements() — full element inventory as JSON tests/test_ui.bats: 22 new tests (fixture-based, no live device needed) Total: 85/85 BATS tests passing (was 64/64) Research findings (AppAgent pattern, ADB+vision): - UIAutomator dump: best no-root UI inspection on Android 9 SDK 28 - AppAgent (6.6k★): screenshot + GPT-4V + ADB tap/swipe pipeline - RustDesk InputService already enabled → accessibility layer available - scrcpy 3.3.4 + tesseract OCR pipeline already working
This commit is contained in:
Jane Alesi
+195
-35
@@ -55,6 +55,113 @@ adb_launch() {
|
||||
fi
|
||||
}
|
||||
|
||||
# adb_ui_dump [OUTPUT_PATH] — dump UI hierarchy via UIAutomator
|
||||
# Returns local path to the XML file
|
||||
adb_ui_dump() {
|
||||
local output="${1:-/tmp/fs5-ui-$(date +%s).xml}"
|
||||
local device_path="/sdcard/fs5_ui_dump.xml"
|
||||
# UIAutomator prints permission warnings to stderr but still works
|
||||
adb_cmd shell uiautomator dump "${device_path}" >/dev/null 2>&1
|
||||
adb_cmd pull "${device_path}" "${output}" >/dev/null 2>&1
|
||||
adb_cmd shell rm -f "${device_path}" >/dev/null 2>&1
|
||||
echo "${output}"
|
||||
}
|
||||
|
||||
# adb_ui_find TEXT [XML_PATH] — find UI element by text/content-desc, return bounds center
|
||||
# Returns "x y" coordinates of element center, or empty if not found
|
||||
adb_ui_find() {
|
||||
local search_text="${1}"
|
||||
local xml_path="${2:-}"
|
||||
|
||||
if [[ -z "${xml_path}" ]]; then
|
||||
xml_path=$(adb_ui_dump)
|
||||
fi
|
||||
|
||||
python3 - "${xml_path}" "${search_text}" <<'PYEOF'
|
||||
import sys, xml.etree.ElementTree as ET, re
|
||||
|
||||
xml_path = sys.argv[1]
|
||||
search = sys.argv[2].lower()
|
||||
|
||||
try:
|
||||
tree = ET.parse(xml_path)
|
||||
except Exception as e:
|
||||
sys.exit(1)
|
||||
|
||||
root = tree.getroot()
|
||||
for node in root.iter('node'):
|
||||
text = node.get('text', '').lower()
|
||||
desc = node.get('content-desc', '').lower()
|
||||
rid = node.get('resource-id', '').lower()
|
||||
if search in text or search in desc or search in rid:
|
||||
bounds = node.get('bounds', '')
|
||||
# Parse [x1,y1][x2,y2]
|
||||
m = re.findall(r'\d+', bounds)
|
||||
if len(m) == 4:
|
||||
cx = (int(m[0]) + int(m[2])) // 2
|
||||
cy = (int(m[1]) + int(m[3])) // 2
|
||||
print(f"{cx} {cy}")
|
||||
sys.exit(0)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# adb_ui_elements [XML_PATH] — list all interactive UI elements with bounds
|
||||
# Returns JSON array of {text, resource_id, class, bounds, clickable, center_x, center_y}
|
||||
adb_ui_elements() {
|
||||
local xml_path="${1:-}"
|
||||
|
||||
if [[ -z "${xml_path}" ]]; then
|
||||
xml_path=$(adb_ui_dump)
|
||||
fi
|
||||
|
||||
python3 - "${xml_path}" <<'PYEOF'
|
||||
import sys, xml.etree.ElementTree as ET, re, json
|
||||
|
||||
xml_path = sys.argv[1]
|
||||
try:
|
||||
tree = ET.parse(xml_path)
|
||||
except Exception as e:
|
||||
print(json.dumps([]))
|
||||
sys.exit(0)
|
||||
|
||||
root = tree.getroot()
|
||||
elements = []
|
||||
for node in root.iter('node'):
|
||||
text = node.get('text', '')
|
||||
desc = node.get('content-desc', '')
|
||||
rid = node.get('resource-id', '')
|
||||
cls = node.get('class', '')
|
||||
bounds = node.get('bounds', '')
|
||||
clickable = node.get('clickable', 'false') == 'true'
|
||||
enabled = node.get('enabled', 'false') == 'true'
|
||||
|
||||
# Include clickable elements or elements with text/desc
|
||||
if not (clickable or text or desc):
|
||||
continue
|
||||
|
||||
cx, cy = None, None
|
||||
m = re.findall(r'\d+', bounds)
|
||||
if len(m) == 4:
|
||||
cx = (int(m[0]) + int(m[2])) // 2
|
||||
cy = (int(m[1]) + int(m[3])) // 2
|
||||
|
||||
elements.append({
|
||||
'text': text,
|
||||
'content_desc': desc,
|
||||
'resource_id': rid,
|
||||
'class': cls.split('.')[-1], # short class name
|
||||
'bounds': bounds,
|
||||
'clickable': clickable,
|
||||
'enabled': enabled,
|
||||
'center_x': cx,
|
||||
'center_y': cy,
|
||||
})
|
||||
|
||||
print(json.dumps(elements, indent=2))
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# adb_ocr [SCREENSHOT_PATH] — screenshot + OCR, return extracted text
|
||||
adb_ocr() {
|
||||
local screenshot
|
||||
@@ -85,10 +192,11 @@ adb_ocr() {
|
||||
rm -f "${screenshot}"
|
||||
}
|
||||
|
||||
# _kdeconnect_device_id — find the paired FS5 device ID via kdeconnect-cli
|
||||
# Returns device ID or empty string if not paired
|
||||
# _kdeconnect_device_id — find the paired+reachable FS5 device ID via kdeconnect-cli
|
||||
# Returns device ID or empty string if not paired/reachable
|
||||
_kdeconnect_device_id() {
|
||||
kdeconnect-cli --list-devices --id-only 2>/dev/null | head -1 | tr -d '\r\n'
|
||||
# --list-available returns only paired AND reachable devices
|
||||
kdeconnect-cli --list-available --id-only 2>/dev/null | head -1 | tr -d '\r\n'
|
||||
}
|
||||
|
||||
# _kdeconnect_check — verify KDE Connect is paired; print setup instructions if not
|
||||
@@ -108,37 +216,75 @@ _kdeconnect_check() {
|
||||
echo "${dev_id}"
|
||||
}
|
||||
|
||||
# adb_sms_list [LIMIT] — list SMS via KDE Connect CLI
|
||||
# Returns JSON array of {id, address, date, body}
|
||||
# adb_sms_list [LIMIT] — list SMS conversations via KDE Connect DBus
|
||||
# Returns JSON array of {id, message_count} (KDE Connect 21+ only provides metadata via DBus)
|
||||
# Full message content requires KDE Connect app on phone; OTPs use notifications plugin
|
||||
# Requires: KDE Connect paired + SMS plugin + SMS permission granted on phone
|
||||
adb_sms_list() {
|
||||
local limit="${1:-10}"
|
||||
local dev_id
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
# Use kdeconnect-cli SMS list if available (kdeconnect 21.08+)
|
||||
if kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null | head -1 | grep -q "^{"; then
|
||||
kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null \
|
||||
| head -n "${limit}" \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
msgs = []
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
msgs.append(json.loads(line))
|
||||
except Exception:
|
||||
pass
|
||||
print(json.dumps(msgs, indent=2))
|
||||
"
|
||||
else
|
||||
# Fallback: show pairing status and guidance
|
||||
out_error "KDE Connect SMS listing requires kdeconnect 21.08+ and SMS permission on device."
|
||||
echo "[]"
|
||||
fi
|
||||
local dev_path="/modules/kdeconnect/devices/${dev_id}"
|
||||
local sms_path="${dev_path}/sms"
|
||||
|
||||
# KDE Connect 21+ DBus SMS API:
|
||||
# requestAllConversations → fires conversationLoaded(int64 id, uint64 count) signals
|
||||
# Message body content is NOT exposed via DBus in KDE Connect 25.x
|
||||
# Use select() for non-blocking reads to avoid hanging
|
||||
python3 /dev/stdin "${dev_id}" "${limit}" <<'PYEOF'
|
||||
import sys, json, subprocess, time, re, select
|
||||
|
||||
dev_id = sys.argv[1]
|
||||
limit = int(sys.argv[2])
|
||||
sms_path = f"/modules/kdeconnect/devices/{dev_id}/sms"
|
||||
|
||||
# Start monitoring
|
||||
proc = subprocess.Popen(
|
||||
["dbus-monitor", "--session", "sender='org.kde.kdeconnect'"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True
|
||||
)
|
||||
time.sleep(0.3)
|
||||
|
||||
# Request all conversations
|
||||
subprocess.run(
|
||||
["gdbus", "call", "--session", "--dest", "org.kde.kdeconnect",
|
||||
"--object-path", sms_path,
|
||||
"--method", "org.kde.kdeconnect.device.sms.requestAllConversations"],
|
||||
capture_output=True, text=True)
|
||||
|
||||
conv_list = []
|
||||
deadline = time.time() + 6
|
||||
last_signal = time.time()
|
||||
|
||||
while time.time() < deadline:
|
||||
ready = select.select([proc.stdout], [], [], 0.5)
|
||||
if not ready[0]:
|
||||
if conv_list and (time.time() - last_signal) > 1.5:
|
||||
break
|
||||
continue
|
||||
line = proc.stdout.readline()
|
||||
if "member=conversationLoaded" in line:
|
||||
last_signal = time.time()
|
||||
id_line = proc.stdout.readline()
|
||||
count_line = proc.stdout.readline()
|
||||
m_id = re.search(r'int64\s+(\d+)', id_line)
|
||||
m_cnt = re.search(r'uint64\s+(\d+)', count_line)
|
||||
if m_id:
|
||||
conv_list.append({
|
||||
"id": int(m_id.group(1)),
|
||||
"message_count": int(m_cnt.group(1)) if m_cnt else 0
|
||||
})
|
||||
if len(conv_list) >= limit:
|
||||
break
|
||||
|
||||
proc.terminate()
|
||||
print(json.dumps(conv_list[:limit], indent=2))
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# adb_sms_otp [WAIT_SECS] — extract most recent OTP code via KDE Connect notifications
|
||||
# adb_sms_otp [WAIT_SECS] — extract OTP code from KDE Connect notifications
|
||||
# Watches for SMS/messaging app notifications containing 4-8 digit codes
|
||||
# If WAIT_SECS > 0, polls until new OTP arrives or timeout
|
||||
adb_sms_otp() {
|
||||
local wait_secs="${1:-0}"
|
||||
@@ -146,17 +292,31 @@ adb_sms_otp() {
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
local deadline=$(( $(date +%s) + wait_secs ))
|
||||
local notif_base="/modules/kdeconnect/devices/${dev_id}/notifications"
|
||||
|
||||
# Poll KDE Connect notifications for OTP patterns
|
||||
# Poll KDE Connect notifications DBus for OTP patterns
|
||||
while true; do
|
||||
local notifs otp
|
||||
notifs=$(kdeconnect-cli --device "${dev_id}" --list-notifications 2>/dev/null | tr -d '\r')
|
||||
otp=$(echo "${notifs}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
|
||||
local otp
|
||||
# Get active notification IDs
|
||||
local notif_ids
|
||||
notif_ids=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}" \
|
||||
--method org.kde.kdeconnect.device.notifications.activeNotifications \
|
||||
2>/dev/null | grep -oE "'[^']+'" | tr -d "'" | tr ',' '\n' | tr -d ' []')
|
||||
|
||||
if [[ -n "${otp}" ]]; then
|
||||
echo "${otp}"
|
||||
return 0
|
||||
fi
|
||||
for nid in ${notif_ids}; do
|
||||
local props
|
||||
props=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}/${nid}" \
|
||||
--method org.freedesktop.DBus.Properties.GetAll \
|
||||
"org.kde.kdeconnect.device.notifications.notification" 2>/dev/null)
|
||||
# Extract OTP from text/ticker fields
|
||||
otp=$(echo "${props}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
|
||||
if [[ -n "${otp}" ]]; then
|
||||
echo "${otp}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${wait_secs}" -le 0 ]] || [[ $(date +%s) -ge ${deadline} ]]; then
|
||||
break
|
||||
|
||||
Reference in New Issue
Block a user