feat(ui): add UIAutomator-based ui subcommand for AI remote control

Add 'fs5 ui' subcommand with 4 operations:
- ui dump [path]      — dump UI hierarchy XML via uiautomator
- ui elements [--json] — list all interactive elements with coords
- ui find <text>      — find element by text/content-desc, return x y
- ui tap <text>       — find element and tap it (ui find + tap)

lib/input.sh:
- adb_ui_dump()     — uiautomator dump → pull XML → cleanup
- adb_ui_find()     — Python XML parse, case-insensitive search
- adb_ui_elements() — full element inventory as JSON

tests/test_ui.bats: 22 new tests (fixture-based, no live device needed)

Total: 85/85 BATS tests passing (was 64/64)

Research findings (AppAgent pattern, ADB+vision):
- UIAutomator dump: best no-root UI inspection on Android 9 SDK 28
- AppAgent (6.6k★): screenshot + GPT-4V + ADB tap/swipe pipeline
- RustDesk InputService already enabled → accessibility layer available
- scrcpy 3.3.4 + tesseract OCR pipeline already working
This commit is contained in:
Jane Alesi
2026-03-02 18:13:44 +01:00
parent aaf008c1bf
commit 9cb6bfb714
3 changed files with 643 additions and 59 deletions
+254 -26
View File
@@ -65,6 +65,10 @@ Remote Control (AI capabilities):
key <keycode> Send Android key event key <keycode> Send Android key event
launch <package> Launch app by package name launch <package> Launch app by package name
ocr Screenshot + extract all text via OCR ocr Screenshot + extract all text via OCR
ui dump [path] Dump UI hierarchy XML (UIAutomator)
ui elements List all interactive UI elements
ui find <text> Find element by text, print coordinates
ui tap <text> Find element by text and tap it
sms list List recent SMS messages sms list List recent SMS messages
sms otp Extract latest OTP code from SMS sms otp Extract latest OTP code from SMS
sms watch Watch for new SMS messages sms watch Watch for new SMS messages
@@ -665,6 +669,225 @@ cmd_launch() {
fi fi
} }
# ── Command: ui ───────────────────────────────────────────────────────────────
cmd_ui() {
local subcmd="${1:-}"
shift || true
case "${subcmd}" in
dump) cmd_ui_dump "$@" ;;
elements) cmd_ui_elements "$@" ;;
find) cmd_ui_find "$@" ;;
tap) cmd_ui_tap "$@" ;;
--help|-h)
cat <<EOF
Usage: fs5 ui <subcommand> [options]
Inspect and interact with the device UI via UIAutomator.
Subcommands:
dump [path] Dump UI hierarchy XML to file
elements [--json] List all interactive UI elements
find <text> Find element by text/desc, print coordinates
tap <text> Find element by text/desc and tap it
EOF
;;
*)
out_error "Unknown ui subcommand: '${subcmd}'. Use: dump, elements, find, tap"
return 1
;;
esac
}
cmd_ui_dump() {
local output="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) [[ -z "${output}" ]] && output="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 ui dump [output-path.xml]
Dump the current UI hierarchy to an XML file via UIAutomator.
If no path given, saves to /tmp/fs5-ui-<timestamp>.xml.
EOF
return 0
fi
adb_check_device
local path
if [[ -n "${output}" ]]; then
path=$(adb_ui_dump "${output}")
else
path=$(adb_ui_dump)
fi
if [[ "${JSON_OUTPUT}" == "true" ]]; then
out_json "path=${path}" "timestamp=$(date -Iseconds)"
else
out_success "UI dump saved: ${path}"
echo "${path}"
fi
}
cmd_ui_elements() {
local xml_path="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
--json) JSON_OUTPUT=true ;;
*) [[ -z "${xml_path}" ]] && xml_path="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 ui elements [xml-path] [--json]
List all interactive UI elements on the current screen.
Optionally provide a previously dumped XML file.
EOF
return 0
fi
adb_check_device
local json_out
if [[ -n "${xml_path}" ]]; then
json_out=$(adb_ui_elements "${xml_path}")
else
json_out=$(adb_ui_elements)
fi
if [[ "${JSON_OUTPUT}" == "true" ]]; then
echo "${json_out}"
else
# Human-readable table
echo "${json_out}" | python3 -c "
import sys, json
elements = json.load(sys.stdin)
print(f'Found {len(elements)} elements:')
for i, e in enumerate(elements):
label = e['text'] or e['content_desc'] or e['resource_id'] or e['class']
click = '✓' if e['clickable'] else ' '
cx, cy = e.get('center_x'), e.get('center_y')
coord = f'({cx},{cy})' if cx is not None else '(?)'
print(f' [{click}] {coord:>12} {label}')
"
fi
}
cmd_ui_find() {
local search_text="" xml_path="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*)
if [[ -z "${search_text}" ]]; then search_text="${arg}"
elif [[ -z "${xml_path}" ]]; then xml_path="${arg}"
fi
;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 ui find <text> [xml-path]
Find a UI element by text or content-desc and print its center coordinates.
Returns "x y" on success, exits 1 if not found.
EOF
return 0
fi
if [[ -z "${search_text}" ]]; then
out_error "Usage: fs5 ui find <text>"
return 1
fi
adb_check_device
local coords
if [[ -n "${xml_path}" ]]; then
coords=$(adb_ui_find "${search_text}" "${xml_path}")
else
coords=$(adb_ui_find "${search_text}")
fi
if [[ -z "${coords}" ]]; then
out_error "Element not found: '${search_text}'"
return 1
fi
if [[ "${JSON_OUTPUT}" == "true" ]]; then
local x y
x="${coords%% *}"
y="${coords##* }"
out_json "text=${search_text}" "x=${x}" "y=${y}"
else
echo "${coords}"
fi
}
cmd_ui_tap() {
local search_text="" xml_path="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*)
if [[ -z "${search_text}" ]]; then search_text="${arg}"
elif [[ -z "${xml_path}" ]]; then xml_path="${arg}"
fi
;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 ui tap <text> [xml-path]
Find a UI element by text or content-desc and tap its center.
Combines ui find + tap in one step.
EOF
return 0
fi
if [[ -z "${search_text}" ]]; then
out_error "Usage: fs5 ui tap <text>"
return 1
fi
adb_check_device
local coords
if [[ -n "${xml_path}" ]]; then
coords=$(adb_ui_find "${search_text}" "${xml_path}")
else
coords=$(adb_ui_find "${search_text}")
fi
if [[ -z "${coords}" ]]; then
out_error "Element not found: '${search_text}'"
return 1
fi
local x y
x="${coords%% *}"
y="${coords##* }"
adb_tap "${x}" "${y}"
if [[ "${JSON_OUTPUT}" == "true" ]]; then
out_json "text=${search_text}" "x=${x}" "y=${y}" "action=tap"
else
out_success "Tapped '${search_text}' at (${x}, ${y})"
fi
}
# ── Command: ocr ────────────────────────────────────────────────────────────── # ── Command: ocr ──────────────────────────────────────────────────────────────
cmd_ocr() { cmd_ocr() {
local show_help=false local show_help=false
@@ -775,40 +998,44 @@ cmd_sms_watch() {
if [[ "${show_help}" == "true" ]]; then if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 sms watch" echo "Usage: fs5 sms watch"
echo "Poll for new SMS messages every 5 seconds. Press Ctrl+C to stop." echo "Watch for new SMS/notification messages via KDE Connect. Press Ctrl+C to stop."
return 0 return 0
fi fi
adb_check_device local dev_id
dev_id=$(_kdeconnect_check) || return 1
out_success "Watching for SMS (Ctrl+C to stop)..." out_success "Watching for SMS via KDE Connect notifications (Ctrl+C to stop)..."
local last_id="" local notif_base="/modules/kdeconnect/devices/${dev_id}/notifications"
last_id=$(adb_cmd shell content query \ local seen_ids=""
--uri content://sms/inbox \
--projection "_id" \
--sort "date DESC" \
2>/dev/null | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2 | tr -d '\r')
while true; do while true; do
local raw new_id local notif_ids
raw=$(adb_cmd shell content query \ notif_ids=$(gdbus call --session --dest org.kde.kdeconnect \
--uri content://sms/inbox \ --object-path "${notif_base}" \
--projection "_id,address,date,body" \ --method org.kde.kdeconnect.device.notifications.activeNotifications \
--sort "date DESC" \ 2>/dev/null | grep -oE "'[^']+'" | tr -d "'" | tr ',' '\n' | tr -d ' []')
2>/dev/null | tr -d '\r' | head -10)
new_id=$(echo "${raw}" | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2) for nid in ${notif_ids}; do
# Skip already-seen notifications
if [[ -n "${new_id}" ]] && [[ "${new_id}" != "${last_id}" ]]; then if echo "${seen_ids}" | grep -qw "${nid}"; then
local ts body address continue
ts=$(date -Iseconds)
address=$(echo "${raw}" | grep "address=" | head -1 | grep -oE 'address=[^,]+' | cut -d= -f2)
body=$(echo "${raw}" | grep "body=" | head -1 | sed 's/.*body=//')
echo "[${ts}] FROM: ${address}"
echo " ${body}"
echo ""
last_id="${new_id}"
fi fi
seen_ids="${seen_ids} ${nid}"
local props app title text ts
props=$(gdbus call --session --dest org.kde.kdeconnect \
--object-path "${notif_base}/${nid}" \
--method org.freedesktop.DBus.Properties.GetAll \
"org.kde.kdeconnect.device.notifications.notification" 2>/dev/null)
ts=$(date -Iseconds)
app=$(echo "${props}" | grep -oP "'appName': <'\K[^']+")
title=$(echo "${props}" | grep -oP "'title': <'\K[^']+")
text=$(echo "${props}" | grep -oP "'text': <'\K[^']+" | head -c 200)
echo "[${ts}] APP: ${app} | ${title}"
echo " ${text}"
echo ""
done
sleep 5 sleep 5
done done
} }
@@ -848,6 +1075,7 @@ main() {
type) cmd_type "${remaining_args[@]+"${remaining_args[@]}"}" ;; type) cmd_type "${remaining_args[@]+"${remaining_args[@]}"}" ;;
key) cmd_key "${remaining_args[@]+"${remaining_args[@]}"}" ;; key) cmd_key "${remaining_args[@]+"${remaining_args[@]}"}" ;;
launch) cmd_launch "${remaining_args[@]+"${remaining_args[@]}"}" ;; launch) cmd_launch "${remaining_args[@]+"${remaining_args[@]}"}" ;;
ui) cmd_ui "${remaining_args[@]+"${remaining_args[@]}"}" ;;
ocr) cmd_ocr "${remaining_args[@]+"${remaining_args[@]}"}" ;; ocr) cmd_ocr "${remaining_args[@]+"${remaining_args[@]}"}" ;;
sms) cmd_sms "${remaining_args[@]+"${remaining_args[@]}"}" ;; sms) cmd_sms "${remaining_args[@]+"${remaining_args[@]}"}" ;;
*) *)
+191 -31
View File
@@ -55,6 +55,113 @@ adb_launch() {
fi fi
} }
# adb_ui_dump [OUTPUT_PATH] — dump UI hierarchy via UIAutomator
# Returns local path to the XML file
adb_ui_dump() {
local output="${1:-/tmp/fs5-ui-$(date +%s).xml}"
local device_path="/sdcard/fs5_ui_dump.xml"
# UIAutomator prints permission warnings to stderr but still works
adb_cmd shell uiautomator dump "${device_path}" >/dev/null 2>&1
adb_cmd pull "${device_path}" "${output}" >/dev/null 2>&1
adb_cmd shell rm -f "${device_path}" >/dev/null 2>&1
echo "${output}"
}
# adb_ui_find TEXT [XML_PATH] — find UI element by text/content-desc, return bounds center
# Returns "x y" coordinates of element center, or empty if not found
adb_ui_find() {
local search_text="${1}"
local xml_path="${2:-}"
if [[ -z "${xml_path}" ]]; then
xml_path=$(adb_ui_dump)
fi
python3 - "${xml_path}" "${search_text}" <<'PYEOF'
import sys, xml.etree.ElementTree as ET, re
xml_path = sys.argv[1]
search = sys.argv[2].lower()
try:
tree = ET.parse(xml_path)
except Exception as e:
sys.exit(1)
root = tree.getroot()
for node in root.iter('node'):
text = node.get('text', '').lower()
desc = node.get('content-desc', '').lower()
rid = node.get('resource-id', '').lower()
if search in text or search in desc or search in rid:
bounds = node.get('bounds', '')
# Parse [x1,y1][x2,y2]
m = re.findall(r'\d+', bounds)
if len(m) == 4:
cx = (int(m[0]) + int(m[2])) // 2
cy = (int(m[1]) + int(m[3])) // 2
print(f"{cx} {cy}")
sys.exit(0)
sys.exit(1)
PYEOF
}
# adb_ui_elements [XML_PATH] — list all interactive UI elements with bounds
# Returns JSON array of {text, resource_id, class, bounds, clickable, center_x, center_y}
adb_ui_elements() {
local xml_path="${1:-}"
if [[ -z "${xml_path}" ]]; then
xml_path=$(adb_ui_dump)
fi
python3 - "${xml_path}" <<'PYEOF'
import sys, xml.etree.ElementTree as ET, re, json
xml_path = sys.argv[1]
try:
tree = ET.parse(xml_path)
except Exception as e:
print(json.dumps([]))
sys.exit(0)
root = tree.getroot()
elements = []
for node in root.iter('node'):
text = node.get('text', '')
desc = node.get('content-desc', '')
rid = node.get('resource-id', '')
cls = node.get('class', '')
bounds = node.get('bounds', '')
clickable = node.get('clickable', 'false') == 'true'
enabled = node.get('enabled', 'false') == 'true'
# Include clickable elements or elements with text/desc
if not (clickable or text or desc):
continue
cx, cy = None, None
m = re.findall(r'\d+', bounds)
if len(m) == 4:
cx = (int(m[0]) + int(m[2])) // 2
cy = (int(m[1]) + int(m[3])) // 2
elements.append({
'text': text,
'content_desc': desc,
'resource_id': rid,
'class': cls.split('.')[-1], # short class name
'bounds': bounds,
'clickable': clickable,
'enabled': enabled,
'center_x': cx,
'center_y': cy,
})
print(json.dumps(elements, indent=2))
PYEOF
}
# adb_ocr [SCREENSHOT_PATH] — screenshot + OCR, return extracted text # adb_ocr [SCREENSHOT_PATH] — screenshot + OCR, return extracted text
adb_ocr() { adb_ocr() {
local screenshot local screenshot
@@ -85,10 +192,11 @@ adb_ocr() {
rm -f "${screenshot}" rm -f "${screenshot}"
} }
# _kdeconnect_device_id — find the paired FS5 device ID via kdeconnect-cli # _kdeconnect_device_id — find the paired+reachable FS5 device ID via kdeconnect-cli
# Returns device ID or empty string if not paired # Returns device ID or empty string if not paired/reachable
_kdeconnect_device_id() { _kdeconnect_device_id() {
kdeconnect-cli --list-devices --id-only 2>/dev/null | head -1 | tr -d '\r\n' # --list-available returns only paired AND reachable devices
kdeconnect-cli --list-available --id-only 2>/dev/null | head -1 | tr -d '\r\n'
} }
# _kdeconnect_check — verify KDE Connect is paired; print setup instructions if not # _kdeconnect_check — verify KDE Connect is paired; print setup instructions if not
@@ -108,37 +216,75 @@ _kdeconnect_check() {
echo "${dev_id}" echo "${dev_id}"
} }
# adb_sms_list [LIMIT] — list SMS via KDE Connect CLI # adb_sms_list [LIMIT] — list SMS conversations via KDE Connect DBus
# Returns JSON array of {id, address, date, body} # Returns JSON array of {id, message_count} (KDE Connect 21+ only provides metadata via DBus)
# Full message content requires KDE Connect app on phone; OTPs use notifications plugin
# Requires: KDE Connect paired + SMS plugin + SMS permission granted on phone
adb_sms_list() { adb_sms_list() {
local limit="${1:-10}" local limit="${1:-10}"
local dev_id local dev_id
dev_id=$(_kdeconnect_check) || return 1 dev_id=$(_kdeconnect_check) || return 1
# Use kdeconnect-cli SMS list if available (kdeconnect 21.08+) local dev_path="/modules/kdeconnect/devices/${dev_id}"
if kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null | head -1 | grep -q "^{"; then local sms_path="${dev_path}/sms"
kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null \
| head -n "${limit}" \ # KDE Connect 21+ DBus SMS API:
| python3 -c " # requestAllConversations → fires conversationLoaded(int64 id, uint64 count) signals
import sys, json # Message body content is NOT exposed via DBus in KDE Connect 25.x
msgs = [] # Use select() for non-blocking reads to avoid hanging
for line in sys.stdin: python3 /dev/stdin "${dev_id}" "${limit}" <<'PYEOF'
line = line.strip() import sys, json, subprocess, time, re, select
if line:
try: dev_id = sys.argv[1]
msgs.append(json.loads(line)) limit = int(sys.argv[2])
except Exception: sms_path = f"/modules/kdeconnect/devices/{dev_id}/sms"
pass
print(json.dumps(msgs, indent=2)) # Start monitoring
" proc = subprocess.Popen(
else ["dbus-monitor", "--session", "sender='org.kde.kdeconnect'"],
# Fallback: show pairing status and guidance stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True
out_error "KDE Connect SMS listing requires kdeconnect 21.08+ and SMS permission on device." )
echo "[]" time.sleep(0.3)
fi
# Request all conversations
subprocess.run(
["gdbus", "call", "--session", "--dest", "org.kde.kdeconnect",
"--object-path", sms_path,
"--method", "org.kde.kdeconnect.device.sms.requestAllConversations"],
capture_output=True, text=True)
conv_list = []
deadline = time.time() + 6
last_signal = time.time()
while time.time() < deadline:
ready = select.select([proc.stdout], [], [], 0.5)
if not ready[0]:
if conv_list and (time.time() - last_signal) > 1.5:
break
continue
line = proc.stdout.readline()
if "member=conversationLoaded" in line:
last_signal = time.time()
id_line = proc.stdout.readline()
count_line = proc.stdout.readline()
m_id = re.search(r'int64\s+(\d+)', id_line)
m_cnt = re.search(r'uint64\s+(\d+)', count_line)
if m_id:
conv_list.append({
"id": int(m_id.group(1)),
"message_count": int(m_cnt.group(1)) if m_cnt else 0
})
if len(conv_list) >= limit:
break
proc.terminate()
print(json.dumps(conv_list[:limit], indent=2))
PYEOF
} }
# adb_sms_otp [WAIT_SECS] — extract most recent OTP code via KDE Connect notifications # adb_sms_otp [WAIT_SECS] — extract OTP code from KDE Connect notifications
# Watches for SMS/messaging app notifications containing 4-8 digit codes
# If WAIT_SECS > 0, polls until new OTP arrives or timeout # If WAIT_SECS > 0, polls until new OTP arrives or timeout
adb_sms_otp() { adb_sms_otp() {
local wait_secs="${1:-0}" local wait_secs="${1:-0}"
@@ -146,17 +292,31 @@ adb_sms_otp() {
dev_id=$(_kdeconnect_check) || return 1 dev_id=$(_kdeconnect_check) || return 1
local deadline=$(( $(date +%s) + wait_secs )) local deadline=$(( $(date +%s) + wait_secs ))
local notif_base="/modules/kdeconnect/devices/${dev_id}/notifications"
# Poll KDE Connect notifications for OTP patterns # Poll KDE Connect notifications DBus for OTP patterns
while true; do while true; do
local notifs otp local otp
notifs=$(kdeconnect-cli --device "${dev_id}" --list-notifications 2>/dev/null | tr -d '\r') # Get active notification IDs
otp=$(echo "${notifs}" | grep -oE '\b[0-9]{4,8}\b' | head -1) local notif_ids
notif_ids=$(gdbus call --session --dest org.kde.kdeconnect \
--object-path "${notif_base}" \
--method org.kde.kdeconnect.device.notifications.activeNotifications \
2>/dev/null | grep -oE "'[^']+'" | tr -d "'" | tr ',' '\n' | tr -d ' []')
for nid in ${notif_ids}; do
local props
props=$(gdbus call --session --dest org.kde.kdeconnect \
--object-path "${notif_base}/${nid}" \
--method org.freedesktop.DBus.Properties.GetAll \
"org.kde.kdeconnect.device.notifications.notification" 2>/dev/null)
# Extract OTP from text/ticker fields
otp=$(echo "${props}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
if [[ -n "${otp}" ]]; then if [[ -n "${otp}" ]]; then
echo "${otp}" echo "${otp}"
return 0 return 0
fi fi
done
if [[ "${wait_secs}" -le 0 ]] || [[ $(date +%s) -ge ${deadline} ]]; then if [[ "${wait_secs}" -le 0 ]] || [[ $(date +%s) -ge ${deadline} ]]; then
break break
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env bats
# tests/test_ui.bats — BATS tests for fs5 ui subcommand (UIAutomator-based)
# Tests use a pre-dumped fixture XML to avoid requiring a live device for most cases.
SCRIPT_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." && pwd)"
FS5="${SCRIPT_DIR}/fs5"
# ── Fixture XML (KDE Connect screen, 15 elements) ────────────────────────────
FIXTURE_XML="${BATS_TMPDIR}/fs5_ui_fixture.xml"
setup_file() {
# Create a minimal UIAutomator XML fixture
cat > "${FIXTURE_XML}" <<'XML'
<?xml version="1.0" encoding="UTF-8"?>
<hierarchy rotation="0">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
<node index="0" text="" resource-id="" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
<node index="0" text="" resource-id="android:id/action_bar_container" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,147]">
<node index="0" text="" resource-id="android:id/action_bar" class="android.support.v7.widget.Toolbar" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,49][720,147]">
<node index="0" text="" resource-id="" class="android.widget.ImageButton" package="org.kde.kdeconnect_tp" content-desc="Open" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,49][98,147]" />
<node index="1" text="ja-manjaro" resource-id="" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[105,68][314,127]" />
<node index="2" text="" resource-id="" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="More options" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[636,49][720,147]" />
</node>
</node>
<node index="1" text="" resource-id="android:id/content" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][720,1280]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/recycler_view" class="androidx.recyclerview.widget.RecyclerView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="true" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[0,147][720,1280]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][720,420]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][380,420]" />
<node index="1" text="Send files" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,295][268,362]" />
</node>
<node index="1" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,147][720,420]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,147][720,420]" />
<node index="1" text="Use as drawing tablet" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,295][720,362]" />
</node>
<node index="2" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,420][720,693]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,420][380,693]" />
<node index="1" text="Presentation remote" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,568][380,635]" />
</node>
<node index="3" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,420][720,693]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,420][720,693]" />
<node index="1" text="Multimedia control" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,568][720,635]" />
</node>
<node index="4" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,693][720,966]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,693][380,966]" />
<node index="1" text="Remote input" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,868][326,935]" />
</node>
<node index="5" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,693][720,966]">
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,693][720,966]" />
<node index="1" text="Run Command" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,868][720,935]" />
</node>
</node>
</node>
</node>
</node>
</node>
</hierarchy>
XML
}
# ── ui --help ─────────────────────────────────────────────────────────────────
@test "ui --help shows subcommands" {
run "${FS5}" ui --help
[ "${status}" -eq 0 ]
[[ "${output}" == *"dump"* ]]
[[ "${output}" == *"elements"* ]]
[[ "${output}" == *"find"* ]]
[[ "${output}" == *"tap"* ]]
}
@test "ui unknown subcommand exits 1" {
run "${FS5}" ui bogus
[ "${status}" -eq 1 ]
[[ "${output}" == *"Unknown ui subcommand"* ]]
}
# ── ui dump ───────────────────────────────────────────────────────────────────
@test "ui dump --help exits 0" {
run "${FS5}" ui dump --help
[ "${status}" -eq 0 ]
[[ "${output}" == *"UIAutomator"* ]]
}
# ── ui elements ───────────────────────────────────────────────────────────────
@test "ui elements with fixture shows element count" {
run "${FS5}" ui elements "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
[[ "${output}" == *"Found"*"elements"* ]]
}
@test "ui elements with fixture shows clickable marker" {
run "${FS5}" ui elements "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
[[ "${output}" == *"[✓]"* ]]
}
@test "ui elements with fixture shows More options" {
run "${FS5}" ui elements "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
[[ "${output}" == *"More options"* ]]
}
@test "ui elements --json returns JSON array" {
run "${FS5}" ui elements "${FIXTURE_XML}" --json
[ "${status}" -eq 0 ]
# Output starts with JSON array
[[ "${output}" == "["* ]]
}
@test "ui elements --json contains center_x field" {
run "${FS5}" ui elements "${FIXTURE_XML}" --json
[ "${status}" -eq 0 ]
[[ "${output}" == *"center_x"* ]]
}
@test "ui elements --json contains clickable field" {
run "${FS5}" ui elements "${FIXTURE_XML}" --json
[ "${status}" -eq 0 ]
[[ "${output}" == *"clickable"* ]]
}
@test "ui elements --help exits 0" {
run "${FS5}" ui elements --help
[ "${status}" -eq 0 ]
}
# ── ui find ───────────────────────────────────────────────────────────────────
@test "ui find by content-desc returns coordinates" {
run "${FS5}" ui find "More options" "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
# Output should be "x y" format
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
}
@test "ui find by text returns coordinates" {
run "${FS5}" ui find "Send files" "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
}
@test "ui find case-insensitive match" {
run "${FS5}" ui find "send files" "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
}
@test "ui find nonexistent element exits 1" {
run "${FS5}" ui find "nonexistent_xyz_abc" "${FIXTURE_XML}"
[ "${status}" -eq 1 ]
[[ "${output}" == *"Element not found"* ]]
}
@test "ui find --json returns JSON with x and y" {
run "${FS5}" ui find "More options" --json "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
[[ "${output}" == *'"x"'* ]]
[[ "${output}" == *'"y"'* ]]
}
@test "ui find --json contains text field" {
run "${FS5}" ui find "More options" --json "${FIXTURE_XML}"
[ "${status}" -eq 0 ]
[[ "${output}" == *'"text"'* ]]
[[ "${output}" == *"More options"* ]]
}
@test "ui find missing argument exits 1" {
run "${FS5}" ui find
[ "${status}" -eq 1 ]
[[ "${output}" == *"Usage"* ]]
}
@test "ui find --help exits 0" {
run "${FS5}" ui find --help
[ "${status}" -eq 0 ]
}
# ── ui tap ────────────────────────────────────────────────────────────────────
@test "ui tap --help exits 0" {
# --help is intercepted by global parse_global_flags (same as 'key --help')
run "${FS5}" ui tap --help
[ "${status}" -eq 0 ]
[[ "${output}" == *"ui tap"* ]]
}
@test "ui tap missing argument exits 1" {
run "${FS5}" ui tap
[ "${status}" -eq 1 ]
[[ "${output}" == *"Usage"* ]]
}
@test "ui tap nonexistent element exits 1" {
run "${FS5}" ui tap "nonexistent_xyz_abc" "${FIXTURE_XML}"
[ "${status}" -eq 1 ]
[[ "${output}" == *"Element not found"* ]]
}