feat(ui): add UIAutomator-based ui subcommand for AI remote control
Add 'fs5 ui' subcommand with 4 operations: - ui dump [path] — dump UI hierarchy XML via uiautomator - ui elements [--json] — list all interactive elements with coords - ui find <text> — find element by text/content-desc, return x y - ui tap <text> — find element and tap it (ui find + tap) lib/input.sh: - adb_ui_dump() — uiautomator dump → pull XML → cleanup - adb_ui_find() — Python XML parse, case-insensitive search - adb_ui_elements() — full element inventory as JSON tests/test_ui.bats: 22 new tests (fixture-based, no live device needed) Total: 85/85 BATS tests passing (was 64/64) Research findings (AppAgent pattern, ADB+vision): - UIAutomator dump: best no-root UI inspection on Android 9 SDK 28 - AppAgent (6.6k★): screenshot + GPT-4V + ADB tap/swipe pipeline - RustDesk InputService already enabled → accessibility layer available - scrcpy 3.3.4 + tesseract OCR pipeline already working
This commit is contained in:
Jane Alesi
@@ -65,6 +65,10 @@ Remote Control (AI capabilities):
|
||||
key <keycode> Send Android key event
|
||||
launch <package> Launch app by package name
|
||||
ocr Screenshot + extract all text via OCR
|
||||
ui dump [path] Dump UI hierarchy XML (UIAutomator)
|
||||
ui elements List all interactive UI elements
|
||||
ui find <text> Find element by text, print coordinates
|
||||
ui tap <text> Find element by text and tap it
|
||||
sms list List recent SMS messages
|
||||
sms otp Extract latest OTP code from SMS
|
||||
sms watch Watch for new SMS messages
|
||||
@@ -665,6 +669,225 @@ cmd_launch() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: ui ───────────────────────────────────────────────────────────────
|
||||
cmd_ui() {
|
||||
local subcmd="${1:-}"
|
||||
shift || true
|
||||
|
||||
case "${subcmd}" in
|
||||
dump) cmd_ui_dump "$@" ;;
|
||||
elements) cmd_ui_elements "$@" ;;
|
||||
find) cmd_ui_find "$@" ;;
|
||||
tap) cmd_ui_tap "$@" ;;
|
||||
--help|-h)
|
||||
cat <<EOF
|
||||
Usage: fs5 ui <subcommand> [options]
|
||||
|
||||
Inspect and interact with the device UI via UIAutomator.
|
||||
|
||||
Subcommands:
|
||||
dump [path] Dump UI hierarchy XML to file
|
||||
elements [--json] List all interactive UI elements
|
||||
find <text> Find element by text/desc, print coordinates
|
||||
tap <text> Find element by text/desc and tap it
|
||||
EOF
|
||||
;;
|
||||
*)
|
||||
out_error "Unknown ui subcommand: '${subcmd}'. Use: dump, elements, find, tap"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
cmd_ui_dump() {
|
||||
local output="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*) [[ -z "${output}" ]] && output="${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui dump [output-path.xml]
|
||||
|
||||
Dump the current UI hierarchy to an XML file via UIAutomator.
|
||||
If no path given, saves to /tmp/fs5-ui-<timestamp>.xml.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local path
|
||||
if [[ -n "${output}" ]]; then
|
||||
path=$(adb_ui_dump "${output}")
|
||||
else
|
||||
path=$(adb_ui_dump)
|
||||
fi
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
out_json "path=${path}" "timestamp=$(date -Iseconds)"
|
||||
else
|
||||
out_success "UI dump saved: ${path}"
|
||||
echo "${path}"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_ui_elements() {
|
||||
local xml_path="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
--json) JSON_OUTPUT=true ;;
|
||||
*) [[ -z "${xml_path}" ]] && xml_path="${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui elements [xml-path] [--json]
|
||||
|
||||
List all interactive UI elements on the current screen.
|
||||
Optionally provide a previously dumped XML file.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local json_out
|
||||
if [[ -n "${xml_path}" ]]; then
|
||||
json_out=$(adb_ui_elements "${xml_path}")
|
||||
else
|
||||
json_out=$(adb_ui_elements)
|
||||
fi
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
echo "${json_out}"
|
||||
else
|
||||
# Human-readable table
|
||||
echo "${json_out}" | python3 -c "
|
||||
import sys, json
|
||||
elements = json.load(sys.stdin)
|
||||
print(f'Found {len(elements)} elements:')
|
||||
for i, e in enumerate(elements):
|
||||
label = e['text'] or e['content_desc'] or e['resource_id'] or e['class']
|
||||
click = '✓' if e['clickable'] else ' '
|
||||
cx, cy = e.get('center_x'), e.get('center_y')
|
||||
coord = f'({cx},{cy})' if cx is not None else '(?)'
|
||||
print(f' [{click}] {coord:>12} {label}')
|
||||
"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_ui_find() {
|
||||
local search_text="" xml_path="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*)
|
||||
if [[ -z "${search_text}" ]]; then search_text="${arg}"
|
||||
elif [[ -z "${xml_path}" ]]; then xml_path="${arg}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui find <text> [xml-path]
|
||||
|
||||
Find a UI element by text or content-desc and print its center coordinates.
|
||||
Returns "x y" on success, exits 1 if not found.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${search_text}" ]]; then
|
||||
out_error "Usage: fs5 ui find <text>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local coords
|
||||
if [[ -n "${xml_path}" ]]; then
|
||||
coords=$(adb_ui_find "${search_text}" "${xml_path}")
|
||||
else
|
||||
coords=$(adb_ui_find "${search_text}")
|
||||
fi
|
||||
|
||||
if [[ -z "${coords}" ]]; then
|
||||
out_error "Element not found: '${search_text}'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
local x y
|
||||
x="${coords%% *}"
|
||||
y="${coords##* }"
|
||||
out_json "text=${search_text}" "x=${x}" "y=${y}"
|
||||
else
|
||||
echo "${coords}"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_ui_tap() {
|
||||
local search_text="" xml_path="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*)
|
||||
if [[ -z "${search_text}" ]]; then search_text="${arg}"
|
||||
elif [[ -z "${xml_path}" ]]; then xml_path="${arg}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui tap <text> [xml-path]
|
||||
|
||||
Find a UI element by text or content-desc and tap its center.
|
||||
Combines ui find + tap in one step.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${search_text}" ]]; then
|
||||
out_error "Usage: fs5 ui tap <text>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local coords
|
||||
if [[ -n "${xml_path}" ]]; then
|
||||
coords=$(adb_ui_find "${search_text}" "${xml_path}")
|
||||
else
|
||||
coords=$(adb_ui_find "${search_text}")
|
||||
fi
|
||||
|
||||
if [[ -z "${coords}" ]]; then
|
||||
out_error "Element not found: '${search_text}'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local x y
|
||||
x="${coords%% *}"
|
||||
y="${coords##* }"
|
||||
adb_tap "${x}" "${y}"
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
out_json "text=${search_text}" "x=${x}" "y=${y}" "action=tap"
|
||||
else
|
||||
out_success "Tapped '${search_text}' at (${x}, ${y})"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: ocr ──────────────────────────────────────────────────────────────
|
||||
cmd_ocr() {
|
||||
local show_help=false
|
||||
@@ -775,40 +998,44 @@ cmd_sms_watch() {
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 sms watch"
|
||||
echo "Poll for new SMS messages every 5 seconds. Press Ctrl+C to stop."
|
||||
echo "Watch for new SMS/notification messages via KDE Connect. Press Ctrl+C to stop."
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
local dev_id
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
out_success "Watching for SMS (Ctrl+C to stop)..."
|
||||
local last_id=""
|
||||
last_id=$(adb_cmd shell content query \
|
||||
--uri content://sms/inbox \
|
||||
--projection "_id" \
|
||||
--sort "date DESC" \
|
||||
2>/dev/null | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2 | tr -d '\r')
|
||||
out_success "Watching for SMS via KDE Connect notifications (Ctrl+C to stop)..."
|
||||
local notif_base="/modules/kdeconnect/devices/${dev_id}/notifications"
|
||||
local seen_ids=""
|
||||
|
||||
while true; do
|
||||
local raw new_id
|
||||
raw=$(adb_cmd shell content query \
|
||||
--uri content://sms/inbox \
|
||||
--projection "_id,address,date,body" \
|
||||
--sort "date DESC" \
|
||||
2>/dev/null | tr -d '\r' | head -10)
|
||||
local notif_ids
|
||||
notif_ids=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}" \
|
||||
--method org.kde.kdeconnect.device.notifications.activeNotifications \
|
||||
2>/dev/null | grep -oE "'[^']+'" | tr -d "'" | tr ',' '\n' | tr -d ' []')
|
||||
|
||||
new_id=$(echo "${raw}" | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2)
|
||||
for nid in ${notif_ids}; do
|
||||
# Skip already-seen notifications
|
||||
if echo "${seen_ids}" | grep -qw "${nid}"; then
|
||||
continue
|
||||
fi
|
||||
seen_ids="${seen_ids} ${nid}"
|
||||
|
||||
if [[ -n "${new_id}" ]] && [[ "${new_id}" != "${last_id}" ]]; then
|
||||
local ts body address
|
||||
local props app title text ts
|
||||
props=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}/${nid}" \
|
||||
--method org.freedesktop.DBus.Properties.GetAll \
|
||||
"org.kde.kdeconnect.device.notifications.notification" 2>/dev/null)
|
||||
ts=$(date -Iseconds)
|
||||
address=$(echo "${raw}" | grep "address=" | head -1 | grep -oE 'address=[^,]+' | cut -d= -f2)
|
||||
body=$(echo "${raw}" | grep "body=" | head -1 | sed 's/.*body=//')
|
||||
echo "[${ts}] FROM: ${address}"
|
||||
echo " ${body}"
|
||||
app=$(echo "${props}" | grep -oP "'appName': <'\K[^']+")
|
||||
title=$(echo "${props}" | grep -oP "'title': <'\K[^']+")
|
||||
text=$(echo "${props}" | grep -oP "'text': <'\K[^']+" | head -c 200)
|
||||
echo "[${ts}] APP: ${app} | ${title}"
|
||||
echo " ${text}"
|
||||
echo ""
|
||||
last_id="${new_id}"
|
||||
fi
|
||||
done
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
@@ -848,6 +1075,7 @@ main() {
|
||||
type) cmd_type "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
key) cmd_key "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
launch) cmd_launch "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
ui) cmd_ui "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
ocr) cmd_ocr "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
sms) cmd_sms "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
*)
|
||||
|
||||
Reference in New Issue
Block a user