feat(ui): add UIAutomator-based ui subcommand for AI remote control
Add 'fs5 ui' subcommand with 4 operations: - ui dump [path] — dump UI hierarchy XML via uiautomator - ui elements [--json] — list all interactive elements with coords - ui find <text> — find element by text/content-desc, return x y - ui tap <text> — find element and tap it (ui find + tap) lib/input.sh: - adb_ui_dump() — uiautomator dump → pull XML → cleanup - adb_ui_find() — Python XML parse, case-insensitive search - adb_ui_elements() — full element inventory as JSON tests/test_ui.bats: 22 new tests (fixture-based, no live device needed) Total: 85/85 BATS tests passing (was 64/64) Research findings (AppAgent pattern, ADB+vision): - UIAutomator dump: best no-root UI inspection on Android 9 SDK 28 - AppAgent (6.6k★): screenshot + GPT-4V + ADB tap/swipe pipeline - RustDesk InputService already enabled → accessibility layer available - scrcpy 3.3.4 + tesseract OCR pipeline already working
This commit is contained in:
Jane Alesi
@@ -65,6 +65,10 @@ Remote Control (AI capabilities):
|
||||
key <keycode> Send Android key event
|
||||
launch <package> Launch app by package name
|
||||
ocr Screenshot + extract all text via OCR
|
||||
ui dump [path] Dump UI hierarchy XML (UIAutomator)
|
||||
ui elements List all interactive UI elements
|
||||
ui find <text> Find element by text, print coordinates
|
||||
ui tap <text> Find element by text and tap it
|
||||
sms list List recent SMS messages
|
||||
sms otp Extract latest OTP code from SMS
|
||||
sms watch Watch for new SMS messages
|
||||
@@ -665,6 +669,225 @@ cmd_launch() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: ui ───────────────────────────────────────────────────────────────
|
||||
cmd_ui() {
|
||||
local subcmd="${1:-}"
|
||||
shift || true
|
||||
|
||||
case "${subcmd}" in
|
||||
dump) cmd_ui_dump "$@" ;;
|
||||
elements) cmd_ui_elements "$@" ;;
|
||||
find) cmd_ui_find "$@" ;;
|
||||
tap) cmd_ui_tap "$@" ;;
|
||||
--help|-h)
|
||||
cat <<EOF
|
||||
Usage: fs5 ui <subcommand> [options]
|
||||
|
||||
Inspect and interact with the device UI via UIAutomator.
|
||||
|
||||
Subcommands:
|
||||
dump [path] Dump UI hierarchy XML to file
|
||||
elements [--json] List all interactive UI elements
|
||||
find <text> Find element by text/desc, print coordinates
|
||||
tap <text> Find element by text/desc and tap it
|
||||
EOF
|
||||
;;
|
||||
*)
|
||||
out_error "Unknown ui subcommand: '${subcmd}'. Use: dump, elements, find, tap"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
cmd_ui_dump() {
|
||||
local output="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*) [[ -z "${output}" ]] && output="${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui dump [output-path.xml]
|
||||
|
||||
Dump the current UI hierarchy to an XML file via UIAutomator.
|
||||
If no path given, saves to /tmp/fs5-ui-<timestamp>.xml.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local path
|
||||
if [[ -n "${output}" ]]; then
|
||||
path=$(adb_ui_dump "${output}")
|
||||
else
|
||||
path=$(adb_ui_dump)
|
||||
fi
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
out_json "path=${path}" "timestamp=$(date -Iseconds)"
|
||||
else
|
||||
out_success "UI dump saved: ${path}"
|
||||
echo "${path}"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_ui_elements() {
|
||||
local xml_path="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
--json) JSON_OUTPUT=true ;;
|
||||
*) [[ -z "${xml_path}" ]] && xml_path="${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui elements [xml-path] [--json]
|
||||
|
||||
List all interactive UI elements on the current screen.
|
||||
Optionally provide a previously dumped XML file.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local json_out
|
||||
if [[ -n "${xml_path}" ]]; then
|
||||
json_out=$(adb_ui_elements "${xml_path}")
|
||||
else
|
||||
json_out=$(adb_ui_elements)
|
||||
fi
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
echo "${json_out}"
|
||||
else
|
||||
# Human-readable table
|
||||
echo "${json_out}" | python3 -c "
|
||||
import sys, json
|
||||
elements = json.load(sys.stdin)
|
||||
print(f'Found {len(elements)} elements:')
|
||||
for i, e in enumerate(elements):
|
||||
label = e['text'] or e['content_desc'] or e['resource_id'] or e['class']
|
||||
click = '✓' if e['clickable'] else ' '
|
||||
cx, cy = e.get('center_x'), e.get('center_y')
|
||||
coord = f'({cx},{cy})' if cx is not None else '(?)'
|
||||
print(f' [{click}] {coord:>12} {label}')
|
||||
"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_ui_find() {
|
||||
local search_text="" xml_path="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*)
|
||||
if [[ -z "${search_text}" ]]; then search_text="${arg}"
|
||||
elif [[ -z "${xml_path}" ]]; then xml_path="${arg}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui find <text> [xml-path]
|
||||
|
||||
Find a UI element by text or content-desc and print its center coordinates.
|
||||
Returns "x y" on success, exits 1 if not found.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${search_text}" ]]; then
|
||||
out_error "Usage: fs5 ui find <text>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local coords
|
||||
if [[ -n "${xml_path}" ]]; then
|
||||
coords=$(adb_ui_find "${search_text}" "${xml_path}")
|
||||
else
|
||||
coords=$(adb_ui_find "${search_text}")
|
||||
fi
|
||||
|
||||
if [[ -z "${coords}" ]]; then
|
||||
out_error "Element not found: '${search_text}'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
local x y
|
||||
x="${coords%% *}"
|
||||
y="${coords##* }"
|
||||
out_json "text=${search_text}" "x=${x}" "y=${y}"
|
||||
else
|
||||
echo "${coords}"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_ui_tap() {
|
||||
local search_text="" xml_path="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*)
|
||||
if [[ -z "${search_text}" ]]; then search_text="${arg}"
|
||||
elif [[ -z "${xml_path}" ]]; then xml_path="${arg}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 ui tap <text> [xml-path]
|
||||
|
||||
Find a UI element by text or content-desc and tap its center.
|
||||
Combines ui find + tap in one step.
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${search_text}" ]]; then
|
||||
out_error "Usage: fs5 ui tap <text>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local coords
|
||||
if [[ -n "${xml_path}" ]]; then
|
||||
coords=$(adb_ui_find "${search_text}" "${xml_path}")
|
||||
else
|
||||
coords=$(adb_ui_find "${search_text}")
|
||||
fi
|
||||
|
||||
if [[ -z "${coords}" ]]; then
|
||||
out_error "Element not found: '${search_text}'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local x y
|
||||
x="${coords%% *}"
|
||||
y="${coords##* }"
|
||||
adb_tap "${x}" "${y}"
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
out_json "text=${search_text}" "x=${x}" "y=${y}" "action=tap"
|
||||
else
|
||||
out_success "Tapped '${search_text}' at (${x}, ${y})"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: ocr ──────────────────────────────────────────────────────────────
|
||||
cmd_ocr() {
|
||||
local show_help=false
|
||||
@@ -775,40 +998,44 @@ cmd_sms_watch() {
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 sms watch"
|
||||
echo "Poll for new SMS messages every 5 seconds. Press Ctrl+C to stop."
|
||||
echo "Watch for new SMS/notification messages via KDE Connect. Press Ctrl+C to stop."
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
local dev_id
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
out_success "Watching for SMS (Ctrl+C to stop)..."
|
||||
local last_id=""
|
||||
last_id=$(adb_cmd shell content query \
|
||||
--uri content://sms/inbox \
|
||||
--projection "_id" \
|
||||
--sort "date DESC" \
|
||||
2>/dev/null | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2 | tr -d '\r')
|
||||
out_success "Watching for SMS via KDE Connect notifications (Ctrl+C to stop)..."
|
||||
local notif_base="/modules/kdeconnect/devices/${dev_id}/notifications"
|
||||
local seen_ids=""
|
||||
|
||||
while true; do
|
||||
local raw new_id
|
||||
raw=$(adb_cmd shell content query \
|
||||
--uri content://sms/inbox \
|
||||
--projection "_id,address,date,body" \
|
||||
--sort "date DESC" \
|
||||
2>/dev/null | tr -d '\r' | head -10)
|
||||
local notif_ids
|
||||
notif_ids=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}" \
|
||||
--method org.kde.kdeconnect.device.notifications.activeNotifications \
|
||||
2>/dev/null | grep -oE "'[^']+'" | tr -d "'" | tr ',' '\n' | tr -d ' []')
|
||||
|
||||
new_id=$(echo "${raw}" | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2)
|
||||
for nid in ${notif_ids}; do
|
||||
# Skip already-seen notifications
|
||||
if echo "${seen_ids}" | grep -qw "${nid}"; then
|
||||
continue
|
||||
fi
|
||||
seen_ids="${seen_ids} ${nid}"
|
||||
|
||||
if [[ -n "${new_id}" ]] && [[ "${new_id}" != "${last_id}" ]]; then
|
||||
local ts body address
|
||||
local props app title text ts
|
||||
props=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}/${nid}" \
|
||||
--method org.freedesktop.DBus.Properties.GetAll \
|
||||
"org.kde.kdeconnect.device.notifications.notification" 2>/dev/null)
|
||||
ts=$(date -Iseconds)
|
||||
address=$(echo "${raw}" | grep "address=" | head -1 | grep -oE 'address=[^,]+' | cut -d= -f2)
|
||||
body=$(echo "${raw}" | grep "body=" | head -1 | sed 's/.*body=//')
|
||||
echo "[${ts}] FROM: ${address}"
|
||||
echo " ${body}"
|
||||
app=$(echo "${props}" | grep -oP "'appName': <'\K[^']+")
|
||||
title=$(echo "${props}" | grep -oP "'title': <'\K[^']+")
|
||||
text=$(echo "${props}" | grep -oP "'text': <'\K[^']+" | head -c 200)
|
||||
echo "[${ts}] APP: ${app} | ${title}"
|
||||
echo " ${text}"
|
||||
echo ""
|
||||
last_id="${new_id}"
|
||||
fi
|
||||
done
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
@@ -848,6 +1075,7 @@ main() {
|
||||
type) cmd_type "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
key) cmd_key "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
launch) cmd_launch "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
ui) cmd_ui "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
ocr) cmd_ocr "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
sms) cmd_sms "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
*)
|
||||
|
||||
+195
-35
@@ -55,6 +55,113 @@ adb_launch() {
|
||||
fi
|
||||
}
|
||||
|
||||
# adb_ui_dump [OUTPUT_PATH] — dump UI hierarchy via UIAutomator
|
||||
# Returns local path to the XML file
|
||||
adb_ui_dump() {
|
||||
local output="${1:-/tmp/fs5-ui-$(date +%s).xml}"
|
||||
local device_path="/sdcard/fs5_ui_dump.xml"
|
||||
# UIAutomator prints permission warnings to stderr but still works
|
||||
adb_cmd shell uiautomator dump "${device_path}" >/dev/null 2>&1
|
||||
adb_cmd pull "${device_path}" "${output}" >/dev/null 2>&1
|
||||
adb_cmd shell rm -f "${device_path}" >/dev/null 2>&1
|
||||
echo "${output}"
|
||||
}
|
||||
|
||||
# adb_ui_find TEXT [XML_PATH] — find UI element by text/content-desc, return bounds center
|
||||
# Returns "x y" coordinates of element center, or empty if not found
|
||||
adb_ui_find() {
|
||||
local search_text="${1}"
|
||||
local xml_path="${2:-}"
|
||||
|
||||
if [[ -z "${xml_path}" ]]; then
|
||||
xml_path=$(adb_ui_dump)
|
||||
fi
|
||||
|
||||
python3 - "${xml_path}" "${search_text}" <<'PYEOF'
|
||||
import sys, xml.etree.ElementTree as ET, re
|
||||
|
||||
xml_path = sys.argv[1]
|
||||
search = sys.argv[2].lower()
|
||||
|
||||
try:
|
||||
tree = ET.parse(xml_path)
|
||||
except Exception as e:
|
||||
sys.exit(1)
|
||||
|
||||
root = tree.getroot()
|
||||
for node in root.iter('node'):
|
||||
text = node.get('text', '').lower()
|
||||
desc = node.get('content-desc', '').lower()
|
||||
rid = node.get('resource-id', '').lower()
|
||||
if search in text or search in desc or search in rid:
|
||||
bounds = node.get('bounds', '')
|
||||
# Parse [x1,y1][x2,y2]
|
||||
m = re.findall(r'\d+', bounds)
|
||||
if len(m) == 4:
|
||||
cx = (int(m[0]) + int(m[2])) // 2
|
||||
cy = (int(m[1]) + int(m[3])) // 2
|
||||
print(f"{cx} {cy}")
|
||||
sys.exit(0)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# adb_ui_elements [XML_PATH] — list all interactive UI elements with bounds
|
||||
# Returns JSON array of {text, resource_id, class, bounds, clickable, center_x, center_y}
|
||||
adb_ui_elements() {
|
||||
local xml_path="${1:-}"
|
||||
|
||||
if [[ -z "${xml_path}" ]]; then
|
||||
xml_path=$(adb_ui_dump)
|
||||
fi
|
||||
|
||||
python3 - "${xml_path}" <<'PYEOF'
|
||||
import sys, xml.etree.ElementTree as ET, re, json
|
||||
|
||||
xml_path = sys.argv[1]
|
||||
try:
|
||||
tree = ET.parse(xml_path)
|
||||
except Exception as e:
|
||||
print(json.dumps([]))
|
||||
sys.exit(0)
|
||||
|
||||
root = tree.getroot()
|
||||
elements = []
|
||||
for node in root.iter('node'):
|
||||
text = node.get('text', '')
|
||||
desc = node.get('content-desc', '')
|
||||
rid = node.get('resource-id', '')
|
||||
cls = node.get('class', '')
|
||||
bounds = node.get('bounds', '')
|
||||
clickable = node.get('clickable', 'false') == 'true'
|
||||
enabled = node.get('enabled', 'false') == 'true'
|
||||
|
||||
# Include clickable elements or elements with text/desc
|
||||
if not (clickable or text or desc):
|
||||
continue
|
||||
|
||||
cx, cy = None, None
|
||||
m = re.findall(r'\d+', bounds)
|
||||
if len(m) == 4:
|
||||
cx = (int(m[0]) + int(m[2])) // 2
|
||||
cy = (int(m[1]) + int(m[3])) // 2
|
||||
|
||||
elements.append({
|
||||
'text': text,
|
||||
'content_desc': desc,
|
||||
'resource_id': rid,
|
||||
'class': cls.split('.')[-1], # short class name
|
||||
'bounds': bounds,
|
||||
'clickable': clickable,
|
||||
'enabled': enabled,
|
||||
'center_x': cx,
|
||||
'center_y': cy,
|
||||
})
|
||||
|
||||
print(json.dumps(elements, indent=2))
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# adb_ocr [SCREENSHOT_PATH] — screenshot + OCR, return extracted text
|
||||
adb_ocr() {
|
||||
local screenshot
|
||||
@@ -85,10 +192,11 @@ adb_ocr() {
|
||||
rm -f "${screenshot}"
|
||||
}
|
||||
|
||||
# _kdeconnect_device_id — find the paired FS5 device ID via kdeconnect-cli
|
||||
# Returns device ID or empty string if not paired
|
||||
# _kdeconnect_device_id — find the paired+reachable FS5 device ID via kdeconnect-cli
|
||||
# Returns device ID or empty string if not paired/reachable
|
||||
_kdeconnect_device_id() {
|
||||
kdeconnect-cli --list-devices --id-only 2>/dev/null | head -1 | tr -d '\r\n'
|
||||
# --list-available returns only paired AND reachable devices
|
||||
kdeconnect-cli --list-available --id-only 2>/dev/null | head -1 | tr -d '\r\n'
|
||||
}
|
||||
|
||||
# _kdeconnect_check — verify KDE Connect is paired; print setup instructions if not
|
||||
@@ -108,37 +216,75 @@ _kdeconnect_check() {
|
||||
echo "${dev_id}"
|
||||
}
|
||||
|
||||
# adb_sms_list [LIMIT] — list SMS via KDE Connect CLI
|
||||
# Returns JSON array of {id, address, date, body}
|
||||
# adb_sms_list [LIMIT] — list SMS conversations via KDE Connect DBus
|
||||
# Returns JSON array of {id, message_count} (KDE Connect 21+ only provides metadata via DBus)
|
||||
# Full message content requires KDE Connect app on phone; OTPs use notifications plugin
|
||||
# Requires: KDE Connect paired + SMS plugin + SMS permission granted on phone
|
||||
adb_sms_list() {
|
||||
local limit="${1:-10}"
|
||||
local dev_id
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
# Use kdeconnect-cli SMS list if available (kdeconnect 21.08+)
|
||||
if kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null | head -1 | grep -q "^{"; then
|
||||
kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null \
|
||||
| head -n "${limit}" \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
msgs = []
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
msgs.append(json.loads(line))
|
||||
except Exception:
|
||||
pass
|
||||
print(json.dumps(msgs, indent=2))
|
||||
"
|
||||
else
|
||||
# Fallback: show pairing status and guidance
|
||||
out_error "KDE Connect SMS listing requires kdeconnect 21.08+ and SMS permission on device."
|
||||
echo "[]"
|
||||
fi
|
||||
local dev_path="/modules/kdeconnect/devices/${dev_id}"
|
||||
local sms_path="${dev_path}/sms"
|
||||
|
||||
# KDE Connect 21+ DBus SMS API:
|
||||
# requestAllConversations → fires conversationLoaded(int64 id, uint64 count) signals
|
||||
# Message body content is NOT exposed via DBus in KDE Connect 25.x
|
||||
# Use select() for non-blocking reads to avoid hanging
|
||||
python3 /dev/stdin "${dev_id}" "${limit}" <<'PYEOF'
|
||||
import sys, json, subprocess, time, re, select
|
||||
|
||||
dev_id = sys.argv[1]
|
||||
limit = int(sys.argv[2])
|
||||
sms_path = f"/modules/kdeconnect/devices/{dev_id}/sms"
|
||||
|
||||
# Start monitoring
|
||||
proc = subprocess.Popen(
|
||||
["dbus-monitor", "--session", "sender='org.kde.kdeconnect'"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True
|
||||
)
|
||||
time.sleep(0.3)
|
||||
|
||||
# Request all conversations
|
||||
subprocess.run(
|
||||
["gdbus", "call", "--session", "--dest", "org.kde.kdeconnect",
|
||||
"--object-path", sms_path,
|
||||
"--method", "org.kde.kdeconnect.device.sms.requestAllConversations"],
|
||||
capture_output=True, text=True)
|
||||
|
||||
conv_list = []
|
||||
deadline = time.time() + 6
|
||||
last_signal = time.time()
|
||||
|
||||
while time.time() < deadline:
|
||||
ready = select.select([proc.stdout], [], [], 0.5)
|
||||
if not ready[0]:
|
||||
if conv_list and (time.time() - last_signal) > 1.5:
|
||||
break
|
||||
continue
|
||||
line = proc.stdout.readline()
|
||||
if "member=conversationLoaded" in line:
|
||||
last_signal = time.time()
|
||||
id_line = proc.stdout.readline()
|
||||
count_line = proc.stdout.readline()
|
||||
m_id = re.search(r'int64\s+(\d+)', id_line)
|
||||
m_cnt = re.search(r'uint64\s+(\d+)', count_line)
|
||||
if m_id:
|
||||
conv_list.append({
|
||||
"id": int(m_id.group(1)),
|
||||
"message_count": int(m_cnt.group(1)) if m_cnt else 0
|
||||
})
|
||||
if len(conv_list) >= limit:
|
||||
break
|
||||
|
||||
proc.terminate()
|
||||
print(json.dumps(conv_list[:limit], indent=2))
|
||||
PYEOF
|
||||
}
|
||||
|
||||
# adb_sms_otp [WAIT_SECS] — extract most recent OTP code via KDE Connect notifications
|
||||
# adb_sms_otp [WAIT_SECS] — extract OTP code from KDE Connect notifications
|
||||
# Watches for SMS/messaging app notifications containing 4-8 digit codes
|
||||
# If WAIT_SECS > 0, polls until new OTP arrives or timeout
|
||||
adb_sms_otp() {
|
||||
local wait_secs="${1:-0}"
|
||||
@@ -146,17 +292,31 @@ adb_sms_otp() {
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
local deadline=$(( $(date +%s) + wait_secs ))
|
||||
local notif_base="/modules/kdeconnect/devices/${dev_id}/notifications"
|
||||
|
||||
# Poll KDE Connect notifications for OTP patterns
|
||||
# Poll KDE Connect notifications DBus for OTP patterns
|
||||
while true; do
|
||||
local notifs otp
|
||||
notifs=$(kdeconnect-cli --device "${dev_id}" --list-notifications 2>/dev/null | tr -d '\r')
|
||||
otp=$(echo "${notifs}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
|
||||
local otp
|
||||
# Get active notification IDs
|
||||
local notif_ids
|
||||
notif_ids=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}" \
|
||||
--method org.kde.kdeconnect.device.notifications.activeNotifications \
|
||||
2>/dev/null | grep -oE "'[^']+'" | tr -d "'" | tr ',' '\n' | tr -d ' []')
|
||||
|
||||
if [[ -n "${otp}" ]]; then
|
||||
echo "${otp}"
|
||||
return 0
|
||||
fi
|
||||
for nid in ${notif_ids}; do
|
||||
local props
|
||||
props=$(gdbus call --session --dest org.kde.kdeconnect \
|
||||
--object-path "${notif_base}/${nid}" \
|
||||
--method org.freedesktop.DBus.Properties.GetAll \
|
||||
"org.kde.kdeconnect.device.notifications.notification" 2>/dev/null)
|
||||
# Extract OTP from text/ticker fields
|
||||
otp=$(echo "${props}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
|
||||
if [[ -n "${otp}" ]]; then
|
||||
echo "${otp}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${wait_secs}" -le 0 ]] || [[ $(date +%s) -ge ${deadline} ]]; then
|
||||
break
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env bats
|
||||
# tests/test_ui.bats — BATS tests for fs5 ui subcommand (UIAutomator-based)
|
||||
# Tests use a pre-dumped fixture XML to avoid requiring a live device for most cases.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." && pwd)"
|
||||
FS5="${SCRIPT_DIR}/fs5"
|
||||
|
||||
# ── Fixture XML (KDE Connect screen, 15 elements) ────────────────────────────
|
||||
FIXTURE_XML="${BATS_TMPDIR}/fs5_ui_fixture.xml"
|
||||
|
||||
setup_file() {
|
||||
# Create a minimal UIAutomator XML fixture
|
||||
cat > "${FIXTURE_XML}" <<'XML'
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<hierarchy rotation="0">
|
||||
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
|
||||
<node index="0" text="" resource-id="" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
|
||||
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
|
||||
<node index="0" text="" resource-id="android:id/action_bar_container" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,147]">
|
||||
<node index="0" text="" resource-id="android:id/action_bar" class="android.support.v7.widget.Toolbar" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,49][720,147]">
|
||||
<node index="0" text="" resource-id="" class="android.widget.ImageButton" package="org.kde.kdeconnect_tp" content-desc="Open" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,49][98,147]" />
|
||||
<node index="1" text="ja-manjaro" resource-id="" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[105,68][314,127]" />
|
||||
<node index="2" text="" resource-id="" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="More options" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[636,49][720,147]" />
|
||||
</node>
|
||||
</node>
|
||||
<node index="1" text="" resource-id="android:id/content" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][720,1280]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/recycler_view" class="androidx.recyclerview.widget.RecyclerView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="true" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[0,147][720,1280]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][720,420]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][380,420]" />
|
||||
<node index="1" text="Send files" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,295][268,362]" />
|
||||
</node>
|
||||
<node index="1" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,147][720,420]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,147][720,420]" />
|
||||
<node index="1" text="Use as drawing tablet" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,295][720,362]" />
|
||||
</node>
|
||||
<node index="2" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,420][720,693]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,420][380,693]" />
|
||||
<node index="1" text="Presentation remote" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,568][380,635]" />
|
||||
</node>
|
||||
<node index="3" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,420][720,693]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,420][720,693]" />
|
||||
<node index="1" text="Multimedia control" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,568][720,635]" />
|
||||
</node>
|
||||
<node index="4" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,693][720,966]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,693][380,966]" />
|
||||
<node index="1" text="Remote input" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,868][326,935]" />
|
||||
</node>
|
||||
<node index="5" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,693][720,966]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,693][720,966]" />
|
||||
<node index="1" text="Run Command" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,868][720,935]" />
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</hierarchy>
|
||||
XML
|
||||
}
|
||||
|
||||
# ── ui --help ─────────────────────────────────────────────────────────────────
|
||||
@test "ui --help shows subcommands" {
|
||||
run "${FS5}" ui --help
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"dump"* ]]
|
||||
[[ "${output}" == *"elements"* ]]
|
||||
[[ "${output}" == *"find"* ]]
|
||||
[[ "${output}" == *"tap"* ]]
|
||||
}
|
||||
|
||||
@test "ui unknown subcommand exits 1" {
|
||||
run "${FS5}" ui bogus
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Unknown ui subcommand"* ]]
|
||||
}
|
||||
|
||||
# ── ui dump ───────────────────────────────────────────────────────────────────
|
||||
@test "ui dump --help exits 0" {
|
||||
run "${FS5}" ui dump --help
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"UIAutomator"* ]]
|
||||
}
|
||||
|
||||
# ── ui elements ───────────────────────────────────────────────────────────────
|
||||
@test "ui elements with fixture shows element count" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"Found"*"elements"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements with fixture shows clickable marker" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"[✓]"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements with fixture shows More options" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"More options"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --json returns JSON array" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}" --json
|
||||
[ "${status}" -eq 0 ]
|
||||
# Output starts with JSON array
|
||||
[[ "${output}" == "["* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --json contains center_x field" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}" --json
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"center_x"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --json contains clickable field" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}" --json
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"clickable"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --help exits 0" {
|
||||
run "${FS5}" ui elements --help
|
||||
[ "${status}" -eq 0 ]
|
||||
}
|
||||
|
||||
# ── ui find ───────────────────────────────────────────────────────────────────
|
||||
@test "ui find by content-desc returns coordinates" {
|
||||
run "${FS5}" ui find "More options" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
# Output should be "x y" format
|
||||
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
|
||||
}
|
||||
|
||||
@test "ui find by text returns coordinates" {
|
||||
run "${FS5}" ui find "Send files" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
|
||||
}
|
||||
|
||||
@test "ui find case-insensitive match" {
|
||||
run "${FS5}" ui find "send files" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
|
||||
}
|
||||
|
||||
@test "ui find nonexistent element exits 1" {
|
||||
run "${FS5}" ui find "nonexistent_xyz_abc" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Element not found"* ]]
|
||||
}
|
||||
|
||||
@test "ui find --json returns JSON with x and y" {
|
||||
run "${FS5}" ui find "More options" --json "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *'"x"'* ]]
|
||||
[[ "${output}" == *'"y"'* ]]
|
||||
}
|
||||
|
||||
@test "ui find --json contains text field" {
|
||||
run "${FS5}" ui find "More options" --json "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *'"text"'* ]]
|
||||
[[ "${output}" == *"More options"* ]]
|
||||
}
|
||||
|
||||
@test "ui find missing argument exits 1" {
|
||||
run "${FS5}" ui find
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Usage"* ]]
|
||||
}
|
||||
|
||||
@test "ui find --help exits 0" {
|
||||
run "${FS5}" ui find --help
|
||||
[ "${status}" -eq 0 ]
|
||||
}
|
||||
|
||||
# ── ui tap ────────────────────────────────────────────────────────────────────
|
||||
@test "ui tap --help exits 0" {
|
||||
# --help is intercepted by global parse_global_flags (same as 'key --help')
|
||||
run "${FS5}" ui tap --help
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"ui tap"* ]]
|
||||
}
|
||||
|
||||
@test "ui tap missing argument exits 1" {
|
||||
run "${FS5}" ui tap
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Usage"* ]]
|
||||
}
|
||||
|
||||
@test "ui tap nonexistent element exits 1" {
|
||||
run "${FS5}" ui tap "nonexistent_xyz_abc" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Element not found"* ]]
|
||||
}
|
||||
Reference in New Issue
Block a user