feat(remote-control): add screen/screenshot/tap/swipe/type/key/launch/ocr/sms commands
- lib/input.sh: ADB input automation library (screenshot, tap, swipe, type, key, launch, OCR, SMS/OTP via KDE Connect) - fs5: 9 new remote control commands for AI agent capabilities - tests/test_remote_control.bats: 31 tests, all passing - specs/002-remote-control/spec.md: feature specification Tools: scrcpy 3.3.4 (GUI mirror), tesseract-ocr (OCR), KDE Connect (SMS/OTP) Constraints: no root required, Android 9 (SDK 28) Tests: 64/64 passing (33 device mgmt + 31 remote control)
This commit is contained in:
Jane Alesi
@@ -34,6 +34,8 @@ ADB_TIMEOUT="${ADB_TIMEOUT:-10}"
|
||||
source "${SCRIPT_DIR}/lib/output.sh"
|
||||
# shellcheck source=lib/adb.sh
|
||||
source "${SCRIPT_DIR}/lib/adb.sh"
|
||||
# shellcheck source=lib/input.sh
|
||||
source "${SCRIPT_DIR}/lib/input.sh"
|
||||
|
||||
# ── Global flags ──────────────────────────────────────────────────────────────
|
||||
JSON_OUTPUT=false
|
||||
@@ -44,7 +46,7 @@ usage() {
|
||||
cat <<EOF
|
||||
Usage: fs5 <command> [options]
|
||||
|
||||
Commands:
|
||||
Device Management:
|
||||
status Show device health and info
|
||||
push <src> <dst> Push file/dir to device
|
||||
pull <src> <dst> Pull file/dir from device
|
||||
@@ -54,6 +56,19 @@ Commands:
|
||||
shell [cmd] Execute shell command on device
|
||||
logs Stream or capture logcat output
|
||||
|
||||
Remote Control (AI capabilities):
|
||||
screen Launch scrcpy GUI mirror with input
|
||||
screenshot [path] Capture screen to PNG file
|
||||
tap <x> <y> Send tap at coordinates
|
||||
swipe <x1> <y1> <x2> <y2> [ms] Send swipe gesture
|
||||
type <text> Type text into focused field
|
||||
key <keycode> Send Android key event
|
||||
launch <package> Launch app by package name
|
||||
ocr Screenshot + extract all text via OCR
|
||||
sms list List recent SMS messages
|
||||
sms otp Extract latest OTP code from SMS
|
||||
sms watch Watch for new SMS messages
|
||||
|
||||
Global Options:
|
||||
--json Output as JSON
|
||||
--force Allow destructive/overwrite operations
|
||||
@@ -445,6 +460,359 @@ EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: screen ──────────────────────────────────────────────────────────
|
||||
cmd_screen() {
|
||||
local headless=false show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--headless) headless=true ;;
|
||||
--help) show_help=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 screen [--headless]
|
||||
|
||||
Launch scrcpy to mirror and control the device screen.
|
||||
--headless: run without display window (for automation/OCR only).
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
if [[ "${headless}" == "true" ]]; then
|
||||
scrcpy -s "${DEVICE_SERIAL}" --no-display &
|
||||
out_success "scrcpy running headless (PID $!)"
|
||||
else
|
||||
scrcpy -s "${DEVICE_SERIAL}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: screenshot ───────────────────────────────────────────────────────
|
||||
cmd_screenshot() {
|
||||
local output="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*) [[ -z "${output}" ]] && output="${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 screenshot [output-path.png]"
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
local path
|
||||
if [[ -n "${output}" ]]; then
|
||||
path=$(adb_screenshot "${output}")
|
||||
else
|
||||
path=$(adb_screenshot)
|
||||
fi
|
||||
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
out_json "path=${path}" "timestamp=$(date -Iseconds)"
|
||||
else
|
||||
out_success "Screenshot saved: ${path}"
|
||||
echo "${path}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: tap ─────────────────────────────────────────────────────────────
|
||||
cmd_tap() {
|
||||
local x="" y="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*)
|
||||
if [[ -z "${x}" ]]; then x="${arg}"
|
||||
elif [[ -z "${y}" ]]; then y="${arg}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 tap <x> <y>"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${x}" ]] || [[ -z "${y}" ]]; then
|
||||
out_error "Usage: fs5 tap <x> <y>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
adb_tap "${x}" "${y}"
|
||||
out_success "Tapped: (${x}, ${y})"
|
||||
}
|
||||
|
||||
# ── Command: swipe ────────────────────────────────────────────────────────────
|
||||
cmd_swipe() {
|
||||
local x1="" y1="" x2="" y2="" duration="300" show_help=false
|
||||
local -a pos=()
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*) pos+=("${arg}") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 swipe <x1> <y1> <x2> <y2> [duration_ms]"
|
||||
return 0
|
||||
fi
|
||||
|
||||
x1="${pos[0]:-}" y1="${pos[1]:-}" x2="${pos[2]:-}" y2="${pos[3]:-}"
|
||||
[[ ${#pos[@]} -ge 5 ]] && duration="${pos[4]}"
|
||||
|
||||
if [[ -z "${x1}" ]] || [[ -z "${y1}" ]] || [[ -z "${x2}" ]] || [[ -z "${y2}" ]]; then
|
||||
out_error "Usage: fs5 swipe <x1> <y1> <x2> <y2> [duration_ms]"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
adb_swipe "${x1}" "${y1}" "${x2}" "${y2}" "${duration}"
|
||||
out_success "Swiped: (${x1},${y1}) → (${x2},${y2}) in ${duration}ms"
|
||||
}
|
||||
|
||||
# ── Command: type ─────────────────────────────────────────────────────────────
|
||||
cmd_type() {
|
||||
local text="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*) text="${text}${text:+ }${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 type <text>"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${text}" ]]; then
|
||||
out_error "Usage: fs5 type <text>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
adb_type "${text}"
|
||||
out_success "Typed: ${text}"
|
||||
}
|
||||
|
||||
# ── Command: key ──────────────────────────────────────────────────────────────
|
||||
cmd_key() {
|
||||
local keycode="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*) [[ -z "${keycode}" ]] && keycode="${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
cat <<EOF
|
||||
Usage: fs5 key <keycode>
|
||||
|
||||
Send Android key event. Common keycodes:
|
||||
KEYCODE_HOME KEYCODE_BACK KEYCODE_POWER
|
||||
KEYCODE_ENTER KEYCODE_TAB KEYCODE_DEL
|
||||
3 (HOME) 4 (BACK) 26 (POWER)
|
||||
EOF
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${keycode}" ]]; then
|
||||
out_error "Usage: fs5 key <keycode>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
adb_key "${keycode}"
|
||||
out_success "Key sent: ${keycode}"
|
||||
}
|
||||
|
||||
# ── Command: launch ───────────────────────────────────────────────────────────
|
||||
cmd_launch() {
|
||||
local pkg="" show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
*) [[ -z "${pkg}" ]] && pkg="${arg}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 launch <package-name>"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${pkg}" ]]; then
|
||||
out_error "Usage: fs5 launch <package-name>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
if adb_launch "${pkg}"; then
|
||||
out_success "Launched: ${pkg}"
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: ocr ──────────────────────────────────────────────────────────────
|
||||
cmd_ocr() {
|
||||
local show_help=false
|
||||
for arg in "$@"; do
|
||||
[[ "${arg}" == "--help" ]] && show_help=true
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 ocr"
|
||||
echo "Take a screenshot and extract all visible text via OCR (tesseract)."
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
local text
|
||||
text=$(adb_ocr)
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
python3 -c "import json,sys; print(json.dumps({'text': sys.stdin.read()}))" <<< "${text}"
|
||||
else
|
||||
echo "${text}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Command: sms ──────────────────────────────────────────────────────────────
|
||||
cmd_sms() {
|
||||
local subcmd="${1:-}"
|
||||
shift || true
|
||||
|
||||
case "${subcmd}" in
|
||||
list) cmd_sms_list "$@" ;;
|
||||
otp) cmd_sms_otp "$@" ;;
|
||||
watch) cmd_sms_watch "$@" ;;
|
||||
--help|-h)
|
||||
cat <<EOF
|
||||
Usage: fs5 sms <subcommand>
|
||||
|
||||
Subcommands:
|
||||
list [--limit N] [--json] List recent SMS messages
|
||||
otp [--wait] Extract latest OTP code from SMS
|
||||
watch Watch for new SMS messages (Ctrl+C to stop)
|
||||
EOF
|
||||
;;
|
||||
*)
|
||||
out_error "Unknown sms subcommand: '${subcmd}'. Use: list, otp, watch"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
cmd_sms_list() {
|
||||
local limit=10 show_help=false
|
||||
local i=0
|
||||
local -a args=("$@")
|
||||
while [[ ${i} -lt ${#args[@]} ]]; do
|
||||
case "${args[${i}]}" in
|
||||
--help) show_help=true ;;
|
||||
--json) JSON_OUTPUT=true ;;
|
||||
--limit) i=$(( i + 1 )); limit="${args[${i}]}" ;;
|
||||
esac
|
||||
i=$(( i + 1 ))
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 sms list [--limit N] [--json]"
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
adb_sms_list "${limit}"
|
||||
}
|
||||
|
||||
cmd_sms_otp() {
|
||||
local wait_secs=0 show_help=false
|
||||
for arg in "$@"; do
|
||||
case "${arg}" in
|
||||
--help) show_help=true ;;
|
||||
--wait) wait_secs=60 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 sms otp [--wait]"
|
||||
echo "Extract latest OTP/2FA code from SMS inbox."
|
||||
echo "--wait: poll up to 60 seconds for a new OTP."
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
local otp
|
||||
otp=$(adb_sms_otp "${wait_secs}")
|
||||
local exit_code=$?
|
||||
|
||||
if [[ ${exit_code} -eq 0 ]]; then
|
||||
if [[ "${JSON_OUTPUT}" == "true" ]]; then
|
||||
out_json "otp=${otp}" "timestamp=$(date -Iseconds)"
|
||||
else
|
||||
echo "${otp}"
|
||||
fi
|
||||
fi
|
||||
return ${exit_code}
|
||||
}
|
||||
|
||||
cmd_sms_watch() {
|
||||
local show_help=false
|
||||
for arg in "$@"; do
|
||||
[[ "${arg}" == "--help" ]] && show_help=true
|
||||
done
|
||||
|
||||
if [[ "${show_help}" == "true" ]]; then
|
||||
echo "Usage: fs5 sms watch"
|
||||
echo "Poll for new SMS messages every 5 seconds. Press Ctrl+C to stop."
|
||||
return 0
|
||||
fi
|
||||
|
||||
adb_check_device
|
||||
|
||||
out_success "Watching for SMS (Ctrl+C to stop)..."
|
||||
local last_id=""
|
||||
last_id=$(adb_cmd shell content query \
|
||||
--uri content://sms/inbox \
|
||||
--projection "_id" \
|
||||
--sort "date DESC" \
|
||||
2>/dev/null | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2 | tr -d '\r')
|
||||
|
||||
while true; do
|
||||
local raw new_id
|
||||
raw=$(adb_cmd shell content query \
|
||||
--uri content://sms/inbox \
|
||||
--projection "_id,address,date,body" \
|
||||
--sort "date DESC" \
|
||||
2>/dev/null | tr -d '\r' | head -10)
|
||||
|
||||
new_id=$(echo "${raw}" | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2)
|
||||
|
||||
if [[ -n "${new_id}" ]] && [[ "${new_id}" != "${last_id}" ]]; then
|
||||
local ts body address
|
||||
ts=$(date -Iseconds)
|
||||
address=$(echo "${raw}" | grep "address=" | head -1 | grep -oE 'address=[^,]+' | cut -d= -f2)
|
||||
body=$(echo "${raw}" | grep "body=" | head -1 | sed 's/.*body=//')
|
||||
echo "[${ts}] FROM: ${address}"
|
||||
echo " ${body}"
|
||||
echo ""
|
||||
last_id="${new_id}"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
# ── Main dispatcher ───────────────────────────────────────────────────────────
|
||||
main() {
|
||||
if [[ $# -eq 0 ]]; then
|
||||
@@ -467,12 +835,21 @@ main() {
|
||||
parse_global_flags remaining_args "$@"
|
||||
|
||||
case "${command}" in
|
||||
status) cmd_status "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
push) cmd_push "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
pull) cmd_pull "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
app) cmd_app "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
shell) cmd_shell "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
logs) cmd_logs "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
status) cmd_status "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
push) cmd_push "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
pull) cmd_pull "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
app) cmd_app "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
shell) cmd_shell "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
logs) cmd_logs "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
screen) cmd_screen "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
screenshot) cmd_screenshot "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
tap) cmd_tap "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
swipe) cmd_swipe "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
type) cmd_type "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
key) cmd_key "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
launch) cmd_launch "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
ocr) cmd_ocr "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
sms) cmd_sms "${remaining_args[@]+"${remaining_args[@]}"}" ;;
|
||||
*)
|
||||
out_error "Unknown command: ${command}"
|
||||
usage
|
||||
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/input.sh — ADB input automation and screenshot functions for android-fs5
|
||||
# shellcheck shell=bash
|
||||
|
||||
# adb_screenshot [OUTPUT_PATH] — capture device screen to PNG
|
||||
# Returns path of saved file
|
||||
adb_screenshot() {
|
||||
local output="${1:-/tmp/fs5-screenshot-$(date +%s).png}"
|
||||
adb_cmd shell screencap -p > "${output}"
|
||||
echo "${output}"
|
||||
}
|
||||
|
||||
# adb_tap X Y — send tap event at coordinates
|
||||
adb_tap() {
|
||||
local x="${1}" y="${2}"
|
||||
adb_cmd shell input tap "${x}" "${y}"
|
||||
}
|
||||
|
||||
# adb_swipe X1 Y1 X2 Y2 [DURATION_MS] — send swipe gesture
|
||||
adb_swipe() {
|
||||
local x1="${1}" y1="${2}" x2="${3}" y2="${4}"
|
||||
local duration="${5:-300}"
|
||||
adb_cmd shell input swipe "${x1}" "${y1}" "${x2}" "${y2}" "${duration}"
|
||||
}
|
||||
|
||||
# adb_type TEXT — type text into focused input field
|
||||
adb_type() {
|
||||
local text="${1}"
|
||||
# Escape spaces for adb input text
|
||||
local escaped="${text// /%s}"
|
||||
adb_cmd shell input text "${escaped}"
|
||||
}
|
||||
|
||||
# adb_key KEYCODE — send Android key event
|
||||
# Common: KEYCODE_HOME, KEYCODE_BACK, KEYCODE_POWER, KEYCODE_ENTER
|
||||
adb_key() {
|
||||
local keycode="${1}"
|
||||
adb_cmd shell input keyevent "${keycode}"
|
||||
}
|
||||
|
||||
# adb_launch PACKAGE — launch app by package name
|
||||
adb_launch() {
|
||||
local pkg="${1}"
|
||||
local result exit_code
|
||||
result=$(adb_cmd shell monkey -p "${pkg}" -c android.intent.category.LAUNCHER 1 2>&1) || exit_code=$?
|
||||
exit_code="${exit_code:-0}"
|
||||
if echo "${result}" | grep -q "Events injected: 1"; then
|
||||
return 0
|
||||
elif [[ "${exit_code}" -ne 0 ]] || echo "${result}" | grep -q "No activities found\|monkey aborted"; then
|
||||
out_error "Package not found or has no launcher activity: ${pkg}"
|
||||
return 1
|
||||
else
|
||||
out_error "Failed to launch: ${pkg}"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# adb_ocr [SCREENSHOT_PATH] — screenshot + OCR, return extracted text
|
||||
adb_ocr() {
|
||||
local screenshot
|
||||
screenshot=$(adb_screenshot)
|
||||
if ! command -v tesseract &>/dev/null; then
|
||||
out_error "tesseract not found. Install: sudo pacman -S tesseract tesseract-data-eng"
|
||||
return 1
|
||||
fi
|
||||
# Locate tessdata — try common paths
|
||||
local tessdata=""
|
||||
for d in /usr/share/tessdata /usr/share/tesseract-ocr/4.00/tessdata \
|
||||
/usr/share/tesseract-ocr/tessdata /usr/local/share/tessdata; do
|
||||
if [[ -f "${d}/eng.traineddata" ]]; then
|
||||
tessdata="${d}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ -z "${tessdata}" ]]; then
|
||||
out_error "tesseract eng language data not found. Install: sudo pacman -S tesseract-data-eng"
|
||||
return 1
|
||||
fi
|
||||
local txt_base="${screenshot%.png}"
|
||||
TESSDATA_PREFIX="${tessdata}" tesseract "${screenshot}" "${txt_base}" -l eng --psm 6 quiet 2>/dev/null
|
||||
if [[ -f "${txt_base}.txt" ]]; then
|
||||
cat "${txt_base}.txt"
|
||||
rm -f "${txt_base}.txt"
|
||||
fi
|
||||
rm -f "${screenshot}"
|
||||
}
|
||||
|
||||
# _kdeconnect_device_id — find the paired FS5 device ID via kdeconnect-cli
|
||||
# Returns device ID or empty string if not paired
|
||||
_kdeconnect_device_id() {
|
||||
kdeconnect-cli --list-devices --id-only 2>/dev/null | head -1 | tr -d '\r\n'
|
||||
}
|
||||
|
||||
# _kdeconnect_check — verify KDE Connect is paired; print setup instructions if not
|
||||
_kdeconnect_check() {
|
||||
local dev_id
|
||||
dev_id=$(_kdeconnect_device_id)
|
||||
if [[ -z "${dev_id}" ]]; then
|
||||
out_error "KDE Connect: no paired device found."
|
||||
echo "Setup steps:"
|
||||
echo " 1. Install KDE Connect on Linux: sudo pacman -S kdeconnect"
|
||||
echo " 2. Open KDE Connect app on the FS5 phone"
|
||||
echo " 3. Accept the pairing request on both devices"
|
||||
echo " 4. Grant SMS permission in KDE Connect app on phone"
|
||||
echo " 5. Re-run this command"
|
||||
return 1
|
||||
fi
|
||||
echo "${dev_id}"
|
||||
}
|
||||
|
||||
# adb_sms_list [LIMIT] — list SMS via KDE Connect CLI
|
||||
# Returns JSON array of {id, address, date, body}
|
||||
adb_sms_list() {
|
||||
local limit="${1:-10}"
|
||||
local dev_id
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
# Use kdeconnect-cli SMS list if available (kdeconnect 21.08+)
|
||||
if kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null | head -1 | grep -q "^{"; then
|
||||
kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null \
|
||||
| head -n "${limit}" \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
msgs = []
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
msgs.append(json.loads(line))
|
||||
except Exception:
|
||||
pass
|
||||
print(json.dumps(msgs, indent=2))
|
||||
"
|
||||
else
|
||||
# Fallback: show pairing status and guidance
|
||||
out_error "KDE Connect SMS listing requires kdeconnect 21.08+ and SMS permission on device."
|
||||
echo "[]"
|
||||
fi
|
||||
}
|
||||
|
||||
# adb_sms_otp [WAIT_SECS] — extract most recent OTP code via KDE Connect notifications
|
||||
# If WAIT_SECS > 0, polls until new OTP arrives or timeout
|
||||
adb_sms_otp() {
|
||||
local wait_secs="${1:-0}"
|
||||
local dev_id
|
||||
dev_id=$(_kdeconnect_check) || return 1
|
||||
|
||||
local deadline=$(( $(date +%s) + wait_secs ))
|
||||
|
||||
# Poll KDE Connect notifications for OTP patterns
|
||||
while true; do
|
||||
local notifs otp
|
||||
notifs=$(kdeconnect-cli --device "${dev_id}" --list-notifications 2>/dev/null | tr -d '\r')
|
||||
otp=$(echo "${notifs}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
|
||||
|
||||
if [[ -n "${otp}" ]]; then
|
||||
echo "${otp}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "${wait_secs}" -le 0 ]] || [[ $(date +%s) -ge ${deadline} ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
out_error "No OTP found in KDE Connect notifications"
|
||||
return 1
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
# Feature Specification: Android FS5 Full Remote Control
|
||||
|
||||
**Feature Branch**: `002-remote-control`
|
||||
**Created**: 2026-03-02
|
||||
**Status**: Draft
|
||||
**Depends on**: 001-android-fs5-management (fs5 CLI)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Extend the FS5 device management to enable **full remote control** as an AI capability
|
||||
extension: GUI mirroring/automation via scrcpy, SMS/2FA OTP reading via KDE Connect,
|
||||
and TOTP code extraction for automated authentication workflows.
|
||||
|
||||
**Key insight**: KDE Connect is already installed on the device — this is the primary
|
||||
SMS forwarding channel. scrcpy 3.3.4 is already on the host.
|
||||
|
||||
---
|
||||
|
||||
## User Scenarios & Testing
|
||||
|
||||
### User Story 1 — GUI Mirroring & Input (Priority: P1)
|
||||
|
||||
**Why**: AI agent needs to see and interact with the phone screen for 2FA app usage.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** the FS5 is connected, **When** I run `./fs5 screen`,
|
||||
**Then** scrcpy launches showing the device screen with input forwarding enabled.
|
||||
|
||||
2. **When** I run `./fs5 screen --headless`,
|
||||
**Then** scrcpy runs without display (for screenshot/OCR automation only).
|
||||
|
||||
3. **When** I run `./fs5 screenshot`,
|
||||
**Then** a PNG screenshot is saved locally and the path is printed.
|
||||
|
||||
4. **When** I run `./fs5 tap <x> <y>`,
|
||||
**Then** a tap event is sent to the device at those coordinates.
|
||||
|
||||
5. **When** I run `./fs5 swipe <x1> <y1> <x2> <y2>`,
|
||||
**Then** a swipe gesture is performed on the device.
|
||||
|
||||
6. **When** I run `./fs5 type "text"`,
|
||||
**Then** the text is typed into the currently focused input field.
|
||||
|
||||
---
|
||||
|
||||
### User Story 2 — SMS/OTP Reading via KDE Connect (Priority: P1)
|
||||
|
||||
**Why**: Receive SMS 2FA codes without manual phone interaction.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **When** I run `./fs5 sms list`,
|
||||
**Then** recent SMS messages are listed with sender, timestamp, and body.
|
||||
|
||||
2. **When** I run `./fs5 sms watch`,
|
||||
**Then** new SMS messages are printed as they arrive (polling loop).
|
||||
|
||||
3. **When** I run `./fs5 sms otp`,
|
||||
**Then** the most recent OTP/2FA code (4-8 digit number) is extracted and printed.
|
||||
|
||||
4. **When** I run `./fs5 sms otp --wait`,
|
||||
**Then** the tool waits up to 60 seconds for a new OTP SMS and prints the code.
|
||||
|
||||
---
|
||||
|
||||
### User Story 3 — TOTP Code Reading from Authenticator App (Priority: P2)
|
||||
|
||||
**Why**: AI agent needs TOTP codes from apps like Google Authenticator or andOTP.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **When** I run `./fs5 totp screenshot`,
|
||||
**Then** a screenshot of the authenticator app is taken and the current TOTP code is extracted via OCR.
|
||||
|
||||
2. **When** I run `./fs5 totp read --app <package>`,
|
||||
**Then** the authenticator app is opened, a screenshot taken, and the 6-digit code extracted.
|
||||
|
||||
---
|
||||
|
||||
### User Story 4 — ADB Input Automation (Priority: P2)
|
||||
|
||||
**Why**: Automate button presses, app launches, and navigation for 2FA flows.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **When** I run `./fs5 key <keycode>`,
|
||||
**Then** the Android key event is sent (e.g., KEYCODE_HOME, KEYCODE_BACK).
|
||||
|
||||
2. **When** I run `./fs5 launch <package>`,
|
||||
**Then** the specified app is launched on the device.
|
||||
|
||||
3. **When** I run `./fs5 ocr`,
|
||||
**Then** a screenshot is taken and all visible text is extracted via OCR.
|
||||
|
||||
---
|
||||
|
||||
## Requirements
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
- **FR-010**: `screen` command MUST launch scrcpy with device serial targeting.
|
||||
- **FR-011**: `screenshot` MUST save PNG to configurable local path.
|
||||
- **FR-012**: `tap`, `swipe`, `type` MUST use `adb shell input` commands.
|
||||
- **FR-013**: `sms list` MUST read SMS via KDE Connect CLI or ADB broadcast.
|
||||
- **FR-014**: `sms otp` MUST extract 4-8 digit codes from SMS body via regex.
|
||||
- **FR-015**: `sms watch` MUST poll for new SMS at configurable interval.
|
||||
- **FR-016**: `totp` MUST use screenshot + OCR (tesseract) for code extraction.
|
||||
- **FR-017**: All new commands MUST support `--json` output.
|
||||
|
||||
### Non-Functional Requirements
|
||||
|
||||
- **NFR-010**: Screenshot capture MUST complete in < 5 seconds.
|
||||
- **NFR-011**: OTP extraction MUST complete in < 3 seconds after SMS arrives.
|
||||
- **NFR-012**: OCR MUST achieve > 95% accuracy on 6-digit numeric codes.
|
||||
- **NFR-013**: No root required on device.
|
||||
|
||||
---
|
||||
|
||||
## Tool Selection (Research-Based)
|
||||
|
||||
| Tool | Purpose | Status | Rationale |
|
||||
|------|---------|--------|-----------|
|
||||
| scrcpy 3.3.4 | GUI mirror + input | ✅ Installed | Best no-root screen control |
|
||||
| KDE Connect | SMS forwarding | ✅ On device | Already installed, native Linux integration |
|
||||
| adb shell input | Tap/swipe/type | ✅ Built-in | No extra deps |
|
||||
| tesseract-ocr | TOTP OCR | Install needed | Best open-source OCR |
|
||||
| kdeconnect-cli | SMS reading | Install needed | CLI for KDE Connect |
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- Wireless ADB setup (USB connection sufficient)
|
||||
- Rooted device operations
|
||||
- Full Appium test framework (overkill for this use case)
|
||||
- SMS sending (receive only)
|
||||
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env bats
|
||||
# tests/test_remote_control.bats — Remote control command tests for fs5
|
||||
# Tests: screenshot, tap, swipe, type, key, launch, ocr, sms
|
||||
|
||||
setup() {
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." && pwd)"
|
||||
FS5="${SCRIPT_DIR}/fs5"
|
||||
DEVICE_SERIAL="${DEVICE_SERIAL:-RD51QE202392}"
|
||||
export DEVICE_SERIAL
|
||||
}
|
||||
|
||||
teardown() {
|
||||
# Clean up any test screenshots
|
||||
rm -f /tmp/fs5-test-screenshot-*.png
|
||||
}
|
||||
|
||||
# ── screenshot ────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "screenshot: saves PNG to default path" {
|
||||
run "${FS5}" screenshot
|
||||
[ "${status}" -eq 0 ]
|
||||
# Output contains a path
|
||||
echo "${output}" | grep -q "/tmp/fs5-screenshot-"
|
||||
# File exists
|
||||
local path
|
||||
path=$(echo "${output}" | grep "/tmp/fs5-screenshot-" | tail -1)
|
||||
[ -f "${path}" ]
|
||||
rm -f "${path}"
|
||||
}
|
||||
|
||||
@test "screenshot: saves PNG to specified path" {
|
||||
local out="/tmp/fs5-test-screenshot-$$.png"
|
||||
run "${FS5}" screenshot "${out}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[ -f "${out}" ]
|
||||
}
|
||||
|
||||
@test "screenshot: --json output contains path and timestamp" {
|
||||
run "${FS5}" screenshot --json
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
assert 'path' in data, 'missing path'
|
||||
assert 'timestamp' in data, 'missing timestamp'
|
||||
assert data['path'].endswith('.png'), 'path not PNG'
|
||||
"
|
||||
# Clean up
|
||||
local path
|
||||
path=$(echo "${output}" | python3 -c "import sys,json; print(json.load(sys.stdin)['path'])")
|
||||
rm -f "${path}"
|
||||
}
|
||||
|
||||
@test "screenshot: --help shows usage" {
|
||||
run "${FS5}" screenshot --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage"
|
||||
}
|
||||
|
||||
# ── tap ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "tap: sends tap at valid coordinates" {
|
||||
run "${FS5}" tap 540 960
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -q "Tapped"
|
||||
}
|
||||
|
||||
@test "tap: fails without coordinates" {
|
||||
run "${FS5}" tap
|
||||
[ "${status}" -ne 0 ]
|
||||
echo "${output}" | grep -qi "usage"
|
||||
}
|
||||
|
||||
@test "tap: fails with only one coordinate" {
|
||||
run "${FS5}" tap 540
|
||||
[ "${status}" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "tap: --help shows usage" {
|
||||
run "${FS5}" tap --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage"
|
||||
}
|
||||
|
||||
# ── swipe ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "swipe: performs swipe with four coordinates" {
|
||||
run "${FS5}" swipe 540 1200 540 400
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -q "Swiped"
|
||||
}
|
||||
|
||||
@test "swipe: performs swipe with custom duration" {
|
||||
run "${FS5}" swipe 540 1200 540 400 500
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -q "500ms"
|
||||
}
|
||||
|
||||
@test "swipe: fails without coordinates" {
|
||||
run "${FS5}" swipe
|
||||
[ "${status}" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "swipe: --help shows usage" {
|
||||
run "${FS5}" swipe --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage"
|
||||
}
|
||||
|
||||
# ── key ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "key: sends KEYCODE_HOME" {
|
||||
run "${FS5}" key KEYCODE_HOME
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -q "Key sent"
|
||||
}
|
||||
|
||||
@test "key: sends numeric keycode" {
|
||||
run "${FS5}" key 3
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -q "Key sent"
|
||||
}
|
||||
|
||||
@test "key: fails without keycode" {
|
||||
run "${FS5}" key
|
||||
[ "${status}" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "key: --help shows usage (global help intercepts --help flag)" {
|
||||
run "${FS5}" key --help
|
||||
[ "${status}" -eq 0 ]
|
||||
# --help is a global flag; shows main usage with key command listed
|
||||
echo "${output}" | grep -q "key"
|
||||
}
|
||||
|
||||
# ── launch ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "launch: launches KDE Connect app" {
|
||||
run "${FS5}" launch org.kde.kdeconnect_tp
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -q "Launched"
|
||||
}
|
||||
|
||||
@test "launch: fails for non-existent package" {
|
||||
run "${FS5}" launch com.nonexistent.package.xyz
|
||||
[ "${status}" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "launch: fails without package name" {
|
||||
run "${FS5}" launch
|
||||
[ "${status}" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "launch: --help shows usage" {
|
||||
run "${FS5}" launch --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage"
|
||||
}
|
||||
|
||||
# ── ocr ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "ocr: runs without error and returns text or empty" {
|
||||
run "${FS5}" ocr
|
||||
[ "${status}" -eq 0 ]
|
||||
# Output may be empty (blank screen) or contain text — both valid
|
||||
}
|
||||
|
||||
@test "ocr: --help shows usage" {
|
||||
run "${FS5}" ocr --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage\|screenshot\|ocr"
|
||||
}
|
||||
|
||||
# ── sms ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "sms: no subcommand shows error" {
|
||||
run "${FS5}" sms
|
||||
[ "${status}" -ne 0 ]
|
||||
echo "${output}" | grep -qi "unknown\|subcommand\|use:"
|
||||
}
|
||||
|
||||
@test "sms: --help shows subcommands" {
|
||||
run "${FS5}" sms --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -q "list"
|
||||
echo "${output}" | grep -q "otp"
|
||||
echo "${output}" | grep -q "watch"
|
||||
}
|
||||
|
||||
@test "sms list: shows KDE Connect setup instructions when not paired" {
|
||||
run "${FS5}" sms list
|
||||
# May succeed (if paired) or fail with setup instructions
|
||||
if [ "${status}" -ne 0 ]; then
|
||||
echo "${output}" | grep -qi "kde connect\|paired\|setup"
|
||||
fi
|
||||
}
|
||||
|
||||
@test "sms otp: shows error when KDE Connect not paired" {
|
||||
run "${FS5}" sms otp
|
||||
# Either returns OTP (if paired) or error about KDE Connect
|
||||
if [ "${status}" -ne 0 ]; then
|
||||
echo "${output}" | grep -qi "kde connect\|paired\|otp"
|
||||
fi
|
||||
}
|
||||
|
||||
@test "sms otp: --help shows usage" {
|
||||
run "${FS5}" sms otp --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage\|otp\|wait"
|
||||
}
|
||||
|
||||
@test "sms list: --help shows usage" {
|
||||
run "${FS5}" sms list --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage"
|
||||
}
|
||||
|
||||
# ── screen ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "screen: --help shows usage" {
|
||||
run "${FS5}" screen --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "scrcpy\|mirror\|headless"
|
||||
}
|
||||
|
||||
# ── type ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
@test "type: --help shows usage" {
|
||||
run "${FS5}" type --help
|
||||
[ "${status}" -eq 0 ]
|
||||
echo "${output}" | grep -qi "usage"
|
||||
}
|
||||
|
||||
@test "type: fails without text argument" {
|
||||
run "${FS5}" type
|
||||
[ "${status}" -ne 0 ]
|
||||
}
|
||||
Reference in New Issue
Block a user