infrastructure/android-fs5
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(remote-control): add screen/screenshot/tap/swipe/type/key/launch/ocr/sms commands

Browse Source 
- lib/input.sh: ADB input automation library (screenshot, tap, swipe, type, key, launch, OCR, SMS/OTP via KDE Connect)
- fs5: 9 new remote control commands for AI agent capabilities
- tests/test_remote_control.bats: 31 tests, all passing
- specs/002-remote-control/spec.md: feature specification

Tools: scrcpy 3.3.4 (GUI mirror), tesseract-ocr (OCR), KDE Connect (SMS/OTP)
Constraints: no root required, Android 9 (SDK 28)

Tests: 64/64 passing (33 device mgmt + 31 remote control)
This commit is contained in:
Jane Alesi
2026-03-02 16:10:11 +01:00
parent e6e5f34455
commit aaf008c1bf
4 changed files with 927 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs5
+384 -7
View File
@@ -34,6 +34,8 @@ ADB_TIMEOUT="${ADB_TIMEOUT:-10}"
source "${SCRIPT_DIR}/lib/output.sh"
# shellcheck source=lib/adb.sh
source "${SCRIPT_DIR}/lib/adb.sh"
# shellcheck source=lib/input.sh
source "${SCRIPT_DIR}/lib/input.sh"
# ── Global flags ──────────────────────────────────────────────────────────────
JSON_OUTPUT=false
@@ -44,7 +46,7 @@ usage() {
cat <<EOF
Usage: fs5 <command> [options]
Commands:
Device Management:
status Show device health and info
push <src> <dst> Push file/dir to device
pull <src> <dst> Pull file/dir from device
@@ -54,6 +56,19 @@ Commands:
shell [cmd] Execute shell command on device
logs Stream or capture logcat output
Remote Control (AI capabilities):
screen Launch scrcpy GUI mirror with input
screenshot [path] Capture screen to PNG file
tap <x> <y> Send tap at coordinates
swipe <x1> <y1> <x2> <y2> [ms] Send swipe gesture
type <text> Type text into focused field
key <keycode> Send Android key event
launch <package> Launch app by package name
ocr Screenshot + extract all text via OCR
sms list List recent SMS messages
sms otp Extract latest OTP code from SMS
sms watch Watch for new SMS messages
Global Options:
--json Output as JSON
--force Allow destructive/overwrite operations
@@ -445,6 +460,359 @@ EOF
fi
}
# ── Command: screen ──────────────────────────────────────────────────────────
cmd_screen() {
local headless=false show_help=false
for arg in "$@"; do
case "${arg}" in
--headless) headless=true ;;
--help) show_help=true ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 screen [--headless]
Launch scrcpy to mirror and control the device screen.
--headless: run without display window (for automation/OCR only).
EOF
return 0
fi
adb_check_device
if [[ "${headless}" == "true" ]]; then
scrcpy -s "${DEVICE_SERIAL}" --no-display &
out_success "scrcpy running headless (PID $!)"
else
scrcpy -s "${DEVICE_SERIAL}"
fi
}
# ── Command: screenshot ───────────────────────────────────────────────────────
cmd_screenshot() {
local output="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) [[ -z "${output}" ]] && output="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 screenshot [output-path.png]"
return 0
fi
adb_check_device
local path
if [[ -n "${output}" ]]; then
path=$(adb_screenshot "${output}")
else
path=$(adb_screenshot)
fi
if [[ "${JSON_OUTPUT}" == "true" ]]; then
out_json "path=${path}" "timestamp=$(date -Iseconds)"
else
out_success "Screenshot saved: ${path}"
echo "${path}"
fi
}
# ── Command: tap ─────────────────────────────────────────────────────────────
cmd_tap() {
local x="" y="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*)
if [[ -z "${x}" ]]; then x="${arg}"
elif [[ -z "${y}" ]]; then y="${arg}"
fi
;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 tap <x> <y>"
return 0
fi
if [[ -z "${x}" ]] || [[ -z "${y}" ]]; then
out_error "Usage: fs5 tap <x> <y>"
return 1
fi
adb_check_device
adb_tap "${x}" "${y}"
out_success "Tapped: (${x}, ${y})"
}
# ── Command: swipe ────────────────────────────────────────────────────────────
cmd_swipe() {
local x1="" y1="" x2="" y2="" duration="300" show_help=false
local -a pos=()
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) pos+=("${arg}") ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 swipe <x1> <y1> <x2> <y2> [duration_ms]"
return 0
fi
x1="${pos[0]:-}" y1="${pos[1]:-}" x2="${pos[2]:-}" y2="${pos[3]:-}"
[[ ${#pos[@]} -ge 5 ]] && duration="${pos[4]}"
if [[ -z "${x1}" ]] || [[ -z "${y1}" ]] || [[ -z "${x2}" ]] || [[ -z "${y2}" ]]; then
out_error "Usage: fs5 swipe <x1> <y1> <x2> <y2> [duration_ms]"
return 1
fi
adb_check_device
adb_swipe "${x1}" "${y1}" "${x2}" "${y2}" "${duration}"
out_success "Swiped: (${x1},${y1}) → (${x2},${y2}) in ${duration}ms"
}
# ── Command: type ─────────────────────────────────────────────────────────────
cmd_type() {
local text="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) text="${text}${text:+ }${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 type <text>"
return 0
fi
if [[ -z "${text}" ]]; then
out_error "Usage: fs5 type <text>"
return 1
fi
adb_check_device
adb_type "${text}"
out_success "Typed: ${text}"
}
# ── Command: key ──────────────────────────────────────────────────────────────
cmd_key() {
local keycode="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) [[ -z "${keycode}" ]] && keycode="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 key <keycode>
Send Android key event. Common keycodes:
KEYCODE_HOME KEYCODE_BACK KEYCODE_POWER
KEYCODE_ENTER KEYCODE_TAB KEYCODE_DEL
3 (HOME) 4 (BACK) 26 (POWER)
EOF
return 0
fi
if [[ -z "${keycode}" ]]; then
out_error "Usage: fs5 key <keycode>"
return 1
fi
adb_check_device
adb_key "${keycode}"
out_success "Key sent: ${keycode}"
}
# ── Command: launch ───────────────────────────────────────────────────────────
cmd_launch() {
local pkg="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) [[ -z "${pkg}" ]] && pkg="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 launch <package-name>"
return 0
fi
if [[ -z "${pkg}" ]]; then
out_error "Usage: fs5 launch <package-name>"
return 1
fi
adb_check_device
if adb_launch "${pkg}"; then
out_success "Launched: ${pkg}"
else
return 1
fi
}
# ── Command: ocr ──────────────────────────────────────────────────────────────
cmd_ocr() {
local show_help=false
for arg in "$@"; do
[[ "${arg}" == "--help" ]] && show_help=true
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 ocr"
echo "Take a screenshot and extract all visible text via OCR (tesseract)."
return 0
fi
adb_check_device
local text
text=$(adb_ocr)
if [[ "${JSON_OUTPUT}" == "true" ]]; then
python3 -c "import json,sys; print(json.dumps({'text': sys.stdin.read()}))" <<< "${text}"
else
echo "${text}"
fi
}
# ── Command: sms ──────────────────────────────────────────────────────────────
cmd_sms() {
local subcmd="${1:-}"
shift || true
case "${subcmd}" in
list) cmd_sms_list "$@" ;;
otp) cmd_sms_otp "$@" ;;
watch) cmd_sms_watch "$@" ;;
--help|-h)
cat <<EOF
Usage: fs5 sms <subcommand>
Subcommands:
list [--limit N] [--json] List recent SMS messages
otp [--wait] Extract latest OTP code from SMS
watch Watch for new SMS messages (Ctrl+C to stop)
EOF
;;
*)
out_error "Unknown sms subcommand: '${subcmd}'. Use: list, otp, watch"
return 1
;;
esac
}
cmd_sms_list() {
local limit=10 show_help=false
local i=0
local -a args=("$@")
while [[ ${i} -lt ${#args[@]} ]]; do
case "${args[${i}]}" in
--help) show_help=true ;;
--json) JSON_OUTPUT=true ;;
--limit) i=$(( i + 1 )); limit="${args[${i}]}" ;;
esac
i=$(( i + 1 ))
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 sms list [--limit N] [--json]"
return 0
fi
adb_check_device
adb_sms_list "${limit}"
}
cmd_sms_otp() {
local wait_secs=0 show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
--wait) wait_secs=60 ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 sms otp [--wait]"
echo "Extract latest OTP/2FA code from SMS inbox."
echo "--wait: poll up to 60 seconds for a new OTP."
return 0
fi
adb_check_device
local otp
otp=$(adb_sms_otp "${wait_secs}")
local exit_code=$?
if [[ ${exit_code} -eq 0 ]]; then
if [[ "${JSON_OUTPUT}" == "true" ]]; then
out_json "otp=${otp}" "timestamp=$(date -Iseconds)"
else
echo "${otp}"
fi
fi
return ${exit_code}
}
cmd_sms_watch() {
local show_help=false
for arg in "$@"; do
[[ "${arg}" == "--help" ]] && show_help=true
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 sms watch"
echo "Poll for new SMS messages every 5 seconds. Press Ctrl+C to stop."
return 0
fi
adb_check_device
out_success "Watching for SMS (Ctrl+C to stop)..."
local last_id=""
last_id=$(adb_cmd shell content query \
--uri content://sms/inbox \
--projection "_id" \
--sort "date DESC" \
2>/dev/null | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2 | tr -d '\r')
while true; do
local raw new_id
raw=$(adb_cmd shell content query \
--uri content://sms/inbox \
--projection "_id,address,date,body" \
--sort "date DESC" \
2>/dev/null | tr -d '\r' | head -10)
new_id=$(echo "${raw}" | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2)
if [[ -n "${new_id}" ]] && [[ "${new_id}" != "${last_id}" ]]; then
local ts body address
ts=$(date -Iseconds)
address=$(echo "${raw}" | grep "address=" | head -1 | grep -oE 'address=[^,]+' | cut -d= -f2)
body=$(echo "${raw}" | grep "body=" | head -1 | sed 's/.*body=//')
echo "[${ts}] FROM: ${address}"
echo " ${body}"
echo ""
last_id="${new_id}"
fi
sleep 5
done
}
# ── Main dispatcher ───────────────────────────────────────────────────────────
main() {
if [[ $# -eq 0 ]]; then
@@ -467,12 +835,21 @@ main() {
parse_global_flags remaining_args "$@"
case "${command}" in
status) cmd_status "${remaining_args[@]+"${remaining_args[@]}"}" ;;
push) cmd_push "${remaining_args[@]+"${remaining_args[@]}"}" ;;
pull) cmd_pull "${remaining_args[@]+"${remaining_args[@]}"}" ;;
app) cmd_app "${remaining_args[@]+"${remaining_args[@]}"}" ;;
shell) cmd_shell "${remaining_args[@]+"${remaining_args[@]}"}" ;;
logs) cmd_logs "${remaining_args[@]+"${remaining_args[@]}"}" ;;
status) cmd_status "${remaining_args[@]+"${remaining_args[@]}"}" ;;
push) cmd_push "${remaining_args[@]+"${remaining_args[@]}"}" ;;
pull) cmd_pull "${remaining_args[@]+"${remaining_args[@]}"}" ;;
app) cmd_app "${remaining_args[@]+"${remaining_args[@]}"}" ;;
shell) cmd_shell "${remaining_args[@]+"${remaining_args[@]}"}" ;;
logs) cmd_logs "${remaining_args[@]+"${remaining_args[@]}"}" ;;
screen) cmd_screen "${remaining_args[@]+"${remaining_args[@]}"}" ;;
screenshot) cmd_screenshot "${remaining_args[@]+"${remaining_args[@]}"}" ;;
tap) cmd_tap "${remaining_args[@]+"${remaining_args[@]}"}" ;;
swipe) cmd_swipe "${remaining_args[@]+"${remaining_args[@]}"}" ;;
type) cmd_type "${remaining_args[@]+"${remaining_args[@]}"}" ;;
key) cmd_key "${remaining_args[@]+"${remaining_args[@]}"}" ;;
launch) cmd_launch "${remaining_args[@]+"${remaining_args[@]}"}" ;;
ocr) cmd_ocr "${remaining_args[@]+"${remaining_args[@]}"}" ;;
sms) cmd_sms "${remaining_args[@]+"${remaining_args[@]}"}" ;;
*)
out_error "Unknown command: ${command}"
usage