feat(remote-control): add screen/screenshot/tap/swipe/type/key/launch/ocr/sms commands

- lib/input.sh: ADB input automation library (screenshot, tap, swipe, type, key, launch, OCR, SMS/OTP via KDE Connect)
- fs5: 9 new remote control commands for AI agent capabilities
- tests/test_remote_control.bats: 31 tests, all passing
- specs/002-remote-control/spec.md: feature specification

Tools: scrcpy 3.3.4 (GUI mirror), tesseract-ocr (OCR), KDE Connect (SMS/OTP)
Constraints: no root required, Android 9 (SDK 28)

Tests: 64/64 passing (33 device mgmt + 31 remote control)
This commit is contained in:
Jane Alesi
2026-03-02 16:10:11 +01:00
parent e6e5f34455
commit aaf008c1bf
4 changed files with 927 additions and 7 deletions
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env bash
# lib/input.sh — ADB input automation and screenshot functions for android-fs5
# shellcheck shell=bash
# adb_screenshot [OUTPUT_PATH] — capture device screen to PNG
# Returns path of saved file
adb_screenshot() {
local output="${1:-/tmp/fs5-screenshot-$(date +%s).png}"
adb_cmd shell screencap -p > "${output}"
echo "${output}"
}
# adb_tap X Y — send tap event at coordinates
adb_tap() {
local x="${1}" y="${2}"
adb_cmd shell input tap "${x}" "${y}"
}
# adb_swipe X1 Y1 X2 Y2 [DURATION_MS] — send swipe gesture
adb_swipe() {
local x1="${1}" y1="${2}" x2="${3}" y2="${4}"
local duration="${5:-300}"
adb_cmd shell input swipe "${x1}" "${y1}" "${x2}" "${y2}" "${duration}"
}
# adb_type TEXT — type text into focused input field
adb_type() {
local text="${1}"
# Escape spaces for adb input text
local escaped="${text// /%s}"
adb_cmd shell input text "${escaped}"
}
# adb_key KEYCODE — send Android key event
# Common: KEYCODE_HOME, KEYCODE_BACK, KEYCODE_POWER, KEYCODE_ENTER
adb_key() {
local keycode="${1}"
adb_cmd shell input keyevent "${keycode}"
}
# adb_launch PACKAGE — launch app by package name
adb_launch() {
local pkg="${1}"
local result exit_code
result=$(adb_cmd shell monkey -p "${pkg}" -c android.intent.category.LAUNCHER 1 2>&1) || exit_code=$?
exit_code="${exit_code:-0}"
if echo "${result}" | grep -q "Events injected: 1"; then
return 0
elif [[ "${exit_code}" -ne 0 ]] || echo "${result}" | grep -q "No activities found\|monkey aborted"; then
out_error "Package not found or has no launcher activity: ${pkg}"
return 1
else
out_error "Failed to launch: ${pkg}"
return 1
fi
}
# adb_ocr [SCREENSHOT_PATH] — screenshot + OCR, return extracted text
adb_ocr() {
local screenshot
screenshot=$(adb_screenshot)
if ! command -v tesseract &>/dev/null; then
out_error "tesseract not found. Install: sudo pacman -S tesseract tesseract-data-eng"
return 1
fi
# Locate tessdata — try common paths
local tessdata=""
for d in /usr/share/tessdata /usr/share/tesseract-ocr/4.00/tessdata \
/usr/share/tesseract-ocr/tessdata /usr/local/share/tessdata; do
if [[ -f "${d}/eng.traineddata" ]]; then
tessdata="${d}"
break
fi
done
if [[ -z "${tessdata}" ]]; then
out_error "tesseract eng language data not found. Install: sudo pacman -S tesseract-data-eng"
return 1
fi
local txt_base="${screenshot%.png}"
TESSDATA_PREFIX="${tessdata}" tesseract "${screenshot}" "${txt_base}" -l eng --psm 6 quiet 2>/dev/null
if [[ -f "${txt_base}.txt" ]]; then
cat "${txt_base}.txt"
rm -f "${txt_base}.txt"
fi
rm -f "${screenshot}"
}
# _kdeconnect_device_id — find the paired FS5 device ID via kdeconnect-cli
# Returns device ID or empty string if not paired
_kdeconnect_device_id() {
kdeconnect-cli --list-devices --id-only 2>/dev/null | head -1 | tr -d '\r\n'
}
# _kdeconnect_check — verify KDE Connect is paired; print setup instructions if not
_kdeconnect_check() {
local dev_id
dev_id=$(_kdeconnect_device_id)
if [[ -z "${dev_id}" ]]; then
out_error "KDE Connect: no paired device found."
echo "Setup steps:"
echo " 1. Install KDE Connect on Linux: sudo pacman -S kdeconnect"
echo " 2. Open KDE Connect app on the FS5 phone"
echo " 3. Accept the pairing request on both devices"
echo " 4. Grant SMS permission in KDE Connect app on phone"
echo " 5. Re-run this command"
return 1
fi
echo "${dev_id}"
}
# adb_sms_list [LIMIT] — list SMS via KDE Connect CLI
# Returns JSON array of {id, address, date, body}
adb_sms_list() {
local limit="${1:-10}"
local dev_id
dev_id=$(_kdeconnect_check) || return 1
# Use kdeconnect-cli SMS list if available (kdeconnect 21.08+)
if kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null | head -1 | grep -q "^{"; then
kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null \
| head -n "${limit}" \
| python3 -c "
import sys, json
msgs = []
for line in sys.stdin:
line = line.strip()
if line:
try:
msgs.append(json.loads(line))
except Exception:
pass
print(json.dumps(msgs, indent=2))
"
else
# Fallback: show pairing status and guidance
out_error "KDE Connect SMS listing requires kdeconnect 21.08+ and SMS permission on device."
echo "[]"
fi
}
# adb_sms_otp [WAIT_SECS] — extract most recent OTP code via KDE Connect notifications
# If WAIT_SECS > 0, polls until new OTP arrives or timeout
adb_sms_otp() {
local wait_secs="${1:-0}"
local dev_id
dev_id=$(_kdeconnect_check) || return 1
local deadline=$(( $(date +%s) + wait_secs ))
# Poll KDE Connect notifications for OTP patterns
while true; do
local notifs otp
notifs=$(kdeconnect-cli --device "${dev_id}" --list-notifications 2>/dev/null | tr -d '\r')
otp=$(echo "${notifs}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
if [[ -n "${otp}" ]]; then
echo "${otp}"
return 0
fi
if [[ "${wait_secs}" -le 0 ]] || [[ $(date +%s) -ge ${deadline} ]]; then
break
fi
sleep 3
done
out_error "No OTP found in KDE Connect notifications"
return 1
}