feat(remote-control): add screen/screenshot/tap/swipe/type/key/launch/ocr/sms commands

- lib/input.sh: ADB input automation library (screenshot, tap, swipe, type, key, launch, OCR, SMS/OTP via KDE Connect)
- fs5: 9 new remote control commands for AI agent capabilities
- tests/test_remote_control.bats: 31 tests, all passing
- specs/002-remote-control/spec.md: feature specification

Tools: scrcpy 3.3.4 (GUI mirror), tesseract-ocr (OCR), KDE Connect (SMS/OTP)
Constraints: no root required, Android 9 (SDK 28)

Tests: 64/64 passing (33 device mgmt + 31 remote control)
This commit is contained in:
Jane Alesi
2026-03-02 16:10:11 +01:00
parent e6e5f34455
commit aaf008c1bf
4 changed files with 927 additions and 7 deletions
+378 -1
View File
@@ -34,6 +34,8 @@ ADB_TIMEOUT="${ADB_TIMEOUT:-10}"
source "${SCRIPT_DIR}/lib/output.sh" source "${SCRIPT_DIR}/lib/output.sh"
# shellcheck source=lib/adb.sh # shellcheck source=lib/adb.sh
source "${SCRIPT_DIR}/lib/adb.sh" source "${SCRIPT_DIR}/lib/adb.sh"
# shellcheck source=lib/input.sh
source "${SCRIPT_DIR}/lib/input.sh"
# ── Global flags ────────────────────────────────────────────────────────────── # ── Global flags ──────────────────────────────────────────────────────────────
JSON_OUTPUT=false JSON_OUTPUT=false
@@ -44,7 +46,7 @@ usage() {
cat <<EOF cat <<EOF
Usage: fs5 <command> [options] Usage: fs5 <command> [options]
Commands: Device Management:
status Show device health and info status Show device health and info
push <src> <dst> Push file/dir to device push <src> <dst> Push file/dir to device
pull <src> <dst> Pull file/dir from device pull <src> <dst> Pull file/dir from device
@@ -54,6 +56,19 @@ Commands:
shell [cmd] Execute shell command on device shell [cmd] Execute shell command on device
logs Stream or capture logcat output logs Stream or capture logcat output
Remote Control (AI capabilities):
screen Launch scrcpy GUI mirror with input
screenshot [path] Capture screen to PNG file
tap <x> <y> Send tap at coordinates
swipe <x1> <y1> <x2> <y2> [ms] Send swipe gesture
type <text> Type text into focused field
key <keycode> Send Android key event
launch <package> Launch app by package name
ocr Screenshot + extract all text via OCR
sms list List recent SMS messages
sms otp Extract latest OTP code from SMS
sms watch Watch for new SMS messages
Global Options: Global Options:
--json Output as JSON --json Output as JSON
--force Allow destructive/overwrite operations --force Allow destructive/overwrite operations
@@ -445,6 +460,359 @@ EOF
fi fi
} }
# ── Command: screen ──────────────────────────────────────────────────────────
cmd_screen() {
local headless=false show_help=false
for arg in "$@"; do
case "${arg}" in
--headless) headless=true ;;
--help) show_help=true ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 screen [--headless]
Launch scrcpy to mirror and control the device screen.
--headless: run without display window (for automation/OCR only).
EOF
return 0
fi
adb_check_device
if [[ "${headless}" == "true" ]]; then
scrcpy -s "${DEVICE_SERIAL}" --no-display &
out_success "scrcpy running headless (PID $!)"
else
scrcpy -s "${DEVICE_SERIAL}"
fi
}
# ── Command: screenshot ───────────────────────────────────────────────────────
cmd_screenshot() {
local output="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) [[ -z "${output}" ]] && output="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 screenshot [output-path.png]"
return 0
fi
adb_check_device
local path
if [[ -n "${output}" ]]; then
path=$(adb_screenshot "${output}")
else
path=$(adb_screenshot)
fi
if [[ "${JSON_OUTPUT}" == "true" ]]; then
out_json "path=${path}" "timestamp=$(date -Iseconds)"
else
out_success "Screenshot saved: ${path}"
echo "${path}"
fi
}
# ── Command: tap ─────────────────────────────────────────────────────────────
cmd_tap() {
local x="" y="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*)
if [[ -z "${x}" ]]; then x="${arg}"
elif [[ -z "${y}" ]]; then y="${arg}"
fi
;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 tap <x> <y>"
return 0
fi
if [[ -z "${x}" ]] || [[ -z "${y}" ]]; then
out_error "Usage: fs5 tap <x> <y>"
return 1
fi
adb_check_device
adb_tap "${x}" "${y}"
out_success "Tapped: (${x}, ${y})"
}
# ── Command: swipe ────────────────────────────────────────────────────────────
cmd_swipe() {
local x1="" y1="" x2="" y2="" duration="300" show_help=false
local -a pos=()
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) pos+=("${arg}") ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 swipe <x1> <y1> <x2> <y2> [duration_ms]"
return 0
fi
x1="${pos[0]:-}" y1="${pos[1]:-}" x2="${pos[2]:-}" y2="${pos[3]:-}"
[[ ${#pos[@]} -ge 5 ]] && duration="${pos[4]}"
if [[ -z "${x1}" ]] || [[ -z "${y1}" ]] || [[ -z "${x2}" ]] || [[ -z "${y2}" ]]; then
out_error "Usage: fs5 swipe <x1> <y1> <x2> <y2> [duration_ms]"
return 1
fi
adb_check_device
adb_swipe "${x1}" "${y1}" "${x2}" "${y2}" "${duration}"
out_success "Swiped: (${x1},${y1}) → (${x2},${y2}) in ${duration}ms"
}
# ── Command: type ─────────────────────────────────────────────────────────────
cmd_type() {
local text="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) text="${text}${text:+ }${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 type <text>"
return 0
fi
if [[ -z "${text}" ]]; then
out_error "Usage: fs5 type <text>"
return 1
fi
adb_check_device
adb_type "${text}"
out_success "Typed: ${text}"
}
# ── Command: key ──────────────────────────────────────────────────────────────
cmd_key() {
local keycode="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) [[ -z "${keycode}" ]] && keycode="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
cat <<EOF
Usage: fs5 key <keycode>
Send Android key event. Common keycodes:
KEYCODE_HOME KEYCODE_BACK KEYCODE_POWER
KEYCODE_ENTER KEYCODE_TAB KEYCODE_DEL
3 (HOME) 4 (BACK) 26 (POWER)
EOF
return 0
fi
if [[ -z "${keycode}" ]]; then
out_error "Usage: fs5 key <keycode>"
return 1
fi
adb_check_device
adb_key "${keycode}"
out_success "Key sent: ${keycode}"
}
# ── Command: launch ───────────────────────────────────────────────────────────
cmd_launch() {
local pkg="" show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
*) [[ -z "${pkg}" ]] && pkg="${arg}" ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 launch <package-name>"
return 0
fi
if [[ -z "${pkg}" ]]; then
out_error "Usage: fs5 launch <package-name>"
return 1
fi
adb_check_device
if adb_launch "${pkg}"; then
out_success "Launched: ${pkg}"
else
return 1
fi
}
# ── Command: ocr ──────────────────────────────────────────────────────────────
cmd_ocr() {
local show_help=false
for arg in "$@"; do
[[ "${arg}" == "--help" ]] && show_help=true
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 ocr"
echo "Take a screenshot and extract all visible text via OCR (tesseract)."
return 0
fi
adb_check_device
local text
text=$(adb_ocr)
if [[ "${JSON_OUTPUT}" == "true" ]]; then
python3 -c "import json,sys; print(json.dumps({'text': sys.stdin.read()}))" <<< "${text}"
else
echo "${text}"
fi
}
# ── Command: sms ──────────────────────────────────────────────────────────────
cmd_sms() {
local subcmd="${1:-}"
shift || true
case "${subcmd}" in
list) cmd_sms_list "$@" ;;
otp) cmd_sms_otp "$@" ;;
watch) cmd_sms_watch "$@" ;;
--help|-h)
cat <<EOF
Usage: fs5 sms <subcommand>
Subcommands:
list [--limit N] [--json] List recent SMS messages
otp [--wait] Extract latest OTP code from SMS
watch Watch for new SMS messages (Ctrl+C to stop)
EOF
;;
*)
out_error "Unknown sms subcommand: '${subcmd}'. Use: list, otp, watch"
return 1
;;
esac
}
cmd_sms_list() {
local limit=10 show_help=false
local i=0
local -a args=("$@")
while [[ ${i} -lt ${#args[@]} ]]; do
case "${args[${i}]}" in
--help) show_help=true ;;
--json) JSON_OUTPUT=true ;;
--limit) i=$(( i + 1 )); limit="${args[${i}]}" ;;
esac
i=$(( i + 1 ))
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 sms list [--limit N] [--json]"
return 0
fi
adb_check_device
adb_sms_list "${limit}"
}
cmd_sms_otp() {
local wait_secs=0 show_help=false
for arg in "$@"; do
case "${arg}" in
--help) show_help=true ;;
--wait) wait_secs=60 ;;
esac
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 sms otp [--wait]"
echo "Extract latest OTP/2FA code from SMS inbox."
echo "--wait: poll up to 60 seconds for a new OTP."
return 0
fi
adb_check_device
local otp
otp=$(adb_sms_otp "${wait_secs}")
local exit_code=$?
if [[ ${exit_code} -eq 0 ]]; then
if [[ "${JSON_OUTPUT}" == "true" ]]; then
out_json "otp=${otp}" "timestamp=$(date -Iseconds)"
else
echo "${otp}"
fi
fi
return ${exit_code}
}
cmd_sms_watch() {
local show_help=false
for arg in "$@"; do
[[ "${arg}" == "--help" ]] && show_help=true
done
if [[ "${show_help}" == "true" ]]; then
echo "Usage: fs5 sms watch"
echo "Poll for new SMS messages every 5 seconds. Press Ctrl+C to stop."
return 0
fi
adb_check_device
out_success "Watching for SMS (Ctrl+C to stop)..."
local last_id=""
last_id=$(adb_cmd shell content query \
--uri content://sms/inbox \
--projection "_id" \
--sort "date DESC" \
2>/dev/null | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2 | tr -d '\r')
while true; do
local raw new_id
raw=$(adb_cmd shell content query \
--uri content://sms/inbox \
--projection "_id,address,date,body" \
--sort "date DESC" \
2>/dev/null | tr -d '\r' | head -10)
new_id=$(echo "${raw}" | grep "_id=" | head -1 | grep -oE '_id=[0-9]+' | cut -d= -f2)
if [[ -n "${new_id}" ]] && [[ "${new_id}" != "${last_id}" ]]; then
local ts body address
ts=$(date -Iseconds)
address=$(echo "${raw}" | grep "address=" | head -1 | grep -oE 'address=[^,]+' | cut -d= -f2)
body=$(echo "${raw}" | grep "body=" | head -1 | sed 's/.*body=//')
echo "[${ts}] FROM: ${address}"
echo " ${body}"
echo ""
last_id="${new_id}"
fi
sleep 5
done
}
# ── Main dispatcher ─────────────────────────────────────────────────────────── # ── Main dispatcher ───────────────────────────────────────────────────────────
main() { main() {
if [[ $# -eq 0 ]]; then if [[ $# -eq 0 ]]; then
@@ -473,6 +841,15 @@ main() {
app) cmd_app "${remaining_args[@]+"${remaining_args[@]}"}" ;; app) cmd_app "${remaining_args[@]+"${remaining_args[@]}"}" ;;
shell) cmd_shell "${remaining_args[@]+"${remaining_args[@]}"}" ;; shell) cmd_shell "${remaining_args[@]+"${remaining_args[@]}"}" ;;
logs) cmd_logs "${remaining_args[@]+"${remaining_args[@]}"}" ;; logs) cmd_logs "${remaining_args[@]+"${remaining_args[@]}"}" ;;
screen) cmd_screen "${remaining_args[@]+"${remaining_args[@]}"}" ;;
screenshot) cmd_screenshot "${remaining_args[@]+"${remaining_args[@]}"}" ;;
tap) cmd_tap "${remaining_args[@]+"${remaining_args[@]}"}" ;;
swipe) cmd_swipe "${remaining_args[@]+"${remaining_args[@]}"}" ;;
type) cmd_type "${remaining_args[@]+"${remaining_args[@]}"}" ;;
key) cmd_key "${remaining_args[@]+"${remaining_args[@]}"}" ;;
launch) cmd_launch "${remaining_args[@]+"${remaining_args[@]}"}" ;;
ocr) cmd_ocr "${remaining_args[@]+"${remaining_args[@]}"}" ;;
sms) cmd_sms "${remaining_args[@]+"${remaining_args[@]}"}" ;;
*) *)
out_error "Unknown command: ${command}" out_error "Unknown command: ${command}"
usage usage
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env bash
# lib/input.sh — ADB input automation and screenshot functions for android-fs5
# shellcheck shell=bash
# adb_screenshot [OUTPUT_PATH] — capture device screen to PNG
# Returns path of saved file
adb_screenshot() {
local output="${1:-/tmp/fs5-screenshot-$(date +%s).png}"
adb_cmd shell screencap -p > "${output}"
echo "${output}"
}
# adb_tap X Y — send tap event at coordinates
adb_tap() {
local x="${1}" y="${2}"
adb_cmd shell input tap "${x}" "${y}"
}
# adb_swipe X1 Y1 X2 Y2 [DURATION_MS] — send swipe gesture
adb_swipe() {
local x1="${1}" y1="${2}" x2="${3}" y2="${4}"
local duration="${5:-300}"
adb_cmd shell input swipe "${x1}" "${y1}" "${x2}" "${y2}" "${duration}"
}
# adb_type TEXT — type text into focused input field
adb_type() {
local text="${1}"
# Escape spaces for adb input text
local escaped="${text// /%s}"
adb_cmd shell input text "${escaped}"
}
# adb_key KEYCODE — send Android key event
# Common: KEYCODE_HOME, KEYCODE_BACK, KEYCODE_POWER, KEYCODE_ENTER
adb_key() {
local keycode="${1}"
adb_cmd shell input keyevent "${keycode}"
}
# adb_launch PACKAGE — launch app by package name
adb_launch() {
local pkg="${1}"
local result exit_code
result=$(adb_cmd shell monkey -p "${pkg}" -c android.intent.category.LAUNCHER 1 2>&1) || exit_code=$?
exit_code="${exit_code:-0}"
if echo "${result}" | grep -q "Events injected: 1"; then
return 0
elif [[ "${exit_code}" -ne 0 ]] || echo "${result}" | grep -q "No activities found\|monkey aborted"; then
out_error "Package not found or has no launcher activity: ${pkg}"
return 1
else
out_error "Failed to launch: ${pkg}"
return 1
fi
}
# adb_ocr [SCREENSHOT_PATH] — screenshot + OCR, return extracted text
adb_ocr() {
local screenshot
screenshot=$(adb_screenshot)
if ! command -v tesseract &>/dev/null; then
out_error "tesseract not found. Install: sudo pacman -S tesseract tesseract-data-eng"
return 1
fi
# Locate tessdata — try common paths
local tessdata=""
for d in /usr/share/tessdata /usr/share/tesseract-ocr/4.00/tessdata \
/usr/share/tesseract-ocr/tessdata /usr/local/share/tessdata; do
if [[ -f "${d}/eng.traineddata" ]]; then
tessdata="${d}"
break
fi
done
if [[ -z "${tessdata}" ]]; then
out_error "tesseract eng language data not found. Install: sudo pacman -S tesseract-data-eng"
return 1
fi
local txt_base="${screenshot%.png}"
TESSDATA_PREFIX="${tessdata}" tesseract "${screenshot}" "${txt_base}" -l eng --psm 6 quiet 2>/dev/null
if [[ -f "${txt_base}.txt" ]]; then
cat "${txt_base}.txt"
rm -f "${txt_base}.txt"
fi
rm -f "${screenshot}"
}
# _kdeconnect_device_id — find the paired FS5 device ID via kdeconnect-cli
# Returns device ID or empty string if not paired
_kdeconnect_device_id() {
kdeconnect-cli --list-devices --id-only 2>/dev/null | head -1 | tr -d '\r\n'
}
# _kdeconnect_check — verify KDE Connect is paired; print setup instructions if not
_kdeconnect_check() {
local dev_id
dev_id=$(_kdeconnect_device_id)
if [[ -z "${dev_id}" ]]; then
out_error "KDE Connect: no paired device found."
echo "Setup steps:"
echo " 1. Install KDE Connect on Linux: sudo pacman -S kdeconnect"
echo " 2. Open KDE Connect app on the FS5 phone"
echo " 3. Accept the pairing request on both devices"
echo " 4. Grant SMS permission in KDE Connect app on phone"
echo " 5. Re-run this command"
return 1
fi
echo "${dev_id}"
}
# adb_sms_list [LIMIT] — list SMS via KDE Connect CLI
# Returns JSON array of {id, address, date, body}
adb_sms_list() {
local limit="${1:-10}"
local dev_id
dev_id=$(_kdeconnect_check) || return 1
# Use kdeconnect-cli SMS list if available (kdeconnect 21.08+)
if kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null | head -1 | grep -q "^{"; then
kdeconnect-cli --device "${dev_id}" --list-sms 2>/dev/null \
| head -n "${limit}" \
| python3 -c "
import sys, json
msgs = []
for line in sys.stdin:
line = line.strip()
if line:
try:
msgs.append(json.loads(line))
except Exception:
pass
print(json.dumps(msgs, indent=2))
"
else
# Fallback: show pairing status and guidance
out_error "KDE Connect SMS listing requires kdeconnect 21.08+ and SMS permission on device."
echo "[]"
fi
}
# adb_sms_otp [WAIT_SECS] — extract most recent OTP code via KDE Connect notifications
# If WAIT_SECS > 0, polls until new OTP arrives or timeout
adb_sms_otp() {
local wait_secs="${1:-0}"
local dev_id
dev_id=$(_kdeconnect_check) || return 1
local deadline=$(( $(date +%s) + wait_secs ))
# Poll KDE Connect notifications for OTP patterns
while true; do
local notifs otp
notifs=$(kdeconnect-cli --device "${dev_id}" --list-notifications 2>/dev/null | tr -d '\r')
otp=$(echo "${notifs}" | grep -oE '\b[0-9]{4,8}\b' | head -1)
if [[ -n "${otp}" ]]; then
echo "${otp}"
return 0
fi
if [[ "${wait_secs}" -le 0 ]] || [[ $(date +%s) -ge ${deadline} ]]; then
break
fi
sleep 3
done
out_error "No OTP found in KDE Connect notifications"
return 1
}
+137
View File
@@ -0,0 +1,137 @@
# Feature Specification: Android FS5 Full Remote Control
**Feature Branch**: `002-remote-control`
**Created**: 2026-03-02
**Status**: Draft
**Depends on**: 001-android-fs5-management (fs5 CLI)
---
## Overview
Extend the FS5 device management to enable **full remote control** as an AI capability
extension: GUI mirroring/automation via scrcpy, SMS/2FA OTP reading via KDE Connect,
and TOTP code extraction for automated authentication workflows.
**Key insight**: KDE Connect is already installed on the device — this is the primary
SMS forwarding channel. scrcpy 3.3.4 is already on the host.
---
## User Scenarios & Testing
### User Story 1 — GUI Mirroring & Input (Priority: P1)
**Why**: AI agent needs to see and interact with the phone screen for 2FA app usage.
**Acceptance Scenarios**:
1. **Given** the FS5 is connected, **When** I run `./fs5 screen`,
**Then** scrcpy launches showing the device screen with input forwarding enabled.
2. **When** I run `./fs5 screen --headless`,
**Then** scrcpy runs without display (for screenshot/OCR automation only).
3. **When** I run `./fs5 screenshot`,
**Then** a PNG screenshot is saved locally and the path is printed.
4. **When** I run `./fs5 tap <x> <y>`,
**Then** a tap event is sent to the device at those coordinates.
5. **When** I run `./fs5 swipe <x1> <y1> <x2> <y2>`,
**Then** a swipe gesture is performed on the device.
6. **When** I run `./fs5 type "text"`,
**Then** the text is typed into the currently focused input field.
---
### User Story 2 — SMS/OTP Reading via KDE Connect (Priority: P1)
**Why**: Receive SMS 2FA codes without manual phone interaction.
**Acceptance Scenarios**:
1. **When** I run `./fs5 sms list`,
**Then** recent SMS messages are listed with sender, timestamp, and body.
2. **When** I run `./fs5 sms watch`,
**Then** new SMS messages are printed as they arrive (polling loop).
3. **When** I run `./fs5 sms otp`,
**Then** the most recent OTP/2FA code (4-8 digit number) is extracted and printed.
4. **When** I run `./fs5 sms otp --wait`,
**Then** the tool waits up to 60 seconds for a new OTP SMS and prints the code.
---
### User Story 3 — TOTP Code Reading from Authenticator App (Priority: P2)
**Why**: AI agent needs TOTP codes from apps like Google Authenticator or andOTP.
**Acceptance Scenarios**:
1. **When** I run `./fs5 totp screenshot`,
**Then** a screenshot of the authenticator app is taken and the current TOTP code is extracted via OCR.
2. **When** I run `./fs5 totp read --app <package>`,
**Then** the authenticator app is opened, a screenshot taken, and the 6-digit code extracted.
---
### User Story 4 — ADB Input Automation (Priority: P2)
**Why**: Automate button presses, app launches, and navigation for 2FA flows.
**Acceptance Scenarios**:
1. **When** I run `./fs5 key <keycode>`,
**Then** the Android key event is sent (e.g., KEYCODE_HOME, KEYCODE_BACK).
2. **When** I run `./fs5 launch <package>`,
**Then** the specified app is launched on the device.
3. **When** I run `./fs5 ocr`,
**Then** a screenshot is taken and all visible text is extracted via OCR.
---
## Requirements
### Functional Requirements
- **FR-010**: `screen` command MUST launch scrcpy with device serial targeting.
- **FR-011**: `screenshot` MUST save PNG to configurable local path.
- **FR-012**: `tap`, `swipe`, `type` MUST use `adb shell input` commands.
- **FR-013**: `sms list` MUST read SMS via KDE Connect CLI or ADB broadcast.
- **FR-014**: `sms otp` MUST extract 4-8 digit codes from SMS body via regex.
- **FR-015**: `sms watch` MUST poll for new SMS at configurable interval.
- **FR-016**: `totp` MUST use screenshot + OCR (tesseract) for code extraction.
- **FR-017**: All new commands MUST support `--json` output.
### Non-Functional Requirements
- **NFR-010**: Screenshot capture MUST complete in < 5 seconds.
- **NFR-011**: OTP extraction MUST complete in < 3 seconds after SMS arrives.
- **NFR-012**: OCR MUST achieve > 95% accuracy on 6-digit numeric codes.
- **NFR-013**: No root required on device.
---
## Tool Selection (Research-Based)
| Tool | Purpose | Status | Rationale |
|------|---------|--------|-----------|
| scrcpy 3.3.4 | GUI mirror + input | ✅ Installed | Best no-root screen control |
| KDE Connect | SMS forwarding | ✅ On device | Already installed, native Linux integration |
| adb shell input | Tap/swipe/type | ✅ Built-in | No extra deps |
| tesseract-ocr | TOTP OCR | Install needed | Best open-source OCR |
| kdeconnect-cli | SMS reading | Install needed | CLI for KDE Connect |
## Out of Scope
- Wireless ADB setup (USB connection sufficient)
- Rooted device operations
- Full Appium test framework (overkill for this use case)
- SMS sending (receive only)
+237
View File
@@ -0,0 +1,237 @@
#!/usr/bin/env bats
# tests/test_remote_control.bats — Remote control command tests for fs5
# Tests: screenshot, tap, swipe, type, key, launch, ocr, sms
setup() {
SCRIPT_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." && pwd)"
FS5="${SCRIPT_DIR}/fs5"
DEVICE_SERIAL="${DEVICE_SERIAL:-RD51QE202392}"
export DEVICE_SERIAL
}
teardown() {
# Clean up any test screenshots
rm -f /tmp/fs5-test-screenshot-*.png
}
# ── screenshot ────────────────────────────────────────────────────────────────
@test "screenshot: saves PNG to default path" {
run "${FS5}" screenshot
[ "${status}" -eq 0 ]
# Output contains a path
echo "${output}" | grep -q "/tmp/fs5-screenshot-"
# File exists
local path
path=$(echo "${output}" | grep "/tmp/fs5-screenshot-" | tail -1)
[ -f "${path}" ]
rm -f "${path}"
}
@test "screenshot: saves PNG to specified path" {
local out="/tmp/fs5-test-screenshot-$$.png"
run "${FS5}" screenshot "${out}"
[ "${status}" -eq 0 ]
[ -f "${out}" ]
}
@test "screenshot: --json output contains path and timestamp" {
run "${FS5}" screenshot --json
[ "${status}" -eq 0 ]
echo "${output}" | python3 -c "
import sys, json
data = json.load(sys.stdin)
assert 'path' in data, 'missing path'
assert 'timestamp' in data, 'missing timestamp'
assert data['path'].endswith('.png'), 'path not PNG'
"
# Clean up
local path
path=$(echo "${output}" | python3 -c "import sys,json; print(json.load(sys.stdin)['path'])")
rm -f "${path}"
}
@test "screenshot: --help shows usage" {
run "${FS5}" screenshot --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage"
}
# ── tap ───────────────────────────────────────────────────────────────────────
@test "tap: sends tap at valid coordinates" {
run "${FS5}" tap 540 960
[ "${status}" -eq 0 ]
echo "${output}" | grep -q "Tapped"
}
@test "tap: fails without coordinates" {
run "${FS5}" tap
[ "${status}" -ne 0 ]
echo "${output}" | grep -qi "usage"
}
@test "tap: fails with only one coordinate" {
run "${FS5}" tap 540
[ "${status}" -ne 0 ]
}
@test "tap: --help shows usage" {
run "${FS5}" tap --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage"
}
# ── swipe ─────────────────────────────────────────────────────────────────────
@test "swipe: performs swipe with four coordinates" {
run "${FS5}" swipe 540 1200 540 400
[ "${status}" -eq 0 ]
echo "${output}" | grep -q "Swiped"
}
@test "swipe: performs swipe with custom duration" {
run "${FS5}" swipe 540 1200 540 400 500
[ "${status}" -eq 0 ]
echo "${output}" | grep -q "500ms"
}
@test "swipe: fails without coordinates" {
run "${FS5}" swipe
[ "${status}" -ne 0 ]
}
@test "swipe: --help shows usage" {
run "${FS5}" swipe --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage"
}
# ── key ───────────────────────────────────────────────────────────────────────
@test "key: sends KEYCODE_HOME" {
run "${FS5}" key KEYCODE_HOME
[ "${status}" -eq 0 ]
echo "${output}" | grep -q "Key sent"
}
@test "key: sends numeric keycode" {
run "${FS5}" key 3
[ "${status}" -eq 0 ]
echo "${output}" | grep -q "Key sent"
}
@test "key: fails without keycode" {
run "${FS5}" key
[ "${status}" -ne 0 ]
}
@test "key: --help shows usage (global help intercepts --help flag)" {
run "${FS5}" key --help
[ "${status}" -eq 0 ]
# --help is a global flag; shows main usage with key command listed
echo "${output}" | grep -q "key"
}
# ── launch ────────────────────────────────────────────────────────────────────
@test "launch: launches KDE Connect app" {
run "${FS5}" launch org.kde.kdeconnect_tp
[ "${status}" -eq 0 ]
echo "${output}" | grep -q "Launched"
}
@test "launch: fails for non-existent package" {
run "${FS5}" launch com.nonexistent.package.xyz
[ "${status}" -ne 0 ]
}
@test "launch: fails without package name" {
run "${FS5}" launch
[ "${status}" -ne 0 ]
}
@test "launch: --help shows usage" {
run "${FS5}" launch --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage"
}
# ── ocr ───────────────────────────────────────────────────────────────────────
@test "ocr: runs without error and returns text or empty" {
run "${FS5}" ocr
[ "${status}" -eq 0 ]
# Output may be empty (blank screen) or contain text — both valid
}
@test "ocr: --help shows usage" {
run "${FS5}" ocr --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage\|screenshot\|ocr"
}
# ── sms ───────────────────────────────────────────────────────────────────────
@test "sms: no subcommand shows error" {
run "${FS5}" sms
[ "${status}" -ne 0 ]
echo "${output}" | grep -qi "unknown\|subcommand\|use:"
}
@test "sms: --help shows subcommands" {
run "${FS5}" sms --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -q "list"
echo "${output}" | grep -q "otp"
echo "${output}" | grep -q "watch"
}
@test "sms list: shows KDE Connect setup instructions when not paired" {
run "${FS5}" sms list
# May succeed (if paired) or fail with setup instructions
if [ "${status}" -ne 0 ]; then
echo "${output}" | grep -qi "kde connect\|paired\|setup"
fi
}
@test "sms otp: shows error when KDE Connect not paired" {
run "${FS5}" sms otp
# Either returns OTP (if paired) or error about KDE Connect
if [ "${status}" -ne 0 ]; then
echo "${output}" | grep -qi "kde connect\|paired\|otp"
fi
}
@test "sms otp: --help shows usage" {
run "${FS5}" sms otp --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage\|otp\|wait"
}
@test "sms list: --help shows usage" {
run "${FS5}" sms list --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage"
}
# ── screen ────────────────────────────────────────────────────────────────────
@test "screen: --help shows usage" {
run "${FS5}" screen --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "scrcpy\|mirror\|headless"
}
# ── type ──────────────────────────────────────────────────────────────────────
@test "type: --help shows usage" {
run "${FS5}" type --help
[ "${status}" -eq 0 ]
echo "${output}" | grep -qi "usage"
}
@test "type: fails without text argument" {
run "${FS5}" type
[ "${status}" -ne 0 ]
}