feat(ui): add UIAutomator-based ui subcommand for AI remote control
Add 'fs5 ui' subcommand with 4 operations: - ui dump [path] — dump UI hierarchy XML via uiautomator - ui elements [--json] — list all interactive elements with coords - ui find <text> — find element by text/content-desc, return x y - ui tap <text> — find element and tap it (ui find + tap) lib/input.sh: - adb_ui_dump() — uiautomator dump → pull XML → cleanup - adb_ui_find() — Python XML parse, case-insensitive search - adb_ui_elements() — full element inventory as JSON tests/test_ui.bats: 22 new tests (fixture-based, no live device needed) Total: 85/85 BATS tests passing (was 64/64) Research findings (AppAgent pattern, ADB+vision): - UIAutomator dump: best no-root UI inspection on Android 9 SDK 28 - AppAgent (6.6k★): screenshot + GPT-4V + ADB tap/swipe pipeline - RustDesk InputService already enabled → accessibility layer available - scrcpy 3.3.4 + tesseract OCR pipeline already working
This commit is contained in:
Jane Alesi
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env bats
|
||||
# tests/test_ui.bats — BATS tests for fs5 ui subcommand (UIAutomator-based)
|
||||
# Tests use a pre-dumped fixture XML to avoid requiring a live device for most cases.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." && pwd)"
|
||||
FS5="${SCRIPT_DIR}/fs5"
|
||||
|
||||
# ── Fixture XML (KDE Connect screen, 15 elements) ────────────────────────────
|
||||
FIXTURE_XML="${BATS_TMPDIR}/fs5_ui_fixture.xml"
|
||||
|
||||
setup_file() {
|
||||
# Create a minimal UIAutomator XML fixture
|
||||
cat > "${FIXTURE_XML}" <<'XML'
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<hierarchy rotation="0">
|
||||
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
|
||||
<node index="0" text="" resource-id="" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
|
||||
<node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,1280]">
|
||||
<node index="0" text="" resource-id="android:id/action_bar_container" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][720,147]">
|
||||
<node index="0" text="" resource-id="android:id/action_bar" class="android.support.v7.widget.Toolbar" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,49][720,147]">
|
||||
<node index="0" text="" resource-id="" class="android.widget.ImageButton" package="org.kde.kdeconnect_tp" content-desc="Open" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,49][98,147]" />
|
||||
<node index="1" text="ja-manjaro" resource-id="" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[105,68][314,127]" />
|
||||
<node index="2" text="" resource-id="" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="More options" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[636,49][720,147]" />
|
||||
</node>
|
||||
</node>
|
||||
<node index="1" text="" resource-id="android:id/content" class="android.widget.FrameLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][720,1280]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/recycler_view" class="androidx.recyclerview.widget.RecyclerView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="true" focused="false" scrollable="true" long-clickable="false" password="false" selected="false" bounds="[0,147][720,1280]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][720,420]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,147][380,420]" />
|
||||
<node index="1" text="Send files" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,295][268,362]" />
|
||||
</node>
|
||||
<node index="1" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,147][720,420]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,147][720,420]" />
|
||||
<node index="1" text="Use as drawing tablet" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,295][720,362]" />
|
||||
</node>
|
||||
<node index="2" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,420][720,693]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,420][380,693]" />
|
||||
<node index="1" text="Presentation remote" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,568][380,635]" />
|
||||
</node>
|
||||
<node index="3" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,420][720,693]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,420][720,693]" />
|
||||
<node index="1" text="Multimedia control" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,568][720,635]" />
|
||||
</node>
|
||||
<node index="4" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,693][720,966]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,693][380,966]" />
|
||||
<node index="1" text="Remote input" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,868][326,935]" />
|
||||
</node>
|
||||
<node index="5" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_item" class="android.widget.LinearLayout" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,693][720,966]">
|
||||
<node index="0" text="" resource-id="org.kde.kdeconnect_tp:id/plugin_icon" class="android.widget.ImageView" package="org.kde.kdeconnect_tp" content-desc="View" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,693][720,966]" />
|
||||
<node index="1" text="Run Command" resource-id="org.kde.kdeconnect_tp:id/plugin_name" class="android.widget.TextView" package="org.kde.kdeconnect_tp" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[380,868][720,935]" />
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</node>
|
||||
</hierarchy>
|
||||
XML
|
||||
}
|
||||
|
||||
# ── ui --help ─────────────────────────────────────────────────────────────────
|
||||
@test "ui --help shows subcommands" {
|
||||
run "${FS5}" ui --help
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"dump"* ]]
|
||||
[[ "${output}" == *"elements"* ]]
|
||||
[[ "${output}" == *"find"* ]]
|
||||
[[ "${output}" == *"tap"* ]]
|
||||
}
|
||||
|
||||
@test "ui unknown subcommand exits 1" {
|
||||
run "${FS5}" ui bogus
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Unknown ui subcommand"* ]]
|
||||
}
|
||||
|
||||
# ── ui dump ───────────────────────────────────────────────────────────────────
|
||||
@test "ui dump --help exits 0" {
|
||||
run "${FS5}" ui dump --help
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"UIAutomator"* ]]
|
||||
}
|
||||
|
||||
# ── ui elements ───────────────────────────────────────────────────────────────
|
||||
@test "ui elements with fixture shows element count" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"Found"*"elements"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements with fixture shows clickable marker" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"[✓]"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements with fixture shows More options" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"More options"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --json returns JSON array" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}" --json
|
||||
[ "${status}" -eq 0 ]
|
||||
# Output starts with JSON array
|
||||
[[ "${output}" == "["* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --json contains center_x field" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}" --json
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"center_x"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --json contains clickable field" {
|
||||
run "${FS5}" ui elements "${FIXTURE_XML}" --json
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"clickable"* ]]
|
||||
}
|
||||
|
||||
@test "ui elements --help exits 0" {
|
||||
run "${FS5}" ui elements --help
|
||||
[ "${status}" -eq 0 ]
|
||||
}
|
||||
|
||||
# ── ui find ───────────────────────────────────────────────────────────────────
|
||||
@test "ui find by content-desc returns coordinates" {
|
||||
run "${FS5}" ui find "More options" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
# Output should be "x y" format
|
||||
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
|
||||
}
|
||||
|
||||
@test "ui find by text returns coordinates" {
|
||||
run "${FS5}" ui find "Send files" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
|
||||
}
|
||||
|
||||
@test "ui find case-insensitive match" {
|
||||
run "${FS5}" ui find "send files" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" =~ ^[0-9]+\ [0-9]+$ ]]
|
||||
}
|
||||
|
||||
@test "ui find nonexistent element exits 1" {
|
||||
run "${FS5}" ui find "nonexistent_xyz_abc" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Element not found"* ]]
|
||||
}
|
||||
|
||||
@test "ui find --json returns JSON with x and y" {
|
||||
run "${FS5}" ui find "More options" --json "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *'"x"'* ]]
|
||||
[[ "${output}" == *'"y"'* ]]
|
||||
}
|
||||
|
||||
@test "ui find --json contains text field" {
|
||||
run "${FS5}" ui find "More options" --json "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *'"text"'* ]]
|
||||
[[ "${output}" == *"More options"* ]]
|
||||
}
|
||||
|
||||
@test "ui find missing argument exits 1" {
|
||||
run "${FS5}" ui find
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Usage"* ]]
|
||||
}
|
||||
|
||||
@test "ui find --help exits 0" {
|
||||
run "${FS5}" ui find --help
|
||||
[ "${status}" -eq 0 ]
|
||||
}
|
||||
|
||||
# ── ui tap ────────────────────────────────────────────────────────────────────
|
||||
@test "ui tap --help exits 0" {
|
||||
# --help is intercepted by global parse_global_flags (same as 'key --help')
|
||||
run "${FS5}" ui tap --help
|
||||
[ "${status}" -eq 0 ]
|
||||
[[ "${output}" == *"ui tap"* ]]
|
||||
}
|
||||
|
||||
@test "ui tap missing argument exits 1" {
|
||||
run "${FS5}" ui tap
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Usage"* ]]
|
||||
}
|
||||
|
||||
@test "ui tap nonexistent element exits 1" {
|
||||
run "${FS5}" ui tap "nonexistent_xyz_abc" "${FIXTURE_XML}"
|
||||
[ "${status}" -eq 1 ]
|
||||
[[ "${output}" == *"Element not found"* ]]
|
||||
}
|
||||
Reference in New Issue
Block a user