visual-agent-processor.sh
7.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
#!/system/bin/sh
# VisualAgent Command Processor
# Monitors /sdcard/VisualAgent/commands/ for JSON command files and executes them
# Usage: su -c "sh /path/to/visual-agent-processor.sh"
#
# NOTE: Before using this script, check if a VisualAgent backend already exists
# on the device. Look for: ~/visual_agent_executor.sh, ~/visual_agent_daemon.sh,
# ~/visual_agent.py, and ~/VisualAgent_README.md. If found, prefer using the
# existing infrastructure. See references/pre-existing-visualagent-infrastructure.md
# in the android-ui-automation skill for the discovery checklist.
#
# This script handles the common case where Termux cannot access /sdcard/ directly
# and must use su for all file operations.
COMMANDS_DIR="/storage/emulated/0/VisualAgent/commands"
RESULTS_DIR="/storage/emulated/0/VisualAgent/results"
PROCESSED_FILE="/storage/emulated/0/VisualAgent/.processed_files"
LOG_FILE="/storage/emulated/0/VisualAgent/visualagent.log"
# Create directories
mkdir -p "$COMMANDS_DIR" "$RESULTS_DIR"
touch "$PROCESSED_FILE"
log_msg() {
local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1"
echo "$msg"
echo "$msg" >> "$LOG_FILE" 2>/dev/null
}
# Process a single command file
process_command() {
local cmd_file="$1"
local filename=$(basename "$cmd_file")
local result_file="$RESULTS_DIR/${filename}.result.json"
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
log_msg "Processing: $filename"
# Read JSON content
local json_content=$(cat "$cmd_file" 2>/dev/null)
if [ -z "$json_content" ]; then
echo "{\"status\":\"error\",\"error\":\"Cannot read command file\",\"timestamp\":\"$timestamp\",\"file\":\"$filename\"}" > "$result_file" 2>/dev/null
return 1
fi
# Extract command type - check both "type" and "command" fields
local cmd_type=$(echo "$json_content" | grep -o '"type"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4)
if [ -z "$cmd_type" ]; then
cmd_type=$(echo "$json_content" | grep -o '"command"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4)
fi
if [ -z "$cmd_type" ]; then
echo "{\"status\":\"error\",\"error\":\"No command type found\",\"timestamp\":\"$timestamp\",\"file\":\"$filename\"}" > "$result_file" 2>/dev/null
return 1
fi
local status="success"
local output=""
local error=""
case "$cmd_type" in
"screenshot")
local path=$(echo "$json_content" | grep -o '"path"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4)
[ -z "$path" ] && path="$RESULTS_DIR/screenshot_$(date +%s).png"
screencap -p "$path" 2>/dev/null
if [ $? -eq 0 ]; then
output="Screenshot saved to $path"
else
status="failed"
error="Screenshot command failed"
fi
;;
"tap")
local x=$(echo "$json_content" | grep -o '"x"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
local y=$(echo "$json_content" | grep -o '"y"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
if [ -n "$x" ] && [ -n "$y" ]; then
input tap $x $y 2>/dev/null
output="Tapped at coordinates ($x, $y)"
else
status="failed"
error="Missing x or y coordinates"
fi
;;
"swipe")
local x1=$(echo "$json_content" | grep -o '"x1"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
local y1=$(echo "$json_content" | grep -o '"y1"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
local x2=$(echo "$json_content" | grep -o '"x2"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
local y2=$(echo "$json_content" | grep -o '"y2"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
local duration=$(echo "$json_content" | grep -o '"duration"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
[ -z "$duration" ] && duration=300
if [ -n "$x1" ] && [ -n "$y1" ] && [ -n "$x2" ] && [ -n "$y2" ]; then
input swipe $x1 $y1 $x2 $y2 $duration 2>/dev/null
output="Swiped from ($x1, $y1) to ($x2, $y2) with duration ${duration}ms"
else
status="failed"
error="Missing coordinates (x1, y1, x2, y2 required)"
fi
;;
"input")
local text=$(echo "$json_content" | grep -o '"text"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4)
if [ -n "$text" ]; then
# Replace spaces with %s for input text command
local encoded_text=$(echo "$text" | sed 's/ /%s/g')
input text "$encoded_text" 2>/dev/null
output="Input text: $text"
else
status="failed"
error="Missing text field"
fi
;;
"start_app")
local package=$(echo "$json_content" | grep -o '"package"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4)
local activity=$(echo "$json_content" | grep -o '"activity"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4)
if [ -n "$package" ]; then
if [ -n "$activity" ]; then
am start -n $package/$activity 2>/dev/null
output="Started activity: $package/$activity"
else
monkey -p $package -c android.intent.category.LAUNCHER 1 2>/dev/null
output="Started app package: $package"
fi
else
status="failed"
error="Missing package field"
fi
;;
*)
status="unknown"
error="Unknown command type: $cmd_type"
;;
esac
# Save result
local result_json="{\"status\":\"$status\",\"command_type\":\"$cmd_type\",\"output\":\"$output\",\"error\":\"$error\",\"timestamp\":\"$timestamp\",\"file\":\"$filename\"}"
echo "$result_json" > "$result_file" 2>/dev/null
log_msg "Result: $status - $output"
return 0
}
# Main loop
log_msg "VisualAgent processor started"
log_msg "Watching: $COMMANDS_DIR"
# Get file list using ls (runs inside su environment, so no glob issues)
file_list=$(ls -1 "$COMMANDS_DIR"/*.json 2>/dev/null)
if [ -z "$file_list" ]; then
log_msg "No command files found"
else
echo "$file_list" | while IFS= read -r cmd_file; do
[ -n "$cmd_file" ] || continue
filename=$(basename "$cmd_file")
# Check if already processed
if grep -q "^${filename}$" "$PROCESSED_FILE" 2>/dev/null; then
log_msg "Skipping already processed: $filename"
continue
fi
# Process command
process_command "$cmd_file"
# Mark as processed
echo "$filename" >> "$PROCESSED_FILE" 2>/dev/null
done
fi
log_msg "VisualAgent processor completed"
# Summary
echo ""
echo "=== VisualAgent Summary ==="
echo "Commands processed: $(grep -c '' "$PROCESSED_FILE" 2>/dev/null || echo 0)"
echo "Results in: $RESULTS_DIR"
ls -lt "$RESULTS_DIR"/*.result.json 2>/dev/null || echo "No results yet"