#!/system/bin/sh # VisualAgent Command Processor # Monitors /sdcard/VisualAgent/commands/ for JSON command files and executes them # Usage: su -c "sh /path/to/visual-agent-processor.sh" # # NOTE: Before using this script, check if a VisualAgent backend already exists # on the device. Look for: ~/visual_agent_executor.sh, ~/visual_agent_daemon.sh, # ~/visual_agent.py, and ~/VisualAgent_README.md. If found, prefer using the # existing infrastructure. See references/pre-existing-visualagent-infrastructure.md # in the android-ui-automation skill for the discovery checklist. # # This script handles the common case where Termux cannot access /sdcard/ directly # and must use su for all file operations. COMMANDS_DIR="/storage/emulated/0/VisualAgent/commands" RESULTS_DIR="/storage/emulated/0/VisualAgent/results" PROCESSED_FILE="/storage/emulated/0/VisualAgent/.processed_files" LOG_FILE="/storage/emulated/0/VisualAgent/visualagent.log" # Create directories mkdir -p "$COMMANDS_DIR" "$RESULTS_DIR" touch "$PROCESSED_FILE" log_msg() { local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1" echo "$msg" echo "$msg" >> "$LOG_FILE" 2>/dev/null } # Process a single command file process_command() { local cmd_file="$1" local filename=$(basename "$cmd_file") local result_file="$RESULTS_DIR/${filename}.result.json" local timestamp=$(date '+%Y-%m-%d %H:%M:%S') log_msg "Processing: $filename" # Read JSON content local json_content=$(cat "$cmd_file" 2>/dev/null) if [ -z "$json_content" ]; then echo "{\"status\":\"error\",\"error\":\"Cannot read command file\",\"timestamp\":\"$timestamp\",\"file\":\"$filename\"}" > "$result_file" 2>/dev/null return 1 fi # Extract command type - check both "type" and "command" fields local cmd_type=$(echo "$json_content" | grep -o '"type"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4) if [ -z "$cmd_type" ]; then cmd_type=$(echo "$json_content" | grep -o '"command"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4) fi if [ -z "$cmd_type" ]; then echo "{\"status\":\"error\",\"error\":\"No command type found\",\"timestamp\":\"$timestamp\",\"file\":\"$filename\"}" > "$result_file" 2>/dev/null return 1 fi local status="success" local output="" local error="" case "$cmd_type" in "screenshot") local path=$(echo "$json_content" | grep -o '"path"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4) [ -z "$path" ] && path="$RESULTS_DIR/screenshot_$(date +%s).png" screencap -p "$path" 2>/dev/null if [ $? -eq 0 ]; then output="Screenshot saved to $path" else status="failed" error="Screenshot command failed" fi ;; "tap") local x=$(echo "$json_content" | grep -o '"x"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') local y=$(echo "$json_content" | grep -o '"y"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') if [ -n "$x" ] && [ -n "$y" ]; then input tap $x $y 2>/dev/null output="Tapped at coordinates ($x, $y)" else status="failed" error="Missing x or y coordinates" fi ;; "swipe") local x1=$(echo "$json_content" | grep -o '"x1"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') local y1=$(echo "$json_content" | grep -o '"y1"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') local x2=$(echo "$json_content" | grep -o '"x2"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') local y2=$(echo "$json_content" | grep -o '"y2"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') local duration=$(echo "$json_content" | grep -o '"duration"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$') [ -z "$duration" ] && duration=300 if [ -n "$x1" ] && [ -n "$y1" ] && [ -n "$x2" ] && [ -n "$y2" ]; then input swipe $x1 $y1 $x2 $y2 $duration 2>/dev/null output="Swiped from ($x1, $y1) to ($x2, $y2) with duration ${duration}ms" else status="failed" error="Missing coordinates (x1, y1, x2, y2 required)" fi ;; "input") local text=$(echo "$json_content" | grep -o '"text"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$text" ]; then # Replace spaces with %s for input text command local encoded_text=$(echo "$text" | sed 's/ /%s/g') input text "$encoded_text" 2>/dev/null output="Input text: $text" else status="failed" error="Missing text field" fi ;; "start_app") local package=$(echo "$json_content" | grep -o '"package"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4) local activity=$(echo "$json_content" | grep -o '"activity"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$package" ]; then if [ -n "$activity" ]; then am start -n $package/$activity 2>/dev/null output="Started activity: $package/$activity" else monkey -p $package -c android.intent.category.LAUNCHER 1 2>/dev/null output="Started app package: $package" fi else status="failed" error="Missing package field" fi ;; *) status="unknown" error="Unknown command type: $cmd_type" ;; esac # Save result local result_json="{\"status\":\"$status\",\"command_type\":\"$cmd_type\",\"output\":\"$output\",\"error\":\"$error\",\"timestamp\":\"$timestamp\",\"file\":\"$filename\"}" echo "$result_json" > "$result_file" 2>/dev/null log_msg "Result: $status - $output" return 0 } # Main loop log_msg "VisualAgent processor started" log_msg "Watching: $COMMANDS_DIR" # Get file list using ls (runs inside su environment, so no glob issues) file_list=$(ls -1 "$COMMANDS_DIR"/*.json 2>/dev/null) if [ -z "$file_list" ]; then log_msg "No command files found" else echo "$file_list" | while IFS= read -r cmd_file; do [ -n "$cmd_file" ] || continue filename=$(basename "$cmd_file") # Check if already processed if grep -q "^${filename}$" "$PROCESSED_FILE" 2>/dev/null; then log_msg "Skipping already processed: $filename" continue fi # Process command process_command "$cmd_file" # Mark as processed echo "$filename" >> "$PROCESSED_FILE" 2>/dev/null done fi log_msg "VisualAgent processor completed" # Summary echo "" echo "=== VisualAgent Summary ===" echo "Commands processed: $(grep -c '' "$PROCESSED_FILE" 2>/dev/null || echo 0)" echo "Results in: $RESULTS_DIR" ls -lt "$RESULTS_DIR"/*.result.json 2>/dev/null || echo "No results yet"