折腾笔记[33]-使用uiautomation自动重复读取图片(被控程序为.net框架)

摘要

基于python使用uiautomation自动操作.net程序.读取目录中png以及查找与其对应的json数据输入软件和点击按钮.

实现

1. 简化版的被控程序(靶点窗口)

配置文件:

<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><OutputType>WinExe</OutputType><TargetFramework>net8.0-windows</TargetFramework><Nullable>enable</Nullable><ImplicitUsings>enable</ImplicitUsings><UseWPF>true</UseWPF></PropertyGroup><ItemGroup><PackageReference Include="TZ.HMI.Common.UI" Version="1.0.0.1" /></ItemGroup><ItemGroup><None Update="处理中.png"><CopyToOutputDirectory>Always</CopyToOutputDirectory></None></ItemGroup></Project><?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"><PropertyGroup /><ItemGroup><ApplicationDefinition Update="App.xaml"><SubType>Designer</SubType></ApplicationDefinition></ItemGroup><ItemGroup><Compile Update="JsonImportWindow.xaml.cs"><SubType>Code</SubType></Compile></ItemGroup><ItemGroup><Page Update="JsonImportWindow.xaml"><SubType>Designer</SubType></Page><Page Update="MainWindow.xaml"><SubType>Designer</SubType></Page></ItemGroup>
</Project>

界面:

<Window x:Class="MockWpf.MainWindow"xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"Title="OCR自动核对" Height="450" Width="800"><Grid><Grid.RowDefinitions><RowDefinition Height="Auto"/><RowDefinition Height="*"/></Grid.RowDefinitions><!-- 顶部按钮区域 --><StackPanel Orientation="Horizontal" Grid.Row="0" Margin="10" HorizontalAlignment="Left"><Button Name="btnLoadImage" Content="读取图片" Width="100" Height="30" Margin="5"/><Button Name="btnOpenFolder" Content="打开文件夹" Width="100" Height="30" Margin="5"/><Button Name="btnPrevImage" Content="上一张" Width="100" Height="30" Margin="5"/><Button Name="btnNextImage" Content="下一张" Width="100" Height="30" Margin="5"/><Button Name="btnDebug" Content="调试" Width="100" Height="30" Margin="5"/><Button Name="btnMesImport" Content="Mes导入" Width="100" Height="30" Margin="5"/></StackPanel><!-- 图片显示区域 --><Image Name="imgDisplay" Grid.Row="1" Margin="10" Stretch="Uniform"/></Grid>
</Window><Window x:Class="MockWpf.JsonImportWindow"xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"Title="Json导入" Height="300" Width="500"WindowStartupLocation="CenterOwner"><Grid Margin="10"><Grid.RowDefinitions><RowDefinition Height="*"/><RowDefinition Height="Auto"/></Grid.RowDefinitions><TextBox x:Name="txtJson"Grid.Row="0"TextWrapping="Wrap"AcceptsReturn="True"ScrollViewer.HorizontalScrollBarVisibility="Auto"ScrollViewer.VerticalScrollBarVisibility="Auto"FontFamily="Consolas"/><Button x:Name="btnOk"Grid.Row="1"Content="确认"Width="80" Height="28"HorizontalAlignment="Right"Margin="0,10,0,0"Click="BtnOk_Click"/></Grid>
</Window>

代码:

using Microsoft.Win32;
using System;
using System.Collections.Generic;
using System.IO;
using System.Windows;
using System.Windows.Media.Imaging;namespace MockWpf
{public partial class MainWindow : Window{private List<string> imageFiles = new List<string>();private int currentIndex = -1;public MainWindow(){InitializeComponent();btnOpenFolder.Click += BtnOpenFolder_Click;btnLoadImage.Click += BtnLoadImage_Click;btnPrevImage.Click += BtnPrevImage_Click;btnNextImage.Click += BtnNextImage_Click;btnDebug.Click += BtnDebug_Click;btnMesImport.Click += BtnMesImport_Click;}private void BtnMesImport_Click(object sender, RoutedEventArgs e){var w = new JsonImportWindow { Owner = this };if (w.ShowDialog() == true)          // 用户点了“确认”{string json = w.JsonText;// TODO: 在这里写真正的 Json 解析/导入逻辑// MessageBox.Show($"已获取到 Json 内容（长度 {json.Length}）\n{json}");}}private void BtnDebug_Click(object sender, RoutedEventArgs e){// 新窗口var win = new Window{Owner = this,Width = 600,Height = 400,WindowStartupLocation = WindowStartupLocation.CenterOwner,WindowStyle = WindowStyle.ToolWindow,ResizeMode = ResizeMode.NoResize,Title = "提示"};// 图片控件var img = new System.Windows.Controls.Image{Stretch = System.Windows.Media.Stretch.Uniform};try{img.Source = new BitmapImage(new Uri("./处理中.png", UriKind.Relative));}catch { /* 图片不存在则留空 */ }win.Content = img;win.Show();// 3 秒后自动关闭var timer = new System.Windows.Threading.DispatcherTimer{Interval = TimeSpan.FromSeconds(3)};timer.Tick += (_, __) =>{timer.Stop();win.Close();};timer.Start();}private void BtnOpenFolder_Click(object sender, RoutedEventArgs e){var dialog = new System.Windows.Forms.FolderBrowserDialog();if (dialog.ShowDialog() == System.Windows.Forms.DialogResult.OK){var folderPath = dialog.SelectedPath;imageFiles = new List<string>(Directory.GetFiles(folderPath, "*.jpg"));imageFiles.AddRange(Directory.GetFiles(folderPath, "*.png"));imageFiles.AddRange(Directory.GetFiles(folderPath, "*.bmp"));if (imageFiles.Count > 0){currentIndex = 0;LoadImage(imageFiles[currentIndex]);}else{MessageBox.Show("未找到图片文件");}}}private void BtnLoadImage_Click(object sender, RoutedEventArgs e){OpenFileDialog openFileDialog = new OpenFileDialog{Filter = "图片文件|*.jpg;*.jpeg;*.png;*.bmp",Title = "选择图片"};if (openFileDialog.ShowDialog() == true){LoadImage(openFileDialog.FileName);}}private void BtnPrevImage_Click(object sender, RoutedEventArgs e){if (imageFiles.Count == 0) return;currentIndex = (currentIndex - 1 + imageFiles.Count) % imageFiles.Count;LoadImage(imageFiles[currentIndex]);}private void BtnNextImage_Click(object sender, RoutedEventArgs e){if (imageFiles.Count == 0) return;currentIndex = (currentIndex + 1) % imageFiles.Count;LoadImage(imageFiles[currentIndex]);}private void LoadImage(string path){try{var bitmap = new BitmapImage();bitmap.BeginInit();bitmap.UriSource = new Uri(path);bitmap.CacheOption = BitmapCacheOption.OnLoad;bitmap.EndInit();imgDisplay.Source = bitmap;}catch (Exception ex){MessageBox.Show($"加载图片失败: {ex.Message}");}}}
}using System;
using System.Windows;namespace MockWpf
{public partial class JsonImportWindow : Window{public string JsonText => txtJson.Text;public JsonImportWindow(){InitializeComponent();}// 点击“确认”后把对话框结果设为 OK，主窗口就能取到 JsonTextprivate void BtnOk_Click(object sender, RoutedEventArgs e){DialogResult = true;// 关闭窗口Close();}}
}

2. python脚本

pyproject.toml

[project]
name = "exp18-pyloid-pyautogui"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.13"
dependencies = ["comtypes>=1.4.12","onnxruntime>=1.22.1","pillow>=11.3.0","psutil>=7.0.0","pyautogui>=0.9.54","pyinstaller>=6.15.0","pyloid>=0.24.8","pyperclip>=1.9.0","pyscreeze>=1.0.1","pywin32>=311","rapidocr>=3.4.0","uiautomation>=2.0.29",
]

脚本:

# -*- coding: utf-8 -*-
import uiautomation as auto
import win32api, win32con, win32gui, win32console, time, sys, os
import datetime
import pyperclipCANDIDATE_TITLES = ['靶点窗口', 'OCR自动核对']
TASKS            = ['Mes导入']
DEFAULT_IMG_DIR  = r'D:\AppTest\测试工作空间'# ---------------- 日志 ----------------
class _RealTimeTee:def __init__(self, log_path: str):# 行缓冲，保证实时落盘self.file = open(log_path, 'a', encoding='utf-8', buffering=1)# 防御：无控制台时退化为只写文件self.console = sys.__stdout__ if sys.__stdout__ is not None else Noneself._write_header(log_path)def _write_header(self, log_path: str):head = f'日志文件：{os.path.abspath(log_path)}\n'self.file.write(head)if self.console:self.console.write(head)def write(self, data: str):self.file.write(data)if self.console:self.console.write(data)self.flush()def flush(self):self.file.flush()if self.console:self.console.flush()def close(self):self.flush()self.file.close()# --------------------------------------------------
# 日志：仅提供 printLog 函数，不替换 sys.stdout
# --------------------------------------------------
_log_file = None          # 全局日志句柄def init_log(log_path: str):"""初始化日志文件，写入头信息"""global _log_file_log_file = open(log_path, 'a', encoding='utf-8', buffering=1)header = f'日志文件：{os.path.abspath(log_path)}\n'printLog(header, end='')          # 屏幕_log_file.write(header)        # 文件def printLog(*args, **kwargs):"""同时打印到屏幕并写入日志"""if _log_file is None:raise RuntimeError('请先调用 init_log() 初始化日志')# 构造输出字符串sep  = kwargs.get('sep', ' ')end  = kwargs.get('end', '\n')msg  = sep.join(map(str, args)) + endprint(msg, end='')             # 屏幕_log_file.write(msg)           # 文件_log_file.flush()def close_log():"""程序结束前关闭日志句柄"""global _log_fileif _log_file:_log_file.close()_log_file = None# ---------------- 控制台工具 ----------------
def alloc_console():if win32console.GetConsoleWindow() == 0:win32console.AllocConsole()sys.stdout = open('CONOUT$', 'w', encoding='utf-8')sys.stdin  = open('CONIN$',  'r', encoding='utf-8')sys.stderr = sys.stdoutdef show_console(visible=True, topmost=False):hwnd = win32console.GetConsoleWindow()if not hwnd:returnwin32gui.ShowWindow(hwnd,win32con.SW_SHOW if visible else win32con.SW_MINIMIZE)if topmost:win32gui.SetWindowPos(hwnd, win32con.HWND_TOPMOST,0, 0, 0, 0,win32con.SWP_NOMOVE | win32con.SWP_NOSIZE)# ---------------- 鼠标/高亮 ----------------
def highlight_rect(rect, sec=3):left, top, right, bottom = map(int, (rect.left, rect.top,rect.right, rect.bottom))hdc = win32gui.GetDC(0)pen = win32gui.CreatePen(win32con.PS_SOLID, 10, win32api.RGB(255, 0, 0))old_pen = win32gui.SelectObject(hdc, pen)win32gui.SelectObject(hdc, win32gui.GetStockObject(win32con.NULL_BRUSH))for _ in range(int(sec * 5)):win32gui.Rectangle(hdc, left, top, right, bottom)time.sleep(0.1)win32gui.SelectObject(hdc, old_pen)win32gui.ReleaseDC(0, hdc)def real_click(rect):x = int((rect.left + rect.right) / 2)y = int((rect.top + rect.bottom) / 2)win32api.SetCursorPos((x, y))win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, 0, 0)time.sleep(0.02)win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, 0, 0)def deep_click(ctrl):rect = ctrl.BoundingRectanglehighlight_rect(rect, 1)sub = ctrl.GetFirstChildControl()while sub:if sub.ControlType in [auto.ButtonControl, auto.TextControl,auto.CustomControl] \and abs(sub.BoundingRectangle.left - rect.left) <= 5 \and abs(sub.BoundingRectangle.top  - rect.top)  <= 5:printLog('  将点击子控件：', sub.Name)rect = sub.BoundingRectanglebreaksub = sub.GetNextSiblingControl()real_click(rect)# ---------------- 业务逻辑 ----------------
def find_window():for ttl in CANDIDATE_TITLES:w = auto.WindowControl(searchDepth=1, Name=ttl)if w.Exists(0.5):printLog('已找到窗口：', ttl)return wreturn Nonedef click_button(win, name):for tp in ['ButtonControl', 'CustomControl', 'TextControl']:btn = getattr(win, tp)(Name=name)if btn.Exists():printLog('找到按钮：', name)deep_click(btn)return TrueprintLog('未找到按钮：', name)return Falsedef wait_disappear(ctrl: auto.Control, timeout: int = 10):for _ in range(int(timeout * 10)):if not ctrl.Exists(0.05):returntime.sleep(0.1)raise RuntimeError(f'等待 {ctrl.Name} 消失超时')# ---------------- 单次完整流程 ----------------
def single_flow(win: auto.WindowControl, png_path: str, json_path: str) -> float:import win32gui as wgimport win32con as wcimport win32api as wat0 = time.perf_counter()img_name = os.path.basename(png_path)# 1. 读取图片（键盘版，原逻辑不变）t1 = time.perf_counter()if not click_button(win, '读取图片'):raise RuntimeError('未找到【读取图片】按钮')def _find_open_dialog():lst = []def enum_cb(hwnd, _):if wg.GetClassName(hwnd) == '#32770' and wg.IsWindowVisible(hwnd):lst.append(hwnd)return Truewg.EnumWindows(enum_cb, None)return lst[0] if lst else Nonefor _ in range(30):h_dlg = _find_open_dialog()if h_dlg:breaktime.sleep(0.1)else:raise RuntimeError('等待文件选择器超时')if wg.IsIconic(h_dlg):wg.ShowWindow(h_dlg, wc.SW_RESTORE)wg.SetForegroundWindow(h_dlg)# Alt+Nwa.keybd_event(wc.VK_MENU, 0, 0, 0); time.sleep(0.02)wa.keybd_event(0x4E, 0, 0, 0);       time.sleep(0.02)wa.keybd_event(0x4E, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.02)wa.keybd_event(wc.VK_MENU, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.2)# Ctrl+A 并输入路径wa.keybd_event(wc.VK_CONTROL, 0, 0, 0); time.sleep(0.02)wa.keybd_event(0x41, 0, 0, 0);          time.sleep(0.02)wa.keybd_event(0x41, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.02)wa.keybd_event(wc.VK_CONTROL, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.1)auto.SendKeys(png_path); time.sleep(0.1)# Alt+Owa.keybd_event(wc.VK_MENU, 0, 0, 0); time.sleep(0.02)wa.keybd_event(0x4F, 0, 0, 0);       time.sleep(0.02)wa.keybd_event(0x4F, 0, wc.KEYEVENTF_KEYUP, 0); time.sleep(0.02)wa.keybd_event(wc.VK_MENU, 0, wc.KEYEVENTF_KEYUP, 0)for _ in range(50):if not wg.IsWindow(h_dlg):breaktime.sleep(0.1)else:raise RuntimeError('文件对话框仍未关闭')t1 = time.perf_counter() - t1# 2. Mes 导入 —— 采用“参考代码”剪贴板方案t2 = time.perf_counter()if not click_button(win, 'Mes导入'):raise RuntimeError('未找到【Mes导入】按钮')# 等待 Json 导入窗口json_dlg = auto.WindowControl(searchDepth=2, Name='Json导入')if not json_dlg.Exists(3):raise RuntimeError('等待 Json导入 窗口超时')# 读取 json 内容try:with open(json_path, 'r', encoding='utf-8') as f:json_txt = f.read()json_ok = Trueexcept Exception as e:json_txt = ''json_ok = FalseprintLog('[警告] 读取 json 失败：', e)# 找到编辑框并聚焦edit_j = json_dlg.EditControl(foundIndex=1)if not edit_j.Exists():# 某些版本用 Custom 包了一层edit_j = json_dlg.CustomControl(foundIndex=1).EditControl(foundIndex=1)edit_j.SetFocus()# 参考代码做法：全选 + 剪贴板写入 + Ctrl+Vauto.SendKeys('{Ctrl}A'); time.sleep(0.05)pyperclip.copy(json_txt); time.sleep(0.05)auto.SendKeys('{Ctrl}V'); time.sleep(0.1)# 确定按钮ok_btn = json_dlg.ButtonControl(Name='确认')if ok_btn.Exists():deep_click(ok_btn)else:json_dlg.SendKeys('{Enter}')wait_disappear(json_dlg, 5)t2 = time.perf_counter() - t2# 3. 调试（原逻辑不变）t3 = time.perf_counter()if not click_button(win, '调试'):raise RuntimeError('未找到【调试】按钮')t3 = time.perf_counter() - t3# 4. 等待提示窗（原逻辑不变）t4 = time.perf_counter()tip = auto.WindowControl(searchDepth=2, SubName='提示')if tip.Exists(1):wait_disappear(tip, 15)t4 = time.perf_counter() - t4total = time.perf_counter() - t0printLog(f'[{img_name}]  读取图片:{t1:.2f}s  'f'Mes导入:{t2:.2f}s  调试:{t3:.2f}s  等待提示:{t4:.2f}s  'f'json成功:{json_ok}  总耗时:{total:.2f}s')return total# ---------------- 主流程 ----------------
def main():alloc_console()show_console(visible=True, topmost=True)# ---- 生成唯一日志文件名 ----log_name = datetime.datetime.now().strftime(r'自动测试_%Y%m%d_%H%M%S.log')init_log(log_name)          # 初始化日志# 询问图片文件夹img_dir = input(f'请输入图片文件夹路径(直接回车使用默认值)：\n{DEFAULT_IMG_DIR}\n> ').strip()if not img_dir:img_dir = DEFAULT_IMG_DIRprintLog('使用文件夹：', img_dir)tasks = []for fn in sorted(os.listdir(img_dir)):if fn.lower().endswith('.png'):json_fn = fn[:-4] + '.json'json_path = os.path.join(img_dir, json_fn)if os.path.isfile(json_path):tasks.append((os.path.join(img_dir, fn), json_path))else:printLog(f'[跳过] {fn} 未找到同名 json')if not tasks:printLog('文件夹内没有可执行的 png/json 对！')show_console(visible=True, topmost=True)input('按回车键退出…')returntotal_cnt = len(tasks)printLog(f'\n共发现 {total_cnt} 组待执行任务，即将开始自动化…\n')show_console(visible=False)win = find_window()if not win:printLog('未找到任何候选窗口：', CANDIDATE_TITLES)show_console(visible=True, topmost=True)input('按回车键退出…')returndone = 0sum_time = 0.0try:for png, json_ in tasks:done += 1cost = single_flow(win, png, json_)sum_time += costprintLog(f'[总进度] 已执行 {done}/{total_cnt} 张  累计耗时 {sum_time:.2f} s')except Exception as e:printLog('\n[异常] 流程中断：', e)finally:show_console(visible=True, topmost=True)printLog(f'\n全部任务执行完毕！ 成功 {done}/{total_cnt} 张  总耗时 {sum_time:.2f} s')input('按回车键退出…')if __name__ == '__main__':main()

打包:

uv run pyinstaller -F -w auto_test.py