Share via

Automation With Windows UI Automation

UIA shipped with .Net 3.0 did not receive the attention it really deserves . UIA can be used to automate native windows applications , WPF applications as well as Silverlight applications . There are two things one must consider when automating

  1. Reference to the Automation Element
  2. The user pattern to be invoked 

That's all . The AutomationElement.RootElement represents the desktop , all other UI elements are children of the desktop . We need to search the tree to find the required automation element . The tree hierarchy can be found by a tool called UISpy that comes as a part of Windows SDK . Then we need to determine what pattern is  to be used , eg  combobox uses  ExpandCollapse Pattern , textbox uses TextPattern and  button uses  Invoke pattern . (MSDN has all information )

Lets see how its done using 2 simple automation actions  

Action  1 : Opening notepad and  putting some junk text in it 

 using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Diagnostics;using System.Windows.Automation;using System.Windows.Automation.Provider;using System.Windows.Automation.Text;using System.Windows.Forms;namespace AutomateNotepad{    class Program    {        static void Main(string[] args)        {            // Make sure that all notepad process are killed            Process[] notepadProcceses = Process.GetProcessesByName("notepad");            foreach (Process notepadProc in notepadProcceses)            {                notepadProc.Kill();             }            // Start a notepad window              Process notepadProcces = Process.Start("notepad");             // Get the reference to the notepad window            AutomationElement notepadWindow = AutomationElement.RootElement.FindFirst(TreeScope.Descendants , new PropertyCondition(AutomationElement.ClassNameProperty,"Notepad")) ;             // Get the reference to the notepad text box            AutomationElement notepadTextBox = AutomationElement.RootElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.ClassNameProperty, "Edit"));            TextPattern putText = notepadTextBox.GetCurrentPattern(TextPattern.Pattern) as TextPattern;            putText.DocumentRange.Select();            SendKeys.SendWait(" You  there ! why are you  looking at me !");            Console.ReadKey();         }    }}

Action 2 : Walk through all the elements in a Silverlight page

The  XAML file of the Silverlight page is  as follows

 

 <UserControl x:Class="SilverlightTestPage.Page"    xmlns="https://schemas.microsoft.com/winfx/2006/xaml/presentation"     xmlns:x="https://schemas.microsoft.com/winfx/2006/xaml"     Width="400" Height="300" xmlns:System="clr-namespace:System;assembly=mscorlib">    <Grid x:Name="LayoutRoot" Background="White">     <Rectangle Height="51" VerticalAlignment="Top" Fill="#FF0F36E2" Stroke="#FF000000"/>     <Rectangle Height="2" HorizontalAlignment="Right" Margin="0,0,8,0" VerticalAlignment="Bottom" Width="0" Fill="#FF0F36E2" Stroke="#FF000000"/>     <TextBlock Height="32" Margin="53,8,126,0" VerticalAlignment="Top" Text="Automation Demo" TextWrapping="Wrap"/>     <Grid HorizontalAlignment="Left" Margin="0,51,0,0" Width="80" Background="#FF904F4F">      <Button x:Name="myButton1" Background="#FF339529" Height="40" Margin="8,27,8,0" VerticalAlignment="Top" Content="Click Me " Click="myButton1_Click">       <Button.BorderBrush>        <LinearGradientBrush EndPoint="0.5,1" StartPoint="0.5,0">         <GradientStop Color="#FFA3AEB9"/>         <GradientStop Color="#FF8399A9" Offset="0.375"/>         <GradientStop Color="#FF718597" Offset="0.375"/>         <GradientStop Color="#FF4DACF3" Offset="1"/>        </LinearGradientBrush>       </Button.BorderBrush>      </Button>     </Grid>     <TextBlock HorizontalAlignment="Left" Margin="0,134,0,147" Width="80" Text="ComboBox1" TextWrapping="Wrap"/>     <ComboBox x:Name="myCombo1"   Height="19" HorizontalAlignment="Left" Margin="0,0,0,114" VerticalAlignment="Bottom" Width="72">      <System:String>String1</System:String>      <System:String>String2</System:String>     </ComboBox>     <CheckBox x:Name="myCheck1" Height="19" HorizontalAlignment="Left" Margin="8,0,0,61" VerticalAlignment="Bottom" Width="20" Content="CheckBox"/>     <TextBlock Height="26" HorizontalAlignment="Left" Margin="0,0,0,84" VerticalAlignment="Bottom" Width="72" Text="Checkbox1" TextWrapping="Wrap"/>     <RadioButton x:Name="myRadio1" Height="15" HorizontalAlignment="Left" Margin="112,91,0,0" VerticalAlignment="Top" Width="17" Content="RadioButton"/>     <TextBlock Height="21" HorizontalAlignment="Left" Margin="93,66,0,0" VerticalAlignment="Top" Width="84" Text="radiobutton1" TextWrapping="Wrap"/>     <TextBox x:Name="myTextBox1" Margin="102,134,126,132" Text="Put Some Text Here" TextWrapping="Wrap"/>     <TextBlock x:Name="myTextBlock1" Height="45" Margin="112,0,51,39" VerticalAlignment="Bottom" Text="TextBlock" TextWrapping="Wrap"/>    </Grid></UserControl>

The code for automating it is as follows

 using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Diagnostics;using System.Windows.Automation;using System.Windows.Automation.Provider;using System.Windows.Automation.Text;using System.Windows.Forms;using System.Threading;namespace WindowsUIAutomationDemo{    class Program    {        static void Main(string[] args)        {            // Start the internet explorer process            Process p = Process.Start("iexplore");            Thread.Sleep(10000);            // Get the handle to the ieWindow            AutomationElement ieWindow = AutomationElement.RootElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.ClassNameProperty,"IEFrame"));            Thread.Sleep(5000);                        if (ieWindow == null)            {                Console.WriteLine("Unable to get handle to the ie main window!");            }            else            {                Thread.Sleep(5000);                // Specify the condition for getting a reference to the edit control                Condition[] condition = new Condition[3];                condition[0] = new PropertyCondition(AutomationElement.ClassNameProperty, "Edit");                condition[1] = new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Edit);                condition[2] = new PropertyCondition(AutomationElement.NameProperty, "Address");                // Getting a reference to the iebar                AutomationElement iebar = ieWindow.FindFirst(TreeScope.Descendants, new AndCondition(condition));                if (iebar == null)                {                    Console.WriteLine("Unable to get handle to the window !");                }                else                {                    // Set focus and go to the address                    iebar.SetFocus();                    SendKeys.SendWait(@"E:\WindowsUIAutomationDemo\SilverlightTestPage\SilverlightTestPage\Bin\Debug\TestPage.html");                    SendKeys.SendWait("{Enter}");                    Thread.Sleep(20000);                    // Get the reference to the silverlight control                     AutomationElement silverlightElement = ieWindow.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.ClassNameProperty, "MicrosoftSilverlight"));                    if (silverlightElement == null)                    {                        Console.WriteLine("Unable to get access to silverligt contorl");                    }                    else                    {                        // click the click me button                        AutomationElement clickButton = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.AutomationIdProperty, "myButton1"));                        InvokePattern invokePattern = clickButton.GetCurrentPattern(InvokePattern.Pattern) as InvokePattern;                        invokePattern.Invoke();                        Thread.Sleep(2000);                        // Expand combo box                        AutomationElement comboBox = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.AutomationIdProperty, "myCombo1"));                        ExpandCollapsePattern expandCollapsePattern = comboBox.GetCurrentPattern(ExpandCollapsePattern.Pattern)  as ExpandCollapsePattern;                        expandCollapsePattern.Expand();                                                Thread.Sleep(2000)  ;                                                // Selct the first item                         AutomationElement firstItemincombo = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.NameProperty, "String1"));                        SelectionItemPattern itemSelection = firstItemincombo.GetCurrentPattern(SelectionItemPattern.Pattern) as SelectionItemPattern;                        itemSelection.Select();                        Thread.Sleep(1000) ;                         expandCollapsePattern.Collapse();                                                //Get the radio checkbox                        AutomationElement checkboxPattern = silverlightElement.FindFirst(TreeScope.Descendants, new PropertyCondition(AutomationElement.AutomationIdProperty, "myCheck1"));                        TogglePattern tooglePattern = checkboxPattern.GetCurrentPattern(TogglePattern.Pattern) as TogglePattern;                        tooglePattern.Toggle();                         Thread.Sleep(1000) ;                         // Get the radio button                         AutomationElement radioButton = silverlightElement.FindFirst(TreeScope.Descendants,new PropertyCondition(AutomationElement.AutomationIdProperty,"myRadio1"));                        SelectionItemPattern radioCheck =radioButton.GetCurrentPattern(SelectionItemPattern.Pattern) as                             SelectionItemPattern;                        radioCheck.Select();                         Thread.Sleep(1000) ;                         // Get the textbox and put something there                         AutomationElement textBox = silverlightElement.FindFirst(TreeScope.Descendants,new PropertyCondition(AutomationElement.AutomationIdProperty,"myTextBox1"));                        ValuePattern valuePattern = textBox.GetCurrentPattern(ValuePattern.Pattern) as ValuePattern;                        valuePattern.SetValue("Yo yo , i am entering text !");                                                                       Thread.Sleep(1000);                     }                                        //AutomationElement silverlightControl = ieWindow.FindFirst(TreeScope.Descendants,new PropertyCondition(                }            }            Console.ReadKey();                  }    }}