From 45da92a86fce93503cc4068590ea8e7a42dac6d4 Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Mon, 10 Jun 2019 11:12:10 +0800 Subject: [PATCH 1/7] #443 - timeout step to also update sikuli timeout --- src/tagui_header.js | 7 ++++++- src/tagui_parse.php | 3 ++- src/test/positive_test.signature | 13 +++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/tagui_header.js b/src/tagui_header.js index a69747f2..1bd5102b 100644 --- a/src/tagui_header.js +++ b/src/tagui_header.js @@ -342,6 +342,11 @@ return fs.read('tagui.sikuli'+ds+'tagui_sikuli.txt').trim(); else return '';} function clear_sikuli_text() {var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; var fs = require('fs'); fs.write('tagui.sikuli'+ds+'tagui_sikuli.txt','','w');} +// for setting timeout in sikuli when looking for ui element +function sikuli_timeout(time_in_seconds) {var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; +var fs = require('fs'); if (fs.exists('tagui.sikuli'+ds+'tagui_sikuli.in')) +sikuli_step('vision setAutoWaitTimeout(' + time_in_seconds.toString() + ')');} + // for initialising integration with R function r_handshake() { // techo('[connecting to R process]'); var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; clear_r_text(); @@ -1175,7 +1180,7 @@ else return call_sikuli(raw_intent.replace(/\\/g,'\\\\').replace(/'/g,'\\\''),'f function timeout_intent(raw_intent) {raw_intent = eval("'" + escape_bs(raw_intent) + "'"); // support dynamic variables var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim(); if (params == '') return "this.echo('ERROR - time in seconds missing for " + raw_intent + "')"; -else return check_chrome_context("casper.options.waitTimeout = " + (parseFloat(params)*1000).toString() + ";");} +else return check_chrome_context("casper.options.waitTimeout = " + (parseFloat(params)*1000).toString() + "; sikuli_timeout(" + parseFloat(params).toString() + ");");} function code_intent(raw_intent) { // code to support dynamic variables not applicable return check_chrome_context(raw_intent);} diff --git a/src/tagui_parse.php b/src/tagui_parse.php index 43522329..cdd9a88b 100755 --- a/src/tagui_parse.php +++ b/src/tagui_parse.php @@ -964,7 +964,8 @@ function vision_intent($raw_intent) {if (strtolower($raw_intent) == "vision begi function timeout_intent($raw_intent) { $params = trim(substr($raw_intent." ",1+strpos($raw_intent." "," "))); if ($params == "") echo "ERROR - " . current_line() . " time in seconds missing for " . $raw_intent . "\n"; -else return "casper.then(function() {"."casper.options.waitTimeout = " . (floatval($params)*1000) . ";" . end_fi()."});"."\n\n";} +else return "casper.then(function() {"."casper.options.waitTimeout = " . (floatval($params)*1000) . +"; sikuli_timeout(" . floatval($params) . ");" . end_fi()."});"."\n\n";} function code_intent($raw_intent) { $params = parse_condition($raw_intent); diff --git a/src/test/positive_test.signature b/src/test/positive_test.signature index ec5d2dab..46ac69a4 100644 --- a/src/test/positive_test.signature +++ b/src/test/positive_test.signature @@ -369,6 +369,11 @@ return fs.read('tagui.sikuli'+ds+'tagui_sikuli.txt').trim(); else return '';} function clear_sikuli_text() {var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; var fs = require('fs'); fs.write('tagui.sikuli'+ds+'tagui_sikuli.txt','','w');} +// for setting timeout in sikuli when looking for ui element +function sikuli_timeout(time_in_seconds) {var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; +var fs = require('fs'); if (fs.exists('tagui.sikuli'+ds+'tagui_sikuli.in')) +sikuli_step('vision setAutoWaitTimeout(' + time_in_seconds.toString() + ')');} + // for initialising integration with R function r_handshake() { // techo('[connecting to R process]'); var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; clear_r_text(); @@ -1202,7 +1207,7 @@ else return call_sikuli(raw_intent.replace(/\\/g,'\\\\').replace(/'/g,'\\\''),'f function timeout_intent(raw_intent) {raw_intent = eval("'" + escape_bs(raw_intent) + "'"); // support dynamic variables var params = ((raw_intent + ' ').substr(1+(raw_intent + ' ').indexOf(' '))).trim(); if (params == '') return "this.echo('ERROR - time in seconds missing for " + raw_intent + "')"; -else return check_chrome_context("casper.options.waitTimeout = " + (parseFloat(params)*1000).toString() + ";");} +else return check_chrome_context("casper.options.waitTimeout = " + (parseFloat(params)*1000).toString() + "; sikuli_timeout(" + parseFloat(params).toString() + ");");} function code_intent(raw_intent) { // code to support dynamic variables not applicable return check_chrome_context(raw_intent);} @@ -2984,11 +2989,11 @@ this.echo('ERROR - cannot find image file for vision step').exit(); else this.echo('ERROR - cannot find for vision step on screen').exit(); this.wait(0);}}); // test timeout -casper.then(function() {casper.options.waitTimeout = 4000;}); +casper.then(function() {casper.options.waitTimeout = 4000; sikuli_timeout(4);}); -casper.then(function() {casper.options.waitTimeout = 5670;}); +casper.then(function() {casper.options.waitTimeout = 5670; sikuli_timeout(5.67);}); -casper.then(function() {casper.options.waitTimeout = 6000;}); +casper.then(function() {casper.options.waitTimeout = 6000; sikuli_timeout(6);}); // test comment // user comment with space From 7faf6194f347de792b2ed43b3cd557c1ef44b435 Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Mon, 10 Jun 2019 11:22:04 +0800 Subject: [PATCH 2/7] #417 - helper scripts to kill processes if Ctrl+C is used --- src/end_processes | 10 ++++++++++ src/end_processes.cmd | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100755 src/end_processes create mode 100644 src/end_processes.cmd diff --git a/src/end_processes b/src/end_processes new file mode 100755 index 00000000..eb71f612 --- /dev/null +++ b/src/end_processes @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# TO KILL INTEGRATION PROCESSES IF CTRL+C WAS USED TO QUIT TAGUI PREMATURELY ~ TEBEL.ORG # + +# checking for existence of files before sending finish signal is important +# otherwise for ongoing tagui iterations, all integrations will be invoked + +if [ -f "tagui_r/tagui_r.in" ]; then echo "finish" > tagui_r/tagui_r.in; fi +if [ -f "tagui_py/tagui_py.in" ]; then echo "finish" > tagui_py/tagui_py.in; fi +if [ -f "tagui.sikuli/tagui_sikuli.in" ]; then echo "finish" > tagui.sikuli/tagui_sikuli.in; fi +if [ -f "tagui_chrome.in" ]; then echo "finish" > tagui_chrome.in; fi diff --git a/src/end_processes.cmd b/src/end_processes.cmd new file mode 100644 index 00000000..35484f9c --- /dev/null +++ b/src/end_processes.cmd @@ -0,0 +1,10 @@ +@echo off +rem # TO KILL INTEGRATION PROCESSES IF CTRL+C WAS USED TO QUIT TAGUI PREMATURELY ~ TEBEL.ORG # + +rem checking for existence of files before sending finish signal is important +rem otherwise for ongoing tagui iterations, all integrations will be invoked + +if exist "tagui_r\tagui_r.in" echo finish > tagui_r\tagui_r.in +if exist "tagui_py\tagui_py.in" echo finish > tagui_py\tagui_py.in +if exist "tagui.sikuli\tagui_sikuli.in" echo finish > tagui.sikuli\tagui_sikuli.in +if exist "tagui_chrome.in" echo finish > tagui_chrome.in From e4ef671771e6de0db8cd5d99300e62a7c7157aa5 Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Mon, 10 Jun 2019 14:48:51 +0800 Subject: [PATCH 3/7] #442 - support usage from non-terminal processes --- src/tagui | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tagui b/src/tagui index a2a24240..9018821c 100755 --- a/src/tagui +++ b/src/tagui @@ -297,14 +297,14 @@ if [ "$chrome_started" == "Darwin" ]; then if ! [ -f "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" ]; then echo "ERROR - cannot find Chrome at \"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome\"" echo "Chrome is at a non-standard location on your macOS, raise an issue on TagUI GitHub page"; exit 1; fi -chrome_process_id="$(ps | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" +chrome_process_id="$(ps -x | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" if [ -n "$chrome_process_id" ]; then kill $chrome_process_id; fi /Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome $chrome_switches $window_size $headless_switch > /dev/null 2>&1 & else if ! type "$chrome_command" > /dev/null; then echo "ERROR - cannot find Chrome command \"$chrome_command\"" echo "update chrome_command setting in tagui/src/tagui and make sure symlink to command is created"; exit 1; fi -chrome_process_id="$(ps | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" +chrome_process_id="$(ps -x | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" if [ -n "$chrome_process_id" ]; then kill $chrome_process_id; fi $chrome_command $chrome_switches $window_size $headless_switch > /dev/null 2>&1 & fi @@ -334,10 +334,10 @@ if [ -f "tagui_chrome.in" ]; then echo "finish" > tagui_chrome.in; fi # kill chrome processes by checking which os the processes are started on if [ "$chrome_started" == "Darwin" ] && [ "$tagui_speed_mode" == false ]; then -chrome_process_id="$(ps | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" +chrome_process_id="$(ps -x | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" if [ -n "$chrome_process_id" ]; then kill $chrome_process_id; fi else if [ -n "$chrome_started" ] && [ "$tagui_speed_mode" == false ]; then -chrome_process_id="$(ps | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" +chrome_process_id="$(ps -x | grep remote-debugging-port=9222 | grep tagui_user_profile | sed -e 's/^[ ]*//' | cut -d' ' -f 1 | sort -nur | head -n 1)" if [ -n "$chrome_process_id" ]; then kill $chrome_process_id; fi fi; fi From c0d0950ecfac4cabb1274480ff9debc392b0ad1f Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Mon, 17 Jun 2019 12:59:56 +0800 Subject: [PATCH 4/7] #443 - timeout step sikulix variable update SikuliX wait_timeout also needs to be changed as part of sikuli_step() when timeout step is used. Otherwise, when present() or visible() is invoked, the SikuliX timeout setting gets reverted back to its internal variable. Also, 'casper' object should be used in displaying the error message instead of 'this'. Otherwise, for the execution context, this would be undefined and error message does not show up. --- src/tagui_header.js | 7 ++++--- src/test/positive_test.signature | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/tagui_header.js b/src/tagui_header.js index 1bd5102b..f7b003b7 100644 --- a/src/tagui_header.js +++ b/src/tagui_header.js @@ -220,14 +220,14 @@ return false;} // friendlier name to use check_tx() in if condition in flow function present(element_locator) {if (!element_locator) return false; if (is_sikuli(element_locator)) {var abs_param = abs_file(element_locator); var fs = require('fs'); -if (!fs.exists(abs_param)) {this.echo('ERROR - cannot find image file for present step').exit();} +if (!fs.exists(abs_param)) {casper.echo('ERROR - cannot find image file for present step').exit();} if (sikuli_step("present " + abs_param)) return true; else return false;} else return check_tx(element_locator);} // friendlier name to check element visibility using elementVisible() function visible(element_locator) {if (!element_locator) return false; if (is_sikuli(element_locator)) {var abs_param = abs_file(element_locator); var fs = require('fs'); -if (!fs.exists(abs_param)) {this.echo('ERROR - cannot find image file for visible step').exit();} +if (!fs.exists(abs_param)) {casper.echo('ERROR - cannot find image file for visible step').exit();} if (sikuli_step("visible " + abs_param)) return true; else return false;} else {var element_located = tx(element_locator); var element_visible = casper.elementVisible(element_located); // if tx() returns xps666('/html') means that the element is not found, so set element_visible to false @@ -345,7 +345,8 @@ var fs = require('fs'); fs.write('tagui.sikuli'+ds+'tagui_sikuli.txt','','w');} // for setting timeout in sikuli when looking for ui element function sikuli_timeout(time_in_seconds) {var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; var fs = require('fs'); if (fs.exists('tagui.sikuli'+ds+'tagui_sikuli.in')) -sikuli_step('vision setAutoWaitTimeout(' + time_in_seconds.toString() + ')');} +sikuli_step('vision setAutoWaitTimeout(' + time_in_seconds.toString() + ')'); +sikuli_step('vision wait_timeout = ' + time_in_seconds.toString());} // for initialising integration with R function r_handshake() { // techo('[connecting to R process]'); diff --git a/src/test/positive_test.signature b/src/test/positive_test.signature index 46ac69a4..5b92bbf9 100644 --- a/src/test/positive_test.signature +++ b/src/test/positive_test.signature @@ -247,14 +247,14 @@ return false;} // friendlier name to use check_tx() in if condition in flow function present(element_locator) {if (!element_locator) return false; if (is_sikuli(element_locator)) {var abs_param = abs_file(element_locator); var fs = require('fs'); -if (!fs.exists(abs_param)) {this.echo('ERROR - cannot find image file for present step').exit();} +if (!fs.exists(abs_param)) {casper.echo('ERROR - cannot find image file for present step').exit();} if (sikuli_step("present " + abs_param)) return true; else return false;} else return check_tx(element_locator);} // friendlier name to check element visibility using elementVisible() function visible(element_locator) {if (!element_locator) return false; if (is_sikuli(element_locator)) {var abs_param = abs_file(element_locator); var fs = require('fs'); -if (!fs.exists(abs_param)) {this.echo('ERROR - cannot find image file for visible step').exit();} +if (!fs.exists(abs_param)) {casper.echo('ERROR - cannot find image file for visible step').exit();} if (sikuli_step("visible " + abs_param)) return true; else return false;} else {var element_located = tx(element_locator); var element_visible = casper.elementVisible(element_located); // if tx() returns xps666('/html') means that the element is not found, so set element_visible to false @@ -372,7 +372,8 @@ var fs = require('fs'); fs.write('tagui.sikuli'+ds+'tagui_sikuli.txt','','w');} // for setting timeout in sikuli when looking for ui element function sikuli_timeout(time_in_seconds) {var ds; if (flow_path.indexOf('/') !== -1) ds = '/'; else ds = '\\'; var fs = require('fs'); if (fs.exists('tagui.sikuli'+ds+'tagui_sikuli.in')) -sikuli_step('vision setAutoWaitTimeout(' + time_in_seconds.toString() + ')');} +sikuli_step('vision setAutoWaitTimeout(' + time_in_seconds.toString() + ')'); +sikuli_step('vision wait_timeout = ' + time_in_seconds.toString());} // for initialising integration with R function r_handshake() { // techo('[connecting to R process]'); From 0ad8cb86ca1153e227458bbe760c74c60dbb4285 Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Thu, 20 Jun 2019 21:35:15 +0800 Subject: [PATCH 5/7] #457 - visual automation images with transparency --- README.md | 61 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 3fb8d699..6d10550f 100755 --- a/README.md +++ b/README.md @@ -236,15 +236,24 @@ To schedule an automation flow with crontab (macOS/Linux), for example at 8am da **Tip** - for Windows, use Task Scheduler (search schedule from Start Menu) or [Z-Cron freeware](https://www.z-cron.com) -### TAGUI WRITER, SCREENSHOTER & EDITOR -TagUI Writer is a Windows app created by Arnaud Degardin / [@adegard](https://github.com/adegard) which makes it easy to write TagUI scripts. By pressing Ctrl + Left-click, a popup menu will appear with the list of TagUI steps for you to paste into your text editor. Arnaud also created a ScreenShoter app which makes it easy to capture snapshots for TagUI visual automation. Lastly, TagUI Editor allows you to edit and run TagUI scripts via AutoHotKey. To download, [click here](https://github.com/adegard/tagui_scripts). +### NATIVE LANGUAGES +To run TagUI flows in human languages or output flow execution in other languages ([see demo run](https://github.com/kelaberetiv/TagUI/issues/68#issuecomment-344380657)) +1. set your default flow language with [tagui_language variable](https://github.com/kelaberetiv/TagUI/blob/master/src/tagui_config.txt) in tagui_config.txt +2. write automation flow in human language base on [language definition .csv files](https://github.com/kelaberetiv/TagUI/tree/master/src/languages) +3. optional - set tagui_language in flow to any other languages as output language
- Click to show a preview of how TagUI Editor can be used to edit and run TagUI scripts + Click to show the 20+ human languages supported by TagUI and how to self-build language definitions + + **Tip** - as Windows Unicode support isn't as straightforward as macOS/Linux, doing this in Windows may require changing system locale, using chcp command, and selecting a font to display native language correctly ([more info](http://www.walkernews.net/2013/05/19/how-to-get-windows-command-prompt-displays-chinese-characters/)) + + TagUI language engine supports over 20 languages and can be modified or extended easily by users to improve accuracy or add more languages. The languages are Bengali, Chinese, English, French, German, Hindi, Hungarian, Indonesian, Italian, Japanese, Korean, Polish, Portuguese, Romanian, Russian, Serbian, Spanish, Tagalog, Tamil, Thai, Vietnamese. This starting set is partly chosen base on the [list of most commonly used languages](https://www.babbel.com/en/magazine/the-10-most-spoken-languages-in-the-world), partly from the countries around where I'm from (Singapore), and partly from countries with a lot of developers. -![TagUI Editor](https://raw.githubusercontent.com/adegard/tagui_scripts/master/TagUI_Editor.gif) + If your native language isn't in the above list, you can also automate building a new human language definition by using this language [build automation flow](https://github.com/kelaberetiv/TagUI/blob/master/src/languages/build) (src/languages/build) that self-builds the vocabulary set using Google Translate. To do that, update [build.csv](https://github.com/kelaberetiv/TagUI/blob/master/src/languages/build.csv) with the languages that you want to build and run `tagui build using chrome` in src/languages folder. Use quiet option to hide the verbose automation output. The generated files are named as their 2-character language codes to prevent overwriting existing language definitions by accident. To use the generated .csv files, rename them to their full language names. See [full list of languages possible](https://cloud.google.com/translate/docs/languages) to be generated by Google Translate. + + Most of the language definitions are automatically self-built using Google Translate (except english.csv, chinese.csv, japanese.csv, german.csv, french.csv which have been vetted), and would be wrong without understanding vocabulary used in UI interaction context. Native language users can update the language definition csv themselves and are invited to submit PRs with correct words to be used. Some languages are very different from English structure (for eg, written from right to left, different order of adjective and noun) and would be impossible to use correctly in TagUI.
@@ -269,7 +278,7 @@ Download from [Chrome Web Store](https://chrome.google.com/webstore/detail/tagui ### VISUAL AUTOMATION -TagUI has built-in integration with [SikuliX (base on OpenCV)](http://sikulix.com) to allow visually identifying the web elements and desktop UI (user interface) elements for interaction by providing their images or (x,y) coordinates. Applicable steps are click, hover, type, select, read, show, save, snap, keyboard, mouse. To use visual automation, Java JDK v8 (64-bit) or later is required. +TagUI has built-in integration with [SikuliX (base on OpenCV)](http://sikulix.com) to allow visually identifying the web elements and desktop UI (user interface) elements for interaction by providing their images or (x,y) coordinates. Applicable steps are click, hover, type, select, read, show, save, snap, keyboard, mouse. To use visual automation, Java JDK v8 (64-bit) or later is needed.
@@ -283,35 +292,19 @@ TagUI has built-in integration with [SikuliX (base on OpenCV)](http://sikulix.co 5. On macOS, if can't find image on screen, may be due to [how the image was captured](https://github.com/kelaberetiv/TagUI/issues/240#issuecomment-405030276) 6. On Linux, requires installing and setting up dependencies by following [this guide](https://sikulix-2014.readthedocs.io/en/latest/newslinux.html#version-1-1-4-special-for-linux-people) - To use visual automation, simply specify an image (in .png or .bmp format) to visually look for in place of the element identifier. Alternatively, you can specify the (x,y) coordinates of the element that you want to interact with. - > Important! The element that corresponds to the image must be visible on the screen for visual automation to succeed. If it's blocked by another window for example, the automation will be unable to find the element. + >Important note - the element that corresponds to the image must be visible on the screen for visual automation to succeed. If it's blocked by another window for example, the automation will be unable to find the element. - To type onto the screen instead of a particular element, use `keyboard text` or `keyboard [modifiers]text` ([examples](https://github.com/kelaberetiv/TagUI/issues/370)). To do a snapshot or an OCR of the whole screen, use `page.png` or `page.bmp` as the element identifier for steps snap / read. Relative paths are supported for image filenames (eg pc.png, images/button.bmp). The usual helper functions visible() / present() can also be used to check whether an image is visible on the screen. - - The keyboard and mouse steps, as well as helper functions mouse_xy(), mouse_x(), mouse_y(), can be used to do complex UI interactions. A screen (real or Xvfb) is needed for visual automation. [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) (optical character recognition) is used for visually retrieving text. Also, by using vision step, you can send [custom SikuliX commands](http://sikulix-2014.readthedocs.io/en/latest/genindex.html) to do things that are not covered by TagUI. + To use visual automation, simply specify an image (in .png or .bmp format) to visually look for in place of the element identifier. Relative paths are supported for image filenames (eg pc.png, images/button.bmp). Alternatively, you can specify the (x,y) coordinates of the element that you want to interact with. ![Sample Visual Automation](https://raw.githubusercontent.com/tebelorg/Tump/master/visual_flow.gif) -
- -### NATIVE LANGUAGES -To run TagUI flows in native languages or output flow execution in other languages ([see demo run](https://github.com/kelaberetiv/TagUI/issues/68#issuecomment-344380657)) -1. set your default flow language with [tagui_language variable](https://github.com/kelaberetiv/TagUI/blob/master/src/tagui_config.txt) in tagui_config.txt -2. write automation flow in native language base on [language definition .csv files](https://github.com/kelaberetiv/TagUI/tree/master/src/languages) -3. optional - set tagui_language in flow to any other languages as output language - -
- - Click to show the 20+ human languages supported by TagUI and how to self-build language definitions - - - **Tip** - as Windows Unicode support isn't as straightforward as macOS/Linux, doing this in Windows may require changing system locale, using chcp command, and selecting a font to display native language correctly ([more info](http://www.walkernews.net/2013/05/19/how-to-get-windows-command-prompt-displays-chinese-characters/)) - - TagUI language engine supports over 20 languages and can be modified or extended easily by users to improve accuracy or add more languages. The languages are Bengali, Chinese, English, French, German, Hindi, Hungarian, Indonesian, Italian, Japanese, Korean, Polish, Portuguese, Romanian, Russian, Serbian, Spanish, Tagalog, Tamil, Thai, Vietnamese. This starting set is partly chosen base on the [list of most commonly used languages](https://www.babbel.com/en/magazine/the-10-most-spoken-languages-in-the-world), partly from the countries around where I'm from (Singapore), and partly from countries with a lot of developers. + To type onto the screen instead of a particular element, use `keyboard text` or `keyboard [modifiers]text` ([examples](https://github.com/kelaberetiv/TagUI/issues/370)). To do a snapshot or an OCR of the whole screen, use `page.png` or `page.bmp` as the element identifier for steps snap / read. The usual helper functions visible() / present() can also be used to check whether an image is visible on the screen. - If your native language isn't in the above list, you can also automate building a new native language definition by using this language [build automation flow](https://github.com/kelaberetiv/TagUI/blob/master/src/languages/build) (src/languages/build) that self-builds the vocabulary set using Google Translate. To do that, update [build.csv](https://github.com/kelaberetiv/TagUI/blob/master/src/languages/build.csv) with the languages that you want to build and run `tagui build using chrome` in src/languages folder. Use quiet option to hide the verbose automation output. The generated files are named as their 2-character language codes to prevent overwriting existing language definitions by accident. To use the generated .csv files, rename them to their full language names. See [full list of languages possible](https://cloud.google.com/translate/docs/languages) to be generated by Google Translate. + >Transparency (0% opacity) is supported in .png images, for eg using an image of an UI element with transparent background to enable clicking on an UI element that appears on different backgrounds on different occasions. + > + >Another example is an image of the window or frame (PDF viewer, MS Word, textbox etc) with the center content of the image set as transparent. This allows using read, show, save, snap steps to perform OCR and save snapshots for application windows, containers, frames, textboxes with varying content. - Most of the language definitions are automatically self-built using Google Translate (except english.csv, chinese.csv, japanese.csv, german.csv, french.csv which have been vetted), and would be wrong without understanding vocabulary used in UI interaction context. Native language users can update the language definition csv themselves and are invited to submit PRs with correct words to be used. Some languages are very different from English structure (for eg, written from right to left, different order of adjective and noun) and would be impossible to use correctly in TagUI. + The keyboard and mouse steps, as well as helper functions mouse_xy(), mouse_x(), mouse_y(), can be used to do complex UI interactions. A screen (real or Xvfb) is needed for visual automation. [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) (optical character recognition) is used for visually retrieving text. Also, by using vision step, you can send [custom SikuliX commands](http://sikulix-2014.readthedocs.io/en/latest/genindex.html) to do things that are not covered by TagUI.
@@ -411,6 +404,18 @@ TagUI scripts are already in natural-language-like syntax to convert to JavaScri +### TAGUI WRITER, SCREENSHOTER & EDITOR +TagUI Writer is a Windows app created by Arnaud Degardin / [@adegard](https://github.com/adegard) which makes it easy to write TagUI scripts. By pressing Ctrl + Left-click, a popup menu will appear with the list of TagUI steps for you to paste into your text editor. Arnaud also created a ScreenShoter app which makes it easy to capture snapshots for TagUI visual automation. Lastly, TagUI Editor allows you to edit and run TagUI scripts via AutoHotKey. To download, [click here](https://github.com/adegard/tagui_scripts). + +
+ + Click to show a preview of how TagUI Editor can be used to edit and run TagUI scripts + + +![TagUI Editor](https://raw.githubusercontent.com/adegard/tagui_scripts/master/TagUI_Editor.gif) + +
+ ### FLOW SAMPLES Following automation flow samples ([tagui/src/samples folder](https://github.com/kelaberetiv/TagUI/tree/master/src/samples)) are included with TagUI From d241a1dc8562bd42bb4890df50df2b79e1a77691 Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Fri, 21 Jun 2019 02:14:53 +0800 Subject: [PATCH 6/7] #459 - update readme on tagui for python package --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d10550f..5e7c6914 100755 --- a/README.md +++ b/README.md @@ -416,6 +416,9 @@ TagUI Writer is a Windows app created by Arnaud Degardin / [@adegard](https://gi +### TAGUI FOR PYTHON +TagUI for Python is a Python package created by TagUI's creator Ken Soh / [@kensoh](https://github.com/kensoh). It brings the digital process automation capabilities of TagUI directly to the Python environment through a simple, expressive and powerful API. To install, `pip install tagui`. For its Python API and architecture, refer to the [project homepage](https://github.com/tebelorg/TagUI-Python). The package is built on TagUI and its architecture is based on integration with TagUI's live mode. At merely 1000 lines of code, one of the project goals is to inspire developers of other programming languages to port it over to their favourite language. + ### FLOW SAMPLES Following automation flow samples ([tagui/src/samples folder](https://github.com/kelaberetiv/TagUI/tree/master/src/samples)) are included with TagUI @@ -919,4 +922,3 @@ TagUI excels in automating user-interface interactions. It's designed to make pr # License TagUI is open-source software released under Apache 2.0 license - From 31ce836e2e6c877bde574365f754be004f6cdde1 Mon Sep 17 00:00:00 2001 From: Ken Soh Date: Fri, 21 Jun 2019 22:04:03 +0800 Subject: [PATCH 7/7] #461 - support defining regions for OCR and snapshot more details in issue - #461 --- src/tagui.sikuli/tagui.py | 33 ++++++++++++++++++++++++++++---- src/tagui_header.js | 3 ++- src/tagui_parse.php | 3 ++- src/test/positive_test.signature | 3 ++- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/tagui.sikuli/tagui.py b/src/tagui.sikuli/tagui.py index 33748abf..94545909 100644 --- a/src/tagui.sikuli/tagui.py +++ b/src/tagui.sikuli/tagui.py @@ -19,7 +19,7 @@ # helper function to detect coordinates locator def is_coordinates ( input_locator ): - if input_locator[:1] == '(' and input_locator[-1:] == ')' and input_locator.count(',') == 1: + if input_locator[:1] == '(' and input_locator[-1:] == ')' and input_locator.count(',') in [1,2]: return True else: return False @@ -32,6 +32,19 @@ def x_coordinate ( input_locator ): def y_coordinate ( input_locator ): return int(input_locator[input_locator.find(',')+1:-1]) +# helper function to return Region from (x1,y1),(x2,y2) +def define_region( input_locator ): + input_tokens = input_locator.split(',') + region_x1_coordinate = int(input_tokens[0].replace('(','')) + region_y1_coordinate = int(input_tokens[1].split('-')[0].replace(')','')) + region_x2_coordinate = int(input_tokens[1].split('-')[1].replace('(','')) + region_y2_coordinate = int(input_tokens[2].replace(')','')) + region_width = abs(region_x2_coordinate - region_x1_coordinate) + region_height = abs(region_y2_coordinate - region_y1_coordinate) + region_origin_x = min(region_x1_coordinate,region_x2_coordinate) + region_origin_y = min(region_y1_coordinate,region_y2_coordinate) + return Region(region_origin_x,region_origin_y,region_width,region_height) + # function to map modifier keys to unicode for use in type() def modifiers_map ( input_keys ): modifier_keys = 0 @@ -222,10 +235,15 @@ def save_intent ( raw_intent ): # function for reading text using Tesseract OCR def text_read ( raw_intent ): params = (raw_intent + ' ')[1+(raw_intent + ' ').find(' '):].strip() - param1 = params[:(params + ' ').find(' ')].strip() + param1 = params.split(' to ')[0].strip() print '[tagui] ACTION - ' + raw_intent - if param1.endswith('page.png') or param1.endswith('page.bmp'): + if is_coordinates(param1): + region_layer = define_region(param1) + temp_text = region_layer.text() + output_sikuli_text(temp_text) + return 1 + elif param1.endswith('page.png') or param1.endswith('page.bmp'): fullscreen_layer = Screen() temp_text = fullscreen_layer.text() output_sikuli_text(temp_text) @@ -244,7 +262,14 @@ def snap_intent ( raw_intent ): param1 = params[:params.find(' to ')].strip() param2 = params[4+params.find(' to '):].strip() print '[tagui] ACTION - snap ' + param1 + ' to ' + param2 - if param1.endswith('page.png') or param1.endswith('page.bmp'): + if is_coordinates(param1): + fullscreen_layer = Screen() + region_layer = define_region(param1) + temp_snap_file = fullscreen_layer.capture(region_layer).getFile() + import shutil + shutil.copy(temp_snap_file,param2) + return 1 + elif param1.endswith('page.png') or param1.endswith('page.bmp'): fullscreen_layer = Screen() temp_snap_file = fullscreen_layer.capture(fullscreen_layer.getBounds()).getFile() import shutil diff --git a/src/tagui_header.js b/src/tagui_header.js index f7b003b7..999e943b 100644 --- a/src/tagui_header.js +++ b/src/tagui_header.js @@ -877,7 +877,8 @@ return input_string.replace(/\\/g,'\\\\');} function is_coordinates(input_params) { // helper function to check if string is (x,y) coordinates if ((input_params.length > 4) && (input_params.substr(0,1) == '(') && (input_params.substr(-1) == ')') -&& (input_params.split(',').length == 2) && (!input_params.match(/[a-z]/i))) return true; else return false;} +&& (input_params.split(',').length == 2 || input_params.split(',').length == 3) +&& (!input_params.match(/[a-z]/i))) return true; else return false;} function is_sikuli(input_params) { // helper function to check if input is meant for sikuli visual automation if (input_params.length > 4 && input_params.substr(-4).toLowerCase() == '.png') return true; // support png and bmp diff --git a/src/tagui_parse.php b/src/tagui_parse.php index cdd9a88b..c8e0ee68 100755 --- a/src/tagui_parse.php +++ b/src/tagui_parse.php @@ -561,7 +561,8 @@ function add_concat($source_string) { // parse string and add missing + concaten function is_coordinates($input_params) { // helper function to check if string is (x,y) coordinates if (strlen($input_params)>4 and substr($input_params,0,1)=='(' and substr($input_params,-1)==')' -and substr_count($input_params,',')==1 and ((strpos($input_params,"'+")!==false and strpos($input_params,"+'")!==false) +and (substr_count($input_params,',')==1 or substr_count($input_params,',')==2) +and ((strpos($input_params,"'+")!==false and strpos($input_params,"+'")!==false) or !preg_match('/[a-zA-Z]/',$input_params))) return true; else return false;} function is_sikuli($input_params) { // helper function to check if input is meant for sikuli visual automation diff --git a/src/test/positive_test.signature b/src/test/positive_test.signature index 5b92bbf9..2b6ae0c8 100644 --- a/src/test/positive_test.signature +++ b/src/test/positive_test.signature @@ -904,7 +904,8 @@ return input_string.replace(/\\/g,'\\\\');} function is_coordinates(input_params) { // helper function to check if string is (x,y) coordinates if ((input_params.length > 4) && (input_params.substr(0,1) == '(') && (input_params.substr(-1) == ')') -&& (input_params.split(',').length == 2) && (!input_params.match(/[a-z]/i))) return true; else return false;} +&& (input_params.split(',').length == 2 || input_params.split(',').length == 3) +&& (!input_params.match(/[a-z]/i))) return true; else return false;} function is_sikuli(input_params) { // helper function to check if input is meant for sikuli visual automation if (input_params.length > 4 && input_params.substr(-4).toLowerCase() == '.png') return true; // support png and bmp