/* Copyright (c) 2017, Sikuli.org, sikulix.com * * This is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this software; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, * USA. */ package org.sikuli.vnc; import org.sikuli.basics.Settings; import org.sikuli.script.*; import java.awt.*; import java.awt.event.KeyEvent; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.Set; import java.util.TreeSet; import static java.awt.event.KeyEvent.*; class VNCRobot implements IRobot { public static final int VNC_POINTER_EVENT_BUTTON_1 = 1 << 0; public static final int VNC_POINTER_EVENT_BUTTON_2 = 1 << 1; public static final int VNC_POINTER_EVENT_BUTTON_3 = 1 << 2; public static final int VNC_POINTER_EVENT_BUTTON_4 = 1 << 3; public static final int VNC_POINTER_EVENT_BUTTON_5 = 1 << 4; public static final int VNC_POINTER_EVENT_BUTTON_6 = 1 << 5; public static final int VNC_POINTER_EVENT_BUTTON_7 = 1 << 6; public static final int VNC_POINTER_EVENT_BUTTON_8 = 1 << 7; private final VNCScreen screen; private int mouseX; private int mouseY; private int mouseButtons; private int autoDelay; private Set<Integer> pressedKeys; private boolean shiftPressed; public VNCRobot(VNCScreen screen) { this.screen = screen; this.autoDelay = 100; this.pressedKeys = new TreeSet<>(); } @Override public ScreenImage captureScreen(Rectangle screenRect) { return screen.capture(screenRect); } @Override public boolean isRemote() { return true; } @Override public IScreen getScreen() { return screen; } @Override public void keyDown(String keys) { for (int i = 0; i < keys.length(); i++) { typeChar(keys.charAt(i), KeyMode.PRESS_ONLY); } } @Override public void keyUp(String keys) { for (int i = 0; i < keys.length(); i++) { typeChar(keys.charAt(i), KeyMode.RELEASE_ONLY); } } @Override public void keyDown(int code) { typeKey(code, KeyMode.PRESS_ONLY); } @Override public void keyUp(int code) { typeCode(keyToXlib(code), KeyMode.RELEASE_ONLY); } @Override public void keyUp() { for (Integer key : new ArrayList<>(pressedKeys)) { typeCode(key, KeyMode.RELEASE_ONLY); } } @Override public void pressModifiers(int modifiers) { typeModifiers(modifiers, KeyMode.PRESS_ONLY); } @Override public void releaseModifiers(int modifiers) { typeModifiers(modifiers, KeyMode.RELEASE_ONLY); } private void typeModifiers(int modifiers, KeyMode keyMode) { if ((modifiers & KeyModifier.CTRL) != 0) typeKey(KeyEvent.VK_CONTROL, keyMode); if ((modifiers & KeyModifier.SHIFT) != 0) typeCode(KeyEvent.VK_SHIFT, keyMode); if ((modifiers & KeyModifier.ALT) != 0) typeCode(KeyEvent.VK_ALT, keyMode); if ((modifiers & KeyModifier.ALTGR) != 0) typeCode(KeyEvent.VK_ALT_GRAPH, keyMode); if ((modifiers & KeyModifier.META) != 0) typeCode(KeyEvent.VK_META, keyMode); } @Override public void typeStarts() { // Nothing to do } @Override public void typeEnds() { // Nothing to do } @Override public void typeKey(int key) { typeKey(key, KeyMode.PRESS_RELEASE); } @Override public void typeChar(char character, KeyMode mode) { if (character > '\ue000' && character < '\ue050') { typeKey(Key.toJavaKeyCode(character)[0], mode); } else { typeCode(charToXlib(character), mode); } } public void typeKey(int key, KeyMode mode) { typeCode(keyToXlib(key), mode); } private void typeCode(int xlibCode, KeyMode mode) { boolean addShift = requiresShift(xlibCode) && !shiftPressed; try { if (mode == KeyMode.PRESS_RELEASE || mode == KeyMode.PRESS_ONLY) { if (addShift) { pressKey(XKeySym.XK_Shift_L); } pressKey(xlibCode); if (xlibCode == XKeySym.XK_Shift_L || xlibCode == XKeySym.XK_Shift_R || xlibCode == XKeySym.XK_Shift_Lock) { shiftPressed = true; } } if (mode == KeyMode.PRESS_RELEASE || mode == KeyMode.RELEASE_ONLY) { releaseKey(xlibCode); if (addShift) { releaseKey(XKeySym.XK_Shift_L); } if (xlibCode == XKeySym.XK_Shift_L || xlibCode == XKeySym.XK_Shift_R || xlibCode == XKeySym.XK_Shift_Lock) { shiftPressed = false; } } } catch (IOException e) { throw new RuntimeException(e); } } private void pressKey(int key) throws IOException { screen.getClient().keyDown(key); pressedKeys.add(key); } private void releaseKey(int key) throws IOException { screen.getClient().keyUp(key); pressedKeys.remove(key); } private int charToXlib(char c) { if (c >= 0x0020 && c <= 0x00FF) { return c; } switch (c) { case '\u0008': return XKeySym.XK_BackSpace; case '\u0009': return XKeySym.XK_Tab; case '\n': return XKeySym.XK_Linefeed; case '\u000b': return XKeySym.XK_Clear; case '\r': return XKeySym.XK_Return; case '\u0013': return XKeySym.XK_Pause; case '\u0014': return XKeySym.XK_Scroll_Lock; case '\u0015': return XKeySym.XK_Sys_Req; case '\u001b': return XKeySym.XK_Escape; case '\u007f': return XKeySym.XK_Delete; default: throw new IllegalArgumentException("Cannot type character " + c); } } private int keyToXlib(int code) { switch (code) { case VK_ENTER: return XKeySym.XK_Return; case VK_BACK_SPACE: return XKeySym.XK_BackSpace; case VK_TAB: return XKeySym.XK_Tab; case VK_CANCEL: return XKeySym.XK_Cancel; case VK_CLEAR: return XKeySym.XK_Clear; case VK_SHIFT: return XKeySym.XK_Shift_L; case VK_CONTROL: return XKeySym.XK_Control_L; case VK_ALT: return XKeySym.XK_Alt_L; case VK_PAUSE: return XKeySym.XK_Pause; case VK_CAPS_LOCK: return XKeySym.XK_Caps_Lock; case VK_ESCAPE: return XKeySym.XK_Escape; case VK_SPACE: return XKeySym.XK_space; case VK_PAGE_UP: return XKeySym.XK_Page_Up; case VK_PAGE_DOWN: return XKeySym.XK_Page_Down; case VK_END: return XKeySym.XK_End; case VK_HOME: return XKeySym.XK_Home; case VK_LEFT: return XKeySym.XK_Left; case VK_UP: return XKeySym.XK_Up; case VK_RIGHT: return XKeySym.XK_Right; case VK_DOWN: return XKeySym.XK_Down; case VK_COMMA: return XKeySym.XK_comma; case VK_MINUS: return XKeySym.XK_minus; case VK_PERIOD: return XKeySym.XK_period; case VK_SLASH: return XKeySym.XK_slash; case VK_0: return XKeySym.XK_0; case VK_1: return XKeySym.XK_1; case VK_2: return XKeySym.XK_2; case VK_3: return XKeySym.XK_3; case VK_4: return XKeySym.XK_4; case VK_5: return XKeySym.XK_5; case VK_6: return XKeySym.XK_6; case VK_7: return XKeySym.XK_7; case VK_8: return XKeySym.XK_8; case VK_9: return XKeySym.XK_9; case VK_SEMICOLON: return XKeySym.XK_semicolon; case VK_EQUALS: return XKeySym.XK_equal; case VK_A: return shiftPressed ? XKeySym.XK_A : XKeySym.XK_a; case VK_B: return shiftPressed ? XKeySym.XK_B : XKeySym.XK_b; case VK_C: return shiftPressed ? XKeySym.XK_C : XKeySym.XK_c; case VK_D: return shiftPressed ? XKeySym.XK_D : XKeySym.XK_d; case VK_E: return shiftPressed ? XKeySym.XK_E : XKeySym.XK_e; case VK_F: return shiftPressed ? XKeySym.XK_F : XKeySym.XK_f; case VK_G: return shiftPressed ? XKeySym.XK_G : XKeySym.XK_g; case VK_H: return shiftPressed ? XKeySym.XK_H : XKeySym.XK_h; case VK_I: return shiftPressed ? XKeySym.XK_I : XKeySym.XK_i; case VK_J: return shiftPressed ? XKeySym.XK_J : XKeySym.XK_j; case VK_K: return shiftPressed ? XKeySym.XK_K : XKeySym.XK_k; case VK_L: return shiftPressed ? XKeySym.XK_L : XKeySym.XK_l; case VK_M: return shiftPressed ? XKeySym.XK_M : XKeySym.XK_m; case VK_N: return shiftPressed ? XKeySym.XK_N : XKeySym.XK_n; case VK_O: return shiftPressed ? XKeySym.XK_O : XKeySym.XK_o; case VK_P: return shiftPressed ? XKeySym.XK_P : XKeySym.XK_p; case VK_Q: return shiftPressed ? XKeySym.XK_Q : XKeySym.XK_q; case VK_R: return shiftPressed ? XKeySym.XK_R : XKeySym.XK_r; case VK_S: return shiftPressed ? XKeySym.XK_S : XKeySym.XK_s; case VK_T: return shiftPressed ? XKeySym.XK_T : XKeySym.XK_t; case VK_U: return shiftPressed ? XKeySym.XK_U : XKeySym.XK_u; case VK_V: return shiftPressed ? XKeySym.XK_V : XKeySym.XK_v; case VK_W: return shiftPressed ? XKeySym.XK_W : XKeySym.XK_w; case VK_X: return shiftPressed ? XKeySym.XK_X : XKeySym.XK_x; case VK_Y: return shiftPressed ? XKeySym.XK_Y : XKeySym.XK_y; case VK_Z: return shiftPressed ? XKeySym.XK_Z : XKeySym.XK_z; case VK_OPEN_BRACKET: return XKeySym.XK_bracketleft; case VK_BACK_SLASH: return XKeySym.XK_backslash; case VK_CLOSE_BRACKET: return XKeySym.XK_bracketright; case VK_NUMPAD0: return XKeySym.XK_KP_0; case VK_NUMPAD1: return XKeySym.XK_KP_1; case VK_NUMPAD2: return XKeySym.XK_KP_2; case VK_NUMPAD3: return XKeySym.XK_KP_3; case VK_NUMPAD4: return XKeySym.XK_KP_4; case VK_NUMPAD5: return XKeySym.XK_KP_5; case VK_NUMPAD6: return XKeySym.XK_KP_6; case VK_NUMPAD7: return XKeySym.XK_KP_7; case VK_NUMPAD8: return XKeySym.XK_KP_8; case VK_NUMPAD9: return XKeySym.XK_KP_9; case VK_MULTIPLY: return XKeySym.XK_KP_Multiply; case VK_ADD: return XKeySym.XK_KP_Add; case VK_SEPARATOR: return XKeySym.XK_KP_Separator; case VK_SUBTRACT: return XKeySym.XK_KP_Subtract; case VK_DECIMAL: return XKeySym.XK_KP_Decimal; case VK_DIVIDE: return XKeySym.XK_KP_Divide; case VK_DELETE: return XKeySym.XK_KP_Delete; case VK_NUM_LOCK: return XKeySym.XK_Num_Lock; case VK_SCROLL_LOCK: return XKeySym.XK_Scroll_Lock; case VK_F1: return XKeySym.XK_F1; case VK_F2: return XKeySym.XK_F2; case VK_F3: return XKeySym.XK_F3; case VK_F4: return XKeySym.XK_F4; case VK_F5: return XKeySym.XK_F5; case VK_F6: return XKeySym.XK_F6; case VK_F7: return XKeySym.XK_F7; case VK_F8: return XKeySym.XK_F8; case VK_F9: return XKeySym.XK_F9; case VK_F10: return XKeySym.XK_F10; case VK_F11: return XKeySym.XK_F11; case VK_F12: return XKeySym.XK_F12; case VK_F13: return XKeySym.XK_F13; case VK_F14: return XKeySym.XK_F14; case VK_F15: return XKeySym.XK_F15; case VK_F16: return XKeySym.XK_F16; case VK_F17: return XKeySym.XK_F17; case VK_F18: return XKeySym.XK_F18; case VK_F19: return XKeySym.XK_F19; case VK_F20: return XKeySym.XK_F20; case VK_F21: return XKeySym.XK_F21; case VK_F22: return XKeySym.XK_F22; case VK_F23: return XKeySym.XK_F23; case VK_F24: return XKeySym.XK_F24; case VK_PRINTSCREEN: return XKeySym.XK_Print; case VK_INSERT: return XKeySym.XK_Insert; case VK_HELP: return XKeySym.XK_Help; case VK_META: return XKeySym.XK_Meta_L; case VK_KP_UP: return XKeySym.XK_KP_Up; case VK_KP_DOWN: return XKeySym.XK_KP_Down; case VK_KP_LEFT: return XKeySym.XK_KP_Left; case VK_KP_RIGHT: return XKeySym.XK_KP_Right; case VK_DEAD_GRAVE: return XKeySym.XK_dead_grave; case VK_DEAD_ACUTE: return XKeySym.XK_dead_acute; case VK_DEAD_CIRCUMFLEX: return XKeySym.XK_dead_circumflex; case VK_DEAD_TILDE: return XKeySym.XK_dead_tilde; case VK_DEAD_MACRON: return XKeySym.XK_dead_macron; case VK_DEAD_BREVE: return XKeySym.XK_dead_breve; case VK_DEAD_ABOVEDOT: return XKeySym.XK_dead_abovedot; case VK_DEAD_DIAERESIS: return XKeySym.XK_dead_diaeresis; case VK_DEAD_ABOVERING: return XKeySym.XK_dead_abovering; case VK_DEAD_DOUBLEACUTE: return XKeySym.XK_dead_doubleacute; case VK_DEAD_CARON: return XKeySym.XK_dead_caron; case VK_DEAD_CEDILLA: return XKeySym.XK_dead_cedilla; case VK_DEAD_OGONEK: return XKeySym.XK_dead_ogonek; case VK_DEAD_IOTA: return XKeySym.XK_dead_iota; case VK_DEAD_VOICED_SOUND: return XKeySym.XK_dead_voiced_sound; case VK_DEAD_SEMIVOICED_SOUND: return XKeySym.XK_dead_semivoiced_sound; case VK_AMPERSAND: return XKeySym.XK_ampersand; case VK_ASTERISK: return XKeySym.XK_asterisk; case VK_QUOTEDBL: return XKeySym.XK_quotedbl; case VK_LESS: return XKeySym.XK_less; case VK_GREATER: return XKeySym.XK_greater; case VK_BRACELEFT: return XKeySym.XK_bracketleft; case VK_BRACERIGHT: return XKeySym.XK_bracketright; case VK_AT: return XKeySym.XK_at; case VK_COLON: return XKeySym.XK_colon; case VK_CIRCUMFLEX: return XKeySym.XK_acircumflex; case VK_DOLLAR: return XKeySym.XK_dollar; case VK_EURO_SIGN: return XKeySym.XK_EuroSign; case VK_EXCLAMATION_MARK: return XKeySym.XK_exclam; case VK_INVERTED_EXCLAMATION_MARK: return XKeySym.XK_exclamdown; case VK_LEFT_PARENTHESIS: return XKeySym.XK_parenleft; case VK_NUMBER_SIGN: return XKeySym.XK_numbersign; case VK_PLUS: return XKeySym.XK_plus; case VK_RIGHT_PARENTHESIS: return XKeySym.XK_parenright; case VK_UNDERSCORE: return XKeySym.XK_underscore; case VK_WINDOWS: return XKeySym.XK_Super_L; case VK_COMPOSE: return XKeySym.XK_Multi_key; case VK_ALT_GRAPH: return XKeySym.XK_ISO_Level3_Shift; case VK_BEGIN: return XKeySym.XK_Begin; } throw new IllegalArgumentException("Cannot type keycode " + code); } private boolean requiresShift(int xlibKeySym) { // This is keyboard layout dependent. // What's encoded here is for a basic US layout switch (xlibKeySym) { case XKeySym.XK_A: case XKeySym.XK_B: case XKeySym.XK_C: case XKeySym.XK_D: case XKeySym.XK_E: case XKeySym.XK_F: case XKeySym.XK_G: case XKeySym.XK_H: case XKeySym.XK_I: case XKeySym.XK_J: case XKeySym.XK_K: case XKeySym.XK_L: case XKeySym.XK_M: case XKeySym.XK_N: case XKeySym.XK_O: case XKeySym.XK_P: case XKeySym.XK_Q: case XKeySym.XK_R: case XKeySym.XK_S: case XKeySym.XK_T: case XKeySym.XK_U: case XKeySym.XK_V: case XKeySym.XK_W: case XKeySym.XK_X: case XKeySym.XK_Y: case XKeySym.XK_Z: case XKeySym.XK_exclam: case XKeySym.XK_at: case XKeySym.XK_numbersign: case XKeySym.XK_dollar: case XKeySym.XK_percent: case XKeySym.XK_asciicircum: case XKeySym.XK_ampersand: case XKeySym.XK_asterisk: case XKeySym.XK_parenleft: case XKeySym.XK_parenright: case XKeySym.XK_underscore: case XKeySym.XK_plus: case XKeySym.XK_braceleft: case XKeySym.XK_braceright: case XKeySym.XK_colon: case XKeySym.XK_quotedbl: case XKeySym.XK_bar: case XKeySym.XK_less: case XKeySym.XK_greater: case XKeySym.XK_question: case XKeySym.XK_asciitilde: case XKeySym.XK_plusminus: return true; default: return false; } } @Override public void mouseMove(int x, int y) { try { screen.getClient().mouseEvent(mouseButtons, x, y); mouseX = x; mouseY = y; } catch (IOException e) { throw new RuntimeException(e); } } @Override public void mouseDown(int buttons) { if ((buttons & Mouse.LEFT) != 0) mouseButtons |= VNC_POINTER_EVENT_BUTTON_1; if ((buttons & Mouse.MIDDLE) != 0) mouseButtons |= VNC_POINTER_EVENT_BUTTON_2; if ((buttons & Mouse.RIGHT) != 0) mouseButtons |= VNC_POINTER_EVENT_BUTTON_3; mouseMove(mouseX, mouseY); } @Override public int mouseUp(int buttons) { if ((buttons & Mouse.LEFT) != 0) mouseButtons &= ~VNC_POINTER_EVENT_BUTTON_1; if ((buttons & Mouse.MIDDLE) != 0) mouseButtons &= ~VNC_POINTER_EVENT_BUTTON_2; if ((buttons & Mouse.RIGHT) != 0) mouseButtons &= ~VNC_POINTER_EVENT_BUTTON_3; mouseMove(mouseX, mouseY); int remainingButtons = 0; if ((mouseButtons & VNC_POINTER_EVENT_BUTTON_1) != 0) remainingButtons |= Mouse.LEFT; if ((mouseButtons & VNC_POINTER_EVENT_BUTTON_2) != 0) remainingButtons |= Mouse.MIDDLE; if ((mouseButtons & VNC_POINTER_EVENT_BUTTON_3) != 0) remainingButtons |= Mouse.RIGHT; return remainingButtons; } @Override public void mouseReset() { mouseButtons = 0; mouseMove(mouseX, mouseY); } @Override public void clickStarts() { // Nothing to do } @Override public void clickEnds() { // Nothing to do } @Override public void smoothMove(Location dest) { smoothMove(new Location(mouseX, mouseY), dest, (long) (Settings.MoveMouseDelay * 1000L)); } @Override public void smoothMove(Location src, Location dest, long duration) { if (duration <= 0) { mouseMove(dest.getX(), dest.getY()); return; } float x = src.getX(); float y = src.getY(); float dx = dest.getX() - src.getX(); float dy = dest.getY() - src.getY(); long start = System.currentTimeMillis(); long elapsed = 0; do { float fraction = (float) elapsed / (float) duration; mouseMove((int) (x + fraction * dx), (int) (y + fraction * dy)); delay(autoDelay); elapsed = System.currentTimeMillis() - start; } while (elapsed < duration); mouseMove(dest.x, dest.y); } @Override public void mouseWheel(int wheelAmt) { if (wheelAmt == Mouse.WHEEL_DOWN) { mouseButtons |= VNC_POINTER_EVENT_BUTTON_5; mouseMove(mouseX, mouseY); mouseButtons &= ~VNC_POINTER_EVENT_BUTTON_5; mouseMove(mouseX, mouseY); } else if (wheelAmt == Mouse.WHEEL_UP) { mouseButtons |= VNC_POINTER_EVENT_BUTTON_4; mouseMove(mouseX, mouseY); mouseButtons &= ~VNC_POINTER_EVENT_BUTTON_4; mouseMove(mouseX, mouseY); } } @Override public void waitForIdle() { // Nothing to do } @Override public void delay(int ms) { try { Thread.sleep(ms); } catch (InterruptedException e) { // ignored } } @Override public void setAutoDelay(int ms) { autoDelay = ms; } @Override public Color getColorAt(int x, int y) { ScreenImage image = captureScreen(new Rectangle(x, y, 1, 1)); return new Color(image.getImage().getRGB(0, 0)); } @Override public void cleanup() { // Nothing to do } }