1 /* 2 * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 package org.jemmy.input; 26 27 28 import java.awt.EventQueue; 29 import java.awt.image.BufferedImage; 30 import java.io.BufferedReader; 31 import java.io.File; 32 import java.io.IOException; 33 import java.io.InputStreamReader; 34 import java.io.ObjectInputStream; 35 import java.io.ObjectOutputStream; 36 import java.io.OptionalDataException; 37 import java.io.PrintWriter; 38 import java.lang.reflect.Field; 39 import java.lang.reflect.InvocationTargetException; 40 import java.lang.reflect.Modifier; 41 import java.net.ServerSocket; 42 import java.net.Socket; 43 import java.net.UnknownHostException; 44 import java.util.Arrays; 45 import java.util.HashSet; 46 import java.util.Set; 47 import java.util.logging.Level; 48 import java.util.logging.Logger; 49 import org.jemmy.JemmyException; 50 import org.jemmy.Rectangle; 51 import org.jemmy.env.Environment; 52 import org.jemmy.env.Timeout; 53 import org.jemmy.image.AWTImage; 54 import org.jemmy.image.Image; 55 import org.jemmy.image.PNGDecoder; 56 import org.jemmy.image.PNGEncoder; 57 import org.jemmy.timing.State; 58 import org.jemmy.timing.Waiter; 59 import org.jemmy.interfaces.Keyboard.KeyboardButton; 60 import org.jemmy.interfaces.Mouse.MouseButton; 61 62 /** 63 * 64 * @author КАМ 65 */ 66 class RobotExecutor { 67 68 private static RobotExecutor instance; 69 private AWTMap awtMap = null; 70 71 public static RobotExecutor get() { 72 if (instance == null) { 73 instance = new RobotExecutor(); 74 } 75 return instance; 76 } 77 78 /** 79 * A reference to the robot instance. 80 */ 81 protected ClassReference robotReference = null; 82 protected Timeout autoDelay; 83 private boolean inited = false; 84 private boolean runInOtherJVM = false; 85 private boolean ready = false; 86 private boolean connectionEstablished = false; 87 private ObjectOutputStream outputStream; 88 private ObjectInputStream inputStream; 89 private Socket socket; 90 private int connectionPort; 91 private String connectionHost; 92 public static final int CONNECTION_TIMEOUT = Integer.parseInt( 93 (String)Environment.getEnvironment().getProperty( 94 AWTRobotInputFactory.OTHER_VM_CONNECTION_TIMEOUT_PROPERTY, 95 Integer.toString(60000 * 15))); // 15 min 96 97 public RobotExecutor() { 98 } 99 100 void setAWTMap(AWTMap awtMap) { 101 this.awtMap = awtMap; 102 } 103 104 AWTMap getAWTMap() { 105 if (awtMap == null) { 106 awtMap = new AWTMap(); 107 } 108 return awtMap; 109 } 110 111 private void ensureInited() { 112 if (!inited) { 113 runInOtherJVM = Boolean.parseBoolean((String)Environment.getEnvironment() 114 .getProperty(AWTRobotInputFactory.OTHER_VM_PROPERTY, 115 Boolean.toString(runInOtherJVM))); 116 inited = true; 117 } 118 } 119 120 public Image createScreenCapture(Rectangle screenRect) { 121 Object result = makeAnOperation("createScreenCapture", new Object[] { 122 new java.awt.Rectangle(screenRect.x, screenRect.y, screenRect.width, 123 screenRect.height) }, 124 new Class[] { java.awt.Rectangle.class }); 125 if (result.getClass().isAssignableFrom(BufferedImage.class)) { 126 return new AWTImage(BufferedImage.class.cast(result)); 127 } else { 128 throw new JemmyException("Screen capture (" + result 129 + ") is not a BufferedImage"); 130 } 131 } 132 133 public Object makeAnOperation(String method, Object[] params, Class[] paramClasses) { 134 ensureInited(); 135 if (runInOtherJVM) { 136 return makeAnOperationRemotely(method, params, paramClasses); 137 } else { 138 return makeAnOperationLocally(method, params, paramClasses); 139 } 140 } 141 142 public void exit() { 143 ensureInited(); 144 if (runInOtherJVM) { 145 ensureConnection(); 146 try { 147 outputStream.writeObject("exit"); 148 connectionEstablished = false; 149 deleteProperties(); 150 } catch (IOException ex) { 151 throw new JemmyException("Failed to invoke exit", ex); 152 } 153 } 154 } 155 156 private Object makeAnOperationLocally(String method, Object[] params, Class[] paramClasses) { 157 if (robotReference == null) { 158 initRobot(); 159 } 160 try { 161 convert(method, params, paramClasses); 162 Object result = robotReference.invokeMethod(method, params, paramClasses); 163 synchronizeRobot(); 164 return result; 165 } catch (InvocationTargetException e) { 166 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 167 } catch (IllegalStateException e) { 168 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 169 } catch (NoSuchMethodException e) { 170 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 171 } catch (IllegalAccessException e) { 172 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 173 } 174 } 175 176 private int convert(Object obj) { 177 if (MouseButton.class.isAssignableFrom(obj.getClass())) { 178 return awtMap.convert((MouseButton)obj); 179 } else if (KeyboardButton.class.isAssignableFrom(obj.getClass())) { 180 return awtMap.convert((KeyboardButton)obj); 181 } else { 182 throw new JemmyException("Unable to recognize object", obj); 183 } 184 } 185 186 private static final Set<String> convertables = new HashSet<String>(Arrays.asList(new String[] {"mousePress", "mouseRelease", "keyPress", "keyRelease"})); 187 188 private void convert(String method, Object[] params, Class[] paramClasses) { 189 if (convertables.contains(method)) 190 for (int i = 0; i < params.length; i++) { 191 params[i] = new Integer(convert(params[i])); 192 paramClasses[i] = Integer.TYPE; 193 } 194 } 195 196 public static void main(String[] args) { 197 System.setProperty("apple.awt.UIElement", "true"); 198 if (args.length != 0 && args.length != 1) { 199 System.err.println("Usage: java ... [-D" + 200 Environment.JEMMY_PROPERTIES_FILE_PROPERTY + "=" + 201 "<.jemmy.properties full path>]" + 202 " RobotExecutor [connectionPort]"); 203 System.exit(-1); 204 } 205 if (args.length == 1) { 206 Environment.getEnvironment().setProperty( 207 AWTRobotInputFactory.OTHER_VM_PORT_PROPERTY, args[0]); 208 } 209 RobotExecutor re = new RobotExecutor(); 210 try { 211 re.server(); 212 } catch (Exception ex) { 213 ex.printStackTrace(System.err); 214 System.err.flush(); 215 System.exit(-1); 216 } 217 } 218 219 private File props; 220 221 private void deleteProperties() { 222 if (props != null) { 223 props.delete(); 224 props = null; 225 } 226 } 227 228 private void prepareProperties() { 229 deleteProperties(); 230 try { 231 props = File.createTempFile(".jemmy.othervm.", ".properties"); 232 props.deleteOnExit(); 233 PrintWriter fw = new PrintWriter(props); 234 for(Field f : AWTRobotInputFactory.class.getDeclaredFields()) { 235 if ((f.getModifiers() & Modifier.FINAL) != 0 && 236 (f.getModifiers() & Modifier.STATIC) != 0 && 237 f.getType().equals(String.class) && 238 f.getName().startsWith("OTHER_VM_") && 239 Environment.getEnvironment().getProperty((String)f.get(null)) != null) { 240 fw.println(f.get(null) + "=" + Environment.getEnvironment().getProperty((String)f.get(null))); 241 } 242 } 243 fw.close(); 244 } catch (IllegalArgumentException ex) { 245 throw new JemmyException("Failed to create temporary properties file: " + props.getAbsolutePath(), ex); 246 } catch (IllegalAccessException ex) { 247 throw new JemmyException("Failed to create temporary properties file: " + props.getAbsolutePath(), ex); 248 } catch (IOException ex) { 249 throw new JemmyException("Failed to create temporary properties file: " + props.getAbsolutePath(), ex); 250 } 251 252 } 253 254 private void startServer() { 255 try { 256 prepareProperties(); 257 ProcessBuilder pb = new ProcessBuilder("java", 258 //"-Xrunjdwp:transport=dt_socket,suspend=y,server=y,address=8000", 259 "-cp", System.getProperty("java.class.path"), 260 "-D" + Environment.JEMMY_PROPERTIES_FILE_PROPERTY + 261 "=" + props.getAbsolutePath(), 262 RobotExecutor.class.getName(), 263 Integer.toString(connectionPort)); 264 // TODO: Improve output 265 // System.out.println("Starting server"); 266 // System.out.println("Command: " + pb.command()); 267 // System.out.flush(); 268 pb.redirectErrorStream(true); 269 final Process p = pb.start(); 270 new Thread() { 271 272 @Override 273 public void run() { 274 BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); 275 while (true) { 276 try { 277 String line = br.readLine(); 278 if (line == null) { 279 break; 280 } 281 System.out.println("SERVER: " + line); 282 } catch (IOException ex) { 283 throw new JemmyException("Exception during other JVM output processing", ex); 284 } 285 } 286 } 287 }.start(); 288 } catch (IOException ex) { 289 throw new JemmyException("Failed to start other JVM", ex); 290 } 291 } 292 293 public void ensureConnection() { 294 ensureInited(); 295 if (runInOtherJVM && !connectionEstablished) { 296 initClientConnection(); 297 } 298 } 299 300 private void initClientConnection() { 301 connectionHost = (String)Environment.getEnvironment().getProperty( 302 AWTRobotInputFactory.OTHER_VM_HOST_PROPERTY, "localhost"); 303 connectionPort = Integer.parseInt((String)Environment.getEnvironment() 304 .getProperty(AWTRobotInputFactory.OTHER_VM_PORT_PROPERTY, 305 "53669")); 306 try { 307 try { 308 socket = new Socket(connectionHost, connectionPort); 309 } catch (IOException ex) { 310 if ("localhost".equalsIgnoreCase(connectionHost) 311 || "127.0.0.1".equals(connectionHost)) { 312 // TODO Improve check for localhost 313 startServer(); 314 Environment.getEnvironment().getTimeout(""); 315 Timeout waitTime = new Timeout("connection wait time", 5 * 60000); 316 socket = new Waiter(waitTime).ensureState(new State<Socket>() { 317 Exception ex; 318 public Socket reached() { 319 Socket socket = null; 320 try { 321 socket = new Socket(connectionHost, connectionPort); 322 } catch (UnknownHostException ex1) { 323 ex = ex1; 324 } catch (Exception ex1) { 325 ex = ex1; 326 } 327 return socket; 328 } 329 330 @Override 331 public String toString() { 332 if (ex != null) { 333 // TODO: Provide better mechanics for exception handling 334 Logger.getLogger(RobotExecutor.class.getName()) 335 .log(Level.INFO, null, ex); 336 } 337 return "Waiting for connection to be established " + 338 "with other JVM (" + connectionHost 339 + ":" + connectionPort + ", exception: " + ex + ")"; 340 } 341 }); 342 } else { 343 throw new JemmyException("Failed to establish socket " + 344 "connection with other JVM (" + connectionHost 345 + ":" + connectionPort + ")", ex); 346 } 347 } 348 outputStream = new ObjectOutputStream(socket.getOutputStream()); 349 inputStream = new ObjectInputStream(socket.getInputStream()); 350 351 connectionEstablished = true; 352 ready = true; 353 354 System.out.println("Connection established!"); 355 setAutoDelay(autoDelay); 356 } catch (IOException ex) { 357 throw new JemmyException("Failed to establish socket connection " + 358 "with other JVM (" + connectionHost + ":" + connectionPort 359 + ")", ex); 360 } 361 } 362 363 public synchronized Object getProperty(String name) { 364 ensureConnection(); 365 try { 366 outputStream.writeObject("getProperty"); 367 outputStream.writeObject(name); 368 Object result = inputStream.readObject(); 369 String response = (String)(inputStream.readObject()); 370 if (!"OK".equals(response)) { 371 throw new JemmyException("Remote operation didn't succeed"); 372 } 373 return result; 374 } catch (ClassNotFoundException ex) { 375 throw new JemmyException("Socket communication with other JVM failed", ex); 376 } catch (OptionalDataException ex) { 377 throw new JemmyException("Socket communication with other JVM " + 378 "failed: OptionalDataException eof = " + ex.eof + ", " + 379 "length = " + ex.length, ex); 380 } catch (IOException ex) { 381 throw new JemmyException("Socket communication with other JVM failed", ex); 382 } 383 } 384 385 private synchronized Object makeAnOperationRemotely(String method, Object[] params, Class[] paramClasses) { 386 ensureConnection(); 387 try { 388 outputStream.writeObject("makeAnOperation"); 389 outputStream.writeObject(method); 390 outputStream.writeObject(params); 391 outputStream.writeObject(paramClasses); 392 Object result; 393 String response = (String)(inputStream.readObject()); 394 if ("image".equals(response)) { 395 result = PNGDecoder.decode(inputStream, false); 396 } else { 397 if (!"OK".equals(response)) { 398 throw new JemmyException("Remote operation didn't succeed"); 399 } 400 result = inputStream.readObject(); 401 } 402 return result; 403 } catch (ClassNotFoundException ex) { 404 throw new JemmyException("Socket communication with other JVM failed", ex); 405 } catch (OptionalDataException ex) { 406 throw new JemmyException("Socket communication with other JVM " + 407 "failed: OptionalDataException eof = " + ex.eof + ", " + 408 "length = " + ex.length, ex); 409 } catch (IOException ex) { 410 throw new JemmyException("Socket communication with other JVM failed", ex); 411 } 412 } 413 414 private void server() { 415 System.out.println("Robot ready!"); 416 System.out.flush(); 417 ServerSocket sc; 418 connectionPort = Integer.parseInt((String)Environment.getEnvironment() 419 .getProperty(AWTRobotInputFactory.OTHER_VM_PORT_PROPERTY, 420 "53669")); 421 while(true) { 422 Thread watchdog = new Thread("RobotExecutor.server watchdog") { 423 424 @Override 425 public void run() { 426 try { 427 Thread.sleep(CONNECTION_TIMEOUT); 428 System.out.println("Exiting server as there is no " + 429 "connection for " + CONNECTION_TIMEOUT / 60000.0 430 + " minutes"); 431 System.out.flush(); 432 System.exit(0); 433 } catch (InterruptedException ex) { 434 // Ignoring exception as it is okay 435 } 436 } 437 438 }; 439 watchdog.start(); 440 System.out.println("Waiting for incoming connection for up to " 441 + CONNECTION_TIMEOUT / 60000.0 + " minutes"); 442 try { 443 sc = new ServerSocket(connectionPort); 444 socket = sc.accept(); 445 watchdog.interrupt(); 446 } catch (IOException ex) { 447 throw new JemmyException("Can't establish connection with client", ex); 448 } 449 System.out.println("Connection established!"); 450 try { 451 inputStream = new ObjectInputStream(socket.getInputStream()); 452 outputStream = new ObjectOutputStream(socket.getOutputStream()); 453 while(true) { 454 String command = (String)inputStream.readObject(); 455 if ("exit".equals(command)) { 456 System.exit(0); 457 } 458 if ("getProperty".equals(command)) { 459 String property = (String)inputStream.readObject(); 460 outputStream.writeObject(Environment.getEnvironment().getProperty(property)); 461 outputStream.writeObject("OK"); 462 } 463 if ("makeAnOperation".equals(command)) { 464 String method = (String)inputStream.readObject(); 465 Object[] params = (Object[])inputStream.readObject(); 466 Class[] paramClasses = (Class[])inputStream.readObject(); 467 Object result = makeAnOperationLocally(method, params, 468 paramClasses); 469 if (result instanceof BufferedImage) { 470 outputStream.writeObject("image"); 471 BufferedImage image = BufferedImage.class.cast(result); 472 new PNGEncoder(outputStream, PNGEncoder.COLOR_MODE) 473 .encode(image, false); 474 } else { 475 outputStream.writeObject("OK"); 476 outputStream.writeObject(result); 477 } 478 } 479 } 480 } catch (ClassNotFoundException ex) { 481 throw new JemmyException("Socket communication with other " + 482 "JVM failed", ex); 483 } catch (IOException ex) { 484 Logger.getLogger(RobotExecutor.class.getName()) 485 .log(Level.SEVERE, null, ex); 486 } finally { 487 if (socket != null) { 488 try { 489 socket.close(); 490 } catch (IOException ex) { 491 Logger.getLogger(RobotExecutor.class.getName()).log( 492 Level.SEVERE, "Exception during socket closing", ex); 493 } 494 } 495 if (sc != null) { 496 try { 497 sc.close(); 498 } catch (IOException ex) { 499 Logger.getLogger(RobotExecutor.class.getName()).log( 500 Level.SEVERE, "Exception during server socket " + 501 "closing", ex); 502 } 503 } 504 } 505 } 506 } 507 508 private void initRobot() { 509 // need to init Robot in dispatch thread because it hangs on Linux 510 // (see http://www.netbeans.org/issues/show_bug.cgi?id=37476) 511 if (EventQueue.isDispatchThread()) { 512 doInitRobot(); 513 } else { 514 try { 515 EventQueue.invokeAndWait(new Runnable() { 516 517 public void run() { 518 doInitRobot(); 519 } 520 }); 521 } catch (InterruptedException ex) { 522 throw new JemmyException("Failed to initialize robot", ex); 523 } catch (InvocationTargetException ex) { 524 throw new JemmyException("Failed to initialize robot", ex); 525 } 526 } 527 } 528 529 private void doInitRobot() { 530 try { 531 ClassReference robotClassReverence = new ClassReference("java.awt.Robot"); 532 robotReference = new ClassReference(robotClassReverence.newInstance(null, null)); 533 if (awtMap == null) { 534 awtMap = new AWTMap(); 535 } 536 setAutoDelay(autoDelay); 537 ready = true; 538 } catch (InvocationTargetException e) { 539 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 540 } catch (IllegalStateException e) { 541 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 542 } catch (NoSuchMethodException e) { 543 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 544 } catch (IllegalAccessException e) { 545 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 546 } catch (ClassNotFoundException e) { 547 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 548 } catch (InstantiationException e) { 549 throw (new JemmyException("Exception during java.awt.Robot accessing", e)); 550 } 551 } 552 553 /** 554 * Calls <code>java.awt.Robot.waitForIdle()</code> method. 555 */ 556 protected void synchronizeRobot() { 557 ensureInited(); 558 if (!runInOtherJVM) { 559 // TODO: It looks like this method is rudimentary 560 if (!EventQueue.isDispatchThread()) { 561 if (robotReference == null) { 562 initRobot(); 563 } 564 try { 565 robotReference.invokeMethod("waitForIdle", null, null); 566 } catch (Exception e) { 567 e.printStackTrace(); 568 } 569 } 570 } 571 } 572 573 public void setAutoDelay(Timeout autoDelay) { 574 this.autoDelay = autoDelay; 575 if (ready) { 576 makeAnOperation("setAutoDelay", new Object[]{new Integer((int) ((autoDelay != null) ? autoDelay.getValue() : 0))}, new Class[]{Integer.TYPE}); 577 } 578 } 579 580 public boolean isRunInOtherJVM() { 581 ensureInited(); 582 return runInOtherJVM; 583 } 584 585 public void setRunInOtherJVM(boolean runInOtherJVM) { 586 if (inited && this.runInOtherJVM && this.connectionEstablished && !runInOtherJVM) { 587 shutdownConnection(); 588 } 589 this.runInOtherJVM = runInOtherJVM; 590 inited = true; 591 ready = false; 592 } 593 594 private void shutdownConnection() { 595 try { 596 outputStream.writeObject("exit"); 597 socket.close(); 598 connectionEstablished = false; 599 } catch (IOException ex) { 600 throw new JemmyException("Failed to shutdown connection", ex); 601 } 602 } 603 }