Modifying mouse event handling to match W3C Specification

THIS IS A POTENTIAL BREAKING CHANGE.

The simulation of mouse movement for the actions command now conforms
to the W3C WebDriver Specification for mouse movement. This means
that offsets in elements are now measured from the center of the
element instead of the top-left, which was the previous behavior.
Additionally, attempting to move the mouse pointer outside the
browser view port will result in a "mouse movement out of bounds"
error. While users should be aware that this might cause code
using user interactions to fail, not that the behavior is now
consistent with the behavior outlined in the specification, and
the behavior of geckodriver in particular.

Author: jimevans

cpp/iedriver/CommandHandlers/ActionsCommandHandler.cpp

@@ -51,7 +51,12 @@ void ActionsCommandHandler::ExecuteInternal(
   }
   status_code = executor.input_manager()->PerformInputSequence(browser_wrapper, actions_parameter_iterator->second);
   if (status_code != WD_SUCCESS) {
-    response->SetErrorResponse(status_code, "Unexpected error performing action sequence.");
+    if (status_code == EMOVETARGETOUTOFBOUNDS) {
+      response->SetErrorResponse(status_code, "The requested mouse movement would be outside the bounds of the current view port.");
+    } else {
+      response->SetErrorResponse(status_code, "Unexpected error performing action sequence.");
+    }
+    return;
   }
   response->SetSuccessResponse(Json::Value::null);
 }

cpp/iedriver/CommandHandlers/ClickElementCommandHandler.cpp

@@ -104,6 +104,22 @@ void ClickElementCommandHandler::ExecuteInternal(const IECommandExecutor& execut
             ::Sleep(double_click_time - milliseconds_since_last_click);
           }
 
+          // Scroll the target element into view before executing the action
+          // sequence.
+          LocationInfo location = {};
+          std::vector<LocationInfo> frame_locations;
+          status_code = element_wrapper->GetLocationOnceScrolledIntoView(executor.input_manager()->scroll_behavior(),
+                                                                         &location,
+                                                                         &frame_locations);
+
+          bool displayed;
+          status_code = element_wrapper->IsDisplayed(true, &displayed);
+          if (status_code != WD_SUCCESS || !displayed) {
+            response->SetErrorResponse(EELEMENTNOTDISPLAYED,
+                                       "Element is not displayed");
+            return;
+          }
+
           IECommandExecutor& mutable_executor = const_cast<IECommandExecutor&>(executor);
           status_code = mutable_executor.input_manager()->PerformInputSequence(browser_wrapper, actions);
           browser_wrapper->set_wait_required(true);

cpp/iedriver/CommandHandlers/SendKeysCommandHandler.cpp

@@ -92,11 +92,13 @@ void SendKeysCommandHandler::ExecuteInternal(
     if (status_code == WD_SUCCESS) {
       CComPtr<IHTMLElement> element(element_wrapper->element());
 
+      // Scroll the target element into view before executing the action
+      // sequence.
       LocationInfo location = {};
       std::vector<LocationInfo> frame_locations;
       element_wrapper->GetLocationOnceScrolledIntoView(executor.input_manager()->scroll_behavior(),
-                                                        &location,
-                                                        &frame_locations);
+                                                       &location,
+                                                       &frame_locations);
 
       CComPtr<IHTMLInputFileElement> file;
       element->QueryInterface<IHTMLInputFileElement>(&file);

cpp/iedriver/Element.cpp

@@ -245,6 +245,15 @@ int Element::GetClickLocation(const ElementScrollBehavior scroll_behavior,
   return status_code;
 }
 
+int Element::GetStaticClickLocation(LocationInfo* click_location) {
+  std::vector<LocationInfo> frame_locations;
+  LocationInfo element_location = {};
+  int result = this->GetLocation(&element_location, &frame_locations);
+  bool document_contains_frames = frame_locations.size() != 0;
+  *click_location = this->CalculateClickPoint(element_location, document_contains_frames);
+  return result;
+}
+
 int Element::GetAttributeValue(const std::string& attribute_name,
                                VARIANT* attribute_value) {
   LOG(TRACE) << "Entering Element::GetAttributeValue";

cpp/iedriver/Element.h

@@ -48,6 +48,7 @@ class Element {
   int GetLocationOnceScrolledIntoView(const ElementScrollBehavior scroll,
                                       LocationInfo* location,
                                       std::vector<LocationInfo>* frame_locations);
+  int GetStaticClickLocation(LocationInfo* click_location);
   int GetClickLocation(const ElementScrollBehavior scroll_behavior,
                        LocationInfo* element_location,
                        LocationInfo* click_location);

cpp/iedriver/InputManager.cpp

@@ -54,6 +54,7 @@ InputManager::InputManager() {
   this->current_input_state_.is_alt_pressed = false;
   this->current_input_state_.is_control_pressed = false;
   this->current_input_state_.is_shift_pressed = false;
+  this->current_input_state_.is_meta_pressed = false;
   this->current_input_state_.is_left_button_pressed = false;
   this->current_input_state_.is_right_button_pressed = false;
   this->current_input_state_.mouse_x = 0;
@@ -239,6 +240,7 @@ InputState InputManager::CloneCurrentInputState(void) {
   current_input_state.is_alt_pressed = this->current_input_state_.is_alt_pressed;
   current_input_state.is_control_pressed = this->current_input_state_.is_control_pressed;
   current_input_state.is_shift_pressed = this->current_input_state_.is_shift_pressed;
+  current_input_state.is_meta_pressed = this->current_input_state_.is_meta_pressed;
   current_input_state.is_left_button_pressed = this->current_input_state_.is_left_button_pressed;
   current_input_state.is_right_button_pressed = this->current_input_state_.is_right_button_pressed;
   current_input_state.mouse_x = this->current_input_state_.mouse_x;
@@ -395,10 +397,13 @@ int InputManager::PointerMoveTo(BrowserHandle browser_wrapper,
     long end_y = start_y;
     if (element_specified) {
       LocationInfo element_location;
-      LocationInfo move_location;
-      status_code = target_element->GetClickLocation(this->scroll_behavior_,
-                                                     &element_location,
-                                                     &move_location);
+      // Note: The caller of the action sequence is responsible for making
+      // sure the target element is in the view port. In particular, the
+      // high-level click and sendKeys implementations do this in their
+      // command handlers. Further note that offsets specified in this
+      // move action will be relative to the center of the element as
+      // calculated here.
+      status_code = target_element->GetStaticClickLocation(&element_location);
       // We can't use the status code alone here. Even though the center of the
       // element may not reachable via the mouse, we might still be able to move
       // to whatever portion of the element *is* visible in the viewport, especially
@@ -420,11 +425,6 @@ int InputManager::PointerMoveTo(BrowserHandle browser_wrapper,
       // move will be at some offset from the element origin.
       end_x = element_location.x;
       end_y = element_location.y;
-      if (!offset_specified) {
-        // No offset was specified, which means move to the center of the element. 
-        end_x = move_location.x;
-        end_y = move_location.y;
-      }
     }
 
     if (origin == "viewport") {
@@ -441,9 +441,20 @@ int InputManager::PointerMoveTo(BrowserHandle browser_wrapper,
       }
     }
 
+
     LOG(DEBUG) << "Queueing SendInput structure for mouse move (origin: " << origin
                << ", x: " << end_x << ", y: " << end_y << ")";
     HWND browser_window_handle = browser_wrapper->GetContentWindowHandle();
+    RECT window_rect;
+    ::GetWindowRect(browser_window_handle, &window_rect);
+    POINT click_point = { end_x, end_y };
+    ::ClientToScreen(browser_window_handle, &click_point);
+    if (click_point.x < window_rect.left ||
+        click_point.x > window_rect.right ||
+        click_point.y < window_rect.top ||
+        click_point.y > window_rect.bottom) {
+      return EMOVETARGETOUTOFBOUNDS;
+    }
     if (end_x == input_state->mouse_x && end_y == input_state->mouse_y) {
       LOG(DEBUG) << "Omitting SendInput structure for mouse move; no movement required (x: "
                  << end_x << ", y: " << end_y << ")";
@@ -685,6 +696,25 @@ void InputManager::AddKeyboardInput(HWND window_handle,
       }
       this->UpdatePressedKeys(WD_KEY_ALT, input_state->is_alt_pressed);
     }
+
+    // If the character represents the Meta (Windows) key, or represents 
+    // the "release all modifiers" key and the Meta key is down, send
+    // the appropriate down or up keystroke for the Meta key.
+    if (character == WD_KEY_META ||
+        character == WD_KEY_R_META ||
+        (character == WD_KEY_NULL && input_state->is_meta_pressed)) {
+      //modifier_key_info.key_code = VK_LWIN;
+      //this->CreateKeyboardInputItem(modifier_key_info,
+      //                              0,
+      //                              input_state->is_meta_pressed);
+      //if (input_state->is_meta_pressed) {
+      //  input_state->is_meta_pressed = false;
+      //} else {
+      //  input_state->is_meta_pressed = true;
+      //}
+      //this->UpdatePressedKeys(WD_KEY_META, input_state->is_meta_pressed);
+    }
+
     return;
   }
 
@@ -803,9 +833,11 @@ bool InputManager::IsModifierKey(wchar_t character) {
   return character == WD_KEY_SHIFT ||
          character == WD_KEY_CONTROL ||
          character == WD_KEY_ALT ||
+         character == WD_KEY_META ||
          character == WD_KEY_R_SHIFT ||
          character == WD_KEY_R_CONTROL ||
          character == WD_KEY_R_ALT ||
+         character == WD_KEY_R_META ||
          character == WD_KEY_NULL;
 }
 
@@ -999,6 +1031,56 @@ KeyInfo InputManager::GetKeyInfo(HWND window_handle, wchar_t character) {
     key_info.scan_code = VK_DIVIDE;
     key_info.is_extended_key = true;
   }
+  else if (character == WD_KEY_R_PAGEUP) {
+    key_info.key_code = VK_PRIOR;
+    key_info.scan_code = VK_PRIOR;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_PAGEDN) {
+    key_info.key_code = VK_NEXT;
+    key_info.scan_code = VK_NEXT;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_END) {  // end
+    key_info.key_code = VK_END;
+    key_info.scan_code = VK_END;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_HOME) {  // home
+    key_info.key_code = VK_HOME;
+    key_info.scan_code = VK_HOME;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_LEFT) {  // left arrow
+    key_info.key_code = VK_LEFT;
+    key_info.scan_code = VK_LEFT;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_UP) {  // up arrow
+    key_info.key_code = VK_UP;
+    key_info.scan_code = VK_UP;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_RIGHT) {  // right arrow
+    key_info.key_code = VK_RIGHT;
+    key_info.scan_code = VK_RIGHT;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_DOWN) {  // down arrow
+    key_info.key_code = VK_DOWN;
+    key_info.scan_code = VK_DOWN;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_INSERT) {  // insert
+    key_info.key_code = VK_INSERT;
+    key_info.scan_code = VK_INSERT;
+    key_info.is_extended_key = true;
+  }
+  else if (character == WD_KEY_R_DELETE) {  // delete
+    key_info.key_code = VK_DELETE;
+    key_info.scan_code = VK_DELETE;
+    key_info.is_extended_key = true;
+  }
   else if (character == WD_KEY_F1) {  // F1
     key_info.key_code = VK_F1;
     key_info.scan_code = VK_F1;
@@ -1047,14 +1129,6 @@ KeyInfo InputManager::GetKeyInfo(HWND window_handle, wchar_t character) {
     key_info.key_code = VK_F12;
     key_info.scan_code = VK_F12;
   }
-  else if (character == WD_KEY_META) {  // Meta
-    key_info.key_code = VK_LWIN;
-    key_info.scan_code = VK_LWIN;
-  }
-  else if (character == WD_KEY_R_META) {  // Meta
-    key_info.key_code = VK_RWIN;
-    key_info.scan_code = VK_RWIN;
-  }
   else if (character == L'\n') {    // line feed
     key_info.key_code = VK_RETURN;
     key_info.scan_code = VK_RETURN;

cpp/iedriver/InputState.h

@@ -25,6 +25,7 @@ struct InputState {
   bool is_shift_pressed;
   bool is_control_pressed;
   bool is_alt_pressed;
+  bool is_meta_pressed;
   bool is_left_button_pressed;
   bool is_right_button_pressed;
   long mouse_x;

cpp/webdriver-server/errorcodes.h

@@ -19,39 +19,40 @@
 
 #define WD_SUCCESS 0
 
-#define EINDEXOUTOFBOUNDS    1
-#define ENOCOLLECTION        2
-#define ENOSTRING            3
-#define ENOSTRINGLENGTH      4
-#define ENOSTRINGWRAPPER     5
-#define ENOSUCHDRIVER        6
-#define ENOSUCHELEMENT       7
-#define ENOSUCHFRAME         8
-#define ENOTIMPLEMENTED      9
-#define EOBSOLETEELEMENT     10
-#define EELEMENTNOTDISPLAYED 11
-#define EELEMENTNOTENABLED   12
-#define EUNHANDLEDERROR      13
-#define EEXPECTEDERROR       14
-#define EELEMENTNOTSELECTED  15
-#define ENOSUCHDOCUMENT      16
-#define EUNEXPECTEDJSERROR   17
-#define ENOSCRIPTRESULT      18
-#define EUNKNOWNSCRIPTRESULT 19
-#define ENOSUCHCOLLECTION    20
-#define ETIMEOUT             21
-#define ENULLPOINTER         22
-#define ENOSUCHWINDOW        23
-#define EINVALIDCOOKIEDOMAIN 24
-#define EUNABLETOSETCOOKIE   25
-#define EUNEXPECTEDALERTOPEN 26
-#define ENOSUCHALERT         27
-#define ESCRIPTTIMEOUT       28
-#define EINVALIDCOORDINATES  29
-#define EINVALIDSELECTOR     32
-#define ECLICKINTERCEPTED    33
-#define EINVALIDARGUMENT     34
-#define ENOSUCHCOOKIE        35
+#define EINDEXOUTOFBOUNDS      1
+#define ENOCOLLECTION          2
+#define ENOSTRING              3
+#define ENOSTRINGLENGTH        4
+#define ENOSTRINGWRAPPER       5
+#define ENOSUCHDRIVER          6
+#define ENOSUCHELEMENT         7
+#define ENOSUCHFRAME           8
+#define ENOTIMPLEMENTED        9
+#define EOBSOLETEELEMENT       10
+#define EELEMENTNOTDISPLAYED   11
+#define EELEMENTNOTENABLED     12
+#define EUNHANDLEDERROR        13
+#define EEXPECTEDERROR         14
+#define EELEMENTNOTSELECTED    15
+#define ENOSUCHDOCUMENT        16
+#define EUNEXPECTEDJSERROR     17
+#define ENOSCRIPTRESULT        18
+#define EUNKNOWNSCRIPTRESULT   19
+#define ENOSUCHCOLLECTION      20
+#define ETIMEOUT               21
+#define ENULLPOINTER           22
+#define ENOSUCHWINDOW          23
+#define EINVALIDCOOKIEDOMAIN   24
+#define EUNABLETOSETCOOKIE     25
+#define EUNEXPECTEDALERTOPEN   26
+#define ENOSUCHALERT           27
+#define ESCRIPTTIMEOUT         28
+#define EINVALIDCOORDINATES    29
+#define EINVALIDSELECTOR       32
+#define ECLICKINTERCEPTED      33
+#define EMOVETARGETOUTOFBOUNDS 34
+#define ENOSUCHCOOKIE          35
+#define EINVALIDARGUMENT       62
 
 #define ERROR_ELEMENT_CLICK_INTERCEPTED "element click intercepted"
 #define ERROR_ELEMENT_NOT_SELECTABLE "element not selectable"
@@ -82,4 +83,4 @@
 #define ERROR_UNKNOWN_METHOD "unknown method"
 #define ERROR_UNSUPPORTED_OPERATION "unsupported operation"
 
-#endif // WEBDRIVER_SERVER_ERRORCODES_H_
+#endif // WEBDRIVER_SERVER_ERRORCODES_H_

cpp/webdriver-server/response.cc

@@ -178,6 +178,8 @@ std::string Response::ConvertErrorCode(const int error_code) {
     return ERROR_INVALID_COOKIE_DOMAIN;
   } else if (error_code == ESCRIPTTIMEOUT) {
     return ERROR_SCRIPT_TIMEOUT;
+  } else if (error_code == EMOVETARGETOUTOFBOUNDS) {
+    return ERROR_MOVE_TARGET_OUT_OF_BOUNDS;
   }
 
   return "";