diff --git a/src/utils/action-event-handler.js b/src/utils/action-event-handler.js
index 80c8d6fb952f7cc4b99caebb9ce649a97c86b703..3e31fafb9097021544fe186ebf8e5829fb5c9914 100644
--- a/src/utils/action-event-handler.js
+++ b/src/utils/action-event-handler.js
@@ -186,8 +186,10 @@ export default class ActionEventHandler {
   }
 
   onPrimaryDown(e) {
-    this.onDown(e, "primary_hand_grab");
-    this.gotPrimaryDown = true;
+    if (!this.gotPrimaryDown) {
+      this.onDown(e, "primary_hand_grab");
+      this.gotPrimaryDown = true;
+    }
   }
 
   onPrimaryUp(e) {