diff --git a/src/utils/action-event-handler.js b/src/utils/action-event-handler.js
index 1884a7313bf88bc345eff106856c270403d2a74f..f6177c3b87c4803cf1e0d18b0436e3a9a0560d8c 100644
--- a/src/utils/action-event-handler.js
+++ b/src/utils/action-event-handler.js
@@ -3,6 +3,7 @@ export default class ActionEventHandler {
     this.scene = scene;
     this.cursor = cursor;
     this.isCursorInteracting = false;
+    this.isCursorInteractingOnGrab = false;
     this.isTeleporting = false;
     this.handThatAlsoDrivesCursor = null;
     this.hovered = false;
@@ -57,6 +58,9 @@ export default class ActionEventHandler {
         return;
       } else {
         this.isCursorInteracting = this.cursor.startInteraction();
+        if (this.isCursorInteracting) {
+          this.isCursorInteractingOnGrab = true;
+        }
         return;
       }
     } else {
@@ -66,8 +70,14 @@ export default class ActionEventHandler {
   }
 
   onRelease(e) {
-    if (this.isCursorInteracting && this.handThatAlsoDrivesCursor && this.handThatAlsoDrivesCursor === e.target) {
+    if (
+      this.isCursorInteracting &&
+      this.isCursorInteractingOnGrab &&
+      this.handThatAlsoDrivesCursor &&
+      this.handThatAlsoDrivesCursor === e.target
+    ) {
       this.isCursorInteracting = false;
+      this.isCursorInteractingOnGrab = false;
       this.cursor.endInteraction();
     } else {
       e.target.emit("hand_release");
@@ -75,6 +85,7 @@ export default class ActionEventHandler {
   }
 
   onPrimaryDown(e) {
+    if (this.isCursorInteractingOnGrab) return;
     if (this.handThatAlsoDrivesCursor && this.handThatAlsoDrivesCursor === e.target) {
       if (this.isCursorInteracting) {
         return;
@@ -94,6 +105,7 @@ export default class ActionEventHandler {
   }
 
   onPrimaryUp(e) {
+    if (this.isCursorInteractingOnGrab) return;
     const isCursorHand = this.handThatAlsoDrivesCursor && this.handThatAlsoDrivesCursor === e.target;
     if (this.isCursorInteracting && isCursorHand) {
       this.isCursorInteracting = false;