diff --git a/src/systems/userinput/devices/app-aware-mouse.js b/src/systems/userinput/devices/app-aware-mouse.js
index 66e006d44c56a98c07a39a498cc68581c4d0e6b9..de5f4acecdfe6ca4ba0400878f078c770c9c455a 100644
--- a/src/systems/userinput/devices/app-aware-mouse.js
+++ b/src/systems/userinput/devices/app-aware-mouse.js
@@ -1,3 +1,4 @@
+import { sets } from "../sets";
 import { paths } from "../paths";
 import { Pose } from "../pose";
 
@@ -35,9 +36,13 @@ export class AppAwareMouseDevice {
       const rawIntersections = [];
       this.cursorController.raycaster.intersectObjects(this.cursorController.targets, true, rawIntersections);
       const intersection = rawIntersections.find(x => x.object.el);
+      const userinput = AFRAME.scenes[0].systems.userinput;
       this.clickedOnAnything =
-        intersection &&
-        intersection.object.el.matches(".pen, .pen *, .video, .video *, .interactable, .interactable *");
+        (intersection &&
+          intersection.object.el.matches(".pen, .pen *, .video, .video *, .interactable, .interactable *")) ||
+        userinput.activeSets.has(sets.cursorHoldingPen) ||
+        userinput.activeSets.has(sets.cursorHoldingInteractable) ||
+        userinput.activeSets.has(sets.cursorHoldingCamera);
     }
     this.prevButtonLeft = buttonLeft;