diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue
index 12e7db7a0..b49971836 100644
--- a/client/src/components/LayerManager.vue
+++ b/client/src/components/LayerManager.vue
@@ -335,7 +335,9 @@ export default defineComponent({
         .search([frame, frame])
         .map((str) => parseInt(str, 10));
       const inlcudesTooltip = visibleModes.includes('tooltip');
-      rectAnnotationLayer.setHoverAnnotations(inlcudesTooltip);
+      // Hidden boxes (kept only as invisible click targets) should not
+      // produce hover tooltips
+      rectAnnotationLayer.setHoverAnnotations(inlcudesTooltip && visibleModes.includes('rectangle'));
       polyAnnotationLayer.setHoverAnnotations(inlcudesTooltip);
       if (!inlcudesTooltip) {
         hoverOvered.value = [];
@@ -449,6 +451,7 @@ export default defineComponent({
       );
 
       if (visibleModes.includes('rectangle')) {
+        rectAnnotationLayer.setClickTargetsOnly(false);
         //We modify rects opacity/thickness if polygons are visible or not
         rectAnnotationLayer.setDrawingOther(visibleModes.includes('Polygon'));
         rectAnnotationLayer.changeData(frameData, comparison.value);
@@ -456,7 +459,11 @@ export default defineComponent({
           overlapLayer.changeData(frameData);
         }
       } else {
-        rectAnnotationLayer.disable();
+        // Keep the hidden boxes around as invisible right-click targets so a
+        // detection can still be right-clicked into edit mode no matter which
+        // of its displays are turned on
+        rectAnnotationLayer.setClickTargetsOnly(true);
+        rectAnnotationLayer.changeData(frameData, comparison.value);
       }
       if (visibleModes.includes('Polygon')) {
         polyAnnotationLayer.setDrawingOther(visibleModes.includes('rectangle'));
diff --git a/client/src/layers/AnnotationLayers/RectangleLayer.ts b/client/src/layers/AnnotationLayers/RectangleLayer.ts
index bb845898d..f38950d2e 100644
--- a/client/src/layers/AnnotationLayers/RectangleLayer.ts
+++ b/client/src/layers/AnnotationLayers/RectangleLayer.ts
@@ -32,6 +32,13 @@ export default class RectangleLayer extends BaseLayer<RectGeoJSData> {
 
   hoverOn: boolean; //to turn over annnotations on
 
+  /**
+   * When box display is off the rectangles are kept as invisible hit targets
+   * so a detection can still be right-clicked into edit mode: nothing is
+   * drawn and left-clicks are ignored, but right-click still works.
+   */
+  clickTargetsOnly: boolean;
+
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   arrowFeatureLayer: any;
 
@@ -39,6 +46,7 @@ export default class RectangleLayer extends BaseLayer<RectGeoJSData> {
     super(params);
     this.drawingOther = false;
     this.hoverOn = false;
+    this.clickTargetsOnly = false;
     //Only initialize once, prevents recreating Layer each edit
     this.initialize();
   }
@@ -55,6 +63,10 @@ export default class RectangleLayer extends BaseLayer<RectGeoJSData> {
          * Rectangle type if only the polygon is visible we use the polygon bounds
          * */
         if (e.mouse.buttonsDown.left) {
+          if (this.clickTargetsOnly) {
+            // Hidden boxes are right-click edit targets only
+            return;
+          }
           if (!e.data.editing || (e.data.editing && !e.data.selected)) {
             if (e.mouse.modifiers.ctrl) {
               this.bus.$emit('annotation-ctrl-clicked', e.data.trackId, false, { ctrl: true });
@@ -135,6 +147,14 @@ export default class RectangleLayer extends BaseLayer<RectGeoJSData> {
     this.drawingOther = val;
   }
 
+  /**
+   * Keep features as invisible right-click edit targets instead of
+   * drawing them (used when box display is toggled off)
+   */
+  setClickTargetsOnly(val: boolean) {
+    this.clickTargetsOnly = val;
+  }
+
   formatData(frameData: FrameDataTrack[], comparisonSets: string[] = []) {
     const arr: RectGeoJSData[] = [];
     frameData.forEach((track: FrameDataTrack) => {
@@ -187,7 +207,8 @@ export default class RectangleLayer extends BaseLayer<RectGeoJSData> {
       .data(this.formattedData)
       .polygon((d: RectGeoJSData) => d.polygon.coordinates[0])
       .draw();
-    const arrowData = this.formattedData.filter((d) => d.rotationArrow);
+    const arrowData = this.clickTargetsOnly
+      ? [] : this.formattedData.filter((d) => d.rotationArrow);
     this.arrowFeatureLayer
       .data(arrowData)
       .line((d: RectGeoJSData) => d.rotationArrow!.coordinates)
@@ -228,6 +249,9 @@ export default class RectangleLayer extends BaseLayer<RectGeoJSData> {
         return this.typeStyling.value.color('');
       },
       fill: (data) => {
+        if (this.clickTargetsOnly) {
+          return false;
+        }
         if (data.set) {
           return this.typeStyling.value.fill(data.set, true);
         }
@@ -255,6 +279,10 @@ export default class RectangleLayer extends BaseLayer<RectGeoJSData> {
         return this.stateStyling.standard.opacity;
       },
       strokeOpacity: (_point, _index, data) => {
+        // Invisible click targets draw nothing
+        if (this.clickTargetsOnly) {
+          return 0.0;
+        }
         // Reduce the rectangle opacity if a polygon is also drawn
         if (_index % 2 === 1 && data.dashed) {
           return 0.0;