Add MediaPipe library integration

src/om1_vlm/anonymizationSys/face_recog_stream/config.py

@@ -120,16 +120,16 @@
 # - Used in: face_tracker.FaceTracker(max_recog_attempts=...)
 # - Hot-tunable: NO
 
-TRACK_BUFFER = 30
+TRACK_BUFFER = 60
 # BoTSORT frames to keep a 'lost' track alive (waiting for re-detection).
 # Higher = better at handling brief occlusions but increases the chance of
 # track-id mixups when two people swap positions.
 # - Used in: face_tracker.FaceTracker(track_buffer=...)
 # - Hot-tunable: NO
-# - Frame-count, scales with FPS: 30 frames @ 15fps = 2 seconds of occlusion
+# - Frame-count, scales with FPS: 60 frames @ 30fps = 2 seconds of occlusion
 #   tolerance. If you change CAMERA_FPS, scale this accordingly to maintain
 #   the same wall-clock duration.
-# - Example: at 15fps, a face occluded for <2s gets the same track_id back
+# - Example: at 30fps, a face occluded for <2s gets the same track_id back
 #   via BoTSORT's Kalman prediction. After 2s, occlusion timeout fires and
 #   a new track_id is assigned on re-detection.
 
@@ -220,18 +220,19 @@
 #   walk-throughs enroll: raise to 1.5s. If robot feels slow: lower
 #   to 0.7-0.8s.
 
-AUTO_ENROLL_MIN_FACE_PX = 26
+AUTO_ENROLL_MIN_FACE_PX = 65
 # Minimum bbox short side for an auto-enroll sample. Below this, samples
 # are dropped — too small to produce reliable embeddings.
 
-# Sized for 640×480 camera (Unitree G1 default). 30 px short side ≈ 6.25%
-# of frame height, equivalent to a person ~2-3m from the camera at ~60° field of view.
+# Sized for the current 1920×1080 camera resolution (~87% of
+# SELFIE_MIN_FACE_PX — see the "slightly looser" note below), equivalent to
+# a person ~2-3m from the camera at ~60° field of view.
 
-# At higher camera resolutions, scale up to maintain the same physical
-# distance threshold:
-#   - 640×480  → 30 px
-#   - 1280×720 → 50 px
-#   - 1920×1080 → 75 px
+# At other camera resolutions, scale to maintain the same physical distance
+# threshold:
+#   - 640×480  → 26 px
+#   - 1280×720 → 43 px
+#   - 1920×1080 → 65 px (current)
 
 # - Used in: auto_enroller.AutoEnroller(min_face_pixels=...)
 # - CLI: --auto-enroll-min-face-px
@@ -415,24 +416,24 @@
 # - cfg key: "selfie_max_samples"
 # - Hot-tunable: YES
 
-SELFIE_MIN_ENGAGEMENT = 0.0025
+SELFIE_MIN_ENGAGEMENT = 0.0023
 # Engagement score floor below which a face isn't considered a selfie
 # target. score = (bbox_area / frame_area) × frontality, in [0, 1].
 
-# Calibrated for 640×480 camera + SELFIE_MIN_FACE_PX=30
+# Calibrated for 1920×1080 camera + SELFIE_MIN_FACE_PX=75
 # ---------------------------------------------------------
 # The minimum sensible engagement is the score of a barely-passing face:
-# 30 px frontal face → score = (30×30) / (640×480) × 1.0 = 0.00293.
+# 75 px frontal face → score = (75×75) / (1920×1080) × 1.0 = 0.00271.
 
-# 0.0025 sits just below this with a small margin to absorb detection
+# 0.0023 sits just below this with a small margin to absorb detection
 # box wobble (1-2 px jitter can shrink bbox area by ~10%).
 
 # Behavior at this threshold:
-#   - 30 px frontal face         → 0.00293 ✓ pass
-#   - 30 px ¾-profile (f=0.85)   → 0.00249 ✗ borderline
-#   - 25 px frontal face         → 0.00203 ✗ too small
-#   - 40 px slight side (f=0.5)  → 0.00260 ✓ pass
-#   - 70 px frontal at 1m        → 0.0159  ✓ pass (typical close selfie)
+#   - 75 px frontal face         → 0.00271 ✓ pass
+#   - 75 px ¾-profile (f=0.85)   → 0.00231 ✓ borderline
+#   - 62 px frontal face         → 0.00185 ✗ too small
+#   - 100 px slight side (f=0.5) → 0.00241 ✓ pass
+#   - 175 px frontal at 1m       → 0.01477 ✓ pass (typical close selfie)
 
 # If you change CAMERA_WIDTH/HEIGHT or SELFIE_MIN_FACE_PX, recompute:
 #     min_engagement = (min_face_px)² / (W × H) × 0.85
@@ -453,17 +454,18 @@
 # - Example: 0.80 means if two faces have engagement scores 0.05 and 0.045,
 #   ratio = 0.9 > 0.80 → ambiguous, skip frame.
 
-SELFIE_MIN_FACE_PX = 30
+SELFIE_MIN_FACE_PX = 75
 # Selfie per-frame quality gate: bbox short side in pixels.
 
-# Sized for 640×480 camera (Unitree G1 default). 30 px short side ≈ 6.25%
-# of frame height, equivalent to a person ~2-3m from the camera at ~60° field of view.
+# Sized for the current 1920×1080 camera resolution. 75 px short side ≈
+# 6.9% of frame height, equivalent to a person ~2-3m from the camera at
+# ~60° field of view.
 
-# At higher camera resolutions (e.g. 1280×720) you can raise this to keep
-# the same physical distance threshold:
+# At other camera resolutions, scale to keep the same physical distance
+# threshold:
 #   - 640×480  → 30 px
 #   - 1280×720 → 50 px
-#   - 1920×1080 → 75 px
+#   - 1920×1080 → 75 px (current)
 
 # - cfg key: "selfie_min_face_px"
 # - Hot-tunable: YES
@@ -616,27 +618,26 @@
 # ============================================================================
 
 CAMERA_DEVICE = "/dev/video0"
-CAMERA_WIDTH = 640
-CAMERA_HEIGHT = 480
-# Camera resolution. Unitree G1's onboard camera default is 640×480.
-
-# All bbox-size thresholds (SELFIE_MIN_FACE_PX=30, AUTO_ENROLL_MIN_FACE_PX=30)
-# are sized for this resolution. If you bump to a higher resolution camera,
-# scale those thresholds proportionally:
-#   640×480  → 30 px (current)
-#   1280×720 → 50 px
-#   1920×1080 → 75 px
-# to keep the same effective "person distance" threshold.
-CAMERA_FPS = 15
-# Capture frame rate. The Unitree G1 onboard camera runs at 15 fps.
+CAMERA_WIDTH = 1920
+CAMERA_HEIGHT = 1080
+
+
+# All bbox-size thresholds (SELFIE_MIN_FACE_PX, AUTO_ENROLL_MIN_FACE_PX) are
+# sized for this resolution. If you change CAMERA_WIDTH/HEIGHT, scale those
+# thresholds proportionally to keep the same effective "person distance"
+# threshold:
+#   640×480   → 30 px
+#   1280×720  → 50 px
+#   1920×1080 → 75 px (current)
+CAMERA_FPS = 30
 
 # All wall-clock-based parameters (recog_interval, min_unknown_sec,
 # re_identify_interval, stale_track_sec) are independent of this — they
 # measure seconds, not frames.
 
 # Frame-count parameters that DO scale with fps:
-#   - TRACK_BUFFER (currently 30 = 2s @ 15fps)
-#   - gc_stale modulo in run.py main loop (15 = ~1s @ 15fps)
+#   - TRACK_BUFFER (currently 60 = 2s @ 30fps)
+#   - gc_stale modulo in run.py main loop (30 = ~1s @ 30fps)
 
 LOCAL_RTSP_URL = "rtsp://localhost:8554/top_camera"
 RAW_LOCAL_RTSP_URL = "rtsp://localhost:8554/top_camera_raw"

src/om1_vlm/anonymizationSys/face_recog_stream/face_tracker.py

@@ -28,8 +28,6 @@
     # results = [TrackResult(track_id=1, raw_name="wendy", tier="confident", ...), ...]
 """
 
-from __future__ import annotations
-
 import logging
 import time
 from collections import Counter
@@ -47,11 +45,6 @@
 log = logging.getLogger(__name__)
 
 
-# ---------------------------------------------------------------------------
-# Public dataclasses
-# ---------------------------------------------------------------------------
-
-
 @dataclass
 class TrackResult:
     """Per-track output each frame.
@@ -96,11 +89,6 @@ class TrackResult:
     is_known: bool = False
 
 
-# ---------------------------------------------------------------------------
-# Internal state
-# ---------------------------------------------------------------------------
-
-
 @dataclass
 class _TrackIdentity:
     """Internal identity state accumulated for one BoTSORT track.
@@ -160,11 +148,6 @@ class _TrackIdentity:
     last_confident_uuid: Optional[str] = None
 
 
-# ---------------------------------------------------------------------------
-# FaceTracker
-# ---------------------------------------------------------------------------
-
-
 class FaceTracker:
     """BoTSORT face tracker with tier-aware recognition and multi-frame voting.
 
@@ -281,10 +264,6 @@ def __init__(
         # Current frame faces (for status queries)
         self._current_faces: list = []
 
-    # ------------------------------------------------------------------
-    # Setup / runtime config
-    # ------------------------------------------------------------------
-
     @staticmethod
     def _init_tracker(track_buffer: int, det_conf: float = 0.5):
         """Initialize BoTSORT with thresholds aligned to detection confidence."""
@@ -446,10 +425,6 @@ def set_sim_thr(self, thr: float) -> None:
         """Back-compat shim — prefer ``set_thresholds(sim_thr=thr)``."""
         self.sim_thr = float(thr)
 
-    # ------------------------------------------------------------------
-    # Main per-frame entry point
-    # ------------------------------------------------------------------
-
     def update(
         self,
         frame: np.ndarray,
@@ -485,6 +460,7 @@ def update(
         self._active_ids = set()
         results: List[TrackResult] = []
         need_recog: List[Tuple[int, np.ndarray, Optional[np.ndarray], float]] = []
+        vvad_track_boxes: List[Tuple[int, Tuple[float, float, float, float]]] = []
 
         for track in tracks:
             x1, y1, x2, y2 = map(int, track[:4])
@@ -498,21 +474,9 @@ def update(
 
             self._active_ids.add(track_id)
 
-            # Feed the speaking-detector buffer. Pad the (tight) SCRFD box so the
-            # crop keeps the full mouth/chin context the VVAD model expects;
-            # clipping the mouth squashes the speaking score. Recognition still
-            # uses the original tight box below.
             if self.vvad is not None:
-                bw, bh = x2 - x1, y2 - y1
-                px = int(bw * 0.20)
-                pyt = int(bh * 0.15)  # a little on top
-                pyb = int(bh * 0.30)  # more on bottom (mouth/chin)
-                cx1, cy1 = max(0, x1 - px), max(0, y1 - pyt)
-                cx2, cy2 = min(W, x2 + px), min(H, y2 + pyb)
-                self.vvad.push(
-                    track_id,
-                    frame[cy1:cy2, cx1:cx2],
-                    kps=track_det_map.get(track_id, {}).get("kps"),
+                vvad_track_boxes.append(
+                    (track_id, (float(x1), float(y1), float(x2), float(y2)))
                 )
 
             # Get or create identity state
@@ -581,17 +545,14 @@ def update(
         self._cleanup_stale()
 
         if self.vvad is not None:
+            self.vvad.push_frame(frame, vvad_track_boxes)
             self.vvad.evict(self._active_ids)
 
         # Build the by-area sorted face list used by /who and unknown capture
         self._current_faces = self._build_current_faces(results)
 
         return results
 
-    # ------------------------------------------------------------------
-    # Re-identify / display
-    # ------------------------------------------------------------------
-
     def _maybe_reset_for_reidentify(self, ident: _TrackIdentity, now: float) -> None:
         """Reset votes if it's been a while since last recognition attempt.
 
@@ -808,10 +769,6 @@ def _build_current_faces(self, results: List[TrackResult]) -> list:
             )
         return sorted(faces, key=lambda f: f["area"], reverse=True)
 
-    # ------------------------------------------------------------------
-    # BoTSORT plumbing
-    # ------------------------------------------------------------------
-
     def _run_tracker(self, dets: np.ndarray, frame: np.ndarray) -> np.ndarray:
         """Run BoTSORT. Returns (M, 7) [x1,y1,x2,y2,track_id,conf,cls]."""
         if dets is None or len(dets) == 0:
@@ -872,10 +829,6 @@ def _match_tracks_to_dets(
 
         return result
 
-    # ------------------------------------------------------------------
-    # Recognition
-    # ------------------------------------------------------------------
-
     def _run_recognition_batch(
         self,
         frame: np.ndarray,
@@ -1208,10 +1161,6 @@ def _set_unknown(self, ident: _TrackIdentity, now: float, *, reason: str) -> Non
             ident.unknown_since = now
         log.debug("Track %d: unknown (%s)", ident.track_id, reason)
 
-    # ------------------------------------------------------------------
-    # Cleanup / introspection
-    # ------------------------------------------------------------------
-
     def _cleanup_stale(self) -> None:
         """Drop identity state for tracks BoTSORT no longer reports."""
         stale = [tid for tid in self._identities if tid not in self._active_ids]

src/om1_vlm/anonymizationSys/face_recog_stream/mediapipe_mesh.py

@@ -0,0 +1,76 @@
+from typing import List, Optional
+
+import cv2
+import mediapipe as mp
+import numpy as np
+
+
+class MediaPipeFaceMesh:
+    """Wrapper around MediaPipe Face Mesh for detecting face landmarks."""
+
+    def __init__(
+        self,
+        max_num_faces: int = 1,
+        min_detection_confidence: float = 0.5,
+        min_tracking_confidence: float = 0.5,
+        refine_landmarks: bool = True,
+    ) -> None:
+        self._mp = mp
+        self._mesh = mp.solutions.face_mesh.FaceMesh(  # type: ignore
+            max_num_faces=max_num_faces,
+            refine_landmarks=refine_landmarks,
+            min_detection_confidence=min_detection_confidence,
+            min_tracking_confidence=min_tracking_confidence,
+        )
+
+    def __call__(self, frame_bgr: np.ndarray) -> Optional[np.ndarray]:
+        """
+        Detect the first face in a BGR frame and return its landmarks.
+
+        Parameters
+        ----------
+        frame_bgr : np.ndarray
+            Input image in BGR format (as read by OpenCV).
+
+        Returns
+        -------
+        Optional[np.ndarray]
+            Detected face represented as an array of shape (N, 2) containing the
+            (x, y) pixel coordinates of the landmarks, or None if no face is detected.
+        """
+        faces = self.all(frame_bgr)
+        return faces[0] if faces else None
+
+    def all(self, frame_bgr: np.ndarray) -> List[np.ndarray]:
+        """
+        Detect all faces in a BGR frame and return their landmarks.
+
+        Parameters
+        ----------
+        frame_bgr : np.ndarray
+            Input image in BGR format (as read by OpenCV).
+
+        Returns
+        -------
+        List[np.ndarray]
+            List of detected faces, each represented as an array of shape (N, 2)
+            containing the (x, y) pixel coordinates of the landmarks.
+        """
+        h, w = frame_bgr.shape[:2]
+        rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
+        rgb.flags.writeable = False
+        res = self._mesh.process(rgb)
+        if not res.multi_face_landmarks:
+            return []
+
+        out = []
+        for face in res.multi_face_landmarks:
+            pts = np.array(
+                [[p.x * w, p.y * h] for p in face.landmark], dtype=np.float64
+            )
+            out.append(pts)
+        return out
+
+    def close(self) -> None:
+        """Release resources."""
+        self._mesh.close()