add sampling of different display models between reference and test

README.md

@@ -25,9 +25,9 @@ The metric is explained in detail in:
 
 If you use the metric in your research, please cite the paper above. 
 
-# Other metrics: ColorVideoVDP-ML, PU-PSNR
+# Other metrics: ColorVideoVDP-ML, ColorVideoVDP-tm, PU-PSNR
 
-The repository also contains code for other metrics, such as PU-PSNR, or ColorVideoVDP-ML. See [metrics.md](/metrics.md) for more information.
+The repository also contains code for other metrics, such as PU-PSNR, ColorVideoVDP-tm, or ColorVideoVDP-ML. See [metrics.md](/metrics.md) for more information.
 
 ## PyTorch quickstart
 1. Start by installing [anaconda](https://docs.anaconda.com/anaconda/install/index.html) or [miniconda](https://docs.conda.io/en/latest/miniconda.html). Then, create a new environment for ColorVideoVDP and activate it:

metrics.md

@@ -6,6 +6,12 @@ The code can run one of the following metrics (selected with `--metrics` or `-m`
 
   The original ColorVideoVDP metric, as described in the [SIGGRAPH paper](https://doi.org/10.1145/3658144).
 
+* `cvvdp-tm`
+
+  This version of ColorVideoVDP works for stimuli scaled at much different absolute luminance range, e.g. due to tone mapping of HDR content. This extension is described in detail in our [SIGGRAPH 2026 paper](https://kenchen10.github.io/projects/tmometric/index.html).
+
+  This metric is backward-compatible, and only requires specification of a new flag, `"diff_sensitivity": "on"`, in the `cvvdp_parameters.json`. 
+
 * `cvvdp-ml-saliency` [experimental]
 
   The extended version of ColorVideoVDP with a machine-learning based regressor and a saliency model, as explained in the [ICME paper](https://www.cl.cam.ac.uk/~rkm38/pdfs/hammou2025_ICME_GC_ColorVideoVDP_ML.pdf).

pycvvdp/cvvdp_metric.py

@@ -223,7 +223,12 @@ def load_config( self, config_paths ):
         self.omega = [0, 5]
 
         self.csf = castleCSF(csf_version=self.csf, device=self.device, config_paths=config_paths)
-
+
+        if 'diff_sensitivity' in parameters:
+            self.diff_sensitivity = True if parameters['diff_sensitivity'] == "on" else False
+        else:
+            self.diff_sensitivity = False
+
         # Mask to block selected channels, used in the ablation stdies [Ysust, RB, YV, Ytrans]
         self.block_channels = torch.as_tensor( parameters['block_channels'], device=self.device, dtype=torch.bool ) if 'block_channels' in parameters else None
 
@@ -705,15 +710,28 @@ def process_block_of_frames(self, R, vid_sz, temp_ch, lpyr, is_image):
             # Compute CSF
             rho = rho_band[bb] # Spatial frequency in cpd
             ch_height, ch_width = logL_bkg.shape[-2], logL_bkg.shape[-1]
-            S = torch.empty((batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device)
-            for cc in range(all_ch):
-                tch = 0 if cc<3 else 1  # Sustained or transient
-                cch = cc if cc<3 else 0 # Y, rg, yv
-                # The sensitivity is always extracted for the reference frame
-                S[:,cc:(cc+1),:,:,:] = self.csf.sensitivity(rho, self.omega[tch], logL_bkg[...,1:2,:,:,:], cch, self.csf_sigma) * 10.0**(self.sensitivity_correction/20.0)
+            if self.diff_sensitivity:
+                # Differential sensitivity sampling
+                S = torch.empty((2,batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device)
+                for ss in range(2):
+                    for cc in range(all_ch):
+                        tch = 0 if cc<3 else 1  # Sustained or transient
+                        cch = cc if cc<3 else 0 # Y, rg, yv
+                        # The sensitivity is extracted for the reference & test frame
+                        S[ss,cc:(cc+1),:,:,:] = self.csf.sensitivity(rho, self.omega[tch], logL_bkg[...,ss,:,:,:], cch, self.csf_sigma) * 10.0**(self.sensitivity_correction/20.0)
+            else:
+                S = torch.empty((batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device)
+                for cc in range(all_ch):
+                    tch = 0 if cc<3 else 1  # Sustained or transient
+                    cch = cc if cc<3 else 0 # Y, rg, yv
+                    # The sensitivity is always extracted for the reference frame
+                    S[:,cc:(cc+1),:,:,:] = self.csf.sensitivity(rho, self.omega[tch], logL_bkg[...,1:2,:,:,:], cch, self.csf_sigma) * 10.0**(self.sensitivity_correction/20.0)
 
             if is_baseband:
-                D = (torch.abs(T_f-R_f) * S)
+                if self.diff_sensitivity:
+                    D = (torch.abs(T_f*S[0,...]-R_f*S[-1,...]))
+                else:
+                    D = (torch.abs(T_f-R_f) * S)
             else:
                 # dimensions: [channel,frame,height,width]
                 D = self.apply_masking_model(T_f, R_f, S)
@@ -828,17 +846,28 @@ def apply_masking_model(self, T, R, S):
             if self.masking_model.startswith( "add" ):
                 zero_tens = torch.as_tensor(0., device=T.device)
                 ch_gain = self.ce_g * torch.reshape( torch.as_tensor( [1, 1.7, 0.237, 1.], device=T.device), (1, 4, 1, 1, 1) )[:,:num_ch,...] 
-                C_t = 1/S
-                T_p = self.diff_sign(T) * torch.maximum( (torch.abs(T)-C_t)*ch_gain + 1, zero_tens )
-                R_p = self.diff_sign(R) * torch.maximum( (torch.abs(R)-C_t)*ch_gain + 1, zero_tens )
+                if self.diff_sensitivity:
+                    C_T = 1/S[0,...]
+                    C_R = 1/S[-1,...]
+                else:
+                    C_T = 1/S
+                    C_R = C_T
+                T_p = self.diff_sign(T) * torch.maximum( (torch.abs(T)-C_T)*ch_gain + 1, zero_tens )
+                R_p = self.diff_sign(R) * torch.maximum( (torch.abs(R)-C_R)*ch_gain + 1, zero_tens )
             else:
+                if self.diff_sensitivity:
+                    S_T = S[0,...]
+                    S_R = S[-1,...]
+                else:
+                    S_T = S
+                    S_R = S_T
                 if self.masking_model.endswith( "mutual-old" ):
-                    T_p = T * S
-                    R_p = R * S
+                    T_p = T * S_T
+                    R_p = R * S_R
                 else:
                     ch_gain = torch.reshape( torch.as_tensor( [1, 1.45, 1, 1.], device=T.device), (1, 4, 1, 1, 1) )[:,:num_ch,...] 
-                    T_p = T * S * ch_gain
-                    R_p = R * S * ch_gain
+                    T_p = T * S_T * ch_gain
+                    R_p = R * S_R * ch_gain
 
             if self.masking_model.endswith( "none" ):
                 D = self.clamp_diffs(torch.abs(T_p-R_p))

pycvvdp/video_source.py

@@ -201,23 +201,34 @@ def numpy2torch_frame(np_array, frame, device, dim_order="HWC" ):
 """
 This video_source uses a photometric display model to convert input content (e.g. sRGB) to luminance maps. 
 """
+def _load_dm(dm, config_paths):
+    if isinstance(dm, str):
+        return vvdp_display_photometry.load(dm, config_paths)
+    elif isinstance(dm, vvdp_display_photometry):
+        return dm
+    else:
+        raise RuntimeError( "display_model must be a string or fvvdp_display_photometry subclass" )
+
 class video_source_dm( video_source ):
 
-    def __init__( self,  display_photometry='sdr_4k_30', config_paths=[] ):
+    def __init__( self,  display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[] ):
 
-#        self.color_trans = ColorTransform(color_space_name)
+        if test_display_photometry is None:
+            test_display_photometry = display_photometry
 
-        if isinstance( display_photometry, str ):
-            self.dm_photometry = vvdp_display_photometry.load(display_photometry, config_paths) 
-        elif isinstance( display_photometry, vvdp_display_photometry ):
-            self.dm_photometry = display_photometry
-        else:
-            raise RuntimeError( "display_model must be a string or fvvdp_display_photometry subclass" )
+        if reference_display_photometry is None:
+            reference_display_photometry = display_photometry
 
-    def apply_dm_and_color_transform(self, frame, target_colorspace):
+        self.test_dm = _load_dm(test_display_photometry, config_paths)
+        self.reference_dm = _load_dm(reference_display_photometry, config_paths)
 
-        I = self.dm_photometry.source_2_target_colorspace(frame, target_colorspace)
+        # backwards compatibility
+        self.dm_photometry = self.test_dm
 
+    def apply_dm_and_color_transform(self, frame, target_colorspace, dm=None):
+        if dm == None:
+            dm = self.dm_photometry
+        I = dm.source_2_target_colorspace(frame, target_colorspace)
         self.check_if_valid(I, target_colorspace)
         return I
 
@@ -240,9 +251,9 @@ class video_source_array( video_source_dm ):
     #   class
     # color_space_name - name of the color space (see
     #   fvvdp_data/color_spaces.json)
-    def __init__( self, test_video, reference_video, fps, dim_order='BCFHW', display_photometry='sdr_4k_30', config_paths=[], ):
+    def __init__( self, test_video, reference_video, fps, dim_order='BCFHW', display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[], ):
 
-        super().__init__(display_photometry=display_photometry, config_paths=config_paths)        
+        super().__init__(display_photometry=display_photometry, test_display_photometry=test_display_photometry, reference_display_photometry=reference_display_photometry, config_paths=config_paths)        
 
         if test_video.shape != reference_video.shape:
             ind = dim_order.find('B')
@@ -312,12 +323,12 @@ def get_batch_size(self):
     # gpuArray.
 
     def get_test_frame( self, frame, device, colorspace ):
-        return self._get_frame(self.test_video, frame, device, colorspace )
+        return self._get_frame(self.test_video, frame, device, colorspace, self.test_dm )
 
     def get_reference_frame( self, frame, device, colorspace ):
-        return self._get_frame(self.reference_video, frame, device, colorspace )
+        return self._get_frame(self.reference_video, frame, device, colorspace, self.reference_dm )
 
-    def _get_frame( self, from_array, frame, device, colorspace ):        
+    def _get_frame( self, from_array, frame, device, colorspace, dm=None ):        
         # Determine the maximum value of the data type storing the
         # image/video
 
@@ -341,7 +352,7 @@ def _get_frame( self, from_array, frame, device, colorspace ):
         else:
             raise RuntimeError( f"Only uint8, uint16 and float32 is currently supported. {from_array.dtype} encountered." )
 
-        I = self.apply_dm_and_color_transform(frame, colorspace)
+        I = self.apply_dm_and_color_transform(frame, colorspace, dm=dm)
 
         return I
 

pycvvdp/video_source_file.py

@@ -337,7 +337,7 @@ def safe_floor(x):
 '''
 class video_source_video_file(video_source_dm):
 
-    def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', config_paths=[], fps=None, frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False, ignore_framerate_mismatch=False ):
+    def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[], fps=None, frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False, ignore_framerate_mismatch=False ):
 
         self.fs_width = -1 if full_screen_resize is None else resize_resolution[0]
         self.fs_height = -1 if full_screen_resize is None else resize_resolution[1]
@@ -357,7 +357,7 @@ def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30',
         self.fps = fps       
         self.ignore_framerate_mismatch = ignore_framerate_mismatch 
 
-        super().__init__(display_photometry=display_photometry, config_paths=config_paths)
+        super().__init__(display_photometry=display_photometry, test_display_photometry=test_display_photometry, reference_display_photometry=reference_display_photometry, config_paths=config_paths)
 
         # Resolutions may be different here because upscaling may happen on the GPU
         # if self.test_vidr.height != self.reference_vidr.height or self.test_vidr.width != self.reference_vidr.width:
@@ -441,19 +441,19 @@ def get_test_frame( self, frame, device, colorspace="Y" ) -> Tensor:
         #print( f"{self.test_fname} - {self.fs_width}x{self.fs_height}" )
         # if not self.last_test_frame is None and frame == self.last_test_frame[0]:
         #     return self.last_test_frame[1]
-        L = self._get_frame( self.test_vidr, frame, device, colorspace )
+        L = self._get_frame( self.test_vidr, frame, device, colorspace, dm=self.test_dm )
         # self.last_test_frame = (frame,L)
         return L
 
     def get_reference_frame( self, frame, device, colorspace="Y" ) -> Tensor:
         self.init_readers()
         # if not self.last_reference_frame is None and frame == self.last_reference_frame[0]:
         #     return self.last_reference_frame[1]
-        L = self._get_frame( self.reference_vidr, frame, device, colorspace )
+        L = self._get_frame( self.reference_vidr, frame, device, colorspace, dm=self.reference_dm )
         # self.reference_test_frame = (frame,L)
         return L
 
-    def _get_frame( self, vid_reader, frame, device, colorspace ):        
+    def _get_frame( self, vid_reader, frame, device, colorspace, dm=None ):        
         self.init_readers()
 
         if frame != (vid_reader.curr_frame+1):
@@ -464,13 +464,13 @@ def _get_frame( self, vid_reader, frame, device, colorspace ):
         if frame_np is None:
             raise vq_exception( f'Could not read frame {frame} of "{vid_reader.fname}". Try passing "--count-frames" or "-nframes".' )
 
-        return self._prepare_frame(frame_np, device, vid_reader.unpack, colorspace)
+        return self._prepare_frame(frame_np, device, vid_reader.unpack, colorspace, dm=dm)
 
-    def _prepare_frame( self, frame_np, device, unpack_fn, colorspace="Y" ):
+    def _prepare_frame( self, frame_np, device, unpack_fn, colorspace="Y", dm=None ):
         frame_t_hwc = unpack_fn(frame_np, device)
         frame_t = reshuffle_dims( frame_t_hwc, in_dims='HWC', out_dims="BCFHW" )
 
-        I = self.apply_dm_and_color_transform(frame_t, colorspace)
+        I = self.apply_dm_and_color_transform(frame_t, colorspace, dm=dm)
 
         return I
 
@@ -483,8 +483,8 @@ class video_source_temp_resample_file(video_source_video_file):
 
     max_fps = 166 # upsample to at most this FPS
 
-    def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', config_paths=[], frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False ):
-        super().__init__(test_fname, reference_fname, display_photometry=display_photometry, config_paths=config_paths, frames=frames, full_screen_resize=full_screen_resize, 
+    def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[], frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False ):
+        super().__init__(test_fname, reference_fname, display_photometry=display_photometry, test_display_photometry=test_display_photometry, reference_display_photometry=reference_display_photometry, config_paths=config_paths, frames=frames, full_screen_resize=full_screen_resize, 
                          resize_resolution=resize_resolution, ffmpeg_cc=ffmpeg_cc, verbose=verbose, ignore_framerate_mismatch=True)
 
 
@@ -528,7 +528,7 @@ def get_video_size(self):
         return super().get_video_size()
 
 
-    def _get_frame( self, vid_reader, frame, device, colorspace ):        
+    def _get_frame( self, vid_reader, frame, device, colorspace, dm=None ):        
 
         frame_ind = int(safe_floor((frame+0.5) * vid_reader.avg_fps/self.resample_fps))
 
@@ -538,7 +538,7 @@ def _get_frame( self, vid_reader, frame, device, colorspace ):
             return self.cache_frame[ce]
         else:
             self.cache_ind[ce] = frame_ind
-            self.cache_frame[ce] = super()._get_frame( vid_reader, frame_ind, device=device, colorspace=colorspace )
+            self.cache_frame[ce] = super()._get_frame( vid_reader, frame_ind, device=device, colorspace=colorspace, dm=dm )
             #self.cache_frame[ce] = self.cache_frame[ce][...,4:-4,4:-4]  # Crop 4 pixels from all the sided because of the dark frame in the test videos
             return self.cache_frame[ce]