From c7635a4365f413cbf85780cc0e00d9820d18a52a Mon Sep 17 00:00:00 2001 From: Kenny Chen Date: Wed, 1 Jul 2026 16:58:20 -0400 Subject: [PATCH 1/4] add support for different display models for test and reference content --- pycvvdp/cvvdp_metric.py | 60 +++++++++++++++++++++++++++--------- pycvvdp/video_source.py | 43 ++++++++++++++++---------- pycvvdp/video_source_file.py | 24 +++++++-------- 3 files changed, 85 insertions(+), 42 deletions(-) diff --git a/pycvvdp/cvvdp_metric.py b/pycvvdp/cvvdp_metric.py index 9d49de6..49fb6c9 100644 --- a/pycvvdp/cvvdp_metric.py +++ b/pycvvdp/cvvdp_metric.py @@ -223,6 +223,9 @@ def load_config( self, config_paths ): self.omega = [0, 5] self.csf = castleCSF(csf_version=self.csf, device=self.device, config_paths=config_paths) + + if 'diff_sensitivity' in parameters: + self.diff_sensitivity = True if parameters['diff_sensitivity'] == "on" else False # Mask to block selected channels, used in the ablation stdies [Ysust, RB, YV, Ytrans] self.block_channels = torch.as_tensor( parameters['block_channels'], device=self.device, dtype=torch.bool ) if 'block_channels' in parameters else None @@ -666,6 +669,11 @@ def process_block_of_frames(self, R, vid_sz, temp_ch, lpyr, is_image): #height, width, N_frames = vid_sz all_ch = 2+temp_ch batch_sz = R.shape[0] + + if self.diff_sensitivity: + dim_S = 2 + else: + dim_S = 1 #torch.autograd.set_detect_anomaly(True) @@ -705,15 +713,28 @@ def process_block_of_frames(self, R, vid_sz, temp_ch, lpyr, is_image): # Compute CSF rho = rho_band[bb] # Spatial frequency in cpd ch_height, ch_width = logL_bkg.shape[-2], logL_bkg.shape[-1] - S = torch.empty((batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device) - for cc in range(all_ch): - tch = 0 if cc<3 else 1 # Sustained or transient - cch = cc if cc<3 else 0 # Y, rg, yv - # The sensitivity is always extracted for the reference frame - S[:,cc:(cc+1),:,:,:] = self.csf.sensitivity(rho, self.omega[tch], logL_bkg[...,1:2,:,:,:], cch, self.csf_sigma) * 10.0**(self.sensitivity_correction/20.0) + if self.diff_sensitivity: + # Differential sensitivity sampling + S = torch.empty((dim_S,batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device) + for ss in range(dim_S): + for cc in range(all_ch): + tch = 0 if cc<3 else 1 # Sustained or transient + cch = cc if cc<3 else 0 # Y, rg, yv + # The sensitivity is extracted for the reference & test frame + S[ss,cc:(cc+1),:,:,:] = self.csf.sensitivity(rho, self.omega[tch], logL_bkg[...,ss,:,:,:], cch, self.csf_sigma) * 10.0**(self.sensitivity_correction/20.0) + else: + S = torch.empty((batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device) + for cc in range(all_ch): + tch = 0 if cc<3 else 1 # Sustained or transient + cch = cc if cc<3 else 0 # Y, rg, yv + # The sensitivity is always extracted for the reference frame + S[:,cc:(cc+1),:,:,:] = self.csf.sensitivity(rho, self.omega[tch], logL_bkg[...,1:2,:,:,:], cch, self.csf_sigma) * 10.0**(self.sensitivity_correction/20.0) if is_baseband: - D = (torch.abs(T_f-R_f) * S) + if self.diff_sensitivity: + D = (torch.abs(T_f*S[0,...]-R_f*S[-1,...])) + else: + D = (torch.abs(T_f-R_f) * S) else: # dimensions: [channel,frame,height,width] D = self.apply_masking_model(T_f, R_f, S) @@ -828,17 +849,28 @@ def apply_masking_model(self, T, R, S): if self.masking_model.startswith( "add" ): zero_tens = torch.as_tensor(0., device=T.device) ch_gain = self.ce_g * torch.reshape( torch.as_tensor( [1, 1.7, 0.237, 1.], device=T.device), (1, 4, 1, 1, 1) )[:,:num_ch,...] - C_t = 1/S - T_p = self.diff_sign(T) * torch.maximum( (torch.abs(T)-C_t)*ch_gain + 1, zero_tens ) - R_p = self.diff_sign(R) * torch.maximum( (torch.abs(R)-C_t)*ch_gain + 1, zero_tens ) + if self.diff_sensitivity: + C_T = 1/S[0,...] + C_R = 1/S[-1,...] + else: + C_T = 1/S + C_R = C_T + T_p = self.diff_sign(T) * torch.maximum( (torch.abs(T)-C_T)*ch_gain + 1, zero_tens ) + R_p = self.diff_sign(R) * torch.maximum( (torch.abs(R)-C_R)*ch_gain + 1, zero_tens ) else: + if self.diff_sensitivity: + S_T = S[0,...] + S_R = S[-1,...] + else: + S_T = S + S_R = S_T if self.masking_model.endswith( "mutual-old" ): - T_p = T * S - R_p = R * S + T_p = T * S_T + R_p = R * S_R else: ch_gain = torch.reshape( torch.as_tensor( [1, 1.45, 1, 1.], device=T.device), (1, 4, 1, 1, 1) )[:,:num_ch,...] - T_p = T * S * ch_gain - R_p = R * S * ch_gain + T_p = T * S_T * ch_gain + R_p = R * S_R * ch_gain if self.masking_model.endswith( "none" ): D = self.clamp_diffs(torch.abs(T_p-R_p)) diff --git a/pycvvdp/video_source.py b/pycvvdp/video_source.py index e2e3af8..0ca7b5e 100644 --- a/pycvvdp/video_source.py +++ b/pycvvdp/video_source.py @@ -201,23 +201,34 @@ def numpy2torch_frame(np_array, frame, device, dim_order="HWC" ): """ This video_source uses a photometric display model to convert input content (e.g. sRGB) to luminance maps. """ +def _load_dm(dm, config_paths): + if isinstance(dm, str): + return vvdp_display_photometry.load(dm, config_paths) + elif isinstance(dm, vvdp_display_photometry): + return dm + else: + raise RuntimeError( "display_model must be a string or fvvdp_display_photometry subclass" ) + class video_source_dm( video_source ): - def __init__( self, display_photometry='sdr_4k_30', config_paths=[] ): + def __init__( self, display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[] ): -# self.color_trans = ColorTransform(color_space_name) + if test_display_photometry is None: + test_display_photometry = display_photometry - if isinstance( display_photometry, str ): - self.dm_photometry = vvdp_display_photometry.load(display_photometry, config_paths) - elif isinstance( display_photometry, vvdp_display_photometry ): - self.dm_photometry = display_photometry - else: - raise RuntimeError( "display_model must be a string or fvvdp_display_photometry subclass" ) + if reference_display_photometry is None: + reference_display_photometry = display_photometry - def apply_dm_and_color_transform(self, frame, target_colorspace): + self.test_dm = _load_dm(test_display_photometry, config_paths) + self.reference_dm = _load_dm(reference_display_photometry, config_paths) - I = self.dm_photometry.source_2_target_colorspace(frame, target_colorspace) + # backwards compatibility + self.dm_photometry = self.test_dm + def apply_dm_and_color_transform(self, frame, target_colorspace, dm=None): + if dm == None: + dm = self.dm_photometry + I = dm.source_2_target_colorspace(frame, target_colorspace) self.check_if_valid(I, target_colorspace) return I @@ -240,9 +251,9 @@ class video_source_array( video_source_dm ): # class # color_space_name - name of the color space (see # fvvdp_data/color_spaces.json) - def __init__( self, test_video, reference_video, fps, dim_order='BCFHW', display_photometry='sdr_4k_30', config_paths=[], ): + def __init__( self, test_video, reference_video, fps, dim_order='BCFHW', display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[], ): - super().__init__(display_photometry=display_photometry, config_paths=config_paths) + super().__init__(display_photometry=display_photometry, test_display_photometry=test_display_photometry, reference_display_photometry=reference_display_photometry, config_paths=config_paths) if test_video.shape != reference_video.shape: ind = dim_order.find('B') @@ -312,12 +323,12 @@ def get_batch_size(self): # gpuArray. def get_test_frame( self, frame, device, colorspace ): - return self._get_frame(self.test_video, frame, device, colorspace ) + return self._get_frame(self.test_video, frame, device, colorspace, self.test_dm ) def get_reference_frame( self, frame, device, colorspace ): - return self._get_frame(self.reference_video, frame, device, colorspace ) + return self._get_frame(self.reference_video, frame, device, colorspace, self.reference_dm ) - def _get_frame( self, from_array, frame, device, colorspace ): + def _get_frame( self, from_array, frame, device, colorspace, dm=None ): # Determine the maximum value of the data type storing the # image/video @@ -341,7 +352,7 @@ def _get_frame( self, from_array, frame, device, colorspace ): else: raise RuntimeError( f"Only uint8, uint16 and float32 is currently supported. {from_array.dtype} encountered." ) - I = self.apply_dm_and_color_transform(frame, colorspace) + I = self.apply_dm_and_color_transform(frame, colorspace, dm=dm) return I diff --git a/pycvvdp/video_source_file.py b/pycvvdp/video_source_file.py index 8a432a3..3b94d34 100644 --- a/pycvvdp/video_source_file.py +++ b/pycvvdp/video_source_file.py @@ -337,7 +337,7 @@ def safe_floor(x): ''' class video_source_video_file(video_source_dm): - def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', config_paths=[], fps=None, frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False, ignore_framerate_mismatch=False ): + def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[], fps=None, frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False, ignore_framerate_mismatch=False ): self.fs_width = -1 if full_screen_resize is None else resize_resolution[0] self.fs_height = -1 if full_screen_resize is None else resize_resolution[1] @@ -357,7 +357,7 @@ def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', self.fps = fps self.ignore_framerate_mismatch = ignore_framerate_mismatch - super().__init__(display_photometry=display_photometry, config_paths=config_paths) + super().__init__(display_photometry=display_photometry, test_display_photometry=test_display_photometry, reference_display_photometry=reference_display_photometry, config_paths=config_paths) # Resolutions may be different here because upscaling may happen on the GPU # if self.test_vidr.height != self.reference_vidr.height or self.test_vidr.width != self.reference_vidr.width: @@ -441,7 +441,7 @@ def get_test_frame( self, frame, device, colorspace="Y" ) -> Tensor: #print( f"{self.test_fname} - {self.fs_width}x{self.fs_height}" ) # if not self.last_test_frame is None and frame == self.last_test_frame[0]: # return self.last_test_frame[1] - L = self._get_frame( self.test_vidr, frame, device, colorspace ) + L = self._get_frame( self.test_vidr, frame, device, colorspace, dm=self.test_dm ) # self.last_test_frame = (frame,L) return L @@ -449,11 +449,11 @@ def get_reference_frame( self, frame, device, colorspace="Y" ) -> Tensor: self.init_readers() # if not self.last_reference_frame is None and frame == self.last_reference_frame[0]: # return self.last_reference_frame[1] - L = self._get_frame( self.reference_vidr, frame, device, colorspace ) + L = self._get_frame( self.reference_vidr, frame, device, colorspace, dm=self.reference_dm ) # self.reference_test_frame = (frame,L) return L - def _get_frame( self, vid_reader, frame, device, colorspace ): + def _get_frame( self, vid_reader, frame, device, colorspace, dm=None ): self.init_readers() if frame != (vid_reader.curr_frame+1): @@ -464,13 +464,13 @@ def _get_frame( self, vid_reader, frame, device, colorspace ): if frame_np is None: raise vq_exception( f'Could not read frame {frame} of "{vid_reader.fname}". Try passing "--count-frames" or "-nframes".' ) - return self._prepare_frame(frame_np, device, vid_reader.unpack, colorspace) + return self._prepare_frame(frame_np, device, vid_reader.unpack, colorspace, dm=dm) - def _prepare_frame( self, frame_np, device, unpack_fn, colorspace="Y" ): + def _prepare_frame( self, frame_np, device, unpack_fn, colorspace="Y", dm=None ): frame_t_hwc = unpack_fn(frame_np, device) frame_t = reshuffle_dims( frame_t_hwc, in_dims='HWC', out_dims="BCFHW" ) - I = self.apply_dm_and_color_transform(frame_t, colorspace) + I = self.apply_dm_and_color_transform(frame_t, colorspace, dm=dm) return I @@ -483,8 +483,8 @@ class video_source_temp_resample_file(video_source_video_file): max_fps = 166 # upsample to at most this FPS - def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', config_paths=[], frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False ): - super().__init__(test_fname, reference_fname, display_photometry=display_photometry, config_paths=config_paths, frames=frames, full_screen_resize=full_screen_resize, + def __init__( self, test_fname, reference_fname, display_photometry='sdr_4k_30', test_display_photometry=None, reference_display_photometry=None, config_paths=[], frames=-1, full_screen_resize=None, resize_resolution=None, ffmpeg_cc=False, verbose=False ): + super().__init__(test_fname, reference_fname, display_photometry=display_photometry, test_display_photometry=test_display_photometry, reference_display_photometry=reference_display_photometry, config_paths=config_paths, frames=frames, full_screen_resize=full_screen_resize, resize_resolution=resize_resolution, ffmpeg_cc=ffmpeg_cc, verbose=verbose, ignore_framerate_mismatch=True) @@ -528,7 +528,7 @@ def get_video_size(self): return super().get_video_size() - def _get_frame( self, vid_reader, frame, device, colorspace ): + def _get_frame( self, vid_reader, frame, device, colorspace, dm=None ): frame_ind = int(safe_floor((frame+0.5) * vid_reader.avg_fps/self.resample_fps)) @@ -538,7 +538,7 @@ def _get_frame( self, vid_reader, frame, device, colorspace ): return self.cache_frame[ce] else: self.cache_ind[ce] = frame_ind - self.cache_frame[ce] = super()._get_frame( vid_reader, frame_ind, device=device, colorspace=colorspace ) + self.cache_frame[ce] = super()._get_frame( vid_reader, frame_ind, device=device, colorspace=colorspace, dm=dm ) #self.cache_frame[ce] = self.cache_frame[ce][...,4:-4,4:-4] # Crop 4 pixels from all the sided because of the dark frame in the test videos return self.cache_frame[ce] From 1afd329b2d40d1c43eef113735e6cb2a13f5c848 Mon Sep 17 00:00:00 2001 From: Kenny Chen Date: Wed, 1 Jul 2026 17:15:20 -0400 Subject: [PATCH 2/4] fix else case for diff_sensitivity --- pycvvdp/cvvdp_metric.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pycvvdp/cvvdp_metric.py b/pycvvdp/cvvdp_metric.py index 49fb6c9..f9d3321 100644 --- a/pycvvdp/cvvdp_metric.py +++ b/pycvvdp/cvvdp_metric.py @@ -226,7 +226,9 @@ def load_config( self, config_paths ): if 'diff_sensitivity' in parameters: self.diff_sensitivity = True if parameters['diff_sensitivity'] == "on" else False - + else: + self.diff_sensitivity = False + # Mask to block selected channels, used in the ablation stdies [Ysust, RB, YV, Ytrans] self.block_channels = torch.as_tensor( parameters['block_channels'], device=self.device, dtype=torch.bool ) if 'block_channels' in parameters else None From 2c671bf2bc7a430202973d915ac2de897ec87fb9 Mon Sep 17 00:00:00 2001 From: Kenny Chen Date: Wed, 1 Jul 2026 18:54:43 -0400 Subject: [PATCH 3/4] dim_S is not needed here --- pycvvdp/cvvdp_metric.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pycvvdp/cvvdp_metric.py b/pycvvdp/cvvdp_metric.py index f9d3321..360c61f 100644 --- a/pycvvdp/cvvdp_metric.py +++ b/pycvvdp/cvvdp_metric.py @@ -671,11 +671,6 @@ def process_block_of_frames(self, R, vid_sz, temp_ch, lpyr, is_image): #height, width, N_frames = vid_sz all_ch = 2+temp_ch batch_sz = R.shape[0] - - if self.diff_sensitivity: - dim_S = 2 - else: - dim_S = 1 #torch.autograd.set_detect_anomaly(True) @@ -717,8 +712,8 @@ def process_block_of_frames(self, R, vid_sz, temp_ch, lpyr, is_image): ch_height, ch_width = logL_bkg.shape[-2], logL_bkg.shape[-1] if self.diff_sensitivity: # Differential sensitivity sampling - S = torch.empty((dim_S,batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device) - for ss in range(dim_S): + S = torch.empty((2,batch_sz,all_ch,block_N_frames,ch_height,ch_width), device=self.device) + for ss in range(2): for cc in range(all_ch): tch = 0 if cc<3 else 1 # Sustained or transient cch = cc if cc<3 else 0 # Y, rg, yv From ab876a1a067119e484677dbab305ee112d017ff7 Mon Sep 17 00:00:00 2001 From: Kenny Chen Date: Wed, 1 Jul 2026 18:59:20 -0400 Subject: [PATCH 4/4] update README and metrics.md --- README.md | 4 ++-- metrics.md | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c9f120..bf6f295 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ The metric is explained in detail in: If you use the metric in your research, please cite the paper above. -# Other metrics: ColorVideoVDP-ML, PU-PSNR +# Other metrics: ColorVideoVDP-ML, ColorVideoVDP-tm, PU-PSNR -The repository also contains code for other metrics, such as PU-PSNR, or ColorVideoVDP-ML. See [metrics.md](/metrics.md) for more information. +The repository also contains code for other metrics, such as PU-PSNR, ColorVideoVDP-tm, or ColorVideoVDP-ML. See [metrics.md](/metrics.md) for more information. ## PyTorch quickstart 1. Start by installing [anaconda](https://docs.anaconda.com/anaconda/install/index.html) or [miniconda](https://docs.conda.io/en/latest/miniconda.html). Then, create a new environment for ColorVideoVDP and activate it: diff --git a/metrics.md b/metrics.md index 6e4f37a..8fdf6fc 100644 --- a/metrics.md +++ b/metrics.md @@ -6,6 +6,12 @@ The code can run one of the following metrics (selected with `--metrics` or `-m` The original ColorVideoVDP metric, as described in the [SIGGRAPH paper](https://doi.org/10.1145/3658144). +* `cvvdp-tm` + + This version of ColorVideoVDP works for stimuli scaled at much different absolute luminance range, e.g. due to tone mapping of HDR content. This extension is described in detail in our [SIGGRAPH 2026 paper](https://kenchen10.github.io/projects/tmometric/index.html). + + This metric is backward-compatible, and only requires specification of a new flag, `"diff_sensitivity": "on"`, in the `cvvdp_parameters.json`. + * `cvvdp-ml-saliency` [experimental] The extended version of ColorVideoVDP with a machine-learning based regressor and a saliency model, as explained in the [ICME paper](https://www.cl.cam.ac.uk/~rkm38/pdfs/hammou2025_ICME_GC_ColorVideoVDP_ML.pdf).