sid,eld,sidd dataset介绍,dng图像处理

文章目录

- SID dataset
- - 1. SID dataset 概述
  - 2. SID 读取和显示代码
  - 3. 一些示例
- SIDD dataset
- ELD dataset
- DNG camera pipeline

SID dataset

1. SID dataset 概述

SID 是Learning to See in the Dark 论文中提出的暗光raw数据集

其中包括两个相机的拍摄数据 Sony alpha7S II 和 Fujifilm X-T2.
下面主要介绍下sony camera的数据

拍摄照度：
室外 0.2-5 lux
室内 0.03-0.3 lux

一共包括short和long两个文件夹：
long文件夹共231个ARW文件，其中20208__00_10s.ARW表示的是场景20208，第00次拍摄，曝光时间是10s
在这里插入图片描述

short文件夹对应的短曝光的拍摄图像, 一共2697张图像
short的曝光时间设置为参考图像的 1/100, 1/250, 1/300。比如参考groundtruth 曝光是10s, 则short图像的曝光可能被设置为 0.1s, 0.04s, 0.033s。相同设置可能被拍摄多张，可以用来开发 burst multiframe相关算法。
在这里插入图片描述

2. SID 读取和显示代码

利用rawpy 库
在这里插入图片描述

import glob
import osimport matplotlib.pyplot as plt
import numpy as np
import rawpyimport colour
from colour_demosaicing import demosaicing_CFA_Bayer_Menon2007
def pack_raw(raw):# pack Bayer image to 4 channelsim = raw.raw_image_visible.astype(np.float32)im = np.maximum(im - 512, 0) / (16383 - 512)  # subtract the black levelim = np.expand_dims(im, axis=2)img_shape = im.shapeH = img_shape[0]W = img_shape[1]out = np.concatenate((im[0:H:2, 0:W:2, :],im[0:H:2, 1:W:2, :],im[1:H:2, 1:W:2, :],im[1:H:2, 0:W:2, :]), axis=2)return outdef pack_raw_bayer(raw):# 和上面的函数功能一样，都是减去black level，然后分成4 channnel图像# pack Bayer image to 4 channelsim = raw.raw_image_visible.astype(np.float32)raw_pattern = raw.raw_patternR = np.where(raw_pattern == 0)G1 = np.where(raw_pattern == 1)B = np.where(raw_pattern == 2)G2 = np.where(raw_pattern == 3)white_point = 16383img_shape = im.shapeH = img_shape[0]W = img_shape[1]out = np.stack((im[R[0][0]:H:2, R[1][0]:W:2],  # RGBGim[G1[0][0]:H:2, G1[1][0]:W:2],im[B[0][0]:H:2, B[1][0]:W:2],im[G2[0][0]:H:2, G2[1][0]:W:2]), axis=0).astype(np.float32)black_level = np.array(raw.black_level_per_channel)[:, None, None].astype(np.float32)out = (out - black_level) / (white_point - black_level)out = np.clip(out, 0, 1)return out
if __name__ == "__main__":input_dir = r'D:\dataset\ELD\sid\Sony\Sony\short'gt_dir = r'D:\dataset\ELD\sid\Sony\Sony\long'train_id = 1in_files = glob.glob(os.path.join(input_dir, '%05d_00*.ARW' % train_id))gt_files = glob.glob(os.path.join(gt_dir, '%05d_00*.ARW' % train_id))print(in_files, gt_files)in_path = in_files[0]gt_path = gt_files[0]print(in_path, gt_path)# 获取曝光时间in_fn = os.path.basename(in_path)gt_fn = os.path.basename(gt_path)in_exposure = float(in_fn[9:-5])gt_exposure = float(gt_fn[9:-5])ratio = min(gt_exposure / in_exposure, 300)print('exp time:', in_exposure, gt_exposure, ratio)raw = rawpy.imread(in_path)gt_raw = rawpy.imread(gt_path)print('raw meta info :', raw.black_level_per_channel, raw.raw_pattern)im = raw.raw_image_visible.astype(np.float32)im = np.maximum(im - 512, 0) / (16383 - 512) * ratio # subtract the black levelim1 = demosaicing_CFA_Bayer_Menon2007(im, 'RGGB')im11 = raw.postprocess(use_camera_wb=True, half_size=False, no_auto_bright=True, output_bps=8)im2 = gt_raw.postprocess(use_camera_wb=True, half_size=False, no_auto_bright=True, output_bps=8)# im1 raw数据乘上一个ratio, 然后 demosaicing, 显示绿色的raw图# im11 对raw数据 应用isp, 由于曝光时间短, 没有乘上ratio, 可能显示sRGB 全黑# im2 是groundtruth 参考图，曝光充分，正常显示sRGB图plt.figure()plt.subplot(131)plt.imshow(im1)plt.subplot(132)plt.imshow(im11)plt.subplot(133)plt.imshow(im2)plt.show()# noise imageim11 = im11 / im11.max()im11 = im11 * ratioim11 = np.clip(im11, 0, 1)plt.figure()plt.imshow(im11)plt.show()# process allgt_files = glob.glob(os.path.join(gt_dir, '*.ARW'))for file in gt_files:print(file)gt_raw = rawpy.imread(file)im2 = gt_raw.postprocess(use_camera_wb=True, half_size=False, no_auto_bright=True, output_bps=8)cv2.imwrite(file[:-4] + '.png', im2[..., ::-1])

3. 一些示例

在这里插入图片描述

gt 和 noise:
在这里插入图片描述

将中间的黑色图乘上ratio显示noise image(只是为了显示，这样操作是不对的):
在这里插入图片描述

SIDD dataset

link:https://www.eecs.yorku.ca/~kamel/sidd/dataset.php
论文 A High-Quality Denoising Dataset for Smartphone Cameras 提出的一个数据集

利用多张有噪声图像，利用对齐技术和fusion技术，生成无噪声图像作为ground truth.

官方网站上有 small , medium, full三个版本的数据集。我下载了medium的数据集， full的太大内存。

medium 版本包含raw 和 sRGB两个文件夹
在这里插入图片描述

命名方式：

<scene-instance-number>_<scene_number>_<smartphone-code>_<ISO-level>_<shutter-speed>_<illuminant-temperature>_<illuminant-brightness-code>

raw:
160 x (2 + 2) = 640 张图像
sRGB:
对应raw的 640张 sRGB 图像。
下载的数据集中有详细说明。

对应的mat数据， wb, ccm, gamma后的图像如下：
在这里插入图片描述

code:

import colour_demosaicing
import scipy.io as sio
import glob
import osimport cv2
import matplotlib.pyplot as plt
import numpy as np
import rawpyimport colour
from colour_demosaicing import demosaicing_CFA_Bayer_Menon2007
import h5pydef extract_metainfo(path='0151_METADATA_RAW_010.MAT'):meta = sio.loadmat(path)['metadata']mat_vals = meta[0][0]mat_keys = mat_vals.dtype.descrkeys = []for item in mat_keys:keys.append(item[0])py_dict = {}for key in keys:py_dict[key] = mat_vals[key]device = py_dict['Model'][0].lower()bitDepth = py_dict['BitDepth'][0][0]if 'iphone' in device or bitDepth != 16:noise = py_dict['UnknownTags'][-2][0][-1][0][:2]iso = py_dict['DigitalCamera'][0, 0]['ISOSpeedRatings'][0][0]pattern = py_dict['SubIFDs'][0][0]['UnknownTags'][0][0][1][0][-1][0]time = py_dict['DigitalCamera'][0, 0]['ExposureTime'][0][0]else:noise = py_dict['UnknownTags'][-1][0][-1][0][:2]iso = py_dict['ISOSpeedRatings'][0][0]pattern = py_dict['UnknownTags'][1][0][-1][0]time = py_dict['ExposureTime'][0][0]  # the 0th row and 0th line itemrgb = ['R', 'G', 'B']pattern = ''.join([rgb[i] for i in pattern])asShotNeutral = py_dict['AsShotNeutral'][0]b_gain, _, r_gain = asShotNeutral# only load ccm1ccm = py_dict['ColorMatrix1'][0].astype(float).reshape((3, 3))return {'device': device,'pattern': pattern,'iso': iso,'noise': noise,'time': time,'wb': np.array([r_gain, 1, b_gain]),'ccm': ccm, }def extract_metainfo2(file):meta = sio.loadmat(file)['metadata']mat_vals = meta[0][0]mat_keys = mat_vals.dtype.descrkeys = []for item in mat_keys:keys.append(item[0])py_dict = {}for key in keys:py_dict[key] = mat_vals[key]return py_dictdef fix_orientation(image, orientation):# 1 = Horizontal(normal)# 2 = Mirror horizontal# 3 = Rotate 180# 4 = Mirror vertical# 5 = Mirror horizontal and rotate 270 CW# 6 = Rotate 90 CW# 7 = Mirror horizontal and rotate 90 CW# 8 = Rotate 270 CWif type(orientation) is list:orientation = orientation[0]if orientation == 1:passelif orientation == 2:image = cv2.flip(image, 0)elif orientation == 3:image = cv2.rotate(image, cv2.ROTATE_180)elif orientation == 4:image = cv2.flip(image, 1)elif orientation == 5:image = cv2.flip(image, 0)image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)elif orientation == 6:image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)elif orientation == 7:image = cv2.flip(image, 0)image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)elif orientation == 8:image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)return imagedef process_mat_img(file, py_dict, pattern='bggr'):''':param file: 输入 mat 文件:return: srgb image'''data = {}f = h5py.File(file)for k, v in f.items():data[k] = np.array(v)data0 = data['x']  # [1000:2000,2000:3000]data = fix_orientation(data0, py_dict['Orientation'])rgb = colour_demosaicing.demosaicing_CFA_Bayer_Menon2007(data, pattern)wb_gain = 1 / py_dict['AsShotNeutral']wb_gain = wb_gain.astype(np.float32).reshape(-1, 1, 3)rgb_wb = rgb * wb_gainrgb_wb = np.clip(rgb_wb, 0, 1)xyz2cam1 = np.reshape(np.asarray(py_dict['ColorMatrix1']),(3, 3))  # 不同光源的标定矩阵，这里xyz2cam1应该是D65， 对应py_dict['CalibrationIlluminant1']xyz2cam2 = np.reshape(np.asarray(py_dict['ColorMatrix2']), (3, 3))# normalize rows (needed?)xyz2cam1 = xyz2cam1 / np.sum(xyz2cam1, axis=1, keepdims=True)xyz2cam2 = xyz2cam2 / np.sum(xyz2cam1, axis=1, keepdims=True)# inversecam2xyz1 = np.linalg.inv(xyz2cam1)cam2xyz2 = np.linalg.inv(xyz2cam2)# for now, use one matrix  # TODO: interpolate btween bothrgb_xyz = rgb_wb.reshape(-1, 3) @ cam2xyz1.Trgb_xyz = rgb_xyz.reshape(rgb_wb.shape)rgb_xyz = np.clip(rgb_xyz, 0.0, 1.0)xyz2srgb = np.array([[3.2404542, -1.5371385, -0.4985314],[-0.9692660, 1.8760108, 0.0415560],[0.0556434, -0.2040259, 1.0572252]])# normalize rows (needed?)rgb_ccm = rgb_xyz.reshape(-1, 3) @ xyz2srgb.Trgb_ccm = rgb_ccm.reshape(rgb_wb.shape)rgb_ccm = np.clip(rgb_ccm, 0.0, 1.0)rgb_gamma = rgb_ccm ** (1 / 2.2)rgb_gamma = np.clip(rgb_gamma, 0, 1)# rgb_gamma_save = np.clip(rgb_gamma * 255, 0, 255).astype(np.uint8)# cv2.imwrite('dd.png', rgb_gamma_save[::-1, :, ::-1])plt.figure()plt.subplot(221)plt.imshow(rgb)plt.subplot(222)plt.imshow(rgb_wb)plt.subplot(223)plt.imshow(rgb_ccm)plt.subplot(224)plt.imshow(rgb_gamma)plt.show()if __name__ == "__main__":file1 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_GT_RAW_010.MAT'file2 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_GT_RAW_011.MAT'file3 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_METADATA_RAW_010.MAT'file4 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_METADATA_RAW_011.MAT'file5 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_NOISY_RAW_010.MAT'file6 = r'D:\dataset\SIDD_Medium_Raw_Parts\SIDD_Medium_Raw\Data\0055_003_N6_00800_01000_5500_N\0055_NOISY_RAW_011.MAT'metainfo = extract_metainfo(file3)print(metainfo)py_dict = extract_metainfo2(file3)# isp: wb, ccm, gammaprocess_mat_img(file5, py_dict, metainfo['pattern'])print('py_dict info:', py_dict)print(py_dict['AsShotNeutral'], py_dict['ColorMatrix1'], py_dict['ColorMatrix2'])print(py_dict['CalibrationIlluminant1'], py_dict['CalibrationIlluminant2'])print(py_dict['Orientation'])print(py_dict['Height'], py_dict['Width'], py_dict['BitDepth'])