Convert events into voxels

[Ruplyn]

Thank you for sharing the data preparation details. I have created events for an image dataset, and the events.h5 file looks as follows:

events/ps/ : (7013772,) bool
events/ts/ : (7013772,) float64
events/xs/ : (7013772,) int16
events/ys/ : (7013772,) int16

Could you please share the code snippet that can convert these events into voxels in the format specified below?
voxels_b/000000/ : (5, 180, 320) float64
voxels_b/000001/ : (5, 180, 320) float64
voxels_b/000002/ : (5, 180, 320) float64
voxels_b/000003/ : (5, 180, 320) float64
voxels_b/000004/ : (5, 180, 320) float64

I tried using [events_contrast_maximization] but ended up generating voxels in a different format, which is incorrect. My dataset contains 26 images, but the voxel file only has 5 lines, and the tensor shape [bins, H, W] is also incorrect.

voxels_f/000000/ : (720, 1280) float64
voxels_f/000001/ : (720, 1280) float64
voxels_f/000002/ : (720, 1280) float64
voxels_f/000003/ : (720, 1280) float64
voxels_f/000004/ : (720, 1280) float64

The generated voxel file does not include all 26 images. Please assist. Thank you.

import random
import esim_py
import os
import matplotlib.pyplot as plt
import numpy as np
import torch

from abc import ABCMeta, abstractmethod
import h5py
import cv2 as cv
import numpy as np
from utils.event_utils import events_to_voxel_torch


image_folder = os.path.join(os.path.dirname(__file__), "data/frames/")
timestamps_file = os.path.join(os.path.dirname(__file__), "data/timestamps.txt")

config = {
    'refractory_period': 1e-4,
    'CT_range': [0.05, 0.5],
    'max_CT': 0.5,
    'min_CT': 0.02,
    'mu': 1,
    'sigma': 0.1,
	'H': 720,
	'W': 1280,
    'log_eps': 1e-3,
    'use_log': True,
}

Cp = random.uniform(config['CT_range'][0], config['CT_range'][1])
Cn = random.gauss(config['mu'], config['sigma']) * Cp
Cp = min(max(Cp, config['min_CT']), config['max_CT'])
Cn = min(max(Cn, config['min_CT']), config['max_CT'])
esim = esim_py.EventSimulator(Cp,
                            Cn,
                            config['refractory_period'],
                            config['log_eps'],
                            config['use_log'])

events = esim.generateFromFolder(image_folder, timestamps_file) # Generate events with shape [N, 4]

xs = torch.tensor(events[:, 0], dtype=torch.int16)
ys = torch.tensor(events[:, 1], dtype=torch.int16)
ts = torch.tensor(events[:, 2], dtype=torch.float32)  # Use float32 for consistency
ps = torch.tensor(events[:, 3], dtype=torch.bool)  


voxel_f = events_to_voxel_torch(xs, ys, ts, ps, 5, device=None, sensor_size=(720, 1280), temporal_bilinear=True)


output_h5_file = "voxel_fx.h5"  # Replace with desired output path
B = 5  # Number of bins in voxel grids
n = 1000  # Number of events per voxel (not sure what to set)
temporal_bilinear = True 

with h5py.File(output_h5_file, 'w') as output_file:
    # Save each voxel grid to HDF5 file with the specified format
    for i, voxel in enumerate(voxel_f):
        print(list(voxel.shape))
        voxel_name = f'voxels_f/{i:06d}'
        output_file.create_dataset(voxel_name, data=voxel.numpy().astype(np.float64))  # Convert to float64

    # Add metadata or attributes if needed
    output_file.attrs['num_voxels'] = len(voxel_f)
    output_file.attrs['B'] = B
    output_file.attrs['n'] = n

    print(f"Voxel data saved to {output_h5_file}")

[DachunKai]

I guess that your timestamp file is incorrect. If you have 26 images, the timestamp file should contain 26 lines, meaning each image timestamp. The generated events (N, 4) should have their timestamps (ts) range between the timestamp of the first frame and the timestamp of the last frame. Below is a snippet of my code for converting bidirectional voxel. Some functions can be seen in our DataPreparation.md. Hope it helps you. Thanks!

def package_bidirectional_event_voxels(self, x, y, t, p, timestamp_list, backward, bins, sensor_size, h5_name, error_txt):
    """
        params:
            x: ndarray, x-position of events
            y: ndarray, y-position of events
            t: ndarray, timestamp of events
            p: ndarray, polarity of events
            backward: bool, if forward or backward
            timestamp_list: list, to split events via timestamp
            bins: voxel num_bins
        returns:
            no return.
    """
    # Step 1: convert data type
    assert x.shape == y.shape == t.shape == p.shape

    x = torch.from_numpy(x.astype(np.int16))
    y = torch.from_numpy(y.astype(np.int16))
    t = torch.from_numpy(t.astype(np.float64))
    p = torch.from_numpy(p.astype(np.int16))

    assert x.shape == y.shape == t.shape == p.shape

    # Step 2: select events between two frames according to timestamp
    temp = t.numpy().tolist()
    output = [
        temp[
            bisect.bisect_left(temp, timestamp_list[i]):bisect.bisect_left(temp, timestamp_list[i+1])
        ]
        for i in range(len(timestamp_list) - 1)
    ]

    # Debug: Check if data error!!!
    assert len(output) == len(timestamp_list) - 1, f"len(output) is {len(output)}, but len(timestamp_list) is {len(timestamp_list)}"
    sum_output = []
    sum = 0
    for i in range(len(output)):
        if len(output[i]) == 0:
            raise ValueError(f"{h5_name} len(output[{i}] == 0)")
        elif len(output[i]) == 1:
            raise ValueError(f"{h5_name} len(output[{i}] == 1)") 
        sum += len(output[i])
        sum_output.append(sum)

    assert len(sum_output) == len(output)

    # Step 3: After checking data, continue.
    start_idx = 0
    for voxel_idx in range(len(timestamp_list) - 1):

        if len(output[voxel_idx]) == 0 or len(output[voxel_idx]) == 1:
            print(f'{h5_name} len(output[{voxel_idx}])): ', len(
                output[voxel_idx]))
            with open(error_txt, 'a+') as f:
                f.write(h5_name + '\n')
            return
        end_idx = start_idx + len(output[voxel_idx])
        

        if end_idx > len(t):
            with open(error_txt, 'a+') as f:
                f.write(f"{h5_name} voxel_idx: {voxel_idx}, start_idx {start_idx} end_idx {end_idx} exceed bound." + '\n')
            print(f"{h5_name} voxel_idx: {voxel_idx}, start_idx {start_idx} end_idx {end_idx} with exceed bound len(t) {len(t)}.")
            return
        
        xs = x[start_idx:end_idx]
        ys = y[start_idx:end_idx]
        ts = t[start_idx:end_idx]
        ps = p[start_idx:end_idx]
        if ts == torch.Size([]) or ts.shape == torch.Size([1]) or ts.shape == torch.Size([0]):
            with open(error_txt, 'a+') as f:
                f.write(f"{h5_name} len(output[{voxel_idx}]) backward {backward} start_idx {start_idx} end_idx {end_idx} is error! Please check the data." + '\n')
            print(f"{h5_name} len(output[{voxel_idx}]) backward {backward} start_idx {start_idx} end_idx {end_idx} is error! Please check the data.")
            return

        if backward:
            t_start = timestamp_list[voxel_idx]
            t_end = timestamp_list[voxel_idx + 1]
            xs = torch.flip(xs, dims=[0])
            ys = torch.flip(ys, dims=[0])
            ts = torch.flip(t_end - ts + t_start, dims=[0])
            ps = torch.flip(-ps, dims=[0])
        voxel = events_to_voxel_torch(
            xs, ys, ts, ps, bins, device=None, sensor_size=sensor_size)
        normed_voxel = voxel_normalization(voxel)
        np_voxel = normed_voxel.numpy()

        if backward:
            self.events_file.create_dataset("voxels_b/{:06d}".format(
                voxel_idx), data=np_voxel, dtype=np.dtype(np.float64), compression="gzip")
        else:
            self.events_file.create_dataset("voxels_f/{:06d}".format(
                voxel_idx), data=np_voxel, dtype=np.dtype(np.float64), compression="gzip")
        start_idx = end_idx

[Ruplyn]

Thank you. Issue was with the timestamp file. Fixed now

[hongsixin]

I guess that your timestamp file is incorrect. If you have 26 images, the timestamp file should contain 26 lines, meaning each image timestamp. The generated events (N, 4) should have their timestamps (ts) range between the timestamp of the first frame and the timestamp of the last frame. Below is a snippet of my code for converting bidirectional voxel. Some functions can be seen in our DataPreparation.md. Hope it helps you. Thanks!

def package_bidirectional_event_voxels(self, x, y, t, p, timestamp_list, backward, bins, sensor_size, h5_name, error_txt):
    """
        params:
            x: ndarray, x-position of events
            y: ndarray, y-position of events
            t: ndarray, timestamp of events
            p: ndarray, polarity of events
            backward: bool, if forward or backward
            timestamp_list: list, to split events via timestamp
            bins: voxel num_bins
        returns:
            no return.
    """
    # Step 1: convert data type
    assert x.shape == y.shape == t.shape == p.shape

    x = torch.from_numpy(x.astype(np.int16))
    y = torch.from_numpy(y.astype(np.int16))
    t = torch.from_numpy(t.astype(np.float64))
    p = torch.from_numpy(p.astype(np.int16))

    assert x.shape == y.shape == t.shape == p.shape

    # Step 2: select events between two frames according to timestamp
    temp = t.numpy().tolist()
    output = [
        temp[
            bisect.bisect_left(temp, timestamp_list[i]):bisect.bisect_left(temp, timestamp_list[i+1])
        ]
        for i in range(len(timestamp_list) - 1)
    ]

    # Debug: Check if data error!!!
    assert len(output) == len(timestamp_list) - 1, f"len(output) is {len(output)}, but len(timestamp_list) is {len(timestamp_list)}"
    sum_output = []
    sum = 0
    for i in range(len(output)):
        if len(output[i]) == 0:
            raise ValueError(f"{h5_name} len(output[{i}] == 0)")
        elif len(output[i]) == 1:
            raise ValueError(f"{h5_name} len(output[{i}] == 1)") 
        sum += len(output[i])
        sum_output.append(sum)

    assert len(sum_output) == len(output)

    # Step 3: After checking data, continue.
    start_idx = 0
    for voxel_idx in range(len(timestamp_list) - 1):

        if len(output[voxel_idx]) == 0 or len(output[voxel_idx]) == 1:
            print(f'{h5_name} len(output[{voxel_idx}])): ', len(
                output[voxel_idx]))
            with open(error_txt, 'a+') as f:
                f.write(h5_name + '\n')
            return
        end_idx = start_idx + len(output[voxel_idx])
        

        if end_idx > len(t):
            with open(error_txt, 'a+') as f:
                f.write(f"{h5_name} voxel_idx: {voxel_idx}, start_idx {start_idx} end_idx {end_idx} exceed bound." + '\n')
            print(f"{h5_name} voxel_idx: {voxel_idx}, start_idx {start_idx} end_idx {end_idx} with exceed bound len(t) {len(t)}.")
            return
        
        xs = x[start_idx:end_idx]
        ys = y[start_idx:end_idx]
        ts = t[start_idx:end_idx]
        ps = p[start_idx:end_idx]
        if ts == torch.Size([]) or ts.shape == torch.Size([1]) or ts.shape == torch.Size([0]):
            with open(error_txt, 'a+') as f:
                f.write(f"{h5_name} len(output[{voxel_idx}]) backward {backward} start_idx {start_idx} end_idx {end_idx} is error! Please check the data." + '\n')
            print(f"{h5_name} len(output[{voxel_idx}]) backward {backward} start_idx {start_idx} end_idx {end_idx} is error! Please check the data.")
            return

        if backward:
            t_start = timestamp_list[voxel_idx]
            t_end = timestamp_list[voxel_idx + 1]
            xs = torch.flip(xs, dims=[0])
            ys = torch.flip(ys, dims=[0])
            ts = torch.flip(t_end - ts + t_start, dims=[0])
            ps = torch.flip(-ps, dims=[0])
        voxel = events_to_voxel_torch(
            xs, ys, ts, ps, bins, device=None, sensor_size=sensor_size)
        normed_voxel = voxel_normalization(voxel)
        np_voxel = normed_voxel.numpy()

        if backward:
            self.events_file.create_dataset("voxels_b/{:06d}".format(
                voxel_idx), data=np_voxel, dtype=np.dtype(np.float64), compression="gzip")
        else:
            self.events_file.create_dataset("voxels_f/{:06d}".format(
                voxel_idx), data=np_voxel, dtype=np.dtype(np.float64), compression="gzip")
        start_idx = end_idx

@DachunKai Hi，when I was processing voxels_f through this code, I encountered an error: ValueError: 000.h5 len(output[0] == 0) or ValueError: 000.h5 len(output[0] == 1). What error is this? Do each image need to have more than 2 events？My code is：

timestamp_list= [i/fps for i in range(len(images))]
events = esim.generateFromFolder(image_folder, timestamps_file)
timestamp_list= [i/fps for i in range(os.listdir(image_folder))]
backward, bins, sensor_size = True, 5, image_size
package_bidirectional_event_voxels(events[:, 0], events[:, 1], events[:, 2], events[:, 3], timestamp_list, backward, bins, sensor_size, h5_name, error_txt)

[DachunKai]

@hongsixin Yes, in our paper, Section 3.1 Equation (1), if there is only one event between two frames, then in Equation (1) $t_{Ne} = t_{0}$, which causes the denominator to be zero. Consequently, the voxel cannot be successfully converted.