brimfile.data

  1import numpy as np
  2import asyncio
  3
  4import warnings
  5
  6from .file_abstraction import FileAbstraction, sync, _async_getitem, _gather_sync
  7from .utils import concatenate_paths, list_objects_matching_pattern, get_object_name, set_object_name
  8from .utils import np_array_to_smallest_int_type, _guess_chunks
  9
 10from .metadata import Metadata
 11
 12from numbers import Number
 13
 14from . import units
 15from .analysis_results import AnalysisResults
 16from .constants import brim_obj_names
 17
 18__docformat__ = "google"
 19
 20
 21class Data:
 22    """
 23    Represents a data group within the brim file.
 24    """
 25    # make AnalysisResults available as an attribute of Data
 26    AnalysisResults = AnalysisResults
 27
 28    def __init__(self, file: FileAbstraction, path: str, *, newly_created = False):
 29        """
 30        Initialize the Data object. This constructor should not be called directly.
 31
 32        Args:
 33            file (File): The parent File object.
 34            path (str): The path to the data group within the file.
 35            newly_created (bool): Whether this data group is being created as new.
 36                            If True, the constructor will not attempt to load spatial mapping.
 37        """
 38        self._file = file
 39        self._path = path
 40        self._group = sync(file.open_group(path))
 41
 42        self._sparse = self._load_sparse_flag()
 43        # the _spatial_map is None for non sparse data but the _spatial_map_px_size should always be valid
 44        self._spatial_map, self._spatial_map_px_size = self._load_spatial_mapping() if not newly_created else (None, None)
 45
 46    def get_name(self):
 47        """
 48        Returns the name of the data group.
 49        """
 50        return sync(get_object_name(self._file, self._path))
 51    
 52    def get_index(self):
 53        """
 54        Returns the index of the data group.
 55        """
 56        return int(self._path.split('/')[-1].split('_')[-1])
 57
 58    def _load_sparse_flag(self) -> bool:
 59        """
 60        Load the 'Sparse' flag for the data group.
 61
 62        Returns:
 63            bool: The value of the 'Sparse' flag, or False if the attribute is not found or invalid.
 64        """
 65        try:
 66            sparse = sync(self._file.get_attr(self._group, 'Sparse'))
 67            if isinstance(sparse, bool):
 68                return sparse
 69            else:
 70                warnings.warn(
 71                    f"Invalid value for 'Sparse' attribute in {self._path}. Expected a boolean, got {type(sparse)}. Defaulting to False.")
 72                return False
 73        except Exception:
 74            # if the attribute is not found, return the default value False
 75            return False
 76
 77    def _load_spatial_mapping(self, load_in_memory: bool=True) -> tuple:
 78        """
 79        Load a spatial mapping in the same format as 'Cartesian visualisation',
 80        irrespectively on whether 'Spatial_map' is defined instead.
 81        -1 is used for "empty" pixels in the image
 82        Args:
 83            load_in_memory (bool): Specify whether the map should be forced to load in memory or just opened as a dataset.
 84        Returns:
 85            The spatial map and the corresponding pixel size as a tuple of 3 Metadata.Item, both in the order z, y, x.
 86            If the spatial mapping is not defined in the file, returns None for the spatial map.
 87            The pixel size is read from the data group for non-sparse data.
 88        """
 89        cv = None
 90        px_size = 3*(Metadata.Item(value=1, units=None),)
 91
 92        cv_path = concatenate_paths(
 93            self._path, brim_obj_names.data.cartesian_visualisation)
 94        sm_path = concatenate_paths(
 95            self._path, brim_obj_names.data.spatial_map)
 96        
 97        if sync(self._file.object_exists(cv_path)):
 98            cv = sync(self._file.open_dataset(cv_path))
 99
100            #read the pixel size from the 'Cartesian visualisation' dataset
101            px_size_val = None
102            px_size_units = None
103            try:
104                px_size_val = sync(self._file.get_attr(cv, 'element_size'))
105                if px_size_val is None or len(px_size_val) != 3:
106                    raise ValueError(
107                        "The 'element_size' attribute of 'Cartesian_visualisation' must be a tuple of 3 elements")
108            except Exception:
109                px_size_val = 3*(1,)
110                warnings.warn(
111                    "No pixel size defined for Cartesian visualisation")            
112            px_size_units = sync(units.of_attribute(
113                    self._file, cv, 'element_size'))
114            px_size = ()
115            for i in range(3):
116                # if px_size_val[i] is not a number, set it to 1 and px_size_units to None
117                if isinstance(px_size_val[i], Number):
118                    px_size += (Metadata.Item(px_size_val[i], px_size_units), )
119                else:
120                    px_size += (Metadata.Item(1, None), )
121                    
122
123            if load_in_memory:
124                cv = np.array(cv)
125                cv = np_array_to_smallest_int_type(cv)
126
127        elif sync(self._file.object_exists(sm_path)):
128            def load_spatial_map_from_file(self):
129                async def load_coordinate_from_sm(coord: str):
130                    res = np.empty(0)  # empty array
131                    try:
132                        res = await self._file.open_dataset(
133                            concatenate_paths(sm_path, coord))
134                        res = await res.to_np_array()
135                        res = np.squeeze(res)  # remove single-dimensional entries
136                    except Exception as e:
137                        # if the coordinate does not exist, return an empty array
138                        pass
139                    if len(res.shape) > 1:
140                        raise ValueError(
141                            f"The 'Spatial_map/{coord}' dataset is not a 1D array as expected")
142                    return res
143
144                def check_coord_array(arr, size):
145                    if arr.size == 0:
146                        return np.zeros(size)
147                    elif arr.size != size:
148                        raise ValueError(
149                            "The 'Spatial_map' dataset is invalid")
150                    return arr
151
152                x, y, z = _gather_sync(
153                    load_coordinate_from_sm('x'),
154                    load_coordinate_from_sm('y'),
155                    load_coordinate_from_sm('z')
156                    )
157                size = max([x.size, y.size, z.size])
158                if size == 0:
159                    raise ValueError("The 'Spatial_map' dataset is empty")
160                x = check_coord_array(x, size)
161                y = check_coord_array(y, size)
162                z = check_coord_array(z, size)
163                return x, y, z
164
165            def calculate_step(x):
166                n = len(np.unique(x))
167                if n == 1:
168                    d = None
169                else:
170                    d = (np.max(x)-np.min(x))/(n-1)
171                return n, d
172
173            x, y, z = load_spatial_map_from_file(self)
174
175            # TODO extend the reconstruction to non-cartesian cases
176
177            nX, dX = calculate_step(x)
178            nY, dY = calculate_step(y)
179            nZ, dZ = calculate_step(z)
180
181            indices = np_array_to_smallest_int_type(np.lexsort((x, y, z)))
182            cv = np.reshape(indices, (nZ, nY, nX))
183
184            px_size_units = sync(units.of_object(self._file, sm_path))
185            px_size = ()
186            for i in range(3):
187                px_sz = (dZ, dY, dX)[i]
188                px_unit = px_size_units
189                if px_sz is None:
190                    px_sz = 1
191                    px_unit = None
192                px_size += (Metadata.Item(px_sz, px_unit),)
193        elif not self._sparse:
194            try:
195                px_sz = sync(self._file.get_attr(self._group, 'element_size'))
196                if len(px_sz) != 3:
197                    raise ValueError(
198                        "The 'element_size' attribute must be a tuple of 3 elements")
199                px_unit = None
200                try:
201                    px_unit = sync(units.of_attribute(self._file, self._group, 'element_size'))
202                except Exception:
203                    warnings.warn("Pixel size unit is not provided for non-sparse data.")
204                px_size = tuple(Metadata.Item(el, px_unit) for el in px_sz)
205            except Exception:
206                warnings.warn("Pixel size is not provided for non-sparse data.")
207
208        return cv, px_size
209
210    def get_PSD(self) -> tuple:
211        """
212        LOW LEVEL FUNCTION
213
214        Retrieve the Power Spectral Density (PSD) and frequency from the current data group.
215        Note: this function exposes the internals of the brim file and thus the interface might change in future versions.
216        Use only if more specialized functions are not working for your application!
217        Returns:
218            tuple: (PSD, frequency, PSD_units, frequency_units)
219                - PSD: A 2D (or more) numpy array containing all the spectra (see [specs](https://github.com/prevedel-lab/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
220                - frequency: A numpy array representing the frequency data (see [specs](https://github.com/prevedel-lab/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
221                - PSD_units: The units of the PSD.
222                - frequency_units: The units of the frequency.
223        """
224        warnings.warn(
225            "Data.get_PSD is deprecated and will be removed in a future release. "
226            "Use Data.get_PSD_as_spatial_map instead.",
227            DeprecationWarning,
228            stacklevel=2,
229        )
230        PSD, frequency = _gather_sync(
231            self._file.open_dataset(concatenate_paths(
232                self._path, brim_obj_names.data.PSD)),
233            self._file.open_dataset(concatenate_paths(
234                self._path, brim_obj_names.data.frequency))
235        )
236        # retrieve the units of the PSD and frequency
237        PSD_units, frequency_units = _gather_sync(
238            units.of_object(self._file, PSD),
239            units.of_object(self._file, frequency)
240        )
241
242        return PSD, frequency, PSD_units, frequency_units
243    
244    def get_PSD_as_spatial_map(self) -> tuple:
245        """
246        Retrieve the Power Spectral Density (PSD) as a spatial map and the frequency from the current data group.
247        Returns:
248            tuple: (PSD, frequency, PSD_units, frequency_units)
249                - PSD: A 4D (or more) numpy array containing all the spectra. Dimensions are z, y, x, [parameters], spectrum.
250                - frequency: A numpy array representing the frequency data, which can be broadcastable to PSD.
251                - PSD_units: The units of the PSD.
252                - frequency_units: The units of the frequency.
253        """
254        PSD, frequency = _gather_sync(
255            self._file.open_dataset(concatenate_paths(
256                self._path, brim_obj_names.data.PSD)),        
257            self._file.open_dataset(concatenate_paths(
258                self._path, brim_obj_names.data.frequency))
259            )        
260        # retrieve the units of the PSD and frequency
261        PSD_units, frequency_units = _gather_sync(
262            units.of_object(self._file, PSD),
263            units.of_object(self._file, frequency)
264        )
265
266        # ensure PSD and frequency are numpy arrays
267        PSD = np.array(PSD)  
268        frequency = np.array(frequency)  # ensure it's a numpy array
269        
270        #if the frequency is not the same for all spectra, broadcast it to match the shape of PSD
271        if frequency.ndim > 1:
272            frequency = np.broadcast_to(frequency, PSD.shape)
273        
274        if self._sparse:
275            if self._spatial_map is None:
276                raise ValueError("The data is defined as sparse, but no spatial mapping is provided.")
277            sm = np.array(self._spatial_map)
278            # reshape the PSD to have the spatial dimensions first      
279            PSD = PSD[sm, ...]
280            # reshape the frequency only if it is not the same for all spectra
281            if frequency.ndim > 1:
282                frequency = frequency[sm, ...]
283
284        return PSD, frequency, PSD_units, frequency_units
285
286    def _get_spectrum(self, index: int | tuple[int, int, int]) -> tuple:
287        """
288        Synchronous wrapper for `_get_spectrum_async` (see doc for `brimfile.data.Data._get_spectrum_async`)
289        """
290        return sync(self._get_spectrum_async(index))
291    async def _get_spectrum_async(self, index: int | tuple[int, int, int]) -> tuple:
292        """
293        Retrieve a spectrum from the data group by its index or coordinates.
294
295        Args:
296            index (int | tuple[int, int, int]): The index (for sparse data) or z, y, x coordinates (for non-sparse data) of the spectrum to retrieve.
297
298        Returns:
299            tuple: (PSD, frequency, PSD_units, frequency_units) for the specified index. 
300                    PSD can be 1D or more (if there are additional parameters);
301                    frequency has the same size as PSD
302        Raises:
303            IndexError: If the index is out of range for the PSD dataset.
304        """
305        if self._sparse and not isinstance(index, int):
306            raise ValueError("For sparse data, index must be an integer.")
307        elif not self._sparse and not (isinstance(index, tuple) and len(index) == 3):
308            raise ValueError("For non-sparse data, index must be a tuple of (z, y, x) coordinates.")
309            
310        # index = -1 corresponds to no spectrum
311        if self._sparse and index < 0:
312            return None, None, None, None
313        elif not self._sparse and any(i < 0 for i in index):
314            return None, None, None, None
315        PSD, frequency = await asyncio.gather(
316            self._file.open_dataset(concatenate_paths(
317                self._path, brim_obj_names.data.PSD)),                       
318            self._file.open_dataset(concatenate_paths(
319                self._path, brim_obj_names.data.frequency))
320            )
321        if self._sparse and index >= PSD.shape[0]:
322            raise IndexError(
323                f"index {index} out of range for PSD with shape {PSD.shape}")
324        elif not self._sparse and any(i >= PSD.shape[j] for j, i in enumerate(index)):
325            raise IndexError(
326                f"index {index} out of range for PSD with shape {PSD.shape}")
327        # retrieve the units of the PSD and frequency
328        PSD_units, frequency_units = await asyncio.gather(
329            units.of_object(self._file, PSD),
330            units.of_object(self._file, frequency)
331        )
332        # add ellipsis to the index to select the spectrum and the corresponding frequency
333        if self._sparse:
334            index = (index, ...)
335        else:
336            index = index + (..., )
337        # map index to the frequency array, considering the broadcasting rules
338        index_frequency = index
339        if frequency.ndim < PSD.ndim:
340            if self._sparse:
341                # given the definition of the brim file format,
342                # if the frequency has less dimensions that PSD,
343                # it can only be because it is the same for all the spatial position (first dimension)
344                index_frequency = (..., )
345            else:
346                unassigned_indices = PSD.ndim - frequency.ndim
347                if unassigned_indices == 3:
348                    # if the frequency has no spatial dimension, it is the same for all the spatial positions
349                    index_frequency = (..., )
350                else:
351                    # if the frequency has some spatial dimensions but not all, we need to add the corresponding indices to the index of the frequency
352                    index_frequency = index[-unassigned_indices:] + (..., )
353        #get the spectrum and the corresponding frequency at the specified index
354        PSD, frequency = await asyncio.gather(
355            _async_getitem(PSD, index),
356            _async_getitem(frequency, index_frequency)
357        )
358        #broadcast the frequency to match the shape of PSD if needed
359        if frequency.ndim < PSD.ndim:
360            frequency = np.broadcast_to(frequency, PSD.shape)
361        return PSD, frequency, PSD_units, frequency_units
362
363    def get_spectrum_in_image(self, coor: tuple) -> tuple:
364        """
365        Retrieve a spectrum from the data group using spatial coordinates.
366
367        Args:
368            coor (tuple): A tuple containing the z, y, x coordinates of the spectrum to retrieve.
369
370        Returns:
371            tuple: A tuple containing the PSD, frequency, PSD_units, frequency_units for the specified coordinates. See `Data._get_spectrum_async` for details.
372        """
373        if len(coor) != 3:
374            raise ValueError("coor must contain 3 values for z, y, x")
375
376        if self._sparse:
377            index = int(self._spatial_map[coor])
378            return self._get_spectrum(index)
379        else:
380            return self._get_spectrum(coor)
381          
382    def get_spectrum_and_all_quantities_in_image(self, ar: 'Data.AnalysisResults', coor: tuple, index_peak: int = 0):
383        """
384        Retrieve the spectrum and all available quantities from the analysis results at a specific spatial coordinate.
385
386        Args:
387            ar (Data.AnalysisResults): The analysis results object to retrieve quantities from.
388            coor (tuple): A tuple containing the z, y, x coordinates in the image.
389            index_peak (int, optional): The index of the peak to retrieve (for multi-peak fits). Defaults to 0.
390
391        Returns:
392            tuple: A tuple containing:
393                - spectrum (tuple): (PSD, frequency, PSD_units, frequency_units) at the specified coordinate
394                - quantities (dict): Dictionary of Metadata.Item in the form result[quantity.name][peak.name]
395        """
396        if len(coor) != 3:
397            raise ValueError("coor must contain 3 values for z, y, x")
398        index = coor
399        if self._sparse:
400            index = int(self._spatial_map[coor])
401        spectrum, quantities = _gather_sync(
402            self._get_spectrum_async(index),
403            ar._get_all_quantities_at_index(index, index_peak)
404        )
405        return spectrum, quantities
406
407    def get_metadata(self):
408        """
409        Returns the metadata associated with the current Data group
410        Note that this contains both the general metadata stored in the file (which might be redifined by the specific data group)
411        and the ones specific for this data group
412        """
413        return Metadata(self._file, self._path)
414
415    def get_num_parameters(self) -> tuple:
416        """
417        Retrieves the number of parameters
418
419        Returns:
420            tuple: The shape of the parameters if they exist, otherwise an empty tuple.
421        """
422        pars, _ = self.get_parameters()
423        return pars.shape if pars is not None else ()
424
425    def get_parameters(self) -> list:
426        """
427        Retrieves the parameters  and their associated names.
428
429        If PSD.ndims > 2, the parameters are stored in a separate dataset.
430
431        Returns:
432            list: A tuple containing the parameters and their names if there are any, otherwise None.
433        """
434        pars_full_path = concatenate_paths(
435            self._path, brim_obj_names.data.parameters)
436        if sync(self._file.object_exists(pars_full_path)):
437            pars = sync(self._file.open_dataset(pars_full_path))
438            pars_names = sync(self._file.get_attr(pars, 'Name'))
439            return (pars, pars_names)
440        return (None, None)
441
442    def create_analysis_results_group(self, data_AntiStokes, data_Stokes=None, *,
443                                          index: int = None, name: str = None, fit_model: 'Data.AnalysisResults.FitModel' = None) -> AnalysisResults:
444        """
445        Adds a new AnalysisResults entry to the current data group.
446        Parameters:
447            data_AntiStokes (dict or list[dict]): see documentation for `brimfile.analysis_results.AnalysisResults.add_data`
448            data_Stokes (dict or list[dict]): same as data_AntiStokes for the Stokes peaks.
449            index (int, optional): The index for the new data entry. If None, the next available index is used. Defaults to None.
450            name (str, optional): The name for the new Analysis group. Defaults to None.
451            fit_model (Data.AnalysisResults.FitModel, optional): The fit model used for the analysis. Defaults to None (no attribute is set).
452        Returns:
453            AnalysisResults: The newly created AnalysisResults object.
454        Raises:
455            IndexError: If the specified index already exists in the dataset.
456            ValueError: If any of the data provided is not valid or consistent
457        """
458        if index is not None:
459            try:
460                self.get_analysis_results(index)
461            except IndexError:
462                pass
463            else:
464                # If the group already exists, raise an error
465                raise IndexError(
466                    f"Analysis {index} already exists in {self._path}")
467        else:
468            ar_groups = self.list_AnalysisResults()
469            indices = [ar['index'] for ar in ar_groups]
470            indices.sort()
471            index = indices[-1] + 1 if indices else 0  # Next available index
472
473        ar = Data.AnalysisResults._create_new(self, index=index, sparse=self._sparse)
474        if name is not None:
475            set_object_name(self._file, ar._path, name)
476        ar.add_data(data_AntiStokes, data_Stokes, fit_model=fit_model)
477
478        return ar
479
480    def list_AnalysisResults(self, retrieve_custom_name=False) -> list:
481        """
482        List all AnalysisResults groups in the current data group. The list is ordered by index.
483
484        Returns:
485            list: A list of dictionaries, each containing:
486                - 'name' (str): The name of the AnalysisResults group.
487                - 'index' (int): The index extracted from the group name.
488                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the AnalysisResults group as returned from utils.get_object_name.
489        """
490
491        analysis_results_groups = []
492
493        matched_objs = list_objects_matching_pattern(
494            self._file, self._group, brim_obj_names.data.analysis_results + r"_(\d+)$")
495        async def _make_dict_item(matched_obj, retrieve_custom_name):
496            name = matched_obj[0]
497            index = int(matched_obj[1])
498            curr_obj_dict = {'name': name, 'index': index}
499            if retrieve_custom_name:
500                ar_path = concatenate_paths(self._path, name)
501                custom_name = await get_object_name(self._file, ar_path)
502                curr_obj_dict['custom_name'] = custom_name
503            return curr_obj_dict
504        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
505        dicts = _gather_sync(*coros)
506        for dict_item in dicts:
507            analysis_results_groups.append(dict_item)
508        # Sort the data groups by index
509        analysis_results_groups.sort(key=lambda x: x['index'])
510
511        return analysis_results_groups
512
513    def get_analysis_results(self, index: int = 0) -> AnalysisResults:
514        """
515        Returns the AnalysisResults at the specified index
516
517        Args:
518            index (int)                
519
520        Raises:
521            IndexError: If there is no analysis with the corresponding index
522        """
523        name = None
524        ls = self.list_AnalysisResults()
525        for el in ls:
526            if el['index'] == index:
527                name = el['name']
528                break
529        if name is None:
530            raise IndexError(f"Analysis {index} not found")
531        path = concatenate_paths(self._path, name)
532        return Data.AnalysisResults(self._file, path, data_group_path=self._path,
533                                    spatial_map=self._spatial_map, spatial_map_px_size=self._spatial_map_px_size, sparse=self._sparse)
534
535    def _add_data(self, PSD: np.ndarray, frequency: np.ndarray, *, scanning: dict = None, freq_units='GHz',
536                  timestamp: np.ndarray = None, compression: FileAbstraction.Compression = FileAbstraction.Compression()):
537        """
538        Add data to the current data group.
539
540        This method adds the provided PSD, frequency, and scanning data to the HDF5 group 
541        associated with this `Data` object. It validates the inputs to ensure they meet 
542        the required specifications before adding them.
543
544        Args:
545            PSD (np.ndarray): A 2D numpy array representing the Power Spectral Density (PSD) data. The last dimension contains the spectra.
546            frequency (np.ndarray): A 1D or 2D numpy array representing the frequency data. 
547                It must be broadcastable to the shape of the PSD array.
548            scanning (dict, optional): A dictionary containing scanning-related data. 
549                Required for sparse data (sparse=True), optional for non-sparse data.
550                For sparse data, must include at least one of 'Spatial_map' or 'Cartesian_visualisation'.
551                It may include the following keys:
552                - 'Spatial_map' (optional): A dictionary containing coordinate arrays:
553                    - 'x', 'y', 'z' (optional): 1D numpy arrays of same length with coordinate values
554                    - 'units' (optional): string with the unit (e.g., 'um')
555                - 'Cartesian_visualisation' (optional): A 3D numpy array (z, y, x) with integer values 
556                   mapping spatial positions to spectra indices. Values must be -1 (invalid/empty pixel) 
557                   or between 0 and PSD.shape[0]-1.
558                - 'Cartesian_visualisation_pixel' (recommended with Cartesian_visualisation): 
559                   Tuple/list of 3 float values (z, y, x) representing pixel size. Unused dimensions can be None.
560                - 'Cartesian_visualisation_pixel_unit' (optional): String for pixel size unit (default: 'um').
561            timestamp (np.ndarray, optional): Timestamps in milliseconds for each spectrum.
562                Must be a 1D array with length equal to PSD.shape[0].
563
564
565        Raises:
566            ValueError: If any of the data provided is not valid or consistent
567        """
568
569        # Check if frequency is broadcastable to PSD
570        try:
571            np.broadcast_shapes(tuple(frequency.shape), tuple(PSD.shape))
572        except ValueError as e:
573            raise ValueError(f"frequency (shape: {frequency.shape}) is not broadcastable to PSD (shape: {PSD.shape}): {e}")
574
575        # Check if at least one of 'Spatial_map' or 'Cartesian_visualisation' is present in the scanning dictionary
576        # This is required for sparse data to establish the spatial mapping
577        has_spatial_mapping = False
578        if scanning is not None:
579            if 'Spatial_map' in scanning:
580                sm = scanning['Spatial_map']
581                size = 0
582
583                def check_coor(coor: str):
584                    if coor in sm:
585                        sm[coor] = np.array(sm[coor])
586                        size1 = sm[coor].size
587                        if size1 != size and size != 0:
588                            raise ValueError(
589                                f"'{coor}' in 'Spatial_map' is invalid!")
590                        return size1
591                size = check_coor('x')
592                size = check_coor('y')
593                size = check_coor('z')
594                if size == 0:
595                    raise ValueError(
596                        "'Spatial_map' should contain at least one x, y or z")
597                has_spatial_mapping = True
598            if 'Cartesian_visualisation' in scanning:
599                cv = scanning['Cartesian_visualisation']
600                if not isinstance(cv, np.ndarray) or cv.ndim != 3:
601                    raise ValueError(
602                        "Cartesian_visualisation must be a 3D numpy array")
603                if not np.issubdtype(cv.dtype, np.integer) or np.min(cv) < -1 or np.max(cv) >= PSD.shape[0]:
604                    raise ValueError(
605                        "Cartesian_visualisation values must be integers between -1 and PSD.shape[0]-1")
606                if 'Cartesian_visualisation_pixel' in scanning:
607                    if len(scanning['Cartesian_visualisation_pixel']) != 3:
608                        raise ValueError(
609                            "Cartesian_visualisation_pixel must always contain 3 values for z, y, x (set to None if not used)")
610                else:
611                    warnings.warn(
612                        "It is recommended to include 'Cartesian_visualisation_pixel' in the scanning dictionary to define pixel size for proper spatial calibration")
613                has_spatial_mapping = True
614        if not has_spatial_mapping and self._sparse:
615            raise ValueError("For sparse data, 'scanning' must be provided and must contain at least one of 'Spatial_map' or 'Cartesian_visualisation'")
616
617        if timestamp is not None:
618            if not isinstance(timestamp, np.ndarray) or timestamp.ndim != 1 or len(timestamp) != PSD.shape[0]:
619                raise ValueError("timestamp is not compatible with PSD")
620
621        # TODO: add and validate additional datasets (i.e. 'Parameters', 'Calibration_index', etc.)
622
623        # Add datasets to the group
624        def determine_chunk_size(arr: np.array) -> tuple:
625            """"
626            Use the same heuristic as the zarr library to determine the chunk size, but without splitting the last dimension
627            """
628            shape = arr.shape
629            typesize = arr.itemsize
630            #if the array is 1D, do not chunk it
631            if len(shape) <= 1:
632                return (shape[0],)
633            target_sizes = _guess_chunks.__kwdefaults__
634            # divide the target size by the last dimension size to get the chunk size for the other dimensions
635            target_sizes = {k: target_sizes[k] // shape[-1] 
636                            for k in target_sizes.keys()}
637            chunks = _guess_chunks(shape[0:-1], typesize, arr.nbytes, **target_sizes)
638            return chunks + (shape[-1],)  # keep the last dimension size unchanged
639        sync(self._file.create_dataset(
640            self._group, brim_obj_names.data.PSD, data=PSD,
641            chunk_size=determine_chunk_size(PSD), compression=compression))
642        freq_ds = sync(self._file.create_dataset(
643            self._group,  brim_obj_names.data.frequency, data=frequency,
644            chunk_size=determine_chunk_size(frequency), compression=compression))
645        units.add_to_object(self._file, freq_ds, freq_units)
646
647        if scanning is not None:
648            if 'Spatial_map' in scanning:
649                sm = scanning['Spatial_map']
650                sm_group = sync(self._file.create_group(concatenate_paths(
651                    self._path, brim_obj_names.data.spatial_map)))
652                if 'units' in sm:
653                    units.add_to_object(self._file, sm_group, sm['units'])
654
655                def add_sm_dataset(coord: str):
656                    if coord in sm:
657                        coord_dts = sync(self._file.create_dataset(
658                            sm_group, coord, data=sm[coord], compression=compression))
659
660                add_sm_dataset('x')
661                add_sm_dataset('y')
662                add_sm_dataset('z')
663            if 'Cartesian_visualisation' in scanning:
664                # convert the Cartesian_visualisation to the smallest integer type
665                cv_arr = np_array_to_smallest_int_type(scanning['Cartesian_visualisation'])
666                cv = sync(self._file.create_dataset(self._group, brim_obj_names.data.cartesian_visualisation,
667                                            data=cv_arr, compression=compression))
668                if 'Cartesian_visualisation_pixel' in scanning:
669                    sync(self._file.create_attr(
670                        cv, 'element_size', scanning['Cartesian_visualisation_pixel']))
671                    if 'Cartesian_visualisation_pixel_unit' in scanning:
672                        px_unit = scanning['Cartesian_visualisation_pixel_unit']
673                    else:
674                        warnings.warn(
675                            "No unit provided for Cartesian_visualisation_pixel, defaulting to 'um'")
676                        px_unit = 'um'
677                    units.add_to_attribute(self._file, cv, 'element_size', px_unit)
678
679        self._spatial_map, self._spatial_map_px_size = self._load_spatial_mapping()
680
681        if timestamp is not None:
682            sync(self._file.create_dataset(
683                self._group, 'Timestamp', data=timestamp, compression=compression))
684
685    @staticmethod
686    def list_data_groups(file: FileAbstraction, retrieve_custom_name=False) -> list:
687        """
688        List all data groups in the brim file. The list is ordered by index.
689
690        Returns:
691            list: A list of dictionaries, each containing:
692                - 'name' (str): The name of the data group in the file.
693                - 'index' (int): The index extracted from the group name.
694                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the data group as returned from utils.get_object_name.
695        """
696
697        data_groups = []
698
699        matched_objs = list_objects_matching_pattern(
700            file, brim_obj_names.Brillouin_base_path, brim_obj_names.data.base_group + r"_(\d+)$")
701        
702        async def _make_dict_item(matched_obj, retrieve_custom_name):
703            name = matched_obj[0]
704            index = int(matched_obj[1])
705            curr_obj_dict = {'name': name, 'index': index}
706            if retrieve_custom_name:
707                path = concatenate_paths(
708                    brim_obj_names.Brillouin_base_path, name)
709                custom_name = await get_object_name(file, path)
710                curr_obj_dict['custom_name'] = custom_name
711            return curr_obj_dict
712        
713        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
714        dicts = _gather_sync(*coros)
715        for dict_item in dicts:
716            data_groups.append(dict_item)        
717        # Sort the data groups by index
718        data_groups.sort(key=lambda x: x['index'])
719
720        return data_groups
721
722    @staticmethod
723    def _get_existing_group_name(file: FileAbstraction, index: int) -> str:
724        """
725        Get the name of an existing data group by index.
726
727        Args:
728            file (File): The parent File object.
729            index (int): The index of the data group.
730
731        Returns:
732            str: The name of the data group, or None if not found.
733        """
734        group_name: str = None
735        data_groups = Data.list_data_groups(file)
736        for dg in data_groups:
737            if dg['index'] == index:
738                group_name = dg['name']
739                break
740        return group_name
741
742    @classmethod
743    def _create_new(cls, file: FileAbstraction, index: int, sparse: bool = False, name: str = None) -> 'Data':
744        """
745        Create a new data group with the specified index.
746
747        Args:
748            file (File): The parent File object.
749            index (int): The index for the new data group.
750            sparse (bool): Whether the data is sparse. See https://github.com/prevedel-lab/Brillouin-standard-file/blob/main/docs/brim_file_specs.md for details. Defaults to False.
751            name (str, optional): The name for the new data group. Defaults to None.
752
753        Returns:
754            Data: The newly created Data object.
755        """
756        group_name = Data._generate_group_name(index)
757        group = sync(file.create_group(concatenate_paths(
758            brim_obj_names.Brillouin_base_path, group_name)))
759        sync(file.create_attr(group, 'Sparse', sparse))
760        if name is not None:
761            set_object_name(file, group, name)
762        return cls(file, concatenate_paths(brim_obj_names.Brillouin_base_path, group_name), newly_created=True)
763
764    @staticmethod
765    def _generate_group_name(index: int, n_digits: int = None) -> str:
766        """
767        Generate a name for a data group based on the index.
768
769        Args:
770            index (int): The index for the data group.
771            n_digits (int, optional): The number of digits to pad the index with. If None no padding is applied. Defaults to None.
772
773        Returns:
774            str: The generated group name.
775
776        Raises:
777            ValueError: If the index is negative.
778        """
779        if index < 0:
780            raise ValueError("index must be positive")
781        num = str(index)
782        if n_digits is not None:
783            num = num.zfill(n_digits)
784        return f"{brim_obj_names.data.base_group}_{num}"