brimfile.data

   1import numpy as np
   2import asyncio
   3
   4import warnings
   5from enum import Enum
   6
   7from .file_abstraction import FileAbstraction, sync, _async_getitem, _gather_sync
   8from .utils import concatenate_paths, list_objects_matching_pattern, get_object_name, set_object_name
   9from .utils import var_to_singleton, np_array_to_smallest_int_type, _guess_chunks
  10
  11from .metadata import Metadata
  12
  13from numbers import Number
  14
  15from . import units
  16from .constants import brim_obj_names
  17
  18__docformat__ = "google"
  19
  20
  21class Data:
  22    """
  23    Represents a data group within the brim file.
  24    """
  25
  26    def __init__(self, file: FileAbstraction, path: str):
  27        """
  28        Initialize the Data object. This constructor should not be called directly.
  29
  30        Args:
  31            file (File): The parent File object.
  32            path (str): The path to the data group within the file.
  33        """
  34        self._file = file
  35        self._path = path
  36        self._group = sync(file.open_group(path))
  37
  38        self._spatial_map, self._spatial_map_px_size = self._load_spatial_mapping()
  39
  40    def get_name(self):
  41        """
  42        Returns the name of the data group.
  43        """
  44        return sync(get_object_name(self._file, self._path))
  45    
  46    def get_index(self):
  47        """
  48        Returns the index of the data group.
  49        """
  50        return int(self._path.split('/')[-1].split('_')[-1])
  51
  52    def _load_spatial_mapping(self, load_in_memory: bool=True) -> tuple:
  53        """
  54        Load a spatial mapping in the same format as 'Cartesian visualisation',
  55        irrespectively on whether 'Spatial_map' is defined instead.
  56        -1 is used for "empty" pixels in the image
  57        Args:
  58            load_in_memory (bool): Specify whether the map should be forced to load in memory or just opened as a dataset.
  59        Returns:
  60            The spatial map and the corresponding pixel size as a tuple of 3 Metadata.Item, both in the order z, y, x.
  61        """
  62        cv = None
  63        px_size = 3*(Metadata.Item(value=1, units=None),)
  64
  65        cv_path = concatenate_paths(
  66            self._path, brim_obj_names.data.cartesian_visualisation)
  67        sm_path = concatenate_paths(
  68            self._path, brim_obj_names.data.spatial_map)
  69        
  70        if sync(self._file.object_exists(cv_path)):
  71            cv = sync(self._file.open_dataset(cv_path))
  72
  73            #read the pixel size from the 'Cartesian visualisation' dataset
  74            px_size_val = None
  75            px_size_units = None
  76            try:
  77                px_size_val = sync(self._file.get_attr(cv, 'element_size'))
  78                if px_size_val is None or len(px_size_val) != 3:
  79                    raise ValueError(
  80                        "The 'element_size' attribute of 'Cartesian_visualisation' must be a tuple of 3 elements")
  81            except Exception:
  82                px_size_val = 3*(1,)
  83                warnings.warn(
  84                    "No pixel size defined for Cartesian visualisation")            
  85            px_size_units = sync(units.of_attribute(
  86                    self._file, cv, 'element_size'))
  87            px_size = ()
  88            for i in range(3):
  89                # if px_size_val[i] is not a number, set it to 1 and px_size_units to None
  90                if isinstance(px_size_val[i], Number):
  91                    px_size += (Metadata.Item(px_size_val[i], px_size_units), )
  92                else:
  93                    px_size += (Metadata.Item(1, None), )
  94                    
  95
  96            if load_in_memory:
  97                cv = np.array(cv)
  98                cv = np_array_to_smallest_int_type(cv)
  99
 100        elif sync(self._file.object_exists(sm_path)):
 101            def load_spatial_map_from_file(self):
 102                async def load_coordinate_from_sm(coord: str):
 103                    res = np.empty(0)  # empty array
 104                    try:
 105                        res = await self._file.open_dataset(
 106                            concatenate_paths(sm_path, coord))
 107                        res = await res.to_np_array()
 108                        res = np.squeeze(res)  # remove single-dimensional entries
 109                    except Exception as e:
 110                        # if the coordinate does not exist, return an empty array
 111                        pass
 112                    if len(res.shape) > 1:
 113                        raise ValueError(
 114                            f"The 'Spatial_map/{coord}' dataset is not a 1D array as expected")
 115                    return res
 116
 117                def check_coord_array(arr, size):
 118                    if arr.size == 0:
 119                        return np.zeros(size)
 120                    elif arr.size != size:
 121                        raise ValueError(
 122                            "The 'Spatial_map' dataset is invalid")
 123                    return arr
 124
 125                x, y, z = _gather_sync(
 126                    load_coordinate_from_sm('x'),
 127                    load_coordinate_from_sm('y'),
 128                    load_coordinate_from_sm('z')
 129                    )
 130                size = max([x.size, y.size, z.size])
 131                if size == 0:
 132                    raise ValueError("The 'Spatial_map' dataset is empty")
 133                x = check_coord_array(x, size)
 134                y = check_coord_array(y, size)
 135                z = check_coord_array(z, size)
 136                return x, y, z
 137
 138            def calculate_step(x):
 139                n = len(np.unique(x))
 140                if n == 1:
 141                    d = None
 142                else:
 143                    d = (np.max(x)-np.min(x))/(n-1)
 144                return n, d
 145
 146            x, y, z = load_spatial_map_from_file(self)
 147
 148            # TODO extend the reconstruction to non-cartesian cases
 149
 150            nX, dX = calculate_step(x)
 151            nY, dY = calculate_step(y)
 152            nZ, dZ = calculate_step(z)
 153
 154            indices = np_array_to_smallest_int_type(np.lexsort((x, y, z)))
 155            cv = np.reshape(indices, (nZ, nY, nX))
 156
 157            px_size_units = sync(units.of_object(self._file, sm_path))
 158            px_size = ()
 159            for i in range(3):
 160                px_sz = (dZ, dY, dX)[i]
 161                px_unit = px_size_units
 162                if px_sz is None:
 163                    px_sz = 1
 164                    px_unit = None
 165                px_size += (Metadata.Item(px_sz, px_unit),)
 166
 167        return cv, px_size
 168
 169    def get_PSD(self) -> tuple:
 170        """
 171        LOW LEVEL FUNCTION
 172
 173        Retrieve the Power Spectral Density (PSD) and frequency from the current data group.
 174        Note: this function exposes the internals of the brim file and thus the interface might change in future versions.
 175        Use only if more specialized functions are not working for your application!
 176        Returns:
 177            tuple: (PSD, frequency, PSD_units, frequency_units)
 178                - PSD: A 2D (or more) numpy array containing all the spectra (see [specs](https://github.com/prevedel-lab/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
 179                - frequency: A numpy array representing the frequency data (see [specs](https://github.com/prevedel-lab/Brillouin-standard-file/blob/main/docs/brim_file_specs.md) for more details).
 180                - PSD_units: The units of the PSD.
 181                - frequency_units: The units of the frequency.
 182        """
 183        PSD, frequency = _gather_sync(
 184            self._file.open_dataset(concatenate_paths(
 185                self._path, brim_obj_names.data.PSD)),
 186            self._file.open_dataset(concatenate_paths(
 187                self._path, brim_obj_names.data.frequency))
 188        )
 189        # retrieve the units of the PSD and frequency
 190        PSD_units, frequency_units = _gather_sync(
 191            units.of_object(self._file, PSD),
 192            units.of_object(self._file, frequency)
 193        )
 194
 195        return PSD, frequency, PSD_units, frequency_units
 196    
 197    def get_PSD_as_spatial_map(self) -> tuple:
 198        """
 199        Retrieve the Power Spectral Density (PSD) as a spatial map and the frequency from the current data group.
 200        Returns:
 201            tuple: (PSD, frequency, PSD_units, frequency_units)
 202                - PSD: A 4D (or more) numpy array containing all the spectra. Dimensions are z, y, x, [parameters], spectrum.
 203                - frequency: A numpy array representing the frequency data, which can be broadcastable to PSD.
 204                - PSD_units: The units of the PSD.
 205                - frequency_units: The units of the frequency.
 206        """
 207        PSD, frequency = _gather_sync(
 208            self._file.open_dataset(concatenate_paths(
 209                self._path, brim_obj_names.data.PSD)),        
 210            self._file.open_dataset(concatenate_paths(
 211                self._path, brim_obj_names.data.frequency))
 212            )        
 213        # retrieve the units of the PSD and frequency
 214        PSD_units, frequency_units = _gather_sync(
 215            units.of_object(self._file, PSD),
 216            units.of_object(self._file, frequency)
 217        )
 218
 219        # ensure PSD and frequency are numpy arrays
 220        PSD = np.array(PSD)  
 221        frequency = np.array(frequency)  # ensure it's a numpy array
 222        
 223        #if the frequency is not the same for all spectra, broadcast it to match the shape of PSD
 224        if frequency.ndim > 1:
 225            frequency = np.broadcast_to(frequency, PSD.shape)
 226        
 227        sm = np.array(self._spatial_map)
 228        # reshape the PSD to have the spatial dimensions first      
 229        PSD = PSD[sm, ...]
 230        # reshape the frequency pnly if it is not the same for all spectra
 231        if frequency.ndim > 1:
 232            frequency = frequency[sm, ...]
 233
 234        return PSD, frequency, PSD_units, frequency_units
 235
 236    def get_spectrum(self, index: int) -> tuple:
 237        """
 238        Synchronous wrapper for `get_spectrum_async` (see doc for `brimfile.data.Data.get_spectrum_async`)
 239        """
 240        return sync(self.get_spectrum_async(index))
 241    async def get_spectrum_async(self, index: int) -> tuple:
 242        """
 243        Retrieve a spectrum from the data group.
 244
 245        Args:
 246            index (int): The index of the spectrum to retrieve.
 247
 248        Returns:
 249            tuple: (PSD, frequency, PSD_units, frequency_units) for the specified index. 
 250                    PSD can be 1D or more (if there are additional parameters);
 251                    frequency has the same size as PSD
 252        Raises:
 253            IndexError: If the index is out of range for the PSD dataset.
 254        """
 255        # index = -1 corresponds to no spectrum
 256        if index < 0:
 257            return None, None, None, None
 258        PSD, frequency = await asyncio.gather(
 259            self._file.open_dataset(concatenate_paths(
 260                self._path, brim_obj_names.data.PSD)),                       
 261            self._file.open_dataset(concatenate_paths(
 262                self._path, brim_obj_names.data.frequency))
 263            )
 264        if index >= PSD.shape[0]:
 265            raise IndexError(
 266                f"index {index} out of range for PSD with shape {PSD.shape}") 
 267        # retrieve the units of the PSD and frequency
 268        PSD_units, frequency_units = await asyncio.gather(
 269            units.of_object(self._file, PSD),
 270            units.of_object(self._file, frequency)
 271        )
 272        # map index to the frequency array, considering the broadcasting rules
 273        index_frequency = (index, ...)
 274        if frequency.ndim < PSD.ndim:
 275            # given the definition of the brim file format,
 276            # if the frequency has less dimensions that PSD,
 277            # it can only be because it is the same for all the spatial position (first dimension)
 278            index_frequency = (..., )
 279        #get the spectrum and the corresponding frequency at the specified index
 280        PSD, frequency = await asyncio.gather(
 281            _async_getitem(PSD, (index,...)),
 282            _async_getitem(frequency, index_frequency)
 283        )
 284        #broadcast the frequency to match the shape of PSD if needed
 285        if frequency.ndim < PSD.ndim:
 286            frequency = np.broadcast_to(frequency, PSD.shape)
 287        return PSD, frequency, PSD_units, frequency_units
 288
 289    def get_spectrum_in_image(self, coor: tuple) -> tuple:
 290        """
 291        Retrieve a spectrum from the data group using spatial coordinates.
 292
 293        Args:
 294            coor (tuple): A tuple containing the z, y, x coordinates of the spectrum to retrieve.
 295
 296        Returns:
 297            tuple: A tuple containing the PSD, frequency, PSD_units, frequency_units for the specified coordinates. See "get_spectrum" for details.
 298        """
 299        if len(coor) != 3:
 300            raise ValueError("coor must contain 3 values for z, y, x")
 301
 302        index = int(self._spatial_map[coor])
 303        return self.get_spectrum(index)    
 304          
 305    def get_spectrum_and_all_quantities_in_image(self, ar: 'Data.AnalysisResults', coor: tuple, index_peak: int = 0):
 306        """
 307            Retrieve the spectrum and all available quantities from the analysis results at a specific spatial coordinate.
 308            TODO complete the documentation
 309        """
 310        if len(coor) != 3:
 311            raise ValueError("coor must contain 3 values for z, y, x")
 312        index = int(self._spatial_map[coor])
 313        spectrum, quantities = _gather_sync(
 314            self.get_spectrum_async(index),
 315            ar._get_all_quantities_at_index(index, index_peak)
 316        )
 317        return spectrum, quantities
 318
 319    class AnalysisResults:
 320        """
 321        Rapresents the analysis results associated with a Data object.
 322        """
 323
 324        class Quantity(Enum):
 325            """
 326            Enum representing the type of analysis results.
 327            """
 328            Shift = "Shift"
 329            Width = "Width"
 330            Amplitude = "Amplitude"
 331            Offset = "Offset"
 332            R2 = "R2"
 333            RMSE = "RMSE"
 334            Cov_matrix = "Cov_matrix"
 335
 336        class PeakType(Enum):
 337            AntiStokes = "AS"
 338            Stokes = "S"
 339            average = "avg"
 340        
 341        class FitModel(Enum):
 342            Undefined = "Undefined"
 343            Lorentzian = "Lorentzian"
 344            DHO = "DHO"
 345            Gaussian = "Gaussian"
 346            Voigt = "Voigt"
 347            Custom = "Custom"
 348
 349        def __init__(self, file: FileAbstraction, full_path: str, spatial_map, spatial_map_px_size):
 350            """
 351            Initialize the AnalysisResults object.
 352
 353            Args:
 354                file (File): The parent File object.
 355                full_path (str): path of the group storing the analysis results
 356            """
 357            self._file = file
 358            self._path = full_path
 359            # self._group = file.open_group(full_path)
 360            self._spatial_map = spatial_map
 361            self._spatial_map_px_size = spatial_map_px_size
 362
 363        def get_name(self):
 364            """
 365            Returns the name of the Analysis group.
 366            """
 367            return sync(get_object_name(self._file, self._path))
 368
 369        @classmethod
 370        def _create_new(cls, data: 'Data', index: int) -> 'Data.AnalysisResults':
 371            """
 372            Create a new AnalysisResults group.
 373
 374            Args:
 375                file (FileAbstraction): The file.
 376                index (int): The index for the new AnalysisResults group.
 377
 378            Returns:
 379                AnalysisResults: The newly created AnalysisResults object.
 380            """
 381            group_name = f"{brim_obj_names.data.analysis_results}_{index}"
 382            ar_full_path = concatenate_paths(data._path, group_name)
 383            group = sync(data._file.create_group(ar_full_path))
 384            return cls(data._file, ar_full_path, data._spatial_map, data._spatial_map_px_size)
 385
 386        def add_data(self, data_AntiStokes=None, data_Stokes=None, fit_model: 'Data.AnalysisResults.FitModel' = None):
 387            """
 388            Adds data for the analysis results for AntiStokes and Stokes peaks to the file.
 389            
 390            Args:
 391                data_AntiStokes (dict or list[dict]): A dictionary containing the analysis results for AntiStokes peaks.
 392                    In case multiple peaks were fitted, it might be a list of dictionaries with each element corresponding to a single peak.
 393                
 394                    Each dictionary may include the following keys (plus the corresponding units,  e.g. 'shift_units'):
 395                        - 'shift': The shift value.
 396                        - 'width': The width value.
 397                        - 'amplitude': The amplitude value.
 398                        - 'offset': The offset value.
 399                        - 'R2': The R-squared value.
 400                        - 'RMSE': The root mean square error value.
 401                        - 'Cov_matrix': The covariance matrix.
 402                data_Stokes (dict or list[dict]): same as `data_AntiStokes` for the Stokes peaks.
 403                fit_model (Data.AnalysisResults.FitModel, optional): The fit model used for the analysis. Defaults to None (no attribute is set).
 404
 405                Both `data_AntiStokes` and `data_Stokes` are optional, but at least one of them must be provided.
 406            """
 407
 408            ar_cls = Data.AnalysisResults
 409            ar_group = sync(self._file.open_group(self._path))
 410
 411            def add_quantity(qt: Data.AnalysisResults.Quantity, pt: Data.AnalysisResults.PeakType, data, index: int = 0):
 412                # TODO: check if the data is valid
 413                sync(self._file.create_dataset(
 414                    ar_group, ar_cls._get_quantity_name(qt, pt, index), data))
 415
 416            def add_data_pt(pt: Data.AnalysisResults.PeakType, data, index: int = 0):
 417                if 'shift' in data:
 418                    add_quantity(ar_cls.Quantity.Shift,
 419                                 pt, data['shift'], index)
 420                    if 'shift_units' in data:
 421                        self._set_units(data['shift_units'],
 422                                        ar_cls.Quantity.Shift, pt, index)
 423                if 'width' in data:
 424                    add_quantity(ar_cls.Quantity.Width,
 425                                 pt, data['width'], index)
 426                    if 'width_units' in data:
 427                        self._set_units(data['width_units'],
 428                                        ar_cls.Quantity.Width, pt, index)
 429                if 'amplitude' in data:
 430                    add_quantity(ar_cls.Quantity.Amplitude,
 431                                 pt, data['amplitude'], index)
 432                    if 'amplitude_units' in data:
 433                        self._set_units(
 434                            data['amplitude_units'], ar_cls.Quantity.Amplitude, pt, index)
 435                if 'offset' in data:
 436                    add_quantity(ar_cls.Quantity.Offset,
 437                                 pt, data['offset'], index)
 438                    if 'offset_units' in data:
 439                        self._set_units(
 440                            data['offset_units'], ar_cls.Quantity.Offset, pt, index)
 441                if 'R2' in data:
 442                    add_quantity(ar_cls.Quantity.R2, pt, data['R2'], index)
 443                    if 'R2_units' in data:
 444                        self._set_units(data['R2_units'],
 445                                        ar_cls.Quantity.R2, pt, index)
 446                if 'RMSE' in data:
 447                    add_quantity(ar_cls.Quantity.RMSE, pt, data['RMSE'], index)
 448                    if 'RMSE_units' in data:
 449                        self._set_units(data['RMSE_units'],
 450                                        ar_cls.Quantity.RMSE, pt, index)
 451                if 'Cov_matrix' in data:
 452                    add_quantity(ar_cls.Quantity.Cov_matrix,
 453                                 pt, data['Cov_matrix'], index)
 454                    if 'Cov_matrix_units' in data:
 455                        self._set_units(
 456                            data['Cov_matrix_units'], ar_cls.Quantity.Cov_matrix, pt, index)
 457
 458            if data_AntiStokes is not None:
 459                data_AntiStokes = var_to_singleton(data_AntiStokes)
 460                for i, d_as in enumerate(data_AntiStokes):
 461                    add_data_pt(ar_cls.PeakType.AntiStokes, d_as, i)
 462            if data_Stokes is not None:
 463                data_Stokes = var_to_singleton(data_Stokes)
 464                for i, d_s in enumerate(data_Stokes):
 465                    add_data_pt(ar_cls.PeakType.Stokes, d_s, i)
 466            if fit_model is not None:
 467                sync(self._file.create_attr(ar_group, 'Fit_model', fit_model.value))
 468
 469        def get_units(self, qt: Quantity, pt: PeakType = PeakType.AntiStokes, index: int = 0) -> str:
 470            """
 471            Retrieve the units of a specified quantity from the data file.
 472
 473            Args:
 474                qt (Quantity): The quantity for which the units are to be retrieved.
 475                pt (PeakType, optional): The type of peak (e.g., Stokes or AntiStokes). Defaults to PeakType.AntiStokes.
 476                index (int, optional): The index of the quantity in case multiple quantities exist. Defaults to 0.
 477
 478            Returns:
 479                str: The units of the specified quantity as a string.
 480            """
 481            dt_name = Data.AnalysisResults._get_quantity_name(qt, pt, index)
 482            full_path = concatenate_paths(self._path, dt_name)
 483            return sync(units.of_object(self._file, full_path))
 484
 485        def _set_units(self, un: str, qt: Quantity, pt: PeakType = PeakType.AntiStokes, index: int = 0) -> str:
 486            """
 487            Set the units of a specified quantity.
 488
 489            Args:
 490                un (str): The units to be set.
 491                qt (Quantity): The quantity for which the units are to be set.
 492                pt (PeakType, optional): The type of peak (e.g., Stokes or AntiStokes). Defaults to PeakType.AntiStokes.
 493                index (int, optional): The index of the quantity in case multiple quantities exist. Defaults to 0.
 494
 495            Returns:
 496                str: The units of the specified quantity as a string.
 497            """
 498            dt_name = Data.AnalysisResults._get_quantity_name(qt, pt, index)
 499            full_path = concatenate_paths(self._path, dt_name)
 500            return units.add_to_object(self._file, full_path, un)
 501        
 502        @property
 503        def fit_model(self) -> 'Data.AnalysisResults.FitModel':
 504            """
 505            Retrieve the fit model used for the analysis.
 506
 507            Returns:
 508                Data.AnalysisResults.FitModel: The fit model used for the analysis.
 509            """
 510            if not hasattr(self, '_fit_model'):
 511                try:
 512                    fit_model_str = sync(self._file.get_attr(self._path, 'Fit_model'))
 513                    self._fit_model = Data.AnalysisResults.FitModel(fit_model_str)
 514                except Exception as e:
 515                    if isinstance(e, ValueError):
 516                        warnings.warn(
 517                            f"Unknown fit model '{fit_model_str}' found in the file.")
 518                    self._fit_model = Data.AnalysisResults.FitModel.Undefined        
 519            return self._fit_model
 520
 521        def save_image_to_OMETiff(self, qt: Quantity, pt: PeakType = PeakType.AntiStokes, index: int = 0, filename: str = None) -> str:
 522            """
 523            Saves the image corresponding to the specified quantity and index to an OMETiff file.
 524
 525            Args:
 526                qt (Quantity): The quantity to retrieve the image for (e.g. shift).
 527                pt (PeakType, optional): The type of peak to consider (default is PeakType.AntiStokes).
 528                index (int, optional): The index of the data to retrieve, if multiple are present (default is 0).
 529                filename (str, optional): The name of the file to save the image to. If None, a default name will be used.
 530
 531            Returns:
 532                str: The path to the saved OMETiff file.
 533            """
 534            try:
 535                import tifffile
 536            except ImportError:
 537                raise ModuleNotFoundError(
 538                    "The tifffile module is required for saving to OME-Tiff. Please install it using 'pip install tifffile'.")
 539            
 540            if filename is None:
 541                filename = f"{qt.value}_{pt.value}_{index}.ome.tif"
 542            if not filename.endswith('.ome.tif'):
 543                filename += '.ome.tif'
 544            img, px_size = self.get_image(qt, pt, index)
 545            if img.ndim > 3:
 546                raise NotImplementedError(
 547                    "Saving images with more than 3 dimensions is not supported yet.")
 548            with tifffile.TiffWriter(filename, bigtiff=True) as tif:
 549                metadata = {
 550                    'axes': 'ZYX',
 551                    'PhysicalSizeX': px_size[2].value,
 552                    'PhysicalSizeXUnit': px_size[2].units,
 553                    'PhysicalSizeY': px_size[1].value,
 554                    'PhysicalSizeYUnit': px_size[1].units,
 555                    'PhysicalSizeZ': px_size[0].value,
 556                    'PhysicalSizeZUnit': px_size[0].units,
 557                }
 558                tif.write(img, metadata=metadata)
 559            return filename
 560
 561        def get_image(self, qt: Quantity, pt: PeakType = PeakType.AntiStokes, index: int = 0) -> tuple:
 562            """
 563            Retrieves an image (spatial map) based on the specified quantity, peak type, and index.
 564
 565            Args:
 566                qt (Quantity): The quantity to retrieve the image for (e.g. shift).
 567                pt (PeakType, optional): The type of peak to consider (default is PeakType.AntiStokes).
 568                index (int, optional): The index of the data to retrieve, if multiple are present (default is 0).
 569
 570            Returns:
 571                A tuple containing the image corresponding to the specified quantity and index and the corresponding pixel size.
 572                The image is a 3D dataset where the dimensions are z, y, x.
 573                If there are additional parameters, more dimensions are added in the order z, y, x, par1, par2, ...
 574                The pixel size is a tuple of 3 Metadata.Item in the order z, y, x.
 575            """
 576            pt_type = Data.AnalysisResults.PeakType
 577            data = None
 578            if pt == pt_type.average:
 579                peaks = self.list_existing_peak_types(index)
 580                match len(peaks):
 581                    case 0:
 582                        raise ValueError(
 583                            "No peaks found for the specified index. Cannot compute average.")
 584                    case 1:
 585                        data = np.array(sync(self._get_quantity(qt, peaks[0], index)))
 586                    case 2:
 587                        data1, data2 = _gather_sync(
 588                            self._get_quantity(qt, peaks[0], index),
 589                            self._get_quantity(qt, peaks[1], index)
 590                            )
 591                        data = (np.abs(data1) + np.abs(data2))/2
 592            else:
 593                data = np.array(sync(self._get_quantity(qt, pt, index)))
 594            sm = np.array(self._spatial_map)
 595            img = data[sm, ...]
 596            img[sm<0, ...] = np.nan  # set invalid pixels to NaN
 597            return img, self._spatial_map_px_size
 598        def get_quantity_at_pixel(self, coord: tuple, qt: Quantity, pt: PeakType = PeakType.AntiStokes, index: int = 0):
 599            """
 600            Synchronous wrapper for `get_quantity_at_pixel_async` (see doc for `brimfile.data.Data.AnalysisResults.get_quantity_at_pixel_async`)
 601            """
 602            return sync(self.get_quantity_at_pixel_async(coord, qt, pt, index))
 603        async def get_quantity_at_pixel_async(self, coord: tuple, qt: Quantity, pt: PeakType = PeakType.AntiStokes, index: int = 0):
 604            """
 605            Retrieves the specified quantity in the image at coord, based on the peak type and index.
 606
 607            Args:
 608                coord (tuple): A tuple of 3 elements corresponding to the z, y, x coordinate in the image
 609                qt (Quantity): The quantity to retrieve the image for (e.g. shift).
 610                pt (PeakType, optional): The type of peak to consider (default is PeakType.AntiStokes).
 611                index (int, optional): The index of the data to retrieve, if multiple peaks are present (default is 0).
 612
 613            Returns:
 614                The requested quantity, which is a scalar or a multidimensional array (depending on whether there are additional parameters in the current Data group)
 615            """
 616            if len(coord) != 3:
 617                raise ValueError(
 618                    "'coord' must have 3 elements corresponding to z, y, x")
 619            i = self._spatial_map[*coord]
 620            assert i.size == 1
 621            if i<0:
 622                return np.nan  # invalid pixel
 623            i = int(i)
 624
 625            pt_type = Data.AnalysisResults.PeakType
 626            value = None
 627            if pt == pt_type.average:
 628                value = None
 629                peaks = await self.list_existing_peak_types_async(index)
 630                match len(peaks):
 631                    case 0:
 632                        raise ValueError(
 633                            "No peaks found for the specified index. Cannot compute average.")
 634                    case 1:
 635                        data = await self._get_quantity(qt, peaks[0], index)
 636                        value = await _async_getitem(data, (i, ...))
 637                    case 2:
 638                        data_p0, data_p1 = await asyncio.gather(
 639                            self._get_quantity(qt, peaks[0], index),
 640                            self._get_quantity(qt, peaks[1], index)
 641                        )
 642                        value1, value2 = await asyncio.gather(
 643                            _async_getitem(data_p0, (i, ...)),
 644                            _async_getitem(data_p1, (i, ...))
 645                        )
 646                        value = (np.abs(value1) + np.abs(value2))/2
 647            else:
 648                data = await self._get_quantity(qt, pt, index)
 649                value = await _async_getitem(data, (i, ...))
 650            return value
 651        def get_all_quantities_in_image(self, coor: tuple, index_peak: int = 0) -> dict:
 652            """
 653            Retrieve all available quantities at a specific spatial coordinate.
 654            see `brimfile.data.Data.AnalysisResults._get_all_quantities_at_index` for more details
 655            TODO complete the documentation
 656            """
 657            if len(coor) != 3:
 658                raise ValueError("coor must contain 3 values for z, y, x")
 659            index = int(self._spatial_map[coor])
 660            return sync(self._get_all_quantities_at_index(index, index_peak))
 661        async def _get_all_quantities_at_index(self, index: int, index_peak: int = 0) -> dict:
 662            """
 663            Retrieve all available quantities for a specific spatial index.
 664            Args:
 665                index (int): The spatial index to retrieve quantities for.
 666                index_peak (int, optional): The index of the data to retrieve, if multiple peaks are present (default is 0).
 667            Returns:
 668                dict: A dictionary of Metadata.Item in the form `result[quantity.name][peak.name] = bls.Metadata.Item(value, units)`
 669            """
 670            async def _get_existing_quantity_at_index_async(self,  pt: Data.AnalysisResults.PeakType = Data.AnalysisResults.PeakType.AntiStokes):
 671                as_cls = Data.AnalysisResults
 672                qts_ls = ()
 673                dts_ls = ()
 674
 675                qts = [qt for qt in as_cls.Quantity]
 676                coros = [self._file.open_dataset(concatenate_paths(self._path, as_cls._get_quantity_name(qt, pt, index_peak))) for qt in qts]
 677                
 678                # open the datasets asynchronously, excluding those that do not exist
 679                opened_dts = await asyncio.gather(*coros, return_exceptions=True)
 680                for i, opened_qt in enumerate(opened_dts):
 681                    if not isinstance(opened_qt, Exception):
 682                        qts_ls += (qts[i],)
 683                        dts_ls += (opened_dts[i],)
 684                # get the values at the specified index
 685                coros_values = [_async_getitem(dt, (index, ...)) for dt in dts_ls]
 686                coros_units = [units.of_object(self._file, dt) for dt in dts_ls]
 687                ret_ls = await asyncio.gather(*coros_values, *coros_units)
 688                n = len(coros_values)
 689                value_ls = [Metadata.Item(ret_ls[i], ret_ls[n+i]) for i in range(n)]
 690                return qts_ls, value_ls
 691            antiStokes, stokes = await asyncio.gather(
 692                _get_existing_quantity_at_index_async(self, Data.AnalysisResults.PeakType.AntiStokes),
 693                _get_existing_quantity_at_index_async(self, Data.AnalysisResults.PeakType.Stokes)
 694            )
 695            res = {}
 696            # combine the results, including the average
 697            for qt in (set(antiStokes[0]) | set(stokes[0])):
 698                res[qt.name] = {}
 699                pts = ()
 700                #Stokes
 701                if qt in stokes[0]:
 702                    res[qt.name][Data.AnalysisResults.PeakType.Stokes.name] = stokes[1][stokes[0].index(qt)]
 703                    pts += (Data.AnalysisResults.PeakType.Stokes,)
 704                #AntiStokes
 705                if qt in antiStokes[0]:
 706                    res[qt.name][Data.AnalysisResults.PeakType.AntiStokes.name] = antiStokes[1][antiStokes[0].index(qt)]
 707                    pts += (Data.AnalysisResults.PeakType.AntiStokes,)
 708                #average getting the units of the first peak
 709                res[qt.name][Data.AnalysisResults.PeakType.average.name] = Metadata.Item(
 710                    np.mean([np.abs(res[qt.name][pt.name].value) for pt in pts]), 
 711                    res[qt.name][pts[0].name].units
 712                    )
 713                if not all(res[qt.name][pt.name].units == res[qt.name][pts[0].name].units for pt in pts):
 714                    warnings.warn(f"The units of {pts} are not consistent.")
 715            return res
 716
 717        @classmethod
 718        def _get_quantity_name(cls, qt: Quantity, pt: PeakType, index: int) -> str:
 719            """
 720            Returns the name of the dataset correponding to the specific Quantity, PeakType and index
 721
 722            Args:
 723                qt (Quantity)   
 724                pt (PeakType)  
 725                intex (int): in case of multiple peaks fitted, the index of the peak to consider       
 726            """
 727            if not pt in (cls.PeakType.AntiStokes, cls.PeakType.Stokes):
 728                raise ValueError("pt has to be either Stokes or AntiStokes")
 729            if qt == cls.Quantity.R2 or qt == cls.Quantity.RMSE or qt == cls.Quantity.Cov_matrix:
 730                name = f"Fit_error_{str(pt.value)}_{index}/{str(qt.value)}"
 731            else:
 732                name = f"{str(qt.value)}_{str(pt.value)}_{index}"
 733            return name
 734
 735        async def _get_quantity(self, qt: Quantity, pt: PeakType = PeakType.AntiStokes, index: int = 0):
 736            """
 737            Retrieve a specific quantity dataset from the file.
 738
 739            Args:
 740                qt (Quantity): The type of quantity to retrieve.
 741                pt (PeakType, optional): The peak type to consider (default is PeakType.AntiStokes).
 742                index (int, optional): The index of the quantity if multiple peaks are available (default is 0).
 743
 744            Returns:
 745                The dataset corresponding to the specified quantity, as stored in the file.
 746
 747            """
 748
 749            dt_name = Data.AnalysisResults._get_quantity_name(qt, pt, index)
 750            full_path = concatenate_paths(self._path, dt_name)
 751            return await self._file.open_dataset(full_path)
 752
 753        def list_existing_peak_types(self, index: int = 0) -> tuple:
 754            """
 755            Synchronous wrapper for `list_existing_peak_types_async` (see doc for `brimfile.data.Data.AnalysisResults.list_existing_peak_types_async`)
 756            """
 757            return sync(self.list_existing_peak_types_async(index)) 
 758        async def list_existing_peak_types_async(self, index: int = 0) -> tuple:
 759            """
 760            Returns a tuple of existing peak types (Stokes and/or AntiStokes) for the specified index.
 761            Args:
 762                index (int, optional): The index of the peak to check (in case of multi-peak fit). Defaults to 0.
 763            Returns:
 764                tuple: A tuple containing `PeakType` members (`Stokes`, `AntiStokes`) that exist for the given index.
 765            """
 766
 767            as_cls = Data.AnalysisResults
 768            shift_s_name = as_cls._get_quantity_name(
 769                as_cls.Quantity.Shift, as_cls.PeakType.Stokes, index)
 770            shift_as_name = as_cls._get_quantity_name(
 771                as_cls.Quantity.Shift, as_cls.PeakType.AntiStokes, index)
 772            ls = ()
 773            coro_as_exists = self._file.object_exists(concatenate_paths(self._path, shift_as_name))
 774            coro_s_exists = self._file.object_exists(concatenate_paths(self._path, shift_s_name))
 775            as_exists, s_exists = await asyncio.gather(coro_as_exists, coro_s_exists)
 776            if as_exists:
 777                ls += (as_cls.PeakType.AntiStokes,)
 778            if s_exists:
 779                ls += (as_cls.PeakType.Stokes,)
 780            return ls
 781
 782        def list_existing_quantities(self,  pt: PeakType = PeakType.AntiStokes, index: int = 0) -> tuple:
 783            """
 784            Synchronous wrapper for `list_existing_quantities_async` (see doc for `brimfile.data.Data.AnalysisResults.list_existing_quantities_async`)
 785            """
 786            return sync(self.list_existing_quantities_async(pt, index))
 787        async def list_existing_quantities_async(self,  pt: PeakType = PeakType.AntiStokes, index: int = 0) -> tuple:
 788            """
 789            Returns a tuple of existing quantities for the specified index.
 790            Args:
 791                index (int, optional): The index of the peak to check (in case of multi-peak fit). Defaults to 0.
 792            Returns:
 793                tuple: A tuple containing `Quantity` members that exist for the given index.
 794            """
 795            as_cls = Data.AnalysisResults
 796            ls = ()
 797
 798            qts = [qt for qt in as_cls.Quantity]
 799            coros = [self._file.object_exists(concatenate_paths(self._path, as_cls._get_quantity_name(qt, pt, index))) for qt in qts]
 800            
 801            qt_exists = await asyncio.gather(*coros)
 802            for i, exists in enumerate(qt_exists):
 803                if exists:
 804                    ls += (qts[i],)
 805            return ls
 806
 807    def get_metadata(self):
 808        """
 809        Returns the metadata associated with the current Data group
 810        Note that this contains both the general metadata stored in the file (which might be redifined by the specific data group)
 811        and the ones specific for this data group
 812        """
 813        return Metadata(self._file, self._path)
 814
 815    def get_num_parameters(self) -> tuple:
 816        """
 817        Retrieves the number of parameters
 818
 819        Returns:
 820            tuple: The shape of the parameters if they exist, otherwise an empty tuple.
 821        """
 822        pars, _ = self.get_parameters()
 823        return pars.shape if pars is not None else ()
 824
 825    def get_parameters(self) -> list:
 826        """
 827        Retrieves the parameters  and their associated names.
 828
 829        If PSD.ndims > 2, the parameters are stored in a separate dataset.
 830
 831        Returns:
 832            list: A tuple containing the parameters and their names if there are any, otherwise None.
 833        """
 834        pars_full_path = concatenate_paths(
 835            self._path, brim_obj_names.data.parameters)
 836        if sync(self._file.object_exists(pars_full_path)):
 837            pars = sync(self._file.open_dataset(pars_full_path))
 838            pars_names = sync(self._file.get_attr(pars, 'Name'))
 839            return (pars, pars_names)
 840        return (None, None)
 841
 842    def create_analysis_results_group(self, data_AntiStokes, data_Stokes=None, index: int = None, name: str = None, fit_model: 'Data.AnalysisResults.FitModel' = None) -> AnalysisResults:
 843        """
 844        Adds a new AnalysisResults entry to the current data group.
 845        Parameters:
 846            data_AntiStokes (dict or list[dict]): contains the same elements as the ones in `AnalysisResults.add_data`,
 847                but all the quantities (i.d. 'shift', 'width', etc.) are 3D, corresponding to the spatial positions (z, y, x).
 848            data_Stokes (dict or list[dict]): same as data_AntiStokes for the Stokes peaks.
 849            index (int, optional): The index for the new data entry. If None, the next available index is used. Defaults to None.
 850            name (str, optional): The name for the new Analysis group. Defaults to None.
 851            fit_model (Data.AnalysisResults.FitModel, optional): The fit model used for the analysis. Defaults to None (no attribute is set).
 852        Returns:
 853            AnalysisResults: The newly created AnalysisResults object.
 854        Raises:
 855            IndexError: If the specified index already exists in the dataset.
 856            ValueError: If any of the data provided is not valid or consistent
 857        """
 858        def flatten_data(data: dict):
 859            if data is None:
 860                return None
 861            data = var_to_singleton(data)
 862            out_data = []
 863            for dn in data:
 864                for k in dn.keys():
 865                    if not k.endswith('_units'):
 866                        d = dn[k]
 867                        if d.ndim != 3 or d.shape != self._spatial_map.shape:
 868                            raise ValueError(
 869                                f"'{k}' must have 3 dimensions (z, y, x) and same shape as the spatial map ({self._spatial_map.shape})")
 870                        dn[k] = np.reshape(d, -1)  # flatten the data
 871                out_data.append(dn)
 872            return out_data
 873        data_AntiStokes = flatten_data(data_AntiStokes)
 874        data_Stokes = flatten_data(data_Stokes)
 875        return self.create_analysis_results_group_raw(data_AntiStokes, data_Stokes, index, name, fit_model=fit_model)
 876
 877    def create_analysis_results_group_raw(self, data_AntiStokes, data_Stokes=None, index: int = None, name: str = None, fit_model: 'Data.AnalysisResults.FitModel' = None) -> AnalysisResults:
 878        """
 879        Adds a new AnalysisResults entry to the current data group.
 880        Parameters:
 881            data_AntiStokes (dict or list[dict]): see documentation for AnalysisResults.add_data
 882            data_Stokes (dict or list[dict]): same as data_AntiStokes for the Stokes peaks.
 883            index (int, optional): The index for the new data entry. If None, the next available index is used. Defaults to None.
 884            name (str, optional): The name for the new Analysis group. Defaults to None.
 885            fit_model (Data.AnalysisResults.FitModel, optional): The fit model used for the analysis. Defaults to None (no attribute is set).
 886        Returns:
 887            AnalysisResults: The newly created AnalysisResults object.
 888        Raises:
 889            IndexError: If the specified index already exists in the dataset.
 890            ValueError: If any of the data provided is not valid or consistent
 891        """
 892        if index is not None:
 893            try:
 894                self.get_analysis_results(index)
 895            except IndexError:
 896                pass
 897            else:
 898                # If the group already exists, raise an error
 899                raise IndexError(
 900                    f"Analysis {index} already exists in {self._path}")
 901        else:
 902            ar_groups = self.list_AnalysisResults()
 903            indices = [ar['index'] for ar in ar_groups]
 904            indices.sort()
 905            index = indices[-1] + 1 if indices else 0  # Next available index
 906
 907        ar = Data.AnalysisResults._create_new(self, index)
 908        if name is not None:
 909            set_object_name(self._file, ar._path, name)
 910        ar.add_data(data_AntiStokes, data_Stokes, fit_model=fit_model)
 911
 912        return ar
 913
 914    def list_AnalysisResults(self, retrieve_custom_name=False) -> list:
 915        """
 916        List all AnalysisResults groups in the current data group. The list is ordered by index.
 917
 918        Returns:
 919            list: A list of dictionaries, each containing:
 920                - 'name' (str): The name of the AnalysisResults group.
 921                - 'index' (int): The index extracted from the group name.
 922                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the AnalysisResults group as returned from utils.get_object_name.
 923        """
 924
 925        analysis_results_groups = []
 926
 927        matched_objs = list_objects_matching_pattern(
 928            self._file, self._group, brim_obj_names.data.analysis_results + r"_(\d+)$")
 929        async def _make_dict_item(matched_obj, retrieve_custom_name):
 930            name = matched_obj[0]
 931            index = int(matched_obj[1])
 932            curr_obj_dict = {'name': name, 'index': index}
 933            if retrieve_custom_name:
 934                ar_path = concatenate_paths(self._path, name)
 935                custom_name = await get_object_name(self._file, ar_path)
 936                curr_obj_dict['custom_name'] = custom_name
 937            return curr_obj_dict
 938        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
 939        dicts = _gather_sync(*coros)
 940        for dict_item in dicts:
 941            analysis_results_groups.append(dict_item)
 942        # Sort the data groups by index
 943        analysis_results_groups.sort(key=lambda x: x['index'])
 944
 945        return analysis_results_groups
 946
 947    def get_analysis_results(self, index: int = 0) -> AnalysisResults:
 948        """
 949        Returns the AnalysisResults at the specified index
 950
 951        Args:
 952            index (int)                
 953
 954        Raises:
 955            IndexError: If there is no analysis with the corresponding index
 956        """
 957        name = None
 958        ls = self.list_AnalysisResults()
 959        for el in ls:
 960            if el['index'] == index:
 961                name = el['name']
 962                break
 963        if name is None:
 964            raise IndexError(f"Analysis {index} not found")
 965        path = concatenate_paths(self._path, name)
 966        return Data.AnalysisResults(self._file, path, self._spatial_map, self._spatial_map_px_size)
 967
 968    def add_data(self, PSD: np.ndarray, frequency: np.ndarray, scanning: dict, freq_units='GHz', timestamp: np.ndarray = None, compression: FileAbstraction.Compression = FileAbstraction.Compression()):
 969        """
 970        Add data to the current data group.
 971
 972        This method adds the provided PSD, frequency, and scanning data to the HDF5 group 
 973        associated with this `Data` object. It validates the inputs to ensure they meet 
 974        the required specifications before adding them.
 975
 976        Args:
 977            PSD (np.ndarray): A 2D numpy array representing the Power Spectral Density (PSD) data. The last dimension contains the spectra.
 978            frequency (np.ndarray): A 1D or 2D numpy array representing the frequency data. 
 979                It must be broadcastable to the shape of the PSD array.
 980            scanning (dict): A dictionary containing scanning-related data. It may include:
 981                - 'Spatial_map' (optional): A dictionary containing (up to) 3 arrays (x, y, z) and a string (units)
 982                - 'Cartesian_visualisation' (optional): A 3D numpy array containing the association between spatial position and spectra.
 983                   It must have integer values between 0 and PSD.shape[0]-1, or -1 for invalid entries.
 984                - 'Cartesian_visualisation_pixel' (optional): A list or array of 3 float values 
 985                  representing the pixel size in the z, y, and x dimensions (unused dimensions can be set to None).
 986                - 'Cartesian_visualisation_pixel_unit' (optional): A string representing the unit of the pixel size (e.g. 'um').
 987            timestamp (np.ndarray): the timestamp associated with each spectrum.
 988                It must be a 1D array with the same length as the PSD array.
 989
 990
 991        Raises:
 992            ValueError: If any of the data provided is not valid or consistent
 993        """
 994
 995        # Check if frequency is broadcastable to PSD
 996        try:
 997            np.broadcast_shapes(tuple(frequency.shape), tuple(PSD.shape))
 998        except ValueError as e:
 999            raise ValueError(f"frequency (shape: {frequency.shape}) is not broadcastable to PSD (shape: {PSD.shape}): {e}")
1000
1001        # define the scanning_is_valid variable to check if at least one of 'Spatial_map' or 'Cartesian_visualisation'
1002        # is present in the scanning dictionary
1003        scanning_is_valid = False
1004        if 'Spatial_map' in scanning:
1005            sm = scanning['Spatial_map']
1006            size = 0
1007
1008            def check_coor(coor: str):
1009                if coor in sm:
1010                    sm[coor] = np.array(sm[coor])
1011                    size1 = sm[coor].size
1012                    if size1 != size and size != 0:
1013                        raise ValueError(
1014                            f"'{coor}' in 'Spatial_map' is invalid!")
1015                    return size1
1016            size = check_coor('x')
1017            size = check_coor('y')
1018            size = check_coor('z')
1019            if size == 0:
1020                raise ValueError(
1021                    "'Spatial_map' should contain at least one x, y or z")
1022            scanning_is_valid = True
1023        if 'Cartesian_visualisation' in scanning:
1024            cv = scanning['Cartesian_visualisation']
1025            if not isinstance(cv, np.ndarray) or cv.ndim != 3:
1026                raise ValueError(
1027                    "Cartesian_visualisation must be a 3D numpy array")
1028            if not np.issubdtype(cv.dtype, np.integer) or np.min(cv) < -1 or np.max(cv) >= PSD.shape[0]:
1029                raise ValueError(
1030                    "Cartesian_visualisation values must be integers between -1 and PSD.shape[0]-1")
1031            if 'Cartesian_visualisation_pixel' in scanning:
1032                if len(scanning['Cartesian_visualisation_pixel']) != 3:
1033                    raise ValueError(
1034                        "Cartesian_visualisation_pixel must always contain 3 values for z, y, x (set to None if not used)")
1035            else:
1036                warnings.warn(
1037                    "It is recommended to add 'Cartesian_visualisation_pixel' to the scanning dictionary, to define the pixel size")
1038            scanning_is_valid = True
1039        if not scanning_is_valid:
1040            raise ValueError("scanning is not valid")
1041
1042        if timestamp is not None:
1043            if not isinstance(timestamp, np.ndarray) or timestamp.ndim != 1 or len(timestamp) != PSD.shape[0]:
1044                raise ValueError("timestamp is not compatible with PSD")
1045
1046        # TODO: add and validate additional datasets (i.e. 'Parameters', 'Calibration_index', etc.)
1047
1048        # Add datasets to the group
1049        def determine_chunk_size(arr: np.array) -> tuple:
1050            """"
1051            Use the same heuristic as the zarr library to determine the chunk size, but without splitting the last dimension
1052            """
1053            shape = arr.shape
1054            typesize = arr.itemsize
1055            #if the array is 1D, do not chunk it
1056            if len(shape) <= 1:
1057                return (shape[0],)
1058            target_sizes = _guess_chunks.__kwdefaults__
1059            # divide the target size by the last dimension size to get the chunk size for the other dimensions
1060            target_sizes = {k: target_sizes[k] // shape[-1] 
1061                            for k in target_sizes.keys()}
1062            chunks = _guess_chunks(shape[0:-1], typesize, arr.nbytes, **target_sizes)
1063            return chunks + (shape[-1],)  # keep the last dimension size unchanged
1064        sync(self._file.create_dataset(
1065            self._group, brim_obj_names.data.PSD, data=PSD,
1066            chunk_size=determine_chunk_size(PSD), compression=compression))
1067        freq_ds = sync(self._file.create_dataset(
1068            self._group,  brim_obj_names.data.frequency, data=frequency,
1069            chunk_size=determine_chunk_size(frequency), compression=compression))
1070        units.add_to_object(self._file, freq_ds, freq_units)
1071
1072        if 'Spatial_map' in scanning:
1073            sm = scanning['Spatial_map']
1074            sm_group = sync(self._file.create_group(concatenate_paths(
1075                self._path, brim_obj_names.data.spatial_map)))
1076            if 'units' in sm:
1077                units.add_to_object(self._file, sm_group, sm['units'])
1078
1079            def add_sm_dataset(coord: str):
1080                if coord in sm:
1081                    coord_dts = sync(self._file.create_dataset(
1082                        sm_group, coord, data=sm[coord], compression=compression))
1083
1084            add_sm_dataset('x')
1085            add_sm_dataset('y')
1086            add_sm_dataset('z')
1087        if 'Cartesian_visualisation' in scanning:
1088            # convert the Cartesian_visualisation to the smallest integer type
1089            cv_arr = np_array_to_smallest_int_type(scanning['Cartesian_visualisation'])
1090            cv = sync(self._file.create_dataset(self._group, brim_obj_names.data.cartesian_visualisation,
1091                                           data=cv_arr, compression=compression))
1092            if 'Cartesian_visualisation_pixel' in scanning:
1093                sync(self._file.create_attr(
1094                    cv, 'element_size', scanning['Cartesian_visualisation_pixel']))
1095                if 'Cartesian_visualisation_pixel_unit' in scanning:
1096                    px_unit = scanning['Cartesian_visualisation_pixel_unit']
1097                else:
1098                    warnings.warn(
1099                        "No unit provided for Cartesian_visualisation_pixel, defaulting to 'um'")
1100                    px_unit = 'um'
1101                units.add_to_attribute(self._file, cv, 'element_size', px_unit)
1102
1103        self._spatial_map, self._spatial_map_px_size = self._load_spatial_mapping()
1104
1105        if timestamp is not None:
1106            sync(self._file.create_dataset(
1107                self._group, 'Timestamp', data=timestamp, compression=compression))
1108
1109    @staticmethod
1110    def list_data_groups(file: FileAbstraction, retrieve_custom_name=False) -> list:
1111        """
1112        List all data groups in the brim file. The list is ordered by index.
1113
1114        Returns:
1115            list: A list of dictionaries, each containing:
1116                - 'name' (str): The name of the data group in the file.
1117                - 'index' (int): The index extracted from the group name.
1118                - 'custom_name' (str, optional): if retrieve_custom_name==True, it contains the name of the data group as returned from utils.get_object_name.
1119        """
1120
1121        data_groups = []
1122
1123        matched_objs = list_objects_matching_pattern(
1124            file, brim_obj_names.Brillouin_base_path, brim_obj_names.data.base_group + r"_(\d+)$")
1125        
1126        async def _make_dict_item(matched_obj, retrieve_custom_name):
1127            name = matched_obj[0]
1128            index = int(matched_obj[1])
1129            curr_obj_dict = {'name': name, 'index': index}
1130            if retrieve_custom_name:
1131                path = concatenate_paths(
1132                    brim_obj_names.Brillouin_base_path, name)
1133                custom_name = await get_object_name(file, path)
1134                curr_obj_dict['custom_name'] = custom_name
1135            return curr_obj_dict
1136        
1137        coros = [_make_dict_item(matched_obj, retrieve_custom_name) for matched_obj in matched_objs]
1138        dicts = _gather_sync(*coros)
1139        for dict_item in dicts:
1140            data_groups.append(dict_item)        
1141        # Sort the data groups by index
1142        data_groups.sort(key=lambda x: x['index'])
1143
1144        return data_groups
1145
1146    @staticmethod
1147    def _get_existing_group_name(file: FileAbstraction, index: int) -> str:
1148        """
1149        Get the name of an existing data group by index.
1150
1151        Args:
1152            file (File): The parent File object.
1153            index (int): The index of the data group.
1154
1155        Returns:
1156            str: The name of the data group, or None if not found.
1157        """
1158        group_name: str = None
1159        data_groups = Data.list_data_groups(file)
1160        for dg in data_groups:
1161            if dg['index'] == index:
1162                group_name = dg['name']
1163                break
1164        return group_name
1165
1166    @classmethod
1167    def _create_new(cls, file: FileAbstraction, index: int, name: str = None) -> 'Data':
1168        """
1169        Create a new data group with the specified index.
1170
1171        Args:
1172            file (File): The parent File object.
1173            index (int): The index for the new data group.
1174            name (str, optional): The name for the new data group. Defaults to None.
1175
1176        Returns:
1177            Data: The newly created Data object.
1178        """
1179        group_name = Data._generate_group_name(index)
1180        group = sync(file.create_group(concatenate_paths(
1181            brim_obj_names.Brillouin_base_path, group_name)))
1182        if name is not None:
1183            set_object_name(file, group, name)
1184        return cls(file, concatenate_paths(brim_obj_names.Brillouin_base_path, group_name))
1185
1186    @staticmethod
1187    def _generate_group_name(index: int, n_digits: int = None) -> str:
1188        """
1189        Generate a name for a data group based on the index.
1190
1191        Args:
1192            index (int): The index for the data group.
1193            n_digits (int, optional): The number of digits to pad the index with. If None no padding is applied. Defaults to None.
1194
1195        Returns:
1196            str: The generated group name.
1197
1198        Raises:
1199            ValueError: If the index is negative.
1200        """
1201        if index < 0:
1202            raise ValueError("index must be positive")
1203        num = str(index)
1204        if n_digits is not None:
1205            num = num.zfill(n_digits)
1206        return f"{brim_obj_names.data.base_group}_{num}"