micro_sam.sample_data

  1r"""
  2Sample microscopy data.
  3
  4You can change the download location for sample data and model weights
  5by setting the environment variable: MICROSAM_CACHEDIR
  6
  7By default sample data is downloaded to a folder named 'micro_sam/sample_data'
  8inside your default cache directory, eg:
  9    * Mac: ~/Library/Caches/<AppName>
 10    * Unix: ~/.cache/<AppName> or the value of the XDG_CACHE_HOME environment variable, if defined.
 11    * Windows: C:\Users\<user>\AppData\Local\<AppAuthor>\<AppName>\Cache
 12
 13"""
 14
 15import os
 16from pathlib import Path
 17from typing import Union
 18
 19import pooch
 20import numpy as np
 21import imageio.v3 as imageio
 22
 23from skimage.measure import label
 24from skimage.transform import resize
 25from skimage.data import binary_blobs
 26
 27from .util import get_cache_directory
 28
 29
 30def fetch_image_series_example_data(save_directory: Union[str, os.PathLike]) -> str:
 31    """Download the sample images for the image series annotator.
 32
 33    Args:
 34        save_directory: Root folder to save the downloaded data.
 35    Returns:
 36        The folder that contains the downloaded data.
 37    """
 38    # This sample dataset is currently not provided to napari by the micro-sam
 39    # plugin, because images are not all the same shape and cannot be combined
 40    # into a single layer
 41    save_directory = Path(save_directory)
 42    os.makedirs(save_directory, exist_ok=True)
 43    print("Example data directory is:", save_directory.resolve())
 44    fname = "image-series.zip"
 45    unpack_filenames = [os.path.join("series", f"im{i}.tif") for i in range(3)]
 46    unpack = pooch.Unzip(members=unpack_filenames)
 47    pooch.retrieve(
 48        url="https://owncloud.gwdg.de/index.php/s/M1zGnfkulWoAhUG/download",
 49        known_hash="92346ca9770bcaf55248efee590718d54c7135b6ebca15d669f3b77b6afc8706",
 50        fname=fname,
 51        path=save_directory,
 52        progressbar=True,
 53        processor=unpack,
 54    )
 55    data_folder = os.path.join(save_directory, f"{fname}.unzip", "series")
 56    assert os.path.exists(data_folder)
 57    return data_folder
 58
 59
 60def sample_data_image_series():
 61    """Provides image series example image to napari.
 62
 63    Opens as three separate image layers in napari (one per image in series).
 64    The third image in the series has a different size and modality.
 65    """
 66    # Return list of tuples
 67    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
 68    # Check the documentation for more information about the
 69    # add_image_kwargs
 70    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
 71    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
 72    data_directory = fetch_image_series_example_data(base_data_directory)
 73    fnames = os.listdir(data_directory)
 74    full_filenames = [os.path.join(data_directory, f) for f in fnames]
 75    full_filenames.sort()
 76    data_and_image_kwargs = [(imageio.imread(f), {"name": f"img-{i}"}) for i, f in enumerate(full_filenames)]
 77    return data_and_image_kwargs
 78
 79
 80def fetch_wholeslide_example_data(save_directory: Union[str, os.PathLike]) -> str:
 81    """Download the sample data for the 2d annotator.
 82
 83    This downloads part of a whole-slide image from the NeurIPS Cell Segmentation Challenge.
 84    See https://neurips22-cellseg.grand-challenge.org/ for details on the data.
 85
 86    Args:
 87        save_directory: Root folder to save the downloaded data.
 88    Returns:
 89        The path of the downloaded image.
 90    """
 91    save_directory = Path(save_directory)
 92    os.makedirs(save_directory, exist_ok=True)
 93    print("Example data directory is:", save_directory.resolve())
 94    fname = "whole-slide-example-image.tif"
 95    pooch.retrieve(
 96        url="https://owncloud.gwdg.de/index.php/s/6ozPtgBmAAJC1di/download",
 97        known_hash="3ddb9c9dcc844429932ab951eb0743d5a1af83ee9b0ab54f06ceb2090a606d36",
 98        fname=fname,
 99        path=save_directory,
100        progressbar=True,
101    )
102    return os.path.join(save_directory, fname)
103
104
105def sample_data_wholeslide():
106    """Provides wholeslide 2d example image to napari."""
107    # Return list of tuples
108    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
109    # Check the documentation for more information about the
110    # add_image_kwargs
111    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
112    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
113    filename = fetch_wholeslide_example_data(base_data_directory)
114    data = imageio.imread(filename)
115    add_image_kwargs = {"name": "wholeslide"}
116    return [(data, add_image_kwargs)]
117
118
119def fetch_livecell_example_data(save_directory: Union[str, os.PathLike]) -> str:
120    """Download the sample data for the 2d annotator.
121
122    This downloads a single image from the LiveCELL dataset.
123    See https://doi.org/10.1038/s41592-021-01249-6 for details on the data.
124
125    Args:
126        save_directory: Root folder to save the downloaded data.
127    Returns:
128        The path of the downloaded image.
129    """
130    save_directory = Path(save_directory)
131    os.makedirs(save_directory, exist_ok=True)
132    fname = "livecell-2d-image.png"
133    pooch.retrieve(
134        url="https://owncloud.gwdg.de/index.php/s/fSaOJIOYjmFBjPM/download",
135        known_hash="4f190983ea672fc333ac26d735d9625d5abb6e4a02bd4d32523127977a31e8fe",
136        fname=fname,
137        path=save_directory,
138        progressbar=True,
139    )
140    return os.path.join(save_directory, fname)
141
142
143def sample_data_livecell():
144    """Provides livecell 2d example image to napari."""
145    # Return list of tuples
146    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
147    # Check the documentation for more information about the
148    # add_image_kwargs
149    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
150    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
151    filename = fetch_livecell_example_data(base_data_directory)
152    data = imageio.imread(filename)
153    add_image_kwargs = {"name": "livecell"}
154    return [(data, add_image_kwargs)]
155
156
157def fetch_hela_2d_example_data(save_directory: Union[str, os.PathLike]) -> Union[str, os.PathLike]:
158    """Download the sample data for the 2d annotator.
159
160    This downloads a single image from the HeLa CTC dataset.
161
162    Args:
163        save_directory: Root folder to save the downloaded data.
164    Returns:
165        The path of the downloaded image.
166    """
167    save_directory = Path(save_directory)
168    os.makedirs(save_directory, exist_ok=True)
169    print("Example data directory is:", save_directory.resolve())
170    fname = "hela-2d-image.png"
171    pooch.retrieve(
172        url="https://owncloud.gwdg.de/index.php/s/2sr1DHQ34tV7WEb/download",
173        known_hash="908fa00e4b273610aa4e0a9c0f22dfa64a524970852f387908f3fa65238259c7",
174        fname=fname,
175        path=save_directory,
176        progressbar=True,
177    )
178    return os.path.join(save_directory, fname)
179
180
181def sample_data_hela_2d():
182    """Provides HeLa 2d example image to napari."""
183    # Return list of tuples
184    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
185    # Check the documentation for more information about the
186    # add_image_kwargs
187    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
188    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
189    filename = fetch_hela_2d_example_data(base_data_directory)
190    data = imageio.imread(filename)
191    add_image_kwargs = {"name": "hela_2d"}
192    return [(data, add_image_kwargs)]
193
194
195def fetch_3d_example_data(save_directory: Union[str, os.PathLike]) -> str:
196    """Download the sample data for the 3d annotator.
197
198    This downloads the Lucchi++ datasets from https://casser.io/connectomics/.
199    It is a dataset for mitochondria segmentation in EM.
200
201    Args:
202        save_directory: Root folder to save the downloaded data.
203    Returns:
204        The folder that contains the downloaded data.
205    """
206    save_directory = Path(save_directory)
207    os.makedirs(save_directory, exist_ok=True)
208    print("Example data directory is:", save_directory.resolve())
209    unpack_filenames = [os.path.join("Lucchi++", "Test_In", f"mask{str(i).zfill(4)}.png") for i in range(165)]
210    unpack = pooch.Unzip(members=unpack_filenames)
211    fname = "lucchi_pp.zip"
212    pooch.retrieve(
213        url="http://www.casser.io/files/lucchi_pp.zip",
214        known_hash="770ce9e98fc6f29c1b1a250c637e6c5125f2b5f1260e5a7687b55a79e2e8844d",
215        fname=fname,
216        path=save_directory,
217        progressbar=True,
218        processor=unpack,
219    )
220    lucchi_dir = save_directory.joinpath(f"{fname}.unzip", "Lucchi++", "Test_In")
221    return str(lucchi_dir)
222
223
224def sample_data_3d():
225    """Provides Lucchi++ 3d example image to napari."""
226    # Return list of tuples
227    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
228    # Check the documentation for more information about the
229    # add_image_kwargs
230    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
231    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
232    data_directory = fetch_3d_example_data(base_data_directory)
233    fnames = os.listdir(data_directory)
234    full_filenames = [os.path.join(data_directory, f) for f in fnames]
235    full_filenames.sort()
236    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
237    add_image_kwargs = {"name": "lucchi++"}
238    return [(data, add_image_kwargs)]
239
240
241def fetch_tracking_example_data(save_directory: Union[str, os.PathLike]) -> str:
242    """Download the sample data for the tracking annotator.
243
244    This data is the cell tracking challenge dataset DIC-C2DH-HeLa.
245    Cell tracking challenge webpage: http://data.celltrackingchallenge.net
246    HeLa cells on a flat glass
247    Dr. G. van Cappellen. Erasmus Medical Center, Rotterdam, The Netherlands
248    Training dataset: http://data.celltrackingchallenge.net/training-datasets/DIC-C2DH-HeLa.zip (37 MB)
249    Challenge dataset: http://data.celltrackingchallenge.net/challenge-datasets/DIC-C2DH-HeLa.zip (41 MB)
250
251    Args:
252        save_directory: Root folder to save the downloaded data.
253    Returns:
254        The folder that contains the downloaded data.
255    """
256    save_directory = Path(save_directory)
257    os.makedirs(save_directory, exist_ok=True)
258    unpack_filenames = [os.path.join("DIC-C2DH-HeLa", "01", f"t{str(i).zfill(3)}.tif") for i in range(84)]
259    unpack = pooch.Unzip(members=unpack_filenames)
260    fname = "DIC-C2DH-HeLa.zip"
261    pooch.retrieve(
262        url="http://data.celltrackingchallenge.net/training-datasets/DIC-C2DH-HeLa.zip",  # 37 MB
263        known_hash="832fed2d05bb7488cf9c51a2994b75f8f3f53b3c3098856211f2d39023c34e1a",
264        fname=fname,
265        path=save_directory,
266        progressbar=True,
267        processor=unpack,
268    )
269    cell_tracking_dir = save_directory.joinpath(f"{fname}.unzip", "DIC-C2DH-HeLa", "01")
270    assert os.path.exists(cell_tracking_dir)
271    return str(cell_tracking_dir)
272
273
274def sample_data_tracking():
275    """Provides tracking example dataset to napari."""
276    # Return list of tuples
277    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
278    # Check the documentation for more information about the
279    # add_image_kwargs
280    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
281    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
282    data_directory = fetch_tracking_example_data(base_data_directory)
283    fnames = os.listdir(data_directory)
284    full_filenames = [os.path.join(data_directory, f) for f in fnames]
285    full_filenames.sort()
286    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
287    add_image_kwargs = {"name": "tracking"}
288    return [(data, add_image_kwargs)]
289
290
291def fetch_tracking_segmentation_data(save_directory: Union[str, os.PathLike]) -> str:
292    """Download groundtruth segmentation for the tracking example data.
293
294    This downloads the groundtruth segmentation for the image data from `fetch_tracking_example_data`.
295
296    Args:
297        save_directory: Root folder to save the downloaded data.
298    Returns:
299        The folder that contains the downloaded data.
300    """
301    save_directory = Path(save_directory)
302    os.makedirs(save_directory, exist_ok=True)
303    print("Example data directory is:", save_directory.resolve())
304    unpack_filenames = [os.path.join("masks", f"mask_{str(i).zfill(4)}.tif") for i in range(84)]
305    unpack = pooch.Unzip(members=unpack_filenames)
306    fname = "hela-ctc-01-gt.zip"
307    pooch.retrieve(
308        url="https://owncloud.gwdg.de/index.php/s/AWxQMblxwR99OjC/download",
309        known_hash="c0644d8ebe1390fb60125560ba15aa2342caf44f50ff0667a0318ea0ac6c958b",
310        fname=fname,
311        path=save_directory,
312        progressbar=True,
313        processor=unpack,
314    )
315    cell_tracking_dir = save_directory.joinpath(f"{fname}.unzip", "masks")
316    assert os.path.exists(cell_tracking_dir)
317    return str(cell_tracking_dir)
318
319
320def sample_data_segmentation():
321    """Provides segmentation example dataset to napari."""
322    # Return list of tuples
323    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
324    # Check the documentation for more information about the
325    # add_image_kwargs
326    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
327    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
328    data_directory = fetch_tracking_segmentation_data(base_data_directory)
329    fnames = os.listdir(data_directory)
330    full_filenames = [os.path.join(data_directory, f) for f in fnames]
331    full_filenames.sort()
332    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
333    add_image_kwargs = {"name": "segmentation"}
334    return [(data, add_image_kwargs)]
335
336
337def synthetic_data(shape, seed=None):
338    """Create synthetic image data and segmentation for training."""
339    ndim = len(shape)
340    assert ndim in (2, 3)
341    image_shape = shape if ndim == 2 else shape[1:]
342    image = binary_blobs(length=image_shape[0], blob_size_fraction=0.05, volume_fraction=0.15, rng=seed)
343
344    if image_shape[1] != image_shape[0]:
345        image = resize(image, image_shape, order=0, anti_aliasing=False, preserve_range=True).astype(image.dtype)
346    if ndim == 3:
347        nz = shape[0]
348        image = np.stack([image] * nz)
349
350    segmentation = label(image)
351    image = image.astype("uint8") * 255
352    return image, segmentation
353
354
355def fetch_nucleus_3d_example_data(save_directory: Union[str, os.PathLike]) -> str:
356    """Download the sample data for 3d segmentation of nuclei.
357
358    This data contains a small crop from a volume from the publication
359    "Efficient automatic 3D segmentation of cell nuclei for high-content screening"
360    https://doi.org/10.1186/s12859-022-04737-4
361
362    Args:
363        save_directory: Root folder to save the downloaded data.
364    Returns:
365        The path of the downloaded image.
366    """
367    save_directory = Path(save_directory)
368    os.makedirs(save_directory, exist_ok=True)
369    print("Example data directory is:", save_directory.resolve())
370    fname = "3d-nucleus-data.tif"
371    pooch.retrieve(
372        url="https://owncloud.gwdg.de/index.php/s/eW0uNCo8gedzWU4/download",
373        known_hash="4946896f747dc1c3fc82fb2e1320226d92f99d22be88ea5f9c37e3ba4e281205",
374        fname=fname,
375        path=save_directory,
376        progressbar=True,
377    )
378    return os.path.join(save_directory, fname)