micro_sam.sample_data

  1r"""Sample microscopy data.
  2
  3You can change the download location for sample data and model weights
  4by setting the environment variable: MICROSAM_CACHEDIR
  5
  6By default sample data is downloaded to a folder named 'micro_sam/sample_data'
  7inside your default cache directory, eg:
  8    * Mac: ~/Library/Caches/<AppName>
  9    * Unix: ~/.cache/<AppName> or the value of the XDG_CACHE_HOME environment variable, if defined.
 10    * Windows: C:\Users\<user>\AppData\Local\<AppAuthor>\<AppName>\Cache
 11"""
 12
 13import os
 14import pooch
 15from pathlib import Path
 16from typing import Union
 17
 18import numpy as np
 19import imageio.v3 as imageio
 20from skimage.measure import label
 21from skimage.transform import resize
 22from skimage.data import binary_blobs
 23
 24from .util import get_cache_directory
 25
 26
 27def fetch_image_series_example_data(save_directory: Union[str, os.PathLike]) -> str:
 28    """Download the sample images for the image series annotator.
 29
 30    Args:
 31        save_directory: Root folder to save the downloaded data.
 32
 33    Returns:
 34        The folder that contains the downloaded data.
 35    """
 36    # This sample dataset is currently not provided to napari by the micro-sam
 37    # plugin, because images are not all the same shape and cannot be combined
 38    # into a single layer
 39    save_directory = Path(save_directory)
 40    os.makedirs(save_directory, exist_ok=True)
 41    print("Example data directory is:", save_directory.resolve())
 42    fname = "image-series.zip"
 43    unpack_filenames = [os.path.join("series", f"im{i}.tif") for i in range(3)]
 44    unpack = pooch.Unzip(members=unpack_filenames)
 45    pooch.retrieve(
 46        url="https://owncloud.gwdg.de/index.php/s/M1zGnfkulWoAhUG/download",
 47        known_hash="92346ca9770bcaf55248efee590718d54c7135b6ebca15d669f3b77b6afc8706",
 48        fname=fname,
 49        path=save_directory,
 50        progressbar=True,
 51        processor=unpack,
 52    )
 53    data_folder = os.path.join(save_directory, f"{fname}.unzip", "series")
 54    assert os.path.exists(data_folder)
 55    return data_folder
 56
 57
 58def sample_data_image_series():
 59    """Provides image series example image to napari.
 60
 61    Opens as three separate image layers in napari (one per image in series).
 62    The third image in the series has a different size and modality.
 63    """
 64    # Return list of tuples
 65    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
 66    # Check the documentation for more information about the
 67    # add_image_kwargs
 68    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
 69    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
 70    data_directory = fetch_image_series_example_data(base_data_directory)
 71    fnames = os.listdir(data_directory)
 72    full_filenames = [os.path.join(data_directory, f) for f in fnames]
 73    full_filenames.sort()
 74    data_and_image_kwargs = [(imageio.imread(f), {"name": f"img-{i}"}) for i, f in enumerate(full_filenames)]
 75    return data_and_image_kwargs
 76
 77
 78def fetch_wholeslide_example_data(save_directory: Union[str, os.PathLike]) -> str:
 79    """Download the sample data for the 2d annotator.
 80
 81    This downloads part of a whole-slide image from the NeurIPS Cell Segmentation Challenge.
 82    See https://neurips22-cellseg.grand-challenge.org/ for details on the data.
 83
 84    Args:
 85        save_directory: Root folder to save the downloaded data.
 86
 87    Returns:
 88        The path of the downloaded image.
 89    """
 90    save_directory = Path(save_directory)
 91    os.makedirs(save_directory, exist_ok=True)
 92    print("Example data directory is:", save_directory.resolve())
 93    fname = "whole-slide-example-image.tif"
 94    pooch.retrieve(
 95        url="https://owncloud.gwdg.de/index.php/s/6ozPtgBmAAJC1di/download",
 96        known_hash="3ddb9c9dcc844429932ab951eb0743d5a1af83ee9b0ab54f06ceb2090a606d36",
 97        fname=fname,
 98        path=save_directory,
 99        progressbar=True,
100    )
101    return os.path.join(save_directory, fname)
102
103
104def sample_data_wholeslide():
105    """Provides wholeslide 2d example image to napari."""
106    # Return list of tuples
107    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
108    # Check the documentation for more information about the
109    # add_image_kwargs
110    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
111    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
112    filename = fetch_wholeslide_example_data(base_data_directory)
113    data = imageio.imread(filename)
114    add_image_kwargs = {"name": "wholeslide"}
115    return [(data, add_image_kwargs)]
116
117
118def fetch_livecell_example_data(save_directory: Union[str, os.PathLike]) -> str:
119    """Download the sample data for the 2d annotator.
120
121    This downloads a single image from the LIVECell dataset.
122    See https://doi.org/10.1038/s41592-021-01249-6 for details on the data.
123
124    Args:
125        save_directory: Root folder to save the downloaded data.
126
127    Returns:
128        The path of the downloaded image.
129    """
130    save_directory = Path(save_directory)
131    os.makedirs(save_directory, exist_ok=True)
132    fname = "livecell-2d-image.png"
133    pooch.retrieve(
134        url="https://owncloud.gwdg.de/index.php/s/fSaOJIOYjmFBjPM/download",
135        known_hash="4f190983ea672fc333ac26d735d9625d5abb6e4a02bd4d32523127977a31e8fe",
136        fname=fname,
137        path=save_directory,
138        progressbar=True,
139    )
140    return os.path.join(save_directory, fname)
141
142
143def sample_data_livecell():
144    """Provides LIVECell 2d example image to napari."""
145    # Return list of tuples
146    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
147    # Check the documentation for more information about the
148    # add_image_kwargs
149    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
150    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
151    filename = fetch_livecell_example_data(base_data_directory)
152    data = imageio.imread(filename)
153    add_image_kwargs = {"name": "livecell"}
154    return [(data, add_image_kwargs)]
155
156
157def fetch_hela_2d_example_data(save_directory: Union[str, os.PathLike]) -> Union[str, os.PathLike]:
158    """Download the sample data for the 2d annotator.
159
160    This downloads a single image from the HeLa CTC dataset.
161
162    Args:
163        save_directory: Root folder to save the downloaded data.
164
165    Returns:
166        The path of the downloaded image.
167    """
168    save_directory = Path(save_directory)
169    os.makedirs(save_directory, exist_ok=True)
170    print("Example data directory is:", save_directory.resolve())
171    fname = "hela-2d-image.png"
172    pooch.retrieve(
173        url="https://owncloud.gwdg.de/index.php/s/2sr1DHQ34tV7WEb/download",
174        known_hash="908fa00e4b273610aa4e0a9c0f22dfa64a524970852f387908f3fa65238259c7",
175        fname=fname,
176        path=save_directory,
177        progressbar=True,
178    )
179    return os.path.join(save_directory, fname)
180
181
182def sample_data_hela_2d():
183    """Provides HeLa 2d example image to napari."""
184    # Return list of tuples
185    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
186    # Check the documentation for more information about the
187    # add_image_kwargs
188    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
189    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
190    filename = fetch_hela_2d_example_data(base_data_directory)
191    data = imageio.imread(filename)
192    add_image_kwargs = {"name": "hela_2d"}
193    return [(data, add_image_kwargs)]
194
195
196def fetch_3d_example_data(save_directory: Union[str, os.PathLike]) -> str:
197    """Download the sample data for the 3d annotator.
198
199    This downloads the Lucchi++ datasets from https://casser.io/connectomics/.
200    It is a dataset for mitochondria segmentation in EM.
201
202    Args:
203        save_directory: Root folder to save the downloaded data.
204
205    Returns:
206        The folder that contains the downloaded data.
207    """
208    save_directory = Path(save_directory)
209    os.makedirs(save_directory, exist_ok=True)
210    print("Example data directory is:", save_directory.resolve())
211    unpack_filenames = [os.path.join("Lucchi++", "Test_In", f"mask{str(i).zfill(4)}.png") for i in range(165)]
212    unpack = pooch.Unzip(members=unpack_filenames)
213    fname = "lucchi_pp.zip"
214    pooch.retrieve(
215        url="http://www.casser.io/files/lucchi_pp.zip",
216        known_hash="770ce9e98fc6f29c1b1a250c637e6c5125f2b5f1260e5a7687b55a79e2e8844d",
217        fname=fname,
218        path=save_directory,
219        progressbar=True,
220        processor=unpack,
221    )
222    lucchi_dir = save_directory.joinpath(f"{fname}.unzip", "Lucchi++", "Test_In")
223    return str(lucchi_dir)
224
225
226def sample_data_3d():
227    """Provides Lucchi++ 3d example image to napari."""
228    # Return list of tuples
229    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
230    # Check the documentation for more information about the
231    # add_image_kwargs
232    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
233    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
234    data_directory = fetch_3d_example_data(base_data_directory)
235    fnames = os.listdir(data_directory)
236    full_filenames = [os.path.join(data_directory, f) for f in fnames]
237    full_filenames.sort()
238    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
239    add_image_kwargs = {"name": "lucchi++"}
240    return [(data, add_image_kwargs)]
241
242
243def fetch_tracking_example_data(save_directory: Union[str, os.PathLike]) -> str:
244    """Download the sample data for the tracking annotator.
245
246    This data is the Cell Tracking Challenge dataset DIC-C2DH-HeLa.
247    Cell Tracking Challenge webpage: http://data.celltrackingchallenge.net
248    HeLa cells on a flat glass
249    Dr. G. van Cappellen. Erasmus Medical Center, Rotterdam, The Netherlands
250    Training dataset: http://data.celltrackingchallenge.net/training-datasets/DIC-C2DH-HeLa.zip (37 MB)
251    Challenge dataset: http://data.celltrackingchallenge.net/challenge-datasets/DIC-C2DH-HeLa.zip (41 MB)
252
253    Args:
254        save_directory: Root folder to save the downloaded data.
255
256    Returns:
257        The folder that contains the downloaded data.
258    """
259    save_directory = Path(save_directory)
260    os.makedirs(save_directory, exist_ok=True)
261    unpack_filenames = [os.path.join("DIC-C2DH-HeLa", "01", f"t{str(i).zfill(3)}.tif") for i in range(84)]
262    unpack = pooch.Unzip(members=unpack_filenames)
263    fname = "DIC-C2DH-HeLa.zip"
264    pooch.retrieve(
265        url="http://data.celltrackingchallenge.net/training-datasets/DIC-C2DH-HeLa.zip",  # 37 MB
266        known_hash="832fed2d05bb7488cf9c51a2994b75f8f3f53b3c3098856211f2d39023c34e1a",
267        fname=fname,
268        path=save_directory,
269        progressbar=True,
270        processor=unpack,
271    )
272    cell_tracking_dir = save_directory.joinpath(f"{fname}.unzip", "DIC-C2DH-HeLa", "01")
273    assert os.path.exists(cell_tracking_dir)
274    return str(cell_tracking_dir)
275
276
277def sample_data_tracking():
278    """Provides tracking example dataset to napari."""
279    # Return list of tuples
280    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
281    # Check the documentation for more information about the
282    # add_image_kwargs
283    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
284    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
285    data_directory = fetch_tracking_example_data(base_data_directory)
286    fnames = os.listdir(data_directory)
287    full_filenames = [os.path.join(data_directory, f) for f in fnames]
288    full_filenames.sort()
289    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
290    add_image_kwargs = {"name": "tracking"}
291    return [(data, add_image_kwargs)]
292
293
294def fetch_tracking_segmentation_data(save_directory: Union[str, os.PathLike]) -> str:
295    """Download groundtruth segmentation for the tracking example data.
296
297    This downloads the groundtruth segmentation for the image data from `fetch_tracking_example_data`.
298
299    Args:
300        save_directory: Root folder to save the downloaded data.
301
302    Returns:
303        The folder that contains the downloaded data.
304    """
305    save_directory = Path(save_directory)
306    os.makedirs(save_directory, exist_ok=True)
307    print("Example data directory is:", save_directory.resolve())
308    unpack_filenames = [os.path.join("masks", f"mask_{str(i).zfill(4)}.tif") for i in range(84)]
309    unpack = pooch.Unzip(members=unpack_filenames)
310    fname = "hela-ctc-01-gt.zip"
311    pooch.retrieve(
312        url="https://owncloud.gwdg.de/index.php/s/AWxQMblxwR99OjC/download",
313        known_hash="c0644d8ebe1390fb60125560ba15aa2342caf44f50ff0667a0318ea0ac6c958b",
314        fname=fname,
315        path=save_directory,
316        progressbar=True,
317        processor=unpack,
318    )
319    cell_tracking_dir = save_directory.joinpath(f"{fname}.unzip", "masks")
320    assert os.path.exists(cell_tracking_dir)
321    return str(cell_tracking_dir)
322
323
324def sample_data_segmentation():
325    """Provides segmentation example dataset to napari."""
326    # Return list of tuples
327    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
328    # Check the documentation for more information about the
329    # add_image_kwargs
330    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
331    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
332    data_directory = fetch_tracking_segmentation_data(base_data_directory)
333    fnames = os.listdir(data_directory)
334    full_filenames = [os.path.join(data_directory, f) for f in fnames]
335    full_filenames.sort()
336    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
337    add_image_kwargs = {"name": "segmentation"}
338    return [(data, add_image_kwargs)]
339
340
341def synthetic_data(shape, seed=None):
342    """Create synthetic image data and segmentation for training."""
343    ndim = len(shape)
344    assert ndim in (2, 3)
345    image_shape = shape if ndim == 2 else shape[1:]
346    image = binary_blobs(length=image_shape[0], blob_size_fraction=0.05, volume_fraction=0.15, rng=seed)
347
348    if image_shape[1] != image_shape[0]:
349        image = resize(image, image_shape, order=0, anti_aliasing=False, preserve_range=True).astype(image.dtype)
350    if ndim == 3:
351        nz = shape[0]
352        image = np.stack([image] * nz)
353
354    segmentation = label(image)
355    image = image.astype("uint8") * 255
356    return image, segmentation
357
358
359def fetch_nucleus_3d_example_data(save_directory: Union[str, os.PathLike]) -> str:
360    """Download the sample data for 3d segmentation of nuclei.
361
362    This data contains a small crop from a volume from the publication
363    "Efficient automatic 3D segmentation of cell nuclei for high-content screening"
364    https://doi.org/10.1186/s12859-022-04737-4
365
366    Args:
367        save_directory: Root folder to save the downloaded data.
368
369    Returns:
370        The path of the downloaded image.
371    """
372    save_directory = Path(save_directory)
373    os.makedirs(save_directory, exist_ok=True)
374    print("Example data directory is:", save_directory.resolve())
375    fname = "3d-nucleus-data.tif"
376    pooch.retrieve(
377        url="https://owncloud.gwdg.de/index.php/s/eW0uNCo8gedzWU4/download",
378        known_hash="4946896f747dc1c3fc82fb2e1320226d92f99d22be88ea5f9c37e3ba4e281205",
379        fname=fname,
380        path=save_directory,
381        progressbar=True,
382    )
383    return os.path.join(save_directory, fname)
384
385
386def fetch_wholeslide_histopathology_example_data(save_directory: Union[str, os.PathLike]) -> str:
387    """Download the sample histpathology data for the 2d annotator.
388
389    This downloads a WSI image from https://openslide.cs.cmu.edu/download/openslide-testdata/,
390    under the "CC0 1.0 Universal" license.
391
392    Args:
393        save_directory: Root folder to save the downloaded data.
394
395    Returns:
396        The path of the downloaded image.
397    """
398    save_directory = Path(save_directory)
399    os.makedirs(save_directory, exist_ok=True)
400    print("Example data directory is:", save_directory.resolve())
401    fname = "whole-slide-histopathology-example-image.svs"
402    pooch.retrieve(
403        url="https://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1.svs",
404        known_hash="00a3d54482cd707abf254fe69dccc8d06b8ff757a1663f1290c23418c480eb30",
405        fname=fname,
406        path=save_directory,
407        progressbar=True,
408    )
409    return os.path.join(save_directory, fname)