micro_sam.sample_data

  1r"""Sample microscopy data.
  2
  3You can change the download location for sample data and model weights
  4by setting the environment variable: MICROSAM_CACHEDIR
  5
  6By default sample data is downloaded to a folder named 'micro_sam/sample_data'
  7inside your default cache directory, eg:
  8    * Mac: ~/Library/Caches/<AppName>
  9    * Unix: ~/.cache/<AppName> or the value of the XDG_CACHE_HOME environment variable, if defined.
 10    * Windows: C:\Users\<user>\AppData\Local\<AppAuthor>\<AppName>\Cache
 11"""
 12
 13import os
 14import pooch
 15from pathlib import Path
 16from typing import Union
 17
 18import numpy as np
 19import imageio.v3 as imageio
 20from skimage.transform import resize
 21from skimage.data import binary_blobs
 22
 23from bioimage_cpp.segmentation import label
 24
 25from .util import get_cache_directory
 26
 27
 28def fetch_image_series_example_data(save_directory: Union[str, os.PathLike]) -> str:
 29    """Download the sample images for the image series annotator.
 30
 31    Args:
 32        save_directory: Root folder to save the downloaded data.
 33
 34    Returns:
 35        The folder that contains the downloaded data.
 36    """
 37    # This sample dataset is currently not provided to napari by the micro-sam
 38    # plugin, because images are not all the same shape and cannot be combined
 39    # into a single layer
 40    save_directory = Path(save_directory)
 41    os.makedirs(save_directory, exist_ok=True)
 42    print("Example data directory is:", save_directory.resolve())
 43    fname = "image-series.zip"
 44    unpack_filenames = [os.path.join("series", f"im{i}.tif") for i in range(3)]
 45    unpack = pooch.Unzip(members=unpack_filenames)
 46    pooch.retrieve(
 47        url="https://owncloud.gwdg.de/index.php/s/M1zGnfkulWoAhUG/download",
 48        known_hash="92346ca9770bcaf55248efee590718d54c7135b6ebca15d669f3b77b6afc8706",
 49        fname=fname,
 50        path=save_directory,
 51        progressbar=True,
 52        processor=unpack,
 53    )
 54    data_folder = os.path.join(save_directory, f"{fname}.unzip", "series")
 55    assert os.path.exists(data_folder)
 56    return data_folder
 57
 58
 59def sample_data_image_series():
 60    """Provides image series example image to napari.
 61
 62    Opens as three separate image layers in napari (one per image in series).
 63    The third image in the series has a different size and modality.
 64    """
 65    # Return list of tuples
 66    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
 67    # Check the documentation for more information about the
 68    # add_image_kwargs
 69    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
 70    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
 71    data_directory = fetch_image_series_example_data(base_data_directory)
 72    fnames = os.listdir(data_directory)
 73    full_filenames = [os.path.join(data_directory, f) for f in fnames]
 74    full_filenames.sort()
 75    data_and_image_kwargs = [(imageio.imread(f), {"name": f"img-{i}"}) for i, f in enumerate(full_filenames)]
 76    return data_and_image_kwargs
 77
 78
 79def fetch_wholeslide_example_data(save_directory: Union[str, os.PathLike]) -> str:
 80    """Download the sample data for the 2d annotator.
 81
 82    This downloads part of a whole-slide image from the NeurIPS Cell Segmentation Challenge.
 83    See https://neurips22-cellseg.grand-challenge.org/ for details on the data.
 84
 85    Args:
 86        save_directory: Root folder to save the downloaded data.
 87
 88    Returns:
 89        The path of the downloaded image.
 90    """
 91    save_directory = Path(save_directory)
 92    os.makedirs(save_directory, exist_ok=True)
 93    print("Example data directory is:", save_directory.resolve())
 94    fname = "whole-slide-example-image.tif"
 95    pooch.retrieve(
 96        url="https://owncloud.gwdg.de/index.php/s/6ozPtgBmAAJC1di/download",
 97        known_hash="3ddb9c9dcc844429932ab951eb0743d5a1af83ee9b0ab54f06ceb2090a606d36",
 98        fname=fname,
 99        path=save_directory,
100        progressbar=True,
101    )
102    return os.path.join(save_directory, fname)
103
104
105def sample_data_wholeslide():
106    """Provides wholeslide 2d example image to napari."""
107    # Return list of tuples
108    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
109    # Check the documentation for more information about the
110    # add_image_kwargs
111    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
112    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
113    filename = fetch_wholeslide_example_data(base_data_directory)
114    data = imageio.imread(filename)
115    add_image_kwargs = {"name": "wholeslide"}
116    return [(data, add_image_kwargs)]
117
118
119def fetch_livecell_example_data(save_directory: Union[str, os.PathLike]) -> str:
120    """Download the sample data for the 2d annotator.
121
122    This downloads a single image from the LIVECell dataset.
123    See https://doi.org/10.1038/s41592-021-01249-6 for details on the data.
124
125    Args:
126        save_directory: Root folder to save the downloaded data.
127
128    Returns:
129        The path of the downloaded image.
130    """
131    save_directory = Path(save_directory)
132    os.makedirs(save_directory, exist_ok=True)
133    fname = "livecell-2d-image.png"
134    pooch.retrieve(
135        url="https://owncloud.gwdg.de/index.php/s/fSaOJIOYjmFBjPM/download",
136        known_hash="4f190983ea672fc333ac26d735d9625d5abb6e4a02bd4d32523127977a31e8fe",
137        fname=fname,
138        path=save_directory,
139        progressbar=True,
140    )
141    return os.path.join(save_directory, fname)
142
143
144def sample_data_livecell():
145    """Provides LIVECell 2d example image to napari."""
146    # Return list of tuples
147    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
148    # Check the documentation for more information about the
149    # add_image_kwargs
150    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
151    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
152    filename = fetch_livecell_example_data(base_data_directory)
153    data = imageio.imread(filename)
154    add_image_kwargs = {"name": "livecell"}
155    return [(data, add_image_kwargs)]
156
157
158def fetch_hela_2d_example_data(save_directory: Union[str, os.PathLike]) -> Union[str, os.PathLike]:
159    """Download the sample data for the 2d annotator.
160
161    This downloads a single image from the HeLa CTC dataset.
162
163    Args:
164        save_directory: Root folder to save the downloaded data.
165
166    Returns:
167        The path of the downloaded image.
168    """
169    save_directory = Path(save_directory)
170    os.makedirs(save_directory, exist_ok=True)
171    print("Example data directory is:", save_directory.resolve())
172    fname = "hela-2d-image.png"
173    pooch.retrieve(
174        url="https://owncloud.gwdg.de/index.php/s/2sr1DHQ34tV7WEb/download",
175        known_hash="908fa00e4b273610aa4e0a9c0f22dfa64a524970852f387908f3fa65238259c7",
176        fname=fname,
177        path=save_directory,
178        progressbar=True,
179    )
180    return os.path.join(save_directory, fname)
181
182
183def sample_data_hela_2d():
184    """Provides HeLa 2d example image to napari."""
185    # Return list of tuples
186    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
187    # Check the documentation for more information about the
188    # add_image_kwargs
189    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
190    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
191    filename = fetch_hela_2d_example_data(base_data_directory)
192    data = imageio.imread(filename)
193    add_image_kwargs = {"name": "hela_2d"}
194    return [(data, add_image_kwargs)]
195
196
197def fetch_3d_example_data(save_directory: Union[str, os.PathLike]) -> str:
198    """Download the sample data for the 3d annotator.
199
200    This downloads the Lucchi++ datasets from https://casser.io/connectomics/.
201    It is a dataset for mitochondria segmentation in EM.
202
203    Args:
204        save_directory: Root folder to save the downloaded data.
205
206    Returns:
207        The folder that contains the downloaded data.
208    """
209    save_directory = Path(save_directory)
210    os.makedirs(save_directory, exist_ok=True)
211    print("Example data directory is:", save_directory.resolve())
212    unpack_filenames = [os.path.join("Lucchi++", "Test_In", f"mask{str(i).zfill(4)}.png") for i in range(165)]
213    unpack = pooch.Unzip(members=unpack_filenames)
214    fname = "lucchi_pp.zip"
215    pooch.retrieve(
216        url="http://www.casser.io/files/lucchi_pp.zip",
217        known_hash="770ce9e98fc6f29c1b1a250c637e6c5125f2b5f1260e5a7687b55a79e2e8844d",
218        fname=fname,
219        path=save_directory,
220        progressbar=True,
221        processor=unpack,
222    )
223    lucchi_dir = save_directory.joinpath(f"{fname}.unzip", "Lucchi++", "Test_In")
224    return str(lucchi_dir)
225
226
227def sample_data_3d():
228    """Provides Lucchi++ 3d example image to napari."""
229    # Return list of tuples
230    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
231    # Check the documentation for more information about the
232    # add_image_kwargs
233    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
234    base_data_directory = os.path.join(get_cache_directory(), "sample_data")
235    data_directory = fetch_3d_example_data(base_data_directory)
236    fnames = os.listdir(data_directory)
237    full_filenames = [os.path.join(data_directory, f) for f in fnames]
238    full_filenames.sort()
239    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
240    add_image_kwargs = {"name": "lucchi++"}
241    return [(data, add_image_kwargs)]
242
243
244def fetch_tracking_example_data(save_directory: Union[str, os.PathLike]) -> str:
245    """Download the sample data for the tracking annotator.
246
247    This data is the Cell Tracking Challenge dataset DIC-C2DH-HeLa.
248    Cell Tracking Challenge webpage: http://data.celltrackingchallenge.net
249    HeLa cells on a flat glass
250    Dr. G. van Cappellen. Erasmus Medical Center, Rotterdam, The Netherlands
251    Training dataset: http://data.celltrackingchallenge.net/training-datasets/DIC-C2DH-HeLa.zip (37 MB)
252    Challenge dataset: http://data.celltrackingchallenge.net/challenge-datasets/DIC-C2DH-HeLa.zip (41 MB)
253
254    Args:
255        save_directory: Root folder to save the downloaded data.
256
257    Returns:
258        The folder that contains the downloaded data.
259    """
260    save_directory = Path(save_directory)
261    os.makedirs(save_directory, exist_ok=True)
262    unpack_filenames = [os.path.join("DIC-C2DH-HeLa", "01", f"t{str(i).zfill(3)}.tif") for i in range(84)]
263    unpack = pooch.Unzip(members=unpack_filenames)
264    fname = "DIC-C2DH-HeLa.zip"
265    pooch.retrieve(
266        url="http://data.celltrackingchallenge.net/training-datasets/DIC-C2DH-HeLa.zip",  # 37 MB
267        known_hash="832fed2d05bb7488cf9c51a2994b75f8f3f53b3c3098856211f2d39023c34e1a",
268        fname=fname,
269        path=save_directory,
270        progressbar=True,
271        processor=unpack,
272    )
273    cell_tracking_dir = save_directory.joinpath(f"{fname}.unzip", "DIC-C2DH-HeLa", "01")
274    assert os.path.exists(cell_tracking_dir)
275    return str(cell_tracking_dir)
276
277
278def sample_data_tracking():
279    """Provides tracking example dataset to napari."""
280    # Return list of tuples
281    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
282    # Check the documentation for more information about the
283    # add_image_kwargs
284    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
285    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
286    data_directory = fetch_tracking_example_data(base_data_directory)
287    fnames = os.listdir(data_directory)
288    full_filenames = [os.path.join(data_directory, f) for f in fnames]
289    full_filenames.sort()
290    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
291    add_image_kwargs = {"name": "tracking"}
292    return [(data, add_image_kwargs)]
293
294
295def fetch_tracking_segmentation_data(save_directory: Union[str, os.PathLike]) -> str:
296    """Download groundtruth segmentation for the tracking example data.
297
298    This downloads the groundtruth segmentation for the image data from `fetch_tracking_example_data`.
299
300    Args:
301        save_directory: Root folder to save the downloaded data.
302
303    Returns:
304        The folder that contains the downloaded data.
305    """
306    save_directory = Path(save_directory)
307    os.makedirs(save_directory, exist_ok=True)
308    print("Example data directory is:", save_directory.resolve())
309    unpack_filenames = [os.path.join("masks", f"mask_{str(i).zfill(4)}.tif") for i in range(84)]
310    unpack = pooch.Unzip(members=unpack_filenames)
311    fname = "hela-ctc-01-gt.zip"
312    pooch.retrieve(
313        url="https://owncloud.gwdg.de/index.php/s/AWxQMblxwR99OjC/download",
314        known_hash="c0644d8ebe1390fb60125560ba15aa2342caf44f50ff0667a0318ea0ac6c958b",
315        fname=fname,
316        path=save_directory,
317        progressbar=True,
318        processor=unpack,
319    )
320    cell_tracking_dir = save_directory.joinpath(f"{fname}.unzip", "masks")
321    assert os.path.exists(cell_tracking_dir)
322    return str(cell_tracking_dir)
323
324
325def sample_data_segmentation():
326    """Provides segmentation example dataset to napari."""
327    # Return list of tuples
328    # [(data1, add_image_kwargs1), (data2, add_image_kwargs2)]
329    # Check the documentation for more information about the
330    # add_image_kwargs
331    # https://napari.org/stable/api/napari.Viewer.html#napari.Viewer.add_image
332    base_data_directory = os.path.join(get_cache_directory(), 'sample_data')
333    data_directory = fetch_tracking_segmentation_data(base_data_directory)
334    fnames = os.listdir(data_directory)
335    full_filenames = [os.path.join(data_directory, f) for f in fnames]
336    full_filenames.sort()
337    data = np.stack([imageio.imread(f) for f in full_filenames], axis=0)
338    add_image_kwargs = {"name": "segmentation"}
339    return [(data, add_image_kwargs)]
340
341
342def synthetic_data(shape, seed=None):
343    """Create synthetic image data and segmentation for training."""
344    ndim = len(shape)
345    assert ndim in (2, 3)
346    image_shape = shape if ndim == 2 else shape[1:]
347    image = binary_blobs(length=image_shape[0], blob_size_fraction=0.05, volume_fraction=0.15, rng=seed)
348
349    if image_shape[1] != image_shape[0]:
350        image = resize(image, image_shape, order=0, anti_aliasing=False, preserve_range=True).astype(image.dtype)
351    if ndim == 3:
352        nz = shape[0]
353        image = np.stack([image] * nz)
354
355    segmentation = label(image)
356    image = image.astype("uint8") * 255
357    return image, segmentation
358
359
360def fetch_nucleus_3d_example_data(save_directory: Union[str, os.PathLike]) -> str:
361    """Download the sample data for 3d segmentation of nuclei.
362
363    This data contains a small crop from a volume from the publication
364    "Efficient automatic 3D segmentation of cell nuclei for high-content screening"
365    https://doi.org/10.1186/s12859-022-04737-4
366
367    Args:
368        save_directory: Root folder to save the downloaded data.
369
370    Returns:
371        The path of the downloaded image.
372    """
373    save_directory = Path(save_directory)
374    os.makedirs(save_directory, exist_ok=True)
375    print("Example data directory is:", save_directory.resolve())
376    fname = "3d-nucleus-data.tif"
377    pooch.retrieve(
378        url="https://owncloud.gwdg.de/index.php/s/eW0uNCo8gedzWU4/download",
379        known_hash="4946896f747dc1c3fc82fb2e1320226d92f99d22be88ea5f9c37e3ba4e281205",
380        fname=fname,
381        path=save_directory,
382        progressbar=True,
383    )
384    return os.path.join(save_directory, fname)
385
386
387def fetch_wholeslide_histopathology_example_data(save_directory: Union[str, os.PathLike]) -> str:
388    """Download the sample histpathology data for the 2d annotator.
389
390    This downloads a WSI image from https://openslide.cs.cmu.edu/download/openslide-testdata/,
391    under the "CC0 1.0 Universal" license.
392
393    Args:
394        save_directory: Root folder to save the downloaded data.
395
396    Returns:
397        The path of the downloaded image.
398    """
399    save_directory = Path(save_directory)
400    os.makedirs(save_directory, exist_ok=True)
401    print("Example data directory is:", save_directory.resolve())
402    fname = "whole-slide-histopathology-example-image.svs"
403    pooch.retrieve(
404        url="https://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1.svs",
405        known_hash="00a3d54482cd707abf254fe69dccc8d06b8ff757a1663f1290c23418c480eb30",
406        fname=fname,
407        path=save_directory,
408        progressbar=True,
409    )
410    return os.path.join(save_directory, fname)