Merge pull request #7 from renerichter/RL

Added padding calculation and normalization by weights
the-lay · Dec 14, 2021 · f68ce75 · f68ce75
2 parents ac10fb6 + c230be4
commit f68ce75
Show file tree

Hide file tree

Showing 16 changed files with 950 additions and 413 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -1,8 +1,10 @@
 ## Bug reports
-Please be as specific as you can, ideally with code to recreate the issue.
+Please include:
+1. Which version you are using (`python -c "import tiler; print(tiler.__version__)"`)
+2. A minimal code example to recreate the issue
 
 
-# Code contribution workflow
+## Code contribution
 First off, thanks for taking the time!  
 Please feel free to contact me with questions.
 
@@ -30,3 +32,4 @@ cd misc
 5. Once you want to share what you've changed, please commit, push make a pull request to the main repo.
 
 6. Github will run lint and tests, but please don't rely on that and test before pull request :)
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Run tests
         run: coverage run -m pytest -v
       - name: Submit coveralls
-        if: github.repository == 'the-lay/tiler'
+        if: github.event_name != 'pull_request'
         run: coveralls
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
diff --git a/README.md b/README.md
@@ -15,7 +15,8 @@ _________________
 ⚠️ **Please note: work in progress, things will change and/or break!** ⚠️
 _________________
 
-This python package provides functions for tiling/patching and subsequent merging of NumPy arrays.
+This python package provides consistent and user-friendly
+functions for tiling/patching and subsequent merging of NumPy arrays.
 
 Such tiling is often required for various heavy image-processing tasks
 such as semantic segmentation in deep learning, especially in domains where images do not fit into GPU memory
@@ -24,12 +25,12 @@ such as semantic segmentation in deep learning, especially in domains where imag
 
 Features
 -------------
- - N-dimensional *(note: currently tile shape must have the same number of dimensions as the array)*
+ - N-dimensional
  - Optional in-place tiling
- - Optional channel dimension, dimension that is not tiled
- - Optional batching
+ - Optional channel dimension (dimension that is not tiled)
+ - Optional tile batching
  - Tile overlapping
- - Access individual tiles with iterator or a getter
+ - Access individual tiles with an iterator or a getter
  - Tile merging, with optional window functions/tapering
 
 

diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/tiler_teaser.png b/docs/tiler_teaser.png
diff --git a/examples/2d_overlap_tile.py b/examples/2d_overlap_tile.py
@@ -12,69 +12,68 @@
 
 # Loading image
 # Photo by Christian Holzinger on Unsplash: https://unsplash.com/photos/CUY_YHhCFl4
-image = np.array(Image.open('example_image.jpg'))  # 1280x1920x3
+image = np.array(Image.open("example_image.jpg"))  # 1280x1920x3
 
-# Padding image
-# Overlap tile strategy assumes we only use the small, non-overlapping, middle part of tile.
-# Assuming we want tiles of size 128x128 and we want to only use middle 64x64,
-# we should pad the image by 32 from each side, using reflect mode
-padded_image = np.pad(image, ((32, 32), (32, 32), (0, 0)), mode='reflect')
-
-# Specifying tiling parameters
+# Overlap tile strategy assumes we only use the small, non-overlapping, middle part of a tile.
+# Assuming we want to use centre 64x64 square, we should specify
 # The overlap should be 0.5, 64 or explicitly (64, 64, 0)
-tiler = Tiler(data_shape=padded_image.shape, tile_shape=(128, 128, 3),
-              overlap=(64, 64, 0), channel_dimension=2)
+tiler = Tiler(
+    data_shape=image.shape,
+    tile_shape=(128, 128, 3),
+    overlap=(64, 64, 0),
+    channel_dimension=2,
+)
+
+# Calculate and apply extra padding, as well as adjust tiling parameters
+new_shape, padding = tiler.calculate_padding()
+tiler.recalculate(data_shape=new_shape)
+padded_image = np.pad(image, padding, mode="reflect")
 
 # Specifying merging parameters
 # You can define overlap-tile window explicitly, i.e.
-# window = np.zeros((128, 128, 3))
-# window[32:-32, 32:-32, :] = 1
-# merger = Merger(tiler=tiler, window=window)
-# or you can use overlap-tile window which will do that automatically based on tiler.overlap
-merger = Merger(tiler=tiler, window='overlap-tile')
+# >>> window = np.zeros((128, 128, 3))
+# >>> window[32:-32, 32:-32, :] = 1
+# >>> merger = Merger(tiler=tiler, window=window)
+# or you can use window="overlap-tile"
+# it will automatically calculate such window based on tiler.overlap and applied padding
+merger = Merger(tiler=tiler, window="overlap-tile")
 
 # Let's define a function that will be applied to each tile
-def process(patch: np.ndarray, sanity_check: bool = True) -> np.ndarray:
+# For this example, let's black out the sides that should be "cropped" by window function
+# as a way to confirm that only the middle parts are being merged
+def process(patch: np.ndarray) -> np.ndarray:
+    patch[:32, :, :] = 0
+    patch[-32:, :, :] = 0
+    patch[:, :32, :] = 0
+    patch[:, -32:, :] = 0
+    return patch
 
-    # One example can be a sanity check
-    # Make the parts that should be removed black
-    # There should not appear any black spots in the final merged image
-    if sanity_check:
-        patch[:32, :, :] = 0
-        patch[-32:, :, :] = 0
-        patch[:, :32, :] = 0
-        patch[:, -32:, :] = 0
-        return patch
-
-    # Another example can be to just modify the whole patch
-    # Using PIL, we adjust the color balance
-    enhancer = ImageEnhance.Color(Image.fromarray(patch))
-    return np.array(enhancer.enhance(5.0))
 
 # Iterate through all the tile and apply the processing function
 # as well as add them back to the merger
-for tile_id, tile in tiler(padded_image):
+for tile_id, tile in tiler(padded_image, progress_bar=True):
     processed_tile = process(tile)
     merger.add(tile_id, processed_tile)
 
-# Merger.merge() returns unpadded from tiler image, but we still need to unpad line#21
-final_image = merger.merge().astype(np.uint8)
-final_unpadded_image = final_image[32:-32, 32:-32, :]
+# Merge processed tiles
+final_image = merger.merge(extra_padding=padding, dtype=image.dtype)
+
+print(f"Sanity check: {np.all(image == final_image)}")
 
 # Show the final merged image, weights and number of times each pixel was seen in tiles
-fig, ax = plt.subplots(3, 2, sharex=True, sharey=True)
-ax[0, 0].set_title('Original image')
+fig, ax = plt.subplots(3, 2)
+ax[0, 0].set_title("Original image")
 ax[0, 0].imshow(image)
-ax[0, 1].set_title('Final unpadded image')
-ax[0, 1].imshow(final_unpadded_image)
+ax[0, 1].set_title("Final merged image")
+ax[0, 1].imshow(final_image)
 
-ax[1, 0].set_title('Padded image')
+ax[1, 0].set_title("Padded image")
 ax[1, 0].imshow(padded_image)
-ax[1, 1].set_title('Merged image')
-ax[1, 1].imshow(final_image)
+ax[1, 1].set_title("Overlap-tile window")
+ax[1, 1].imshow(merger.window)
 
-ax[2, 0].set_title('Weights sum')
+ax[2, 0].set_title("Weights sum")
 ax[2, 0].imshow(merger.weights_sum[:, :, 0], vmin=0, vmax=merger.weights_sum.max())
-ax[2, 1].set_title('Pixel visits')
+ax[2, 1].set_title("Pixel visits")
 ax[2, 1].imshow(merger.data_visits[:, :, 0], vmin=0, vmax=merger.data_visits.max())
 plt.show()
diff --git a/examples/3d_overlap_tile.py b/examples/3d_overlap_tile.py
@@ -16,23 +16,27 @@
 volume[:75] *= np.linspace(3, 10, 75)[:, None, None]
 volume[75:] *= np.linspace(10, 3, 75)[:, None, None]
 
-# Let's assume we want to use tiles of size 48x48x48 and only the middle 20x20x20 for the final image
-# That means we need to pad the image by 14 from each side
-# To extrapolate missing context let's use reflect mode
-padded_volume = np.pad(volume, 14, mode='reflect')
+# Let's assume we want to use tiles of size 48x48x48 and only the middle 20x20x20 contribute to the final image
+# The overlap then should be 28 (48-20) voxels
+tiler = Tiler(
+    data_shape=volume.shape,
+    tile_shape=(48, 48, 48),
+    overlap=(28, 28, 28),
+)
 
-# Specifying tiling
-# The overlap should be 28 voxels
-tiler = Tiler(data_shape=padded_volume.shape,
-              tile_shape=(48, 48, 48),
-              overlap=(28, 28, 28))
+# Calculate and apply extra padding, as well as adjust tiling parameters
+new_shape, padding = tiler.calculate_padding()
+tiler.recalculate(data_shape=new_shape)
+padded_volume = np.pad(volume, padding, mode="reflect")
 
-# Window function for merging
-window = np.zeros((48, 48, 48))
-window[14:-14, 14:-14, 14:-14] = 1
-
-# Specifying merging
-merger = Merger(tiler=tiler, window=window)
+# Specifying merging parameters
+# You can define overlap-tile window explicitly, i.e.
+# >>> window = np.zeros((48, 48, 48))
+# >>> window[14:-14, 14:-14, 14:-14] = 1
+# >>> merger = Merger(tiler=tiler, window=window)
+# or you can use window="overlap-tile"
+# it will automatically calculate such window based on tiler.overlap and applied padding
+merger = Merger(tiler=tiler, window="overlap-tile")
 
 # Let's define a function that will be applied to each tile
 # For this example, let's black out the sides that should be "cropped" by window function
@@ -46,20 +50,25 @@ def process(patch: np.ndarray) -> np.ndarray:
     patch[:, :, -14:] = 0
     return patch
 
-# Iterate through all the tiles and apply the processing function and merge everything back
+
+# Apply the processing function to each tile and add it to the merger
 for tile_id, tile in tiler(padded_volume, progress_bar=True):
     processed_tile = process(tile)
     merger.add(tile_id, processed_tile)
 
-final_volume = merger.merge()
-final_unpadded_volume = final_volume[14:-14, 14:-14, 14:-14]
+# Merge processed tiles
+final_volume = merger.merge(extra_padding=padding)
+
+print(f"Sanity check: {np.all(volume == final_volume)}")
 
-# Show all the
-with napari.gui_qt():
-    v = napari.Viewer()
-    v.add_image(volume, name='Original volume')
-    v.add_image(padded_volume, name='Padded volume')
-    v.add_image(final_volume, name='Final volume')
-    v.add_image(final_unpadded_volume, name='Final unpadded volume')
-    v.add_image(merger.weights_sum, name='Merger weights sum')
-    v.add_image(merger.data_visits, name='Merger data visits')
+# Show all the produced volumes
+v = napari.Viewer()
+# v.add_image(merger.window, name="window")
+# v.add_image(tile, name="tile")
+# v.add_image(processed_tile, name="processed_tile")
+v.add_image(volume, name="Original volume")
+v.add_image(padded_volume, name="Padded volume")
+v.add_image(final_volume, name="Final merged volume")
+v.add_image(merger.weights_sum, name="Merger weights sum")
+v.add_image(merger.data_visits, name="Merger data visits")
+v.show(block=True)
diff --git a/examples/batch_processing.py b/examples/batch_processing.py
@@ -4,7 +4,7 @@
 
 # Loading image
 # Photo by Christian Holzinger on Unsplash: https://unsplash.com/photos/CUY_YHhCFl4
-image = np.array(Image.open('example_image.jpg'))  # 1280x1920x3
+image = np.array(Image.open("example_image.jpg"))  # 1280x1920x3
 
 # Setup Tiler and Merger
 tiler = Tiler(data_shape=image.shape, tile_shape=(200, 200, 3), channel_dimension=2)
@@ -13,25 +13,25 @@
 # Example 1: process all tiles one by one, i.e. batch_size=0
 for tile_i, tile in tiler(image, batch_size=0):
     merger.add(tile_i, tile)
-result_bs0 = merger.merge().astype(np.uint8)
+result_bs0 = merger.merge(dtype=image.dtype)
 
 # Example 2: process all tiles in batches of 1, i.e. batch_size=1
 merger.reset()
 for batch_i, batch in tiler(image, batch_size=1):
     merger.add_batch(batch_i, 1, batch)
-result_bs1 = merger.merge().astype(np.uint8)
+result_bs1 = merger.merge(dtype=image.dtype)
 
 # Example 3: process all tiles in batches of 10, i.e. batch_size=10
 merger.reset()
 for batch_i, batch in tiler(image, batch_size=10):
     merger.add_batch(batch_i, 10, batch)
-result_bs10 = merger.merge().astype(np.uint8)
+result_bs10 = merger.merge(dtype=image.dtype)
 
 # Example 4: process all tiles in batches of 10, but drop the batch that has <batch_size tiles, drop_last=True
 merger.reset()
 for batch_i, batch in tiler(image, batch_size=10, drop_last=True):
     merger.add_batch(batch_i, 10, batch)
-result_bs10 = merger.merge().astype(np.uint8)
+result_bs10 = merger.merge(dtype=image.dtype)
 
 assert np.all(result_bs0 == result_bs1)
 assert np.all(result_bs0 == result_bs10)
diff --git a/misc/docs.sh b/misc/docs.sh
@@ -11,4 +11,4 @@ rm docs/index.html
 mv docs/tiler.html docs/index.html
 
 # remove unused search index
-rm docs/search.js
+rm -f docs/search.js
diff --git a/misc/teaser/teaser_image.py b/misc/teaser/teaser_image.py
@@ -3,25 +3,24 @@
 import matplotlib.pyplot as plt
 from tiler import Tiler, Merger
 
-# Warning: ugly code, but explicit.
-# Moreover, it is not really merging the tiles back together.
-# You can find example of actual 2D overlap tiling and merging in examples/2d_overlap_tile.py
-
 
 #### Generate images
 # Original image by Marco Bianchetti on Unsplash
 # https://unsplash.com/photos/8blA_V0MI9I
 image = np.array(Image.open('original_image.jpg'))
 
-# Calculate tile shape to have exactly 3x3 tiles
+# Calculate tile shape to have exactly 3x3 tiles with 0.4 overlap
 full_image_shape = np.array(image.shape)
 full_image_shape[:2] = full_image_shape[:2] * 1.4
 tile_shape = full_image_shape[:2] // 3
 tile_shape = tuple(tile_shape) + (3, )
 
-# Tile and merge
+# Tile original image and merge it back
 tiler = Tiler(image.shape, tile_shape, overlap=0.4, channel_dimension=2, mode='reflect')
-tiles = [tile.astype(np.uint8) for _, tile in tiler(image)]
+tiles = np.array([tile for _, tile in tiler(image)])
+merger = Merger(tiler)
+merger.add_batch(0, len(tiler), tiles)
+merged_image = merger.merge(dtype=tiles.dtype)
 
 
 #### Plot images
@@ -121,9 +120,7 @@
 merged_img.set_xticks([])
 merged_img.set_yticks([])
 merged_img.axis('off')
-merged_img.imshow(image)
+merged_img.imshow(merged_image)
 
 # fig.show()
 plt.savefig('tiler_teaser.png')
-
-