Skip to content

Commit

Permalink
Merge pull request #1625 from girder/zarr-sink-axis-values
Browse files Browse the repository at this point in the history 
Zarr Sink Frame Values
  • Loading branch information
@annehaley
annehaley authored Oct 22, 2024
2 parents f4517c4 + 3876620 commit a4160ea
Show file tree
Hide file tree
Showing 7 changed files with 629 additions and 57 deletions.
59 changes: 53 additions & 6 deletions docs/getting_started.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,25 +334,60 @@ You can also composite a multi-frame image into a false-color output:
Writing an Image
----------------

If you wish to visualize numpy data, large_image can write a tiled tiff. This requires a tile source that supports writing to be installed. As of this writing, the ``large-image-source-zarr`` and ``large-image-source-vips`` sources supports this. If both are installed, the ``large-image-source-zarr`` is the default.
If you wish to visualize numpy data, ``large_image`` can write a tiled image.
This requires a tile source that supports writing to be installed.
As of this writing, the ``large-image-source-zarr`` and ``large-image-source-vips`` sources both support this.
If both are installed, the ``large-image-source-zarr`` is the default.
Some of the API options available for ``large-image-source-zarr`` are not available for ``large-image-source-vips``.

.. code-block:: python
import large_image
source = large_image.new()
for nparray, x, y in fancy_algorithm():
# We could optionally add a mask to limit the output
source.addTile(nparray, x, y)
source.write('/tmp/sample.tiff', lossy=False)
The ``large-image-source-zarr`` can be used to store multiple frame data with arbitrary axes.
Multiple Frames
~~~~~~~~~~~~~~~

``large-image-source-zarr`` can be used to store multiframe data with arbitrary axes.
The example below demonstrates the creation of an image with five axes: T, Z, Y, X, S.

.. code-block:: python
import large_image
time_values = [0.5, 1.5, 2.5, 3.5]
z_values = [3, 6, 9]
tile_pos_values = [0, 1024, 2048, 3072, 4096]
source = large_image.new()
for nparray, x, y, time, param1 in fancy_algorithm():
source.addTile(nparray, x, y, time=time, p1=param1)
for t_index, t_value in enumerate(time_values):
for z_index, z_value in enumerate(z_values):
for y_value in tile_pos_values:
for x_value in tile_pos_values:
# tile is a numpy array with shape (1024, 1024, 3)
# this shape corresponds to the following axes, respectively: (Y, X, S)
tile = get_my_data_tile(x_value, y_value, z_value, t_value)
source.addTile(
tile,
x_value,
y_value,
z=z_index,
time=t_index,
# z_value and t_value are optional parameters to store the
# true values at the provided z index and t index
z_value=z_value,
time_value=t_value,
)
source.frameUnits = dict(t='ms', z='cm')
# The writer supports a variety of formats
source.write('/tmp/sample.zarr.zip', lossy=False)
Expand All @@ -361,31 +396,37 @@ You may also choose to read tiles from one source and write modified tiles to a
.. code-block:: python
import large_image
original_source = large_image.open('path/to/original/image.tiff')
new_source = large_image.new()
for frame in original_source.getMetadata().get('frames', []):
for tile in original_source.tileIterator(frame=frame['Frame'], format='numpy'):
t, x, y = tile['tile'], tile['x'], tile['y']
tile_data, x, y = tile['tile'], tile['x'], tile['y']
kwargs = {
'z': frame['IndexZ'],
'c': frame['IndexC'],
}
modified_tile = modify_tile(t)
modified_tile = modify_tile(tile_data)
new_source.addTile(modified_tile, x=x, y=y, **kwargs)
new_source.write('path/to/new/image.tiff', lossy=False)
Multiple processes
~~~~~~~~~~~~~~~~~~

In some cases, it may be beneficial to write to a single image from multiple processes or threads:

.. code-block:: python
import large_image
import multiprocessing
# Important: Must be a pickleable function
def add_tile_to_source(tilesource, nparray, position):
tilesource.addTile(
nparray,
**position
)
source = large_image.new()
# Important: Maximum size must be allocated before any multiprocess concurrency
add_tile_to_source(source, np.zeros(1, 1, 3), dict(x=max_x, y=max_y, z=max_z))
Expand All @@ -397,4 +438,10 @@ In some cases, it may be beneficial to write to a single image from multiple pro
)
source.write('/tmp/sample.zarr.zip', lossy=False)
More examples
~~~~~~~~~~~~~

To see more examples of using ``large-image-source-zarr`` to write images, see :doc:`notebooks` and the `Zarr Sink Tests <https://github.com/girder/large_image/blob/master/test/test_sink.py>`_.

.. _Girder: https://girder.readthedocs.io/en/latest/
91 changes: 62 additions & 29 deletions docs/notebooks/zarr_sink_example.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "4ba28d02",
"metadata": {},
"outputs": [],
Expand All @@ -37,7 +37,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"id": "63c0c38f",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -164,7 +164,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "bf6d0e9480cb43ef9851d7bd3ca7e356",
"model_id": "97c6787b148c4b4f9639d623f32f8a3a",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -191,7 +191,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"id": "0e75e6de",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -228,10 +228,22 @@
"data": {
"application/json": {
"IndexRange": {
"IndexI": 3
"IndexFOOTPRINT": 3
},
"IndexStride": {
"IndexI": 1
"IndexFOOTPRINT": 1
},
"ValueFOOTPRINT": {
"datatype": "int64",
"max": 50,
"min": 1,
"uniform": true,
"units": null,
"values": [
1,
10,
50
]
},
"bandCount": 3,
"channelmap": {
Expand All @@ -246,19 +258,22 @@
"Channel": "Band 1",
"Frame": 0,
"Index": 0,
"IndexI": 0
"IndexFOOTPRINT": 0,
"ValueFOOTPRINT": 1
},
{
"Channel": "Band 1",
"Frame": 1,
"Index": 1,
"IndexI": 1
"IndexFOOTPRINT": 1,
"ValueFOOTPRINT": 10
},
{
"Channel": "Band 1",
"Frame": 2,
"Index": 2,
"IndexI": 2
"IndexFOOTPRINT": 2,
"ValueFOOTPRINT": 50
}
],
"levels": 6,
Expand All @@ -281,16 +296,34 @@
" 'mm_y': 0,\n",
" 'dtype': 'float64',\n",
" 'bandCount': 3,\n",
" 'frames': [{'Frame': 0, 'IndexI': 0, 'Index': 0, 'Channel': 'Band 1'},\n",
" {'Frame': 1, 'IndexI': 1, 'Index': 1, 'Channel': 'Band 1'},\n",
" {'Frame': 2, 'IndexI': 2, 'Index': 2, 'Channel': 'Band 1'}],\n",
" 'IndexRange': {'IndexI': 3},\n",
" 'IndexStride': {'IndexI': 1},\n",
" 'frames': [{'Frame': 0,\n",
" 'IndexFOOTPRINT': 0,\n",
" 'ValueFOOTPRINT': 1,\n",
" 'Index': 0,\n",
" 'Channel': 'Band 1'},\n",
" {'Frame': 1,\n",
" 'IndexFOOTPRINT': 1,\n",
" 'ValueFOOTPRINT': 10,\n",
" 'Index': 1,\n",
" 'Channel': 'Band 1'},\n",
" {'Frame': 2,\n",
" 'IndexFOOTPRINT': 2,\n",
" 'ValueFOOTPRINT': 50,\n",
" 'Index': 2,\n",
" 'Channel': 'Band 1'}],\n",
" 'ValueFOOTPRINT': {'values': [1, 10, 50],\n",
" 'uniform': True,\n",
" 'units': None,\n",
" 'min': 1,\n",
" 'max': 50,\n",
" 'datatype': 'int64'},\n",
" 'IndexRange': {'IndexFOOTPRINT': 3},\n",
" 'IndexStride': {'IndexFOOTPRINT': 1},\n",
" 'channels': ['Band 1'],\n",
" 'channelmap': {'Band 1': 0}}"
]
},
"execution_count": 7,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -315,7 +348,7 @@
"\n",
" # add modified tile to sink\n",
" # specify tile x, tile y, and any arbitrary frame parameters\n",
" sink.addTile(processed_tile, x=tile['x'], y=tile['y'], i=i)\n",
" sink.addTile(processed_tile, x=tile['x'], y=tile['y'], footprint=i, footprint_value=footprint_size)\n",
"# view metadata\n",
"sink.getMetadata()"
]
Expand All @@ -329,7 +362,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9b8e6175005d4af89cb4cfada7b72983",
"model_id": "fb60c2b967274a99a632f32f99389705",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -398,10 +431,10 @@
"data": {
"application/json": {
"IndexRange": {
"IndexI": 3
"IndexFOOTPRINT": 3
},
"IndexStride": {
"IndexI": 1
"IndexFOOTPRINT": 1
},
"bandCount": 3,
"channelmap": {
Expand All @@ -416,19 +449,19 @@
"Channel": "Band 1",
"Frame": 0,
"Index": 0,
"IndexI": 0
"IndexFOOTPRINT": 0
},
{
"Channel": "Band 1",
"Frame": 1,
"Index": 1,
"IndexI": 1
"IndexFOOTPRINT": 1
},
{
"Channel": "Band 1",
"Frame": 2,
"Index": 2,
"IndexI": 2
"IndexFOOTPRINT": 2
}
],
"levels": 4,
Expand All @@ -451,16 +484,16 @@
" 'mm_y': None,\n",
" 'dtype': 'uint16',\n",
" 'bandCount': 3,\n",
" 'frames': [{'Channel': 'Band 1', 'Frame': 0, 'Index': 0, 'IndexI': 0},\n",
" {'Channel': 'Band 1', 'Frame': 1, 'Index': 1, 'IndexI': 1},\n",
" {'Channel': 'Band 1', 'Frame': 2, 'Index': 2, 'IndexI': 2}],\n",
" 'IndexRange': {'IndexI': 3},\n",
" 'IndexStride': {'IndexI': 1},\n",
" 'frames': [{'Channel': 'Band 1', 'Frame': 0, 'Index': 0, 'IndexFOOTPRINT': 0},\n",
" {'Channel': 'Band 1', 'Frame': 1, 'Index': 1, 'IndexFOOTPRINT': 1},\n",
" {'Channel': 'Band 1', 'Frame': 2, 'Index': 2, 'IndexFOOTPRINT': 2}],\n",
" 'IndexRange': {'IndexFOOTPRINT': 3},\n",
" 'IndexStride': {'IndexFOOTPRINT': 1},\n",
" 'channels': ['Band 1'],\n",
" 'channelmap': {'Band 1': 0}}"
]
},
"execution_count": 10,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -506,7 +539,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "30812bb388a0426da9806e62bf5e8711",
"model_id": "a2d3df88fbd44079877b8f314ed97e6f",
"version_major": 2,
"version_minor": 0
},
Expand Down
9 changes: 6 additions & 3 deletions examples/algorithm_progression.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,15 @@ def applyAlgorithm(self, sink, params):
**tiparams,
):
scaled = tile.get('scaled', 1)
axisparams = {
p['axis']: iteration_id[i] for i, p in enumerate(self.param_order.values())}
axisparams.update({
p['axis'] + '_value': params[i] for i, p in enumerate(self.param_order.values())})
if self.overlay:
self.addTile(
tilesink,
tile['tile'], int(tile['x'] * scaled), int(tile['y'] * scaled),
**{p['axis']: iteration_id[i] for i, p in enumerate(
self.param_order.values())})
**axisparams)
altered_data = self.algorithm(tile['tile'], *params)
mask = None
if self.overlay:
Expand All @@ -104,7 +107,7 @@ def applyAlgorithm(self, sink, params):
self.addTile(
tilesink,
altered_data, int(tile['x'] * scaled), int(tile['y'] * scaled), mask=mask,
**{p['axis']: iteration_id[i] for i, p in enumerate(self.param_order.values())})
**axisparams)
if time.time() - lastlogtime > 10:
sys.stdout.write(
f'Processed {tile["tile_position"]["position"] + 1} of '
Expand Down
27 changes: 26 additions & 1 deletion girder/girder_large_image/web_client/vue/components/FrameSelector.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,31 @@ export default Vue.extend({
currentStyle() {
const curStyle = this.style[this.currentModeId];
return curStyle ? JSON.stringify(curStyle, null, null) : '';
},
sliderLabels() {
const labels = {};
labels.IndexC = this.imageMetadata.channels;
Object.entries(this.metadata).forEach(([key, info]) => {
if (key.includes('Value')) {
const labelKey = key.replace('Value', 'Index');
if (info.values) {
if (info.uniform) {
labels[labelKey] = info.values.map((v) => {
if (typeof v === 'number') return Number(v.toPrecision(5));
return v;
});
} else {
// non-uniform values have a value for every frame
// labels will change with currentFrame, so only populate current label
let currentLabel = info.values[this.currentFrame];
if (typeof currentLabel === 'number') currentLabel = Number(currentLabel.toPrecision(5));
labels[labelKey] = new Array(this.indexInfo[labelKey].range + 1).fill('');
labels[labelKey][this.indexInfo[labelKey].current] = currentLabel;
}
}
}
});
return labels;
}
},
watch: {
Expand Down Expand Up @@ -328,7 +353,7 @@ export default Vue.extend({
:current-value="indexInfo[index].current"
:value-max="indexInfo[index].range"
:label="index.replace('Index', '')"
:slider-labels="index === 'IndexC' ? imageMetadata.channels : []"
:slider-labels="sliderLabels[index]"
:max-merge="indexInfo[index].maxMerge || false"
@updateMaxMerge="(v) => updateMaxMergeAxis({index, maxMerge: v})"
@updateValue="(v) => updateAxisSlider({index, frame: v})"
Expand Down
Loading

0 comments on commit a4160ea

Please sign in to comment.