Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add 'batch' option to set command #35

Closed
wants to merge 1 commit into from
Closed

Add 'batch' option to set command #35

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 122 additions & 65 deletions src/omero_cli_render.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from builtins import str
from builtins import range
from builtins import object
import concurrent.futures
import sys
import time
import json
Expand Down Expand Up @@ -397,6 +398,11 @@ def _configure(self, parser):
help="Local file or OriginalFile:ID which specifies the "
"rendering settings")

set_cmd.add_argument(
"--batch", type=int, default=1,
help="Batch size to process simultaneously"
)

test.add_argument(
"--force", action="store_true",
help="Force creation of pixel data file in binary "
Expand All @@ -415,7 +421,7 @@ def _lookup(self, gateway, type, oid):
self.ctx.die(110, "No such %s: %s" % (type, oid))
return obj

def render_images(self, gateway, object, batch=100):
def load_images(self, gateway, object, batch=100):
"""
Get the images.

Expand All @@ -430,12 +436,12 @@ def render_images(self, gateway, object, batch=100):

if isinstance(object, list):
for x in object:
for rv in self.render_images(gateway, x, batch):
for rv in self.load_images(gateway, x, batch):
yield rv
elif isinstance(object, Screen):
scr = self._lookup(gateway, "Screen", object.id)
for plate in scr.listChildren():
for rv in self.render_images(gateway, plate._obj, batch):
for rv in self.load_images(gateway, plate._obj, batch):
yield rv
elif isinstance(object, Plate):
plt = self._lookup(gateway, "Plate", object.id)
Expand All @@ -456,7 +462,7 @@ def render_images(self, gateway, object, batch=100):
elif isinstance(object, Project):
prj = self._lookup(gateway, "Project", object.id)
for ds in prj.listChildren():
for rv in self.render_images(gateway, ds._obj, batch):
for rv in self.load_images(gateway, ds._obj, batch):
yield rv

elif isinstance(object, Dataset):
Expand Down Expand Up @@ -486,7 +492,7 @@ def render_images(self, gateway, object, batch=100):
def info(self, args):
""" Implements the 'info' command """
first = True
for img in self.render_images(self.gateway, args.object, batch=1):
for img in self.load_images(self.gateway, args.object, batch=1):
ro = RenderObject(img)
if args.style == 'plain':
self.ctx.out(str(ro))
Expand All @@ -506,8 +512,8 @@ def info(self, args):
@gateway_required
def copy(self, args):
""" Implements the 'copy' command """
for src_img in self.render_images(self.gateway, args.object, batch=1):
for targets in self.render_images(self.gateway, args.target):
for src_img in self.load_images(self.gateway, args.object, batch=1):
for targets in self.load_images(self.gateway, args.target):
batch = dict()
for target in targets:
if target.id == src_img.id:
Expand All @@ -532,7 +538,7 @@ def copy(self, args):
self._generate_thumbs(list(batch.values()))

def update_channel_names(self, gateway, obj, namedict):
for targets in self.render_images(gateway, obj):
for targets in self.load_images(gateway, obj):
iids = [img.id for img in targets]
self._update_channel_names(self, iids, namedict)

Expand All @@ -550,13 +556,10 @@ def _generate_thumbs(self, images):
self.ctx.dbg("Image:%s got thumbnail in %2.2fs" % (
img.id, stop - start))

def _read_default_planes(self, img, data, ignore_errors=False):
def _read_default_planes(self, img, def_z=None, def_t=None,
ignore_errors=False):
"""Read and validate the default planes"""

# Read values from dictionary
def_z = data['z'] if 'z' in data else None
def_t = data['t'] if 't' in data else None

# Minimal validation: default planes should be 1-indexed integers
if (def_z is not None) and (def_z < 1 or int(def_z) != def_z):
self.ctx.die(
Expand Down Expand Up @@ -639,69 +642,123 @@ def _read_channels(self, data):
colourlist.append(c.color)
return (namedict, cindices, rangelist, colourlist)

def _set_rendering_settings(self, img, disable, def_z, def_t, cindices,
rangelist, colourlist, greyscale, skipthumbs,
ignore_errors):
"""
Set the rendering settings for an image
:param img: The image
:param disable: Flag if channels not in cindices should be disabled
:param def_z: The default z
:param def_t: The default t
:param cindices: The channel indices
:param rangelist: The window start end ranges
:param colourlist: The channel colors
:param greyscale: Flag if greyscale
:param skipthumbs: Flag to skip thumbnail generation
:param ignore_errors: Flag to ignore errors
:return: The image id
"""
(def_z, def_t) = self._read_default_planes(
img, def_z, def_t, ignore_errors=ignore_errors)

reactivatechannels = []

if not disable:
# Calling set_active_channels will disable channels which
# are not specified, have to keep track of them and
# re-activate them later again
imgchannels = img.getChannels()
for ci, ch in enumerate(imgchannels, 1):
if ci not in cindices and -ci not in cindices \
and ch.isActive():
reactivatechannels.append(ci)

img.set_active_channels(
cindices, windows=rangelist, colors=colourlist)

if greyscale is not None:
if greyscale:
img.setGreyscaleRenderingModel()
else:
img.setColorRenderingModel()

if len(reactivatechannels) > 0:
img.set_active_channels(reactivatechannels)

if def_z:
img.setDefaultZ(def_z - 1)
if def_t:
img.setDefaultT(def_t - 1)

try:
img.saveDefaults()
self.ctx.dbg(
"Updated rendering settings for Image:%s" % img.id)
if not skipthumbs:
self._generate_thumbs([img])
return img.id
except Exception as e:
self.ctx.err('ERROR: %s' % e)
if not ignore_errors:
self.ctx.die(141)
finally:
img._closeRE()

@gateway_required
def set(self, args):
""" Implements the 'set' command """
data = self._load_rendering_settings(
args.channels, session=self.client.getSession())

(namedict, cindices, rangelist, colourlist) = self._read_channels(
data)

greyscale = data.get('greyscale', None)
if greyscale is not None:
self.ctx.dbg('greyscale=%s' % greyscale)

iids = []
for img in self.render_images(self.gateway, args.object, batch=1):
iids.append(img.id)

(def_z, def_t) = self._read_default_planes(
img, data, ignore_errors=args.ignore_errors)

reactivatechannels = []
if not args.disable:
# Calling set_active_channels will disable channels which
# are not specified, have to keep track of them and
# re-activate them later again
imgchannels = img.getChannels()
for ci, ch in enumerate(imgchannels, 1):
if ci not in cindices and -ci not in cindices\
and ch.isActive():
reactivatechannels.append(ci)

img.set_active_channels(
cindices, windows=rangelist, colors=colourlist)

if greyscale is not None:
if greyscale:
img.setGreyscaleRenderingModel()
else:
img.setColorRenderingModel()

if len(reactivatechannels) > 0:
img.set_active_channels(reactivatechannels)

if def_z:
img.setDefaultZ(def_z - 1)
if def_t:
img.setDefaultT(def_t - 1)

try:
img.saveDefaults()
self.ctx.dbg(
"Updated rendering settings for Image:%s" % img.id)
if not args.skipthumbs:
self._generate_thumbs([img])
except Exception as e:
self.ctx.err('ERROR: %s' % e)
finally:
img._closeRE()

if not iids:
self.ctx.die(113, "ERROR: No images found for %s %d" %
(args.object.__class__.__name__, args.object.id._val))
def_z = data['z'] if 'z' in data else None
def_t = data['t'] if 't' in data else None

if namedict:
self._update_channel_names(self.gateway, iids, namedict)
# Turn off cache usage in order to make this work in parallel
self.gateway.cacheServices(False)

for img_batch in self.load_images(self.gateway, args.object,
batch=args.batch):
if not isinstance(img_batch, list):
img_batch = [img_batch]

with concurrent.futures.ThreadPoolExecutor() as executor:
future_image = {executor.submit(self._set_rendering_settings,
img, args.disable, def_z,
def_t, cindices, rangelist,
colourlist, greyscale,
args.skipthumbs,
args.ignore_errors):
img for img in img_batch}

for future in concurrent.futures.as_completed(future_image):
img = future_image[future]
try:
future.result()
except Exception as exc:
self.ctx.err('ERROR: Image %d: %s' % (img.id, exc))
if not args.ignore_errors:
self.ctx.die(142)

if namedict:
try:
batch_iids = [img.id for img in img_batch]
self._update_channel_names(self.gateway, batch_iids,
namedict)
except Exception as exc:
self.ctx.err('ERROR: Could not set channel name:%s'
% exc)
if not args.ignore_errors:
self.ctx.die(143)

self.gateway.cacheServices(True)

def edit(self, args):
self.ctx.die(112, "ERROR: 'edit' command has been renamed to 'set'")
Expand All @@ -710,7 +767,7 @@ def edit(self, args):
def test(self, args):
""" Implements the 'test' command """
self.gateway.SERVICE_OPTS.setOmeroGroup('-1')
for img in self.render_images(self.gateway, args.object, batch=1):
for img in self.load_images(self.gateway, args.object, batch=1):
self.test_per_pixel(
self.client, img.getPrimaryPixels().id, args.force, args.thumb)

Expand Down