diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8c57179..18185fa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,16 @@ +0.3.7 +----- +**ragavi-gains** + + - Fixed legend display bug + - ``--gain-type`` argument is now optional + - Static and interactive files can be generated simultaneously by specifying both ``--plotname`` and ``--htmlname`` + +ous +**general** + + - Fixed bug in phase wrapping + 0.3.6 ----- **ragavi-vis** diff --git a/README.rst b/README.rst index 74b3a9f..1a3a591 100644 --- a/README.rst +++ b/README.rst @@ -90,14 +90,13 @@ To use ragavi gain plotter .. code-block:: bash - $ ragavi-gains -t /path/to/your/table -g table_type (K / B/ F/ G/ D) + $ ragavi-gains -t /path/to/your/table -Multiple tables can be plotted on the same document simply by adding them in a space separated list to the :code:`-t` / :code:`--table` switch. -They must however be accompanied by their respective gain table type in the :code:`-g` switch. e.g +Multiple tables can be plotted on the same document simply by adding them in a space separated list to the :code:`-t` / :code:`--table` switch e.g .. code-block:: bash - $ ragavi-gains -t delay/table/1/ bandpass/table/2 flux/table/3 -g K B F + $ ragavi-gains -t delay/table/1/ bandpass/table/2 flux/table/3 For the visibility plotter, the name-space :code:`ragavi-vis` is used. Help can be obtained by running diff --git a/docs/source/gains.rst b/docs/source/gains.rst index 191079d..9d585d5 100644 --- a/docs/source/gains.rst +++ b/docs/source/gains.rst @@ -10,15 +10,15 @@ Currently, gain tables supported for visualisation by ``ragavi-gains`` are : * Gain calibration (G tables) * D-Jones Leakage tables (D tables) -Mandatory fields are :code:`--table`, :code:`--gain_type` +Mandatory argument is :code:`--table`. -If a field name is not specified to ``ragavi-vis`` all the fields will be plotted by default +If a field name is not specified to ``ragavi-vis`` all the fields will be plotted by default. This is the same for correlations and spectral windows. -It is possible to place multiple gain table plots of different [or same] types into a single HTML document using ``ragavi`` This can be done by specifying the table names and gain types as space separate list as below +It is possible to place multiple gain table plots of different [or same] types into a single HTML document using ``ragavi`` This can be done by specifying the table names as a space separate list as below .. code-block:: bash - $ragavi-gains --table table/one/name table/two/name table/three/name table/four/name --gain_types B G D F --fields 0 + $ragavi-gains --table table/one/name table/two/name table/three/name table/four/name --fields 0 This will yield an output HTML file, with plots in the order @@ -47,7 +47,7 @@ To use ``ragavi-gains`` in a notebook environment, run in a notebook cell from ragavi.ragavi import plot_table #specifying function arguments - args = dict(mytabs=[], gain_types=[], cmap='viridis', doplot='ri', corr=1, ant='1,2,3,9,10') + args = dict(mytabs=[], cmap='viridis', doplot='ri', corr=1, ant='1,2,3,9,10') #inline plotting will be done plot_table(**args) @@ -74,17 +74,14 @@ It is possible to generate PNG and SVG with ``ragavi-gains`` via two methods. Th export PATH=$PATH:/absolute/path/to/gecko -With this setup, one can now supply to ``ragavi-gains`` a ``--plotname`` value, which will result in the generation of PNG/SVG files, depending on the file extension provided. If, for example the plotname provided is ``foo.png``, ``ragavi-gains`` will assume the desired output should be PNG. The same applies for SVG. +With this setup, one can now supply to ``ragavi-gains`` a ``--plotname`` value, which will result in the generation of PNG/SVG files, depending on the file extension provided. If, for example, the plotname provided is ``foo.png``, ``ragavi-gains`` will assume the desired output should be PNG. The same applies for SVG. If both ``--plotname`` and ``--htmlname`` are provided, ``ragavi`` will generate both static (PNG) and interactive (HTML) outputs simulaneously. -It is necessary to point out that by default, ``ragavi`` uses the canvas image backend for interactive plots, due to performance issues associated with SVG image backend as stated in the `docs`_. +It is necessary to point out that by default, ``ragavi`` uses the canvas image backend for interactive plots, due to performance issues associated with SVG image backend as stated in the Bokeh `docs`_. The default plots generated are always in HTML format. -API -*** -.. autoclass:: ragavi.ragavi.DataCoreProcessor - :members: blackbox, act - +Useful function +**************** .. autofunction:: ragavi.ragavi.plot_table diff --git a/ragavi/arguments.py b/ragavi/arguments.py index 8aaac19..f9da4a1 100644 --- a/ragavi/arguments.py +++ b/ragavi/arguments.py @@ -202,15 +202,6 @@ def gains_argparser(): parser = MyParser(usage="%(prog)s [options] ", description="A Radio Astronomy Gains and Visibility Inspector") required = parser.add_argument_group("Required arguments") - required.add_argument("-g", "--gaintype", nargs='+', type=str, - metavar=' ', dest="gain_types", required=True, - choices=['B', 'D', 'G', 'K', 'F'], - help="""Type of table(s) to be plotted. Can be - specified as a single character e.g. "B" if a - single table has been provided or space - separated list e.g B D G if multiple tables have - been specified. Valid choices are B D G K & F""", - default=[]) required.add_argument("-t", "--table", dest="mytabs", nargs='+', type=str, metavar=(' '), required=True, help="""Table(s) to plot. Multiple tables can be @@ -267,6 +258,15 @@ def gains_argparser(): help="""Plot complex values as amplitude & phase (ap) or real and imaginary (ri). Defaults to ap.""", default="ap") + pconfig.add_argument("-g", "--gaintype", nargs='*', type=str, + metavar=' ', dest="gain_types", + choices=['B', 'D', 'G', 'K', 'F'], + help="""Type of table(s) to be plotted. Can be + specified as a single character e.g. "B" if a + single table has been provided or space + separated list e.g B D G if multiple tables have + been specified. Valid choices are B D G K & F""", + default=[]) pconfig.add_argument("-kx", "--k-xaxis", dest="kx", type=str, metavar='', choices=["time", "antenna"], help="""Choose the x-xaxis for the K table. Valid diff --git a/ragavi/plotting.py b/ragavi/plotting.py index 84c5617..545c2cb 100644 --- a/ragavi/plotting.py +++ b/ragavi/plotting.py @@ -105,10 +105,11 @@ def create_bk_fig(x=None, xlab=None, x_min=None, x_max=None, # define the axes ranges x_range = DataRange1d(name="p_x_range", only_visible=True) + y_range = DataRange1d(name="p_y_range", only_visible=True) + if x_min != None and x_max != None and x_name.lower() in ["channel", "frequency"]: x_range = Range1d(name="p_x_range", start=x_min, end=x_max) - - y_range = DataRange1d(name="p_y_range", only_visible=True) + y_range.only_visible = False # define items to add on the plot p_htool = HoverTool(tooltips=[(x_name, "$x"), diff --git a/ragavi/ragavi.py b/ragavi/ragavi.py index 81c98c4..aa596d2 100644 --- a/ragavi/ragavi.py +++ b/ragavi/ragavi.py @@ -90,6 +90,10 @@ def __init__(self, xds_table_obj, ms_name, gtype, yaxis, chan=None, self.xaxis = self.set_xaxis() self.yaxis = yaxis + def __repr__(self): + return "DataCoreProcessor(%r, %r, %r, %r)" % ( + self.xds_table_obj, self.ms_name, self.gtype, self.yaxis) + def set_xaxis(self): if self.gtype in ['B', 'D']: xaxis = "channel" @@ -123,11 +127,12 @@ def process_data(self, ydata, yaxis=None): elif yaxis == "imaginary": y = vu.calc_imaginary(ydata) elif yaxis == "phase": - y = vu.calc_phase(ydata, wrap=False) + y = vu.calc_phase(ydata, unwrap=False) elif yaxis == "real": y = vu.calc_real(ydata) elif yaxis == "delay" or yaxis == "error": y = ydata + return y def flatten_bandpass_data(self, y, x=None): @@ -151,17 +156,21 @@ def flatten_bandpass_data(self, y, x=None): """ + logger.debug(f"Flattening bandpass: In y shape {str(y.shape)}") if y.ndim > 1: # no of unique times available in the bandpass # last index because y is transposed to (chan, time) nrows = y.shape[1] if x is not None: + logger.debug(f"Flattening bandpass: In x shape {str(x.shape)}") x = np.tile(x, (nrows)) + logger.debug(f"Out x shape: {str(x.shape)}") return x else: # TODO check if this flattening works proper y = y.T.flatten() + logger.debug(f"Out y shape: {str(y.shape)}") return y def get_errors(self): @@ -171,6 +180,8 @@ def get_errors(self): errors: :obj:`xarray.DataArray` Data from the PARAMERR column """ + logger.debug("DCP: Collecting parameter errors") + errors = self.xds_table_obj.PARAMERR return errors @@ -184,7 +195,6 @@ def get_xaxis_data(self): x_label : :obj:`str` Label to appear on the x-axis of the plots. """ - if self.xaxis in ["antenna", "antenna1"]: xdata = self.xds_table_obj.ANTENNA1 x_label = "Antenna" @@ -198,6 +208,7 @@ def get_xaxis_data(self): else: logger.error("Invalid xaxis name") return + return xdata, x_label def prep_xaxis_data(self, xdata, freq=None): @@ -256,7 +267,6 @@ def get_yaxis_data(self): except KeyError: logger.exception("Column '{}'' not Found".format(datacol)) return sys.exit(-1) - return ydata, y_label def prep_yaxis_data(self, ydata, yaxis=None): @@ -289,6 +299,7 @@ def prep_yaxis_data(self, ydata, yaxis=None): # if flagging enabled return a list of DataArrays otherwise return a # single dataarray if self.flag: + logger.debug("Flagging active") # if flagging is activated select data only where flag mask is 0 processed = self.process_data(ydata) y = processed.where(flags == False) @@ -315,12 +326,16 @@ def blackbox(self): A named tuple containing all processed x-axis data, errors and label, as well as both pairs of y-axis data, their error margins and labels. Items from this tuple can be gotten by using the dot notation. """ + logger.debug("DCP: Running blackbox") + + logger.debug("DCP: Getting x-axis data") Data = namedtuple("Data", "x x_label y y_label y_err") x = self.x_only() xdata = x.x xlabel = x.x_label + logger.debug("DCP: Getting y-axis data") y = self.y_only() ydata = y.y ylabel = y.y_label @@ -336,6 +351,8 @@ def blackbox(self): d = Data(x=xdata, x_label=xlabel, y=ydata, y_label=ylabel, y_err=(hi, lo)) + + logger.debug("Blackbox blackout") return d def act(self): @@ -409,6 +426,8 @@ def get_table(tab_name, antenna=None, fid=None, spwid=None, where=[], """ + logger.debug("Getting calibration table") + # defining part of the gain table schema tab_schema = {"CPARAM": {"dims": ("chan", "corr")}, "FLAG": {"dims": ("chan", "corr")}, @@ -436,14 +455,25 @@ def get_table(tab_name, antenna=None, fid=None, spwid=None, where=[], if group_cols is None: group_cols = ["SPECTRAL_WINDOW_ID", "FIELD_ID", "ANTENNA1"] + logger.debug(f"Grouping data in: {str(group_cols)}") + logger.debug(f"TAQL selection: {where}") + try: - tab_objs = xm.xds_from_table(tab_name, taql_where=where, - table_schema=tab_schema, - group_cols=group_cols) - return tab_objs - except: - logger.exception("""Invalid ANTENNA id, FIELD_ID, - SPECTRAL_WINDOW_ID or TAQL clause""") + tab_objs, t_keywords = xm.xds_from_table(tab_name, taql_where=where, + table_schema=tab_schema, + group_cols=group_cols, + table_keywords=True) + + # get table type from VisCal + g_type = t_keywords["VisCal"].split()[0] + + logger.info(f"Table type: {g_type} Jones") + + logger.debug(f"{len(tab_objs)} groups created from table") + return tab_objs, g_type + + except Exception as ex: + logger.error(ex, exc_info=True) sys.exit(-1) @@ -469,6 +499,8 @@ def get_time_range(tab_name, unix_time=True): ms.close() + logger.debug(f"Table time range. Initial: {str(i_time)}, final: {str(f_time)}") + return i_time, f_time @@ -493,7 +525,9 @@ def get_tooltip_data(xds_table_obj, gtype, freqs): """ scan_no = xds_table_obj.SCAN_NUMBER.values - spw_id = [xds_table_obj.SPECTRAL_WINDOW_ID] * scan_no.size + scan_no = scan_no.astype(np.uint16) + spw_id = np.full(scan_no.shape, xds_table_obj.SPECTRAL_WINDOW_ID, + dtype=np.uint8) # get the number of channels nchan = freqs.size @@ -755,9 +789,8 @@ def spw_select_callback(): } //Make the extra y-axes visible only if corresponding spw selected - if (cb_obj.active.includes(sp) && ex_ax[sp].name==`spw${sp}`){ + if (cb_obj.active.includes(sp)){ ex_ax[sp].visible=true; - console.log("Trueuee"); } else{ ex_ax[sp].visible=false; @@ -844,12 +877,17 @@ def legend_toggle_callback(): """ code = """ //Show only the legend for the first item + let n; if (cb_obj.active.includes(0)){ - legs[0].visible = true; + for (n=0; n 1: t_now = datetime.now().strftime("%Y%m%d_%H%M%S") html_name = html_name.replace(t_name, t_now) - save_html(html_name, final_layout) - - logger.info("Rendered: {}.html".format(html_name)) + save_html(html_name, final_layout) return 0 @@ -2019,16 +2122,10 @@ def plot_table(**kwargs): Parameters ---------- - **Required** - gaintype: :obj:`str`, :obj:`list` - Cal-table (list of caltypes) type to be plotted. Can be either - 'B'-bandpass, 'D'- D jones leakages, G'-gains, 'K'-delay or 'F'-flux. - Default is none - table : :obj:`str` or :obj:`list` required + table : :obj:`str` or :obj:`list` The table (list of tables) to be plotted. - **Optional** - ant : :obj:`str` + ant : :obj:`str, optional` Plot only specific antennas, or comma-separated list of antennas. corr : :obj:`int, optional` Correlation index to plot. Can be a single integer or comma separated diff --git a/ragavi/utils.py b/ragavi/utils.py index 29b8047..59ba1d3 100644 --- a/ragavi/utils.py +++ b/ragavi/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging import os +import re import sys import textwrap import warnings @@ -33,6 +34,8 @@ def calc_amplitude(ydata): amplitude : :obj:`xarray.DataArray` :attr:`ydata` converted to an amplitude """ + logger.debug("Setting up amplitude") + amplitude = da.absolute(ydata) return amplitude @@ -50,6 +53,8 @@ def calc_imaginary(ydata): imag : :obj:`xarray.DataArray` Imaginary part of :attr:`ydata` """ + logger.debug("Setting up imaginary") + imag = ydata.imag return imag @@ -67,11 +72,13 @@ def calc_real(ydata): real : :obj:`xarray.DataArray` Real part of :attr:`ydata` """ + logger.debug("Setting up real") + real = ydata.real return real -def calc_phase(ydata, wrap=True): +def calc_phase(ydata, unwrap=False): """Convert complex data to angle in degrees Parameters @@ -86,15 +93,18 @@ def calc_phase(ydata, wrap=True): phase: `xarray.DataArray` :attr:`ydata` data converted to degrees """ + logger.debug("Setting up wrapped phase") + + # np.angle already returns a phase wrapped between (-pi and pi] + # https://numpy.org/doc/1.18/reference/generated/numpy.angle.html phase = xr.apply_ufunc(da.angle, ydata, dask="allowed", kwargs=dict(deg=True)) - if wrap: + + if unwrap: # using an alternative method to avoid warnings - try: - phase = phase.reduce(np.unwrap) - except TypeError: - # this is for python2 compat - phase = xr.apply_ufunc(np.unwrap, phase, dask="allowed") + logger.debug("Unwrapping enabled. Unwrapping angles") + + phase = phase.reduce(np.unwrap) return phase @@ -111,6 +121,7 @@ def calc_uvdist(uvw): uvdist : :obj:`xarray.DataArray` uv distance in meters """ + logger.debug("Setting up UV Distance (metres)") u = uvw.isel(uvw=0) v = uvw.isel(uvw=1) uvdist = da.sqrt(da.square(u) + da.square(v)) @@ -133,6 +144,8 @@ def calc_uvwave(uvw, freq): uv distance in wavelength for specific frequency """ + logger.debug("Setting up UV Wavelengths (lambdas)") + # speed of light C = 3e8 @@ -154,11 +167,14 @@ def calc_unique_bls(n_ants=None): Available antennas Returns ------- - Number of unique baselines + pq: :obj:`int` + Number of unique baselines """ - return int(0.5 * n_ants * (n_ants - 1)) + pq = int(0.5 * n_ants * (n_ants - 1)) + logger.debug(f"Number of unique baselines: {str(pq)}.") + return pq ######################################################################## @@ -179,11 +195,15 @@ def get_antennas(ms_name): A :obj:`xarray.DataArray` containing names for all the antennas available. """ + logger.debug("Getting antenna names") + subname = "::".join((ms_name, "ANTENNA")) ant_subtab = list(xm.xds_from_table(subname)) ant_subtab = ant_subtab[0] ant_names = ant_subtab.NAME # ant_subtab("close") + + logger.debug(f"Antennas found: {str(ant_names.values)}") return ant_names @@ -200,10 +220,14 @@ def get_fields(ms_name): field_names : :obj:`xarray.DataArray` String names for the available fields """ + logger.debug("Getting antenna names") + subname = "::".join((ms_name, "FIELD")) field_subtab = list(xm.xds_from_table(subname)) field_subtab = field_subtab[0] field_names = field_subtab.NAME + + logger.debug(f"Fields found: {str(field_names.values)}") return field_names @@ -226,6 +250,9 @@ def get_frequencies(ms_name, spwid=None, chan=None, cbin=None): frequencies : :obj:`xarray.DataArray` Channel centre frequencies for specified spectral window or all the frequencies for all spectral windows if one is not specified """ + + logger.debug("Gettting Frequencies for selected SPWS and channels") + subname = "::".join((ms_name, "SPECTRAL_WINDOW")) # if averaging is true, it shall be done before selection @@ -239,6 +266,7 @@ def get_frequencies(ms_name, spwid=None, chan=None, cbin=None): else: from ragavi.averaging import get_averaged_spws # averages done per spectral window, scan and field + logger.info("Channel averaging active") spw_subtab = get_averaged_spws(subname, cbin, chan_select=chan) # concat all spws into a single data array @@ -251,6 +279,9 @@ def get_frequencies(ms_name, spwid=None, chan=None, cbin=None): if spwid is not None: # if multiple SPWs due to slicer, select the desired one(S) frequencies = frequencies.sel(row=spwid) + + logger.debug( + f"Frequency table shape (spws, chans): {str(frequencies.shape)}") return frequencies @@ -267,6 +298,8 @@ def get_polarizations(ms_name): cor2stokes: :obj:`list` Returns a list containing the types of correlation """ + + logger.debug("Getting Stokes' types") # Stokes types in this case are 1 based and NOT 0 based. stokes_types = ["I", "Q", "U", "V", "RR", "RL", "LR", "LL", "XX", "XY", "YX", "YY", "RX", "RY", "LX", "LY", "XR", "XL", "YR", @@ -284,6 +317,8 @@ def get_polarizations(ms_name): # Select corr_type name from the stokes types cor2stokes = [stokes_types[typ] for typ in corr_types] + logger.debug(f"Found stokes: {str(cor2stokes)}") + return cor2stokes @@ -303,7 +338,12 @@ def get_flags(xds_table_obj, corr=None, chan=slice(0, None)): Data array containing values from FLAG column selected by correlation if index is available. """ + logger.debug("Getting flags") flags = xds_table_obj.FLAG + + if np.all(flags.values): + logger.debug(f"All flags are active for {xds_table_obj.attrs}") + if corr is None: return flags.sel(chan=chan) else: @@ -329,6 +369,8 @@ def name_2id(tab_name, field_name): field_id : :obj:`int` Integer field id """ + logger.debug("Converting field names to FIELD_IDs") + field_names = get_fields(tab_name).data.compute() # make the sup field name uppercase @@ -336,8 +378,12 @@ def name_2id(tab_name, field_name): if field_name in field_names: field_id = np.where(field_names == field_name)[0][0] + + logger.debug(f"Field name {field_name} --> found in ID {field_id}") + return int(field_id) else: + logger.debug(f"FIELD_ID of {field_name} not found") return -1 @@ -363,6 +409,8 @@ def resolve_ranges(inp): # takes care of 5:8 or 5,6,7 or 5: res = "[{}]".format(inp) + + logger.debug(f"Resolved range {inp} --> {res} for TAQL selection") return res @@ -381,44 +429,30 @@ def slice_data(inp): """ if inp is None: - # start = 0 - # stop = None - # sl = slice(start, stop) sl = slice(0, None) - return sl - - if inp.isdigit(): + elif inp.isdigit(): sl = int(inp) - return sl - # check where the string is a comma separated list - if ',' not in inp: - if '~' in inp: - splits = inp.replace('~', ':').split(':') - else: - splits = inp.split(':') - splits = [None if x == '' else int(x) for x in splits] - - len_splits = len(splits) - start, stop, step = None, None, 1 - - if len_splits == 1: - start = int(splits[0]) - elif len_splits == 2: - start, stop = splits - elif len_splits == 3: - start, stop, step = splits - - # since ~ only alters the end value, we can then only change the stop - # value - if '~' in inp: - stop += 1 - - sl = slice(start, stop, step) - else: - # assuming string is a comma separated list - inp = inp.replace(' ', '') - splits = [int(x) for x in inp.split(',')] - sl = np.array(splits) + elif ',' in inp: + # assume a comma separated list + sl = np.array(inp.split(","), dtype=int) + elif '~' in inp or ':' in inp: + splits = re.split(r"(~|:)", inp) + n_splits = [] + for x in splits: + if x.isdigit(): + x = int(x) + elif x == '': + x = None + n_splits.append(x) + + if '~' in n_splits: + n_splits[n_splits.index('~') + 1] += 1 + n_splits = [':' if _ == '~' else _ for _ in n_splits] + + # get every second item on list + sl = slice(*n_splits[::2]) + + logger.debug(f"Created slicer / (fancy) index {str(sl)} from input {inp}") return sl @@ -488,11 +522,14 @@ def wrap_warning_text(message, category, filename, lineno, file=None, warnings.formatwarning = wrap_warning_text -def __config_logger(): +def __config_logger(level="info"): """Configure the logger for ragavi and catch all warnings output by sys.stdout. """ logfile_name = "ragavi.log" + # numeric value of logging level + level_num = getattr(logging, level.upper(), None) + # capture only a single instance of a matching repeated warning warnings.filterwarnings("module") @@ -507,18 +544,18 @@ def __config_logger(): # create logger named ragavi logger = logging.getLogger("ragavi") - logger.setLevel(logging.INFO) + logger.setLevel(level_num) # warnings logger w_logger = logging.getLogger("py.warnings") - w_logger.setLevel(logging.INFO) + w_logger.setLevel(level_num) # console handler c_handler = logging.StreamHandler() f_handler = logging.FileHandler(logfile_name) - c_handler.setLevel(logging.INFO) - f_handler.setLevel(logging.INFO) + c_handler.setLevel(level_num) + f_handler.setLevel(level_num) # setting the format for the logging messages start = " (O_o) ".center(cols, "=") diff --git a/ragavi/visibilities.py b/ragavi/visibilities.py index 51e4925..72a388f 100644 --- a/ragavi/visibilities.py +++ b/ragavi/visibilities.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import division - import sys from collections import namedtuple, OrderedDict @@ -91,6 +89,10 @@ def __init__(self, xds_table_obj, ms_name, xaxis, yaxis, self.flag = flag self.cbin = cbin + def __repr__(self): + return "DataCoreProcessor(%r, %r, %r, %r)" % ( + self.xds_table_obj, self.ms_name, self.xaxis, self.yaxis) + def get_xaxis_data(self): """Get x-axis data. This function also returns the relevant x-axis label. Returns @@ -101,6 +103,8 @@ def get_xaxis_data(self): Label to appear on the x-axis of the plots. """ + logger.debug("DCP: Getting x-axis data") + if self.xaxis == "antenna1": xdata = self.xds_table_obj.ANTENNA1 x_label = "Antenna1" @@ -142,6 +146,8 @@ def get_xaxis_data(self): logger.error("Invalid xaxis name") return + logger.debug("Done") + return xdata, x_label def get_yaxis_data(self): @@ -154,6 +160,9 @@ def get_yaxis_data(self): y_label: :obj:`str` Label to appear on the y-axis of the plots. """ + + logger.debug("DCP: Getting y-axis data") + if self.yaxis == "amplitude": y_label = "Amplitude" elif self.yaxis == "imaginary": @@ -169,6 +178,8 @@ def get_yaxis_data(self): logger.exception("Column '{}' not Found".format(self.datacol)) return sys.exit(-1) + logger.debug("Done") + return ydata, y_label def prep_xaxis_data(self, xdata, freq=None): @@ -189,6 +200,8 @@ def prep_xaxis_data(self, xdata, freq=None): prepdx: :obj:`xarray.DataArray` Prepared :attr:`xdata` """ + logger.debug("DCP: Preping x-axis data") + if self.xaxis == "channel" or self.xaxis == "frequency": prepdx = xdata / 1e9 elif self.xaxis in ["phase", "amplitude", "real", "imaginary"]: @@ -201,6 +214,9 @@ def prep_xaxis_data(self, xdata, freq=None): prepdx = vu.calc_uvwave(xdata, freq) / 1e3 elif self.xaxis == "antenna1" or self.xaxis == "antenna2" or self.xaxis == "scan": prepdx = xdata + + logger.debug("Done") + return prepdx def prep_yaxis_data(self, ydata, yaxis=None): @@ -222,6 +238,9 @@ def prep_yaxis_data(self, ydata, yaxis=None): y: :obj:`xarray.DataArray` Processed :attr:`ydata` data. """ + + logger.debug("DCP: Preping y-axis data") + flags = vu.get_flags(self.xds_table_obj) # Doing this because some of xaxis data must be processed here @@ -238,14 +257,20 @@ def prep_yaxis_data(self, ydata, yaxis=None): "Please use -nf or --no-flagged to deactivate flagging if you still wish to generate this plot.") logger.warning(" Exiting.") sys.exit(0) - processed = self.process_data(ydata, yaxis=yaxis, wrap=True) + + processed = self.process_data(ydata, yaxis=yaxis, unwrap=False) + + logger.debug("Applying flags") + y = processed.where(flags == False) else: - y = self.process_data(ydata, yaxis=yaxis) + y = self.process_data(ydata, yaxis=yaxis, unwrap=False) + + logger.debug("Done") return y - def process_data(self, ydata, wrap=True, yaxis=None): + def process_data(self, ydata, unwrap=False, yaxis=None): """Abstraction for processing y-data passes it to the processing function. Parameters @@ -260,6 +285,8 @@ def process_data(self, ydata, wrap=True, yaxis=None): y : :obj:`xarray.DataArray` Processed :obj:`ydata` """ + logger.debug("DCP: Calculating y-axis data") + if yaxis is None: yaxis = self.yaxis if yaxis == "amplitude": @@ -267,9 +294,12 @@ def process_data(self, ydata, wrap=True, yaxis=None): elif yaxis == "imaginary": y = vu.calc_imaginary(ydata) elif yaxis == "phase": - y = vu.calc_phase(ydata, wrap=wrap) + y = vu.calc_phase(ydata, unwrap=unwrap) elif yaxis == "real": y = vu.calc_real(ydata) + + logger.debug("Done") + return y def blackbox(self): @@ -411,6 +441,8 @@ def gen_image(df, x_min, x_max, y_min, y_max, c_height, c_width, cat=None, if add_xaxis: # some formatting on the x and y axes + logger.debug("Formatting x-axis") + if x_name.lower() in ["channel", "frequency"]: try: p_title = fig.above.pop() @@ -428,12 +460,16 @@ def gen_image(df, x_min, x_max, y_min, y_max, c_height, c_width, cat=None, xaxis.formatter = PrintfTickFormatter(format=u"%f\u00b0") if add_yaxis: + logger.debug("Formatting y-axis") + if y_name.lower() == "phase": yaxis = fig.select(name="p_y_axis")[0] yaxis.formatter = PrintfTickFormatter(format=u"%f\u00b0") # only to be added in iteration mode if add_subtitle: + + logger.debug("Formatting and adding sub-title") # getting the iteration axis chunk_attrs = xds_table_obj.attrs @@ -486,7 +522,9 @@ def gen_image(df, x_min, x_max, y_min, y_max, c_height, c_width, cat=None, fig.add_glyph(p_txtt_src, p_txtt) + logger.debug("Adding glyph to figure") fig.add_glyph(cds, image_glyph) + return fig @@ -496,6 +534,9 @@ def gen_grid(df, x_min, x_max, y_min, y_max, c_height, c_width, cat=None, x_name=None, xlab=None, y_name=None, ylab=None, xds_table_obj=None): """ Generate bokeh grid of figures""" + + logger.debug("Preparing bokeh grid generation") + n_grid = [] nrows = int(np.ceil(cat_vals.size / ncols)) @@ -547,6 +588,8 @@ def gen_grid(df, x_min, x_max, y_min, y_max, c_height, c_width, cat=None, logger.info("Creating Bokeh grid") for i, c_val in enumerate(cat_vals): + + logger.debug(f"Item: {i}/{cat_vals.size}, iterating: {cat}") # some text title for the iterated columns p_txtt = Text(x="x", y="y", text="text", text_font="monospace", text_font_style="bold", text_font_size="10pt", @@ -629,6 +672,8 @@ def gen_grid(df, x_min, x_max, y_min, y_max, c_height, c_width, cat=None, n_grid.tags = [ncols, nrows] # final_grid = gridplot(children=[title_div, n_grid], ncols=1) + logger.debug("Bokeh grid done") + return n_grid @@ -683,6 +728,8 @@ def make_cbar(cats, category, cmap=None): b_ticker = BasicTicker(desired_num_ticks=ncats, num_minor_ticks=0) + logger.debug("Creating colour bar") + cbar = ColorBar(color_mapper=cmapper, label_standoff=12, border_line_color=None, ticker=b_ticker, location=(0, 0), title=category, title_standoff=5, @@ -690,6 +737,8 @@ def make_cbar(cats, category, cmap=None): title_text_font="monospace", minor_tick_line_width=0, title_text_font_style="normal") + logger.debug("Done") + return cbar @@ -898,6 +947,7 @@ def antenna_iter(ms_name, columns, **kwargs): A list containing data for each individual antenna. This list is ordered by antenna and SPW. """ + logger.debug("Creating antenna iterable") taql_where = kwargs.get("taql_where", "") table_schema = kwargs.get("table_schema", None) @@ -914,6 +964,9 @@ def antenna_iter(ms_name, columns, **kwargs): for d in range(n_spws): for a in range(n_ants): + + logger.debug(f"Spw: {d}, antenna: {a}") + sel_str = taql_where + \ f"ANTENNA1=={a} || ANTENNA2=={a} && DATA_DESC_ID=={d}" @@ -929,6 +982,7 @@ def antenna_iter(ms_name, columns, **kwargs): outp.append(sub) + logger.debug("Done") return outp @@ -948,14 +1002,22 @@ def corr_iter(subs): # NOTE: can also be used for chan iteration. Will require name change """ + + logger.debug("Creating correlation iterable") + outp = [] n_corrs = subs[0].corr.size for sub in subs: for c in range(n_corrs): + + logger.debug(f"Corr: {c}") + nsub = sub.copy(deep=True).sel(corr=c) nsub.attrs["Corr"] = c outp.append(nsub) + + logger.debug("Done") return outp @@ -1003,6 +1065,8 @@ def get_ms(ms_name, ants=None, cbin=None, chan_select=None, chunks=None, tab_objs: :obj:`list` A list containing the specified table objects as :obj:`xarray.Dataset` """ + logger.debug("Starting MS acquisition") + group_cols = set() # Always group by DDID @@ -1065,6 +1129,10 @@ def get_ms(ms_name, ants=None, cbin=None, chan_select=None, chunks=None, sel_cols = list(sel_cols) group_cols = list(group_cols) + logger.debug(f"Selected columns: {', '.join(sel_cols)}") + logger.debug(f"Grouping by: {', '.join(group_cols)}") + logger.debug(f"TAQL selection: {where}") + xds_inputs = dict(chunks=chunks, taql_where=where, columns=sel_cols, group_cols=group_cols, table_schema=ms_schema) @@ -1090,9 +1158,11 @@ def get_ms(ms_name, ants=None, cbin=None, chan_select=None, chunks=None, tab_objs = corr_iter(tab_objs) # select channels if chan_select is not None: + logger.debug("Selecting channels") tab_objs = [_.sel(chan=chan_select) for _ in tab_objs] # select corrs if corr_select is not None: + logger.debug("Selecting correlations") tab_objs = [_.sel(corr=corr_select) for _ in tab_objs] # get some info about the data @@ -1103,9 +1173,8 @@ def get_ms(ms_name, ants=None, cbin=None, chan_select=None, chunks=None, logger.info("Number of Partitions: {}".format(chunk_p)) return tab_objs - except: - logger.error( - "Invalid DATA_DESC_ID, FIELD_ID, SCAN_NUMBER or TAQL clause") + except Exception as ex: + logger.error(ex.args[0]) sys.exit(-1) @@ -1185,7 +1254,7 @@ def create_categorical_df(it_axis, x_data, y_data, xds_table_obj): logger.error("Specified data column not found.") sys.exit(-1) - logger.info("Creating Dataframe") + logger.info("Starting creation of categorical dataframe") xy_df = create_df(x_data, y_data, iter_data=iter_data)[[x_data.name, y_data.name, @@ -1233,9 +1302,12 @@ def create_df(x, y, iter_data=None): var_names[i_name] = ("row", iter_data) logger.info("Creating dataframe") + new_ds = xr.Dataset(data_vars=var_names) new_ds = new_ds.to_dask_dataframe() new_ds = new_ds.dropna() + + logger.info("Done") return new_ds @@ -1260,6 +1332,7 @@ def massage_data(x, y, get_y=False, iter_ax=None): y: :obj:`dask.array` Data for the y-axis """ + logger.debug("Flattening x-axis and y-axis data") iter_cols = ["ANTENNA1", "ANTENNA2", "FIELD_ID", "SCAN_NUMBER", "DATA_DESC_ID", "Baseline"] @@ -1269,6 +1342,9 @@ def massage_data(x, y, get_y=False, iter_ax=None): y_dims = set(y.dims) x_dims = set(x.dims) + logger.debug(f"x shape: {str(x.shape)}") + logger.debug(f"y shape: {str(y.shape)}") + # find dims that are not available in x and only avail in y req_x_dims = y_dims - x_dims @@ -1305,10 +1381,14 @@ def massage_data(x, y, get_y=False, iter_ax=None): except ValueError: # for the non-xarray dask data nx = nx.repeat(np.prod(sizes)).rechunk(y.data.ravel().chunks) + + logger.debug(f"New x shape: {str(nx.shape)}") + if get_y: # flatten y data # get_data also ny = y.data.ravel() + logger.debug(f"New y shape: {str(ny.shape)}") return nx, ny else: return nx @@ -1340,6 +1420,8 @@ def get_colname(inp, data_col=None): col_name = aliases[inp] else: col_name = None + + logger.debug(f"Column name for {inp} --> {col_name}") return col_name @@ -1356,7 +1438,7 @@ def validate_axis_inputs(inp): Returns ------- - inp: :obj:`str` + oup: :obj:`str` Validated string """ alts = {} @@ -1380,15 +1462,21 @@ def validate_axis_inputs(inp): # convert to proper name if in other name if inp in alts: - inp = alts[inp] + oup = alts[inp] + else: + oup = inp - return inp + logger.debug(f"Alias for axis {inp} --> {oup}") + return oup def link_grid_plots(plot_list): """Link all the plots in the X and Y axes """ + + logger.debug(f"Linking {len(plot_list)} generated plots") + if not isinstance(plot_list[0], Plot): plots = [] plot_list = plot_list[0].children @@ -1404,6 +1492,8 @@ def link_grid_plots(plot_list): plots[i].x_range = init_xr plots[i].y_range = init_yr + logger.debug("Done") + return 0 diff --git a/setup.py b/setup.py index afff470..fe3e480 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def requirements(): return [pname.strip() for pname in f.readlines()] setup(name='ragavi', - version="0.3.6", + version="0.3.7", description='Radio Astronomy Gain and Visibility Inspector', long_description=readme(), url='https://github.com/ratt-ru/ragavi',