From 28f8fb44825a913316f230e119129f6a31319bb0 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Sat, 16 Nov 2024 08:32:21 +0000 Subject: [PATCH 1/4] First pass at importable package --- LICENSE | 28 ++++ pyproject.toml | 104 ++++++++++++ ruff.toml | 36 +---- src/genie_python/genie_cachannel_wrapper.py | 19 ++- src/genie_python/genie_dae.py | 6 +- src/genie_python/genie_plot.py | 6 +- src/genie_python/genie_scisoft_plot.py | 84 ---------- .../ibex_websocket_backend.py | 4 +- .../typings/CaChannel/CaChannel.pyi | 148 ++++++++---------- .../typings/CaChannel/__init__.pyi | 2 - src/genie_python/typings/CaChannel/ca.pyi | 25 ++- src/genie_python/utilities.py | 3 - 12 files changed, 240 insertions(+), 225 deletions(-) create mode 100644 LICENSE create mode 100644 pyproject.toml delete mode 100644 src/genie_python/genie_scisoft_plot.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..ca51081d --- /dev/null +++ b/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2024, ISIS Experiment Controls Computing + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..36ea37f4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,104 @@ +[build-system] +requires = ["setuptools", "setuptools_scm>=8"] +build-backend = "setuptools.build_meta" + + +[project] +name = "genie_python" +dynamic = ["version"] +description = "Instrument control & scripting for the ISIS Neutron & Muon source" +readme = "README.md" +requires-python = ">=3.11" +license = {file = "LICENSE"} + +authors = [ + {name = "ISIS Experiment Controls", email = "ISISExperimentControls@stfc.ac.uk" } +] +maintainers = [ + {name = "ISIS Experiment Controls", email = "ISISExperimentControls@stfc.ac.uk" } +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 3.11", +] + +dependencies = [ + # EPICS Channel access lib + "CaChannel", + # Caffi provides some EPICS CA definitions if CaChannel is falling back to using caffi backend + # (not used if CaChannel._ca is available) + "caffi", + # Send log messages to graylog + "graypy", + # genie_python will install ipython completers + "ipython", + # Getting user details from database + "mysql-connector-python", + # Array support for CA calls + "numpy", + # EPICS PV access lib + "p4p", + # Used to find process by name to kill it + "psutil", + # For checking user scripts on g.load_script() + "pylint", + # For setting windows job-object flags + "pywin32;platform_system=='Windows'", +] + +[project.optional-dependencies] + +# Make plotting an optional dependency as: +# - It fixes a *specific* matplotlib version +# - It depends on a couple of heavyweight libs (py4j and tornado) that aren't necessary otherwise +plot = [ + # When updating, check plotting works in GUI. Must keep pinned to a specific, tested version. + "matplotlib==3.9.2", + # Python <-> Java communication, to spawn matplotlib plots in GUI + "py4j", + # Tornado webserver used by custom backend + "tornado", +] + +doc = [ + "sphinx", + "sphinx_rtd_theme", + "myst_parser", + "sphinx-autobuild", +] + +dev = [ + "genie_python[plot,doc]", + "mock", + "parameterized", + "pyhamcrest", + "pyright", + "ruff>=0.6", +] + +[project.urls] +"Homepage" = "https://github.com/isiscomputinggroup/genie" +"Bug Reports" = "https://github.com/isiscomputinggroup/genie/issues" +"Source" = "https://github.com/isiscomputinggroup/genie" + +[tool.pyright] +include = ["src", "tests"] +reportConstantRedefinition = true +reportDeprecated = true +reportInconsistentConstructor = true +reportMissingParameterType = true +reportMissingTypeArgument = true +reportUnnecessaryCast = true +reportUnnecessaryComparison = true +reportUnnecessaryContains = true +reportUnnecessaryIsInstance = true +reportUntypedBaseClass = true +reportUntypedClassDecorator = true +reportUntypedFunctionDecorator = true + +[tool.setuptools_scm] + +[tool.build_sphinx] diff --git a/ruff.toml b/ruff.toml index 80026774..1a80d10a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,32 +1,8 @@ # Exclude a variety of commonly ignored directories. exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".ipynb_checkpoints", - ".mypy_cache", - ".nox", - ".pants.d", - ".pyenv", - ".pytest_cache", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - ".vscode", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "venv", - "uk.ac.stfc.isis.ibex.opis", + ".pyi", "snmp-mibs", + "tests/test_scripts", ] # Set the maximum line length to 100. @@ -52,7 +28,7 @@ ignore = [ "ANN401", # explicit ANY type is allowed ] [lint.per-file-ignores] -"{**/tests/**,/tests/**,**/*tests.py,tests/**,*tests.py,*test.py,**/*test.py,common_tests/**,**/test_modules**}" = [ +"tests/*" = [ "N802", "D100", "D101", @@ -64,9 +40,3 @@ ignore = [ [lint.pydocstyle] # Use Google-style docstrings. convention = "google" - -[format] -quote-style = "double" -indent-style = "space" -docstring-code-format = true -line-ending = "auto" \ No newline at end of file diff --git a/src/genie_python/genie_cachannel_wrapper.py b/src/genie_python/genie_cachannel_wrapper.py index 7cc29695..c3387c90 100644 --- a/src/genie_python/genie_cachannel_wrapper.py +++ b/src/genie_python/genie_cachannel_wrapper.py @@ -11,7 +11,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, TypeVar from CaChannel import CaChannel, CaChannelException, ca -from CaChannel._ca import AlarmCondition, AlarmSeverity, dbf_type_to_DBR_STS, dbf_type_to_DBR_TIME + +try: + from CaChannel._ca import ( + AlarmCondition, + AlarmSeverity, + dbf_type_to_DBR_STS, + dbf_type_to_DBR_TIME, + ) +except ImportError: + from caffi.ca import AlarmCondition, AlarmSeverity, dbf_type_to_DBR_STS, dbf_type_to_DBR_TIME if TYPE_CHECKING: from genie_python.genie import PVValue @@ -112,7 +121,13 @@ def installHandlers(chan: CaChannel) -> None: # noqa N802 # callbacks CaChannel itself delays creation of the context, so if we just installed the # handlers now we would get a default non-preemptive CA context created. chan.poll() - chan.replace_printf_handler(CaChannelWrapper.printfHandler) + try: + chan.replace_printf_handler(CaChannelWrapper.printfHandler) + except AttributeError: + # If we can't replace the printf handler, ignore that error - it is not crucial. + # It probably means we are using default CaChannel, as opposed to ISIS' special build. + # Cope with both cases. + pass chan.add_exception_event(CaChannelWrapper.CAExceptionHandler) # noinspection PyPep8Naming diff --git a/src/genie_python/genie_dae.py b/src/genie_python/genie_dae.py index ab4f1487..d9e69251 100644 --- a/src/genie_python/genie_dae.py +++ b/src/genie_python/genie_dae.py @@ -18,7 +18,11 @@ import numpy as np import numpy.typing as npt import psutil -from CaChannel._ca import AlarmCondition, AlarmSeverity + +try: + from CaChannel._ca import AlarmCondition, AlarmSeverity +except ImportError: + from caffi.ca import AlarmCondition, AlarmSeverity from genie_python.genie_cachannel_wrapper import CaChannelWrapper from genie_python.genie_change_cache import ChangeCache diff --git a/src/genie_python/genie_plot.py b/src/genie_python/genie_plot.py index 8541a86a..5148b3cd 100644 --- a/src/genie_python/genie_plot.py +++ b/src/genie_python/genie_plot.py @@ -3,8 +3,6 @@ from builtins import object, str from functools import wraps -import matplotlib.pyplot as pyplt - def _plotting_func(func): """ @@ -16,6 +14,8 @@ def _plotting_func(func): @wraps(func) def wrapper(*args, **kwargs): + import matplotlib.pyplot as pyplt + was_interactive = pyplt.isinteractive() pyplt.ioff() result = func(*args, **kwargs) @@ -29,6 +29,8 @@ def wrapper(*args, **kwargs): class SpectraPlot(object): @_plotting_func def __init__(self, api, spectrum, period, dist): + import matplotlib.pyplot as pyplt + self.api = api self.spectra = [] self.fig = pyplt.figure() diff --git a/src/genie_python/genie_scisoft_plot.py b/src/genie_python/genie_scisoft_plot.py deleted file mode 100644 index a9046e67..00000000 --- a/src/genie_python/genie_scisoft_plot.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import absolute_import - -from builtins import object - -import matplotlib.dates as dates -import scisoftpy as dnp - -# Plot using a scisoftpy's AnalysisRPC plot server, e.g. to display in a Java application - - -class SePlot(object): - def __init__(self): - self.title = "Se plot" - # TODO: Set date/time formatting for x-axis - - def add_plot(self, data, name=""): - x = dates.date2num(data[0]) - dnp.plot.addline(_configure_x(x, "Date"), _configure_y(data[1], "y", name), self.title) - # TODO: Set line names - - -class SpectraPlot(object): - def __init__(self, api, spectrum, period, dist): - self.api = api - self.spectra = [] - self.x_label = "Time" - self.y_label = "Counts" - self.title = "Spectrum %s" % spectrum - - self.add_spectrum(spectrum, period, dist) - - def add_spectrum(self, spectrum, period, dist): - plot = dnp.plot.plot if not self.spectra else dnp.plot.addline - self.spectra.append((spectrum, period, dist)) - data = self.api.dae.get_spectrum(spectrum, period, dist) - name = "Spect %s " % spectrum - - plot( - _configure_x(data["time"], self.x_label), - _configure_y(data["signal"], self.y_label, name), - self.title, - ) - - def refresh(self): - # TODO - pass - - def delete_plot(self, plotNum): - # TODO - pass - - -class GeniePlot(object): - def __init__(self, x_label="X", y_label="Y", title=None): - self.x_label = x_label - self.y_label = y_label - self.title = title - self.plot_count = 0 - - def add_plot(self, x_data, y_data, name=""): - plot = dnp.plot.plot if not self.plot_count else dnp.plot.addline - self.plot_count += 1 - plot( - _configure_x(x_data, self.x_label), _configure_y(y_data, self.y_label, name), self.title - ) - - def update_data(self, x_data, y_data, plotnum=0): - # TODO - pass - - def delete_plot(self, plotnum): - # TODO - # self.plot_count -= 1 - pass - - -def _configure_x(x, axis_label="x"): - # format for plotting on the x-axis - return {axis_label: dnp.array(x)} - - -def _configure_y(y, axis_label="y", line_label=""): - # format for plotting on the y-axis - return [{axis_label: (dnp.array(y), line_label)}] diff --git a/src/genie_python/matplotlib_backend/ibex_websocket_backend.py b/src/genie_python/matplotlib_backend/ibex_websocket_backend.py index d34b4ffe..29b0b04e 100644 --- a/src/genie_python/matplotlib_backend/ibex_websocket_backend.py +++ b/src/genie_python/matplotlib_backend/ibex_websocket_backend.py @@ -277,9 +277,7 @@ def pyplot_show(cls, *args, **kwargs): with IBEX_BACKEND_LOCK: WebAggApplication.initialize(port=_web_backend_port) worker_thread = threading.Thread( - target=WebAggApplication.start, - daemon=True, - name="ibex_websocket_backend" + target=WebAggApplication.start, daemon=True, name="ibex_websocket_backend" ) worker_thread.start() diff --git a/src/genie_python/typings/CaChannel/CaChannel.pyi b/src/genie_python/typings/CaChannel/CaChannel.pyi index 94fed59f..ee43e55e 100644 --- a/src/genie_python/typings/CaChannel/CaChannel.pyi +++ b/src/genie_python/typings/CaChannel/CaChannel.pyi @@ -2,23 +2,17 @@ This type stub file was generated by pyright. """ -import sys -from typing import TypeAlias, Tuple, Callable, TypeVar, Any +from typing import Any, Callable, Tuple, TypeAlias, TypeVar + from numpy import NDArray __all__ = ["CaChannelException", "CaChannel"] T = TypeVar("T") class CaChannelException(Exception): - def __init__(self, status) -> None: - ... - - def __int__(self) -> int: - ... - - def __str__(self) -> str: - ... - + def __init__(self, status) -> None: ... + def __int__(self) -> int: ... + def __str__(self) -> str: ... PVBaseValue: TypeAlias = bool | int | float | str PVValue: TypeAlias = PVBaseValue | list[PVBaseValue] | NDArray | None @@ -54,6 +48,7 @@ class CaChannel: >>> chan.getw(ca.DBR_STRING) 'Done' """ + __context = ... __callbacks = ... __context_lock = ... @@ -61,26 +56,16 @@ class CaChannel: ca_timeout = ... last_value: PVValue = ... @staticmethod - def get_thread_id(thread) -> str: # -> str: + def get_thread_id(thread) -> str: # -> str: ... - @staticmethod - def create_context(): + def create_context(): ... + def attach_ca_context(func): # -> _Wrapped[Callable[..., Any], Any, Callable[..., Any], Any]: ... - - def attach_ca_context(func): # -> _Wrapped[Callable[..., Any], Any, Callable[..., Any], Any]: - ... - - def __init__(self, pvName=...) -> None: - ... - - def __del__(self) -> None: - ... - + def __init__(self, pvName=...) -> None: ... + def __del__(self) -> None: ... @staticmethod - def version(): - ... - + def version(): ... def setTimeout(self, timeout: float) -> None: """Set the timeout for this channel object. It overrides the class timeout. @@ -92,8 +77,8 @@ class CaChannel: 10.0 """ ... - - def getTimeout(self)-> float: + + def getTimeout(self) -> float: """Retrieve the timeout set for this channel object. :return: timeout in seconds for this channel instance @@ -104,10 +89,14 @@ class CaChannel: True """ ... - + @classmethod @attach_ca_context - def replace_printf_handler(cls, callback: Callable[[str, Tuple[T, ...]], None]=..., user_args:Tuple[T, ...]|None=...)-> None: + def replace_printf_handler( + cls, + callback: Callable[[str, Tuple[T, ...]], None] = ..., + user_args: Tuple[T, ...] | None = ..., + ) -> None: """ Install or replace the callback used for formatted CA diagnostic message output. The default is to send to stderr. @@ -125,10 +114,10 @@ class CaChannel: .. versionadded:: 3.1 """ ... - + @classmethod @attach_ca_context - def add_exception_event(cls, callback=..., user_args=...): # -> None: + def add_exception_event(cls, callback=..., user_args=...): # -> None: """ Install or replace the currently installed CA context global exception handler callback. @@ -159,8 +148,8 @@ class CaChannel: .. versionadded:: 3.1 """ ... - - def replace_access_rights_event(self, callback=..., user_args=...): # -> None: + + def replace_access_rights_event(self, callback=..., user_args=...): # -> None: """ Install or replace the access rights state change callback handler for the specified channel. @@ -188,9 +177,9 @@ class CaChannel: .. versionadded:: 3.0 """ ... - + @attach_ca_context - def search_and_connect(self, pvName, callback, *user_args): # -> None: + def search_and_connect(self, pvName, callback, *user_args): # -> None: """Attempt to establish a connection to a process variable. :param pvName: process variable name @@ -228,9 +217,9 @@ class CaChannel: >>> chan.clear_channel() """ ... - + @attach_ca_context - def search(self, pvName=...): # -> None: + def search(self, pvName=...): # -> None: """Attempt to establish a connection to a process variable. :param pvName: process variable name @@ -248,8 +237,8 @@ class CaChannel: """ ... - - def clear_channel(self): # -> None: + + def clear_channel(self): # -> None: """Close a channel created by one of the search functions. Clearing a channel does not cause its connection handler to be called. @@ -263,8 +252,8 @@ class CaChannel: """ ... - - def change_connection_event(self, callback, *user_args): # -> None: + + def change_connection_event(self, callback, *user_args): # -> None: """Change the connection callback function :param callback: function called when connection completes and connection status changes later on. @@ -287,8 +276,8 @@ class CaChannel: """ ... - - def array_put(self, value, req_type=..., count=...): # -> None: + + def array_put(self, value, req_type=..., count=...): # -> None: """Write a value or array of values to a channel :param value: data to be written. For multiple values use a list or tuple @@ -330,8 +319,8 @@ class CaChannel: [49, 50, 51, 0] """ ... - - def array_put_callback(self, value, req_type, count, callback, *user_args): # -> None: + + def array_put_callback(self, value, req_type, count, callback, *user_args): # -> None: """Write a value or array of values to a channel and execute the user supplied callback after the put has completed. @@ -389,8 +378,8 @@ class CaChannel: cawavec put completed """ ... - - def getValue(self): # -> dict[Any, Any] | None: + + def getValue(self): # -> dict[Any, Any] | None: """ Return the value(s) after :meth:`array_get` has completed. @@ -418,8 +407,8 @@ class CaChannel: pv_value 1 """ ... - - def array_get(self, req_type=..., count=..., **keywords): # -> None: + + def array_get(self, req_type=..., count=..., **keywords): # -> None: """Read a value or array of values from a channel. The new value is not available until a subsequent :meth:`pend_io` returns :data:`ca.ECA_NORMAL`. @@ -454,8 +443,8 @@ class CaChannel: 123.0 """ ... - - def array_get_callback(self, req_type, count, callback, *user_args, **keywords): # -> None: + + def array_get_callback(self, req_type, count, callback, *user_args, **keywords): # -> None: """Read a value or array of values from a channel and execute the user supplied callback after the get has completed. @@ -573,8 +562,10 @@ class CaChannel: pv_value 0 """ ... - - def add_masked_array_event(self, req_type, count, mask, callback, *user_args, **keywords): # -> None: + + def add_masked_array_event( + self, req_type, count, mask, callback, *user_args, **keywords + ): # -> None: """Specify a callback function to be executed whenever changes occur to a PV. Creates a new event id and stores it on self.__evid. Only one event registered per CaChannel object. @@ -625,8 +616,8 @@ class CaChannel: >>> chan.clear_event() """ ... - - def clear_event(self): # -> None: + + def clear_event(self): # -> None: """Remove previously installed callback function. .. note:: All remote operation requests such as the above are accumulated (buffered) and not forwarded to @@ -634,9 +625,9 @@ class CaChannel: is called. This allows several requests to be efficiently sent over the network in one message. """ ... - + @attach_ca_context - def pend_io(self, timeout=...): # -> None: + def pend_io(self, timeout=...): # -> None: """ Flush the send buffer and wait until outstanding queries (:meth:`search`, :meth:`array_get`) complete or the specified timeout expires. @@ -646,7 +637,7 @@ class CaChannel: """ ... - + @attach_ca_context def pend_event(self, timeout=...): """Flush the send buffer and process background activity (connect/get/put/monitor callbacks) for ``timeout`` seconds. @@ -657,7 +648,7 @@ class CaChannel: :return: :data:`ca.ECA_TIMEOUT` """ ... - + @attach_ca_context def poll(self): """Flush the send buffer and execute any outstanding background activity. @@ -667,16 +658,16 @@ class CaChannel: .. note:: It is an alias to ``pend_event(1e-12)``. """ ... - + @attach_ca_context - def flush_io(self): # -> None: + def flush_io(self): # -> None: """ Flush the send buffer and does not execute outstanding background activity. :raises CaChannelException: if error happens """ ... - + def field_type(self): """ Native type of the PV in the server, :data:`ca.DBF_XXXX`. @@ -692,7 +683,7 @@ class CaChannel: True """ ... - + def element_count(self): """ Maximum array element count of the PV in the server. @@ -703,7 +694,7 @@ class CaChannel: 1 """ ... - + def name(self): """ Channel name specified when the channel was created. @@ -714,7 +705,7 @@ class CaChannel: 'catest' """ ... - + def state(self): """ Current state of the CA connection. @@ -737,7 +728,7 @@ class CaChannel: """ ... - + def host_name(self): """ Host name that hosts the process variable. @@ -748,7 +739,7 @@ class CaChannel: """ ... - + def read_access(self): """Access right to read the channel. @@ -758,9 +749,9 @@ class CaChannel: >>> chan.searchw() >>> chan.read_access() True - """ + """ ... - + def write_access(self): """Access right to write the channel. @@ -772,8 +763,8 @@ class CaChannel: True """ ... - - def searchw(self, pvName=...): # -> None: + + def searchw(self, pvName=...): # -> None: """ Attempt to establish a connection to a process variable. @@ -790,8 +781,8 @@ class CaChannel: CaChannelException: User specified timeout on IO operation expired """ ... - - def putw(self, value, req_type=...): # -> None: + + def putw(self, value, req_type=...): # -> None: """ Write a value or array of values to a channel. @@ -843,7 +834,7 @@ class CaChannel: ['string 1', 'string 2', ''] """ ... - + def getw(self, req_type=..., count=..., **keywords) -> dict[Any, Any] | PVValue: """Read the value from a channel. @@ -898,8 +889,5 @@ class CaChannel: """ ... - - -if __name__ == "__main__": - ... +if __name__ == "__main__": ... diff --git a/src/genie_python/typings/CaChannel/__init__.pyi b/src/genie_python/typings/CaChannel/__init__.pyi index f39f06e3..e6328caa 100644 --- a/src/genie_python/typings/CaChannel/__init__.pyi +++ b/src/genie_python/typings/CaChannel/__init__.pyi @@ -3,8 +3,6 @@ This type stub file was generated by pyright. """ from .CaChannel import CaChannel, CaChannelException -from . import ca -from ._version import __version__, version_info USE_NUMPY = ... diff --git a/src/genie_python/typings/CaChannel/ca.pyi b/src/genie_python/typings/CaChannel/ca.pyi index ad0b4c49..4b12884b 100644 --- a/src/genie_python/typings/CaChannel/ca.pyi +++ b/src/genie_python/typings/CaChannel/ca.pyi @@ -4,17 +4,17 @@ This type stub file was generated by pyright. import os from enum import Enum +from typing import TypeAlias + from caffi.ca import * -from caffi.macros import * from caffi.constants import * -from ._ca import * -from typing import TypeAlias +from caffi.macros import * from numpy import NDArray -if os.environ.get('CACHANNEL_BACKEND') == 'caffi': - ... -else: - ... +from ._ca import * + +if os.environ.get("CACHANNEL_BACKEND") == "caffi": ... +else: ... ECA_TIMEOUT: int DBR_STRING: str @@ -26,11 +26,6 @@ CA_OP_CONN_DOWN: int PVBaseValue: TypeAlias = bool | int | float | str PVValue: TypeAlias = PVBaseValue | list[PVBaseValue] | NDArray | None -def dbr_type_is_CHAR(PVValue)->bool: - ... - -def dbr_type_is_ENUM(PVValue)->bool: - ... - -def dbr_type_is_STRING(PVValue)->bool: - ... \ No newline at end of file +def dbr_type_is_CHAR(PVValue) -> bool: ... +def dbr_type_is_ENUM(PVValue) -> bool: ... +def dbr_type_is_STRING(PVValue) -> bool: ... diff --git a/src/genie_python/utilities.py b/src/genie_python/utilities.py index 030c6683..dc988f19 100644 --- a/src/genie_python/utilities.py +++ b/src/genie_python/utilities.py @@ -10,8 +10,6 @@ from datetime import timedelta from functools import wraps -from future.utils import python_2_unicode_compatible - try: from nicos import session @@ -47,7 +45,6 @@ def cleanup_subprocs_on_process_exit(): raise OSError(f"cleanup_subprocs_on_process_exit() failed: {err}") -@python_2_unicode_compatible class PVReadException(Exception): """ Exception to throw when there is a problem reading a PV. From 6ffcf337ed6f37aab4cb591817f2d04af9dc65b4 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Sat, 16 Nov 2024 09:17:28 +0000 Subject: [PATCH 2/4] Add CI --- .github/workflows/lint_and_test.yml | 45 +++++++++++++++++++++ .github/workflows/linter.yml | 8 ---- pyproject.toml | 26 +++++++++++- src/genie_python/genie_cachannel_wrapper.py | 7 ++++ src/genie_python/genie_logging.py | 3 +- tests/test_genie.py | 4 +- tests/test_mysql_abstraction_layer.py | 15 ++++--- tests/test_simulation.py | 8 ++-- tests/test_utilities.py | 6 +-- 9 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/lint_and_test.yml delete mode 100644 .github/workflows/linter.yml diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml new file mode 100644 index 00000000..32c0c21b --- /dev/null +++ b/.github/workflows/lint_and_test.yml @@ -0,0 +1,45 @@ +env: + PYTHONUNBUFFERED: "TRUE" +name: test +on: [pull_request, workflow_call] +jobs: + lint: + uses: ISISComputingGroup/reusable-workflows/.github/workflows/linters.yml@main + with: + compare-branch: origin/main + python-ver: '3.11' + unit-tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "windows-latest"] + version: ['3.11'] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.version }} + - name: Install base requirements + run: pip install -e . + - name: Verify package is importable + run: | + python -c "from genie_python.genie_startup import *" + python -c "from genie_python import genie as g" + - name: Install dev requirements + run: pip install -e .[dev] + - name: Run unit tests + run: python -m pytest + results: + if: ${{ always() }} + runs-on: ubuntu-latest + name: Final Results + needs: [lint, unit-tests] + steps: + - run: exit 1 + # see https://stackoverflow.com/a/67532120/4907315 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + }} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index be2dd3d0..00000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Linter -on: [pull_request] -jobs: - call-workflow: - uses: ISISComputingGroup/reusable-workflows/.github/workflows/linters.yml@main - with: - compare-branch: origin/master - requirements-path: "package_builder/requirements.txt" diff --git a/pyproject.toml b/pyproject.toml index 36ea37f4..86e08b26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,8 +43,10 @@ dependencies = [ "p4p", # Used to find process by name to kill it "psutil", - # For checking user scripts on g.load_script() + # For linting user scripts on g.load_script() "pylint", + # For type-checking user scripts on g.load_script() + "pyright", # For setting windows job-object flags "pywin32;platform_system=='Windows'", ] @@ -75,7 +77,8 @@ dev = [ "mock", "parameterized", "pyhamcrest", - "pyright", + "pytest", + "pytest-cov", "ruff>=0.6", ] @@ -84,6 +87,25 @@ dev = [ "Bug Reports" = "https://github.com/isiscomputinggroup/genie/issues" "Source" = "https://github.com/isiscomputinggroup/genie" +[tool.pytest.ini_options] +testpaths = "tests" +addopts = "--cov --cov-report=html -vv" + +[tool.coverage.run] +branch = true +source = ["src"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "if typing.TYPE_CHECKING:", + "@abstractmethod", +] + +[tool.coverage.html] +directory = "coverage_html_report" + [tool.pyright] include = ["src", "tests"] reportConstantRedefinition = true diff --git a/src/genie_python/genie_cachannel_wrapper.py b/src/genie_python/genie_cachannel_wrapper.py index c3387c90..a703d990 100644 --- a/src/genie_python/genie_cachannel_wrapper.py +++ b/src/genie_python/genie_cachannel_wrapper.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function +import os import threading from builtins import object from collections.abc import Callable @@ -229,6 +230,7 @@ def get_chan(name: str, timeout: float = EXIST_TIMEOUT) -> CaChannel: Raises: UnableToConnectToPVException if it was unable to connect to the channel """ + try: lock = CACHE_LOCK.lock except AttributeError: @@ -363,6 +365,11 @@ def connect_to_pv(ca_channel: CaChannel) -> None: Raises: UnableToConnectToPVException: If cannot connect to PV. """ + if os.getenv("GITHUB_ACTIONS"): + # genie_python does some PV accesses on import. To avoid them timing out and making CI + # builds really slow, shortcut every PV to "non-existent" here. + raise UnableToConnectToPVException("", "In CI") + event = Event() try: ca_channel.search_and_connect(None, CaChannelWrapper.putCB, event) diff --git a/src/genie_python/genie_logging.py b/src/genie_python/genie_logging.py index b8b2a463..720f6195 100644 --- a/src/genie_python/genie_logging.py +++ b/src/genie_python/genie_logging.py @@ -1,5 +1,6 @@ """Logging module for genie""" +import getpass import logging import logging.config @@ -93,7 +94,7 @@ def get_log_file_dir(): if os.name == "nt": return os.path.join("C:", os.sep, "Instrument", "Var", "logs", "genie_python") else: - return os.path.join("/tmp/{}/genie_python".format(os.getlogin())) + return os.path.join("/tmp/{}/genie_python".format(getpass.getuser())) @staticmethod def get_log_file_path(): diff --git a/tests/test_genie.py b/tests/test_genie.py index bbaf3876..5b273406 100644 --- a/tests/test_genie.py +++ b/tests/test_genie.py @@ -536,6 +536,7 @@ class TestChangeScriptDir(unittest.TestCase): def setUp(self): self.default_dir = r"C:/scripts/" genie.change_script_dir(self.default_dir) + genie_python.genie_api_setup._exceptions_raised = True @patch("genie_python.utilities._correct_path_casing_existing") def test_GIVEN_defaults_set_WHEN_get_path_THEN_path_is_default(self, correct_casing): @@ -624,7 +625,8 @@ def test_GIVEN_defaults_WHEN_change_dir_but_root_dir_does_not_exist_THEN_script_ ): self.setup_mocks(r"c:/scripts", correct_casing, make_dirs_mock) - genie.change_script_dir(r"D:/scripts/test") + with self.assertRaises(OSError): + genie.change_script_dir(r"D:/scripts/test") result = genie.get_script_dir() assert_that(result, is_(r"C:/scripts/")) diff --git a/tests/test_mysql_abstraction_layer.py b/tests/test_mysql_abstraction_layer.py index 64adfee1..e27681c2 100644 --- a/tests/test_mysql_abstraction_layer.py +++ b/tests/test_mysql_abstraction_layer.py @@ -16,6 +16,7 @@ import unittest from typing import List +from unittest.mock import patch from hamcrest import assert_that, equal_to from mock import Mock @@ -103,9 +104,10 @@ class TestConnectionLayer(unittest.TestCase): ] ) def test_GIVEN_close_error_WHEN_execute_command_THEN_connection_is_closed(self, raise_it): - sql = mysql_abstraction_layer.SQLAbstraction( - dbid="exp_data", user="report", password="$report" - ) + with patch("genie_python.mysql_abstraction_layer.SQLAbstraction._start_connection_pool"): + sql = mysql_abstraction_layer.SQLAbstraction( + dbid="exp_data", user="report", password="$report" + ) sql._get_connection = Mock(return_value=SQLConnectorStub(raise_it)) exception_raised = False try: @@ -127,9 +129,10 @@ def test_GIVEN_close_error_WHEN_execute_command_THEN_connection_is_closed(self, def test_GIVEN_close_error_WHEN_query_returning_cursor_THEN_connection_is_closed( self, raise_it ): - sql = mysql_abstraction_layer.SQLAbstraction( - dbid="exp_data", user="report", password="$report" - ) + with patch("genie_python.mysql_abstraction_layer.SQLAbstraction._start_connection_pool"): + sql = mysql_abstraction_layer.SQLAbstraction( + dbid="exp_data", user="report", password="$report" + ) sql._get_connection = Mock(return_value=SQLConnectorStub(raise_it)) exception_raised = False try: diff --git a/tests/test_simulation.py b/tests/test_simulation.py index 0fe9e920..f2d4955c 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -216,21 +216,21 @@ def test_GIVEN_running_state_WHEN_pause_run_THEN_pause_run(self): def test_GIVEN_one_block_WHEN_cset_runcontrol_and_wait__true_THEN_exception(self): create_dummy_blocks(["a"]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( Exception, "Cannot enable or disable runcontrol at the same time as setting a wait" ): genie.cset(a=1, runcontrol=True, wait=True) def test_GIVEN_multiple_blocks_WHEN_cset_runcontrol_THEN_exception(self): create_dummy_blocks(["a", "b"]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( Exception, "Runcontrol and wait can only be changed for one block at a time" ): genie.cset(a=1, b=2, runcontrol=True) def test_GIVEN_multiple_blocks_WHEN_cset_wait_THEN_exception(self): create_dummy_blocks(["a", "b"]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( Exception, "Runcontrol and wait can only be changed for one block at a time" ): genie.cset(a=1, b=2, wait=True) @@ -240,7 +240,7 @@ def test_GIVEN_period_WHEN_set_period_to_higher_value_THEN_exception(self): period = genie.get_number_periods() # Assert - with self.assertRaisesRegexp( + with self.assertRaisesRegex( Exception, "Cannot set period as it is higher than the number of periods" ): genie.change_period(period + 1) diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 2db84ba5..6464bef3 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -282,7 +282,7 @@ def test_GIVEN_invalid_json_WHEN_get_pv_THEN_list_raise(self): api = Mock() api.get_pv_value = Mock(return_value=compressed_list) - self.assertRaisesRegexp( + self.assertRaisesRegex( PVReadException, "Can not unmarshal.*", get_json_pv_value, "name", api ) @@ -301,7 +301,7 @@ def test_GIVEN_invalid_compressed_string_WHEN_get_pv_THEN_raise(self): api = Mock() api.get_pv_value = Mock(return_value=(invalid_string)) - self.assertRaisesRegexp( + self.assertRaisesRegex( PVReadException, "Can not decompress.*", get_json_pv_value, "name", api ) @@ -309,7 +309,7 @@ def test_GIVEN_pv_can_not_be_read_WHEN_get_pv_THEN_raise(self): api = Mock() api.get_pv_value = Mock(side_effect=Exception()) - self.assertRaisesRegexp(PVReadException, "Can not read.*", get_json_pv_value, "name", api) + self.assertRaisesRegex(PVReadException, "Can not read.*", get_json_pv_value, "name", api) def test_GIVEN_unicode_with_only_ascii_WHEN_converted_THEN_no_change(self): # Arrange From 95aa11e124e4e0bc64c7296f5635d27a2a36b435 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Sat, 16 Nov 2024 14:48:54 +0000 Subject: [PATCH 3/4] Add release workflow --- .github/workflows/release.yml | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..66f0530c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,92 @@ +name: Publish Python distribution to PyPI +on: push +jobs: + lint-and-test: + if: github.ref_type == 'tag' + name: Run lint & tests + uses: ./.github/workflows/lint_and_test.yml + build: + needs: lint-and-test + if: github.ref_type == 'tag' + name: build distribution + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + publish-to-pypi: + name: >- + Publish Python distribution to PyPI + if: github.ref_type == 'tag' + needs: [lint-and-test, build] + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/p/genie_python + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + github-release: + name: >- + Sign the Python distribution with Sigstore + and upload them to GitHub Release + needs: [lint-and-test, build, publish-to-pypi] + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' From 07f55344b6b35b156c5fcc22323b104213f02220 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Sat, 16 Nov 2024 15:07:50 +0000 Subject: [PATCH 4/4] Add doc workflow --- .github/workflows/documentation.yml | 8 ++++++++ .gitignore | 2 ++ {docs => doc}/Makefile | 0 {docs => doc}/conf.py | 2 +- {docs => doc}/genie_python.rst | 0 {docs => doc}/make.bat | 0 {docs => doc}/make_doc.sh | 0 docs/requirements.txt | 1 - src/genie_python/genie.py | 8 ++++---- src/genie_python/genie_api_setup.py | 3 +++ 10 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/documentation.yml rename {docs => doc}/Makefile (100%) rename {docs => doc}/conf.py (99%) rename {docs => doc}/genie_python.rst (100%) rename {docs => doc}/make.bat (100%) rename {docs => doc}/make_doc.sh (100%) mode change 100755 => 100644 delete mode 100644 docs/requirements.txt diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000..3bda9711 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,8 @@ +name: sphinx + +on: [push, pull_request, workflow_call] + +jobs: + call_sphinx_builder: + uses: ISISComputingGroup/reusable-workflows/.github/workflows/sphinx.yml@main + secrets: inherit diff --git a/.gitignore b/.gitignore index 7b6caf34..f4841ee0 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ + +_build/ diff --git a/docs/Makefile b/doc/Makefile similarity index 100% rename from docs/Makefile rename to doc/Makefile diff --git a/docs/conf.py b/doc/conf.py similarity index 99% rename from docs/conf.py rename to doc/conf.py index 2c0cf354..eecfedb3 100644 --- a/docs/conf.py +++ b/doc/conf.py @@ -20,7 +20,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath("../Lib/site-packages/genie_python/")) +sys.path.insert(0, os.path.abspath("../src/genie_python/")) # -- General configuration ------------------------------------------------ diff --git a/docs/genie_python.rst b/doc/genie_python.rst similarity index 100% rename from docs/genie_python.rst rename to doc/genie_python.rst diff --git a/docs/make.bat b/doc/make.bat similarity index 100% rename from docs/make.bat rename to doc/make.bat diff --git a/docs/make_doc.sh b/doc/make_doc.sh old mode 100755 new mode 100644 similarity index 100% rename from docs/make_doc.sh rename to doc/make_doc.sh diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 6966869c..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sphinx diff --git a/src/genie_python/genie.py b/src/genie_python/genie.py index a581a48e..590c8885 100644 --- a/src/genie_python/genie.py +++ b/src/genie_python/genie.py @@ -1079,10 +1079,9 @@ def get_time_since_begin(get_timedelta: bool = False) -> float | datetime.timede otherwise return seconds (defaults to false) Returns - integer: the time since start in seconds if get_datetime is False - timedelta: the time since begin as a datetime.timedelta object - (Year-Month-Day Hour:Minute:Second) - if get_datetime is True + integer: the time since start in seconds if get_datetime is False, + or timedelta, the time since begin as a datetime.timedelta object + (Year-Month-Day Hour:Minute:Second) if get_datetime is True """ return _genie_api.dae.get_time_since_begin(get_timedelta) @@ -2392,6 +2391,7 @@ def get_version() -> str: def set_dae_simulation_mode(mode: bool, skip_required_runstates: bool = False) -> None: """ Sets the DAE into simulation mode. + Args: mode: True to set the DAE into simulated mode, False to set the DAE into non-simulated (hardware) mode diff --git a/src/genie_python/genie_api_setup.py b/src/genie_python/genie_api_setup.py index ca309c34..3dcfb97c 100644 --- a/src/genie_python/genie_api_setup.py +++ b/src/genie_python/genie_api_setup.py @@ -1,6 +1,7 @@ from __future__ import print_function import ctypes +import functools import glob import inspect import os @@ -363,6 +364,7 @@ def log_command_and_handle_exception(f: Callable[P, T]) -> Callable[P, T]: accidentally put into a genie function """ + @functools.wraps(f) def decorator(*args: P.args, **kwargs: P.kwargs) -> T: log_args = {"kwargs": kwargs} arg_names = getfullargspec(f).args @@ -380,6 +382,7 @@ def decorator(*args: P.args, **kwargs: P.kwargs) -> T: except Exception as e: command_exception = traceback.format_exc() _handle_exception(e) + return None # type: ignore finally: end = time.time() time_taken = end - start