diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 1ec1d6fc39d1..1f4dabcf05ed 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -9254,7 +9254,7 @@ export interface components { * Type * @enum {string} */ - type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav" | "dropbox" | "googledrive"; + type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav" | "dropbox" | "googledrive" | "elabftw"; /** Variables */ variables?: | ( @@ -17599,7 +17599,7 @@ export interface components { * Type * @enum {string} */ - type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav" | "dropbox" | "googledrive"; + type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav" | "dropbox" | "googledrive" | "elabftw"; /** Uri Root */ uri_root: string; /** diff --git a/client/src/components/FileSources/FileSourceTypeSpan.vue b/client/src/components/FileSources/FileSourceTypeSpan.vue index 890649e4cf86..8d6f5d6324b7 100644 --- a/client/src/components/FileSources/FileSourceTypeSpan.vue +++ b/client/src/components/FileSources/FileSourceTypeSpan.vue @@ -12,6 +12,7 @@ const MESSAGES = { webdav: "This is a remote file source plugin based on the WebDAV protocol.", dropbox: "This is a file source plugin that connects with the commercial Dropbox service.", googledrive: "This is a file source plugin that connects with the commercial Google Drive service.", + elabftw: "This is a remote file source that connects with an eLabFTW instance.", }; interface Props { diff --git a/doc/source/admin/data.md b/doc/source/admin/data.md index 15fbecbade4a..f0c4e827d08b 100644 --- a/doc/source/admin/data.md +++ b/doc/source/admin/data.md @@ -373,6 +373,19 @@ configuration). ![](file_source_dropbox_configuration.png) +#### `elabftw` + +The syntax for the `configuration` section of `elabftw` templates looks like this. + +![](file_source_elabftw_configuration_template.png) + +At runtime, after the `configuration` template is expanded, the resulting dictionary +passed to Galaxy's file source plugin infrastructure looks like this and should match a subset +of what you'd be able to add directly to `file_sources_conf.yml` (Galaxy's global file source +configuration). + +![](file_source_elabftw_configuration.png) + ### YAML Syntax ![galaxy.files.templates.models](file_source_templates.png) diff --git a/doc/source/admin/file_source_elabftw_configuration.png b/doc/source/admin/file_source_elabftw_configuration.png new file mode 100644 index 000000000000..f7dacc2f520c Binary files /dev/null and b/doc/source/admin/file_source_elabftw_configuration.png differ diff --git a/doc/source/admin/file_source_elabftw_configuration_template.png b/doc/source/admin/file_source_elabftw_configuration_template.png new file mode 100644 index 000000000000..a830dfaa1e36 Binary files /dev/null and b/doc/source/admin/file_source_elabftw_configuration_template.png differ diff --git a/doc/source/admin/gen_diagrams.py b/doc/source/admin/gen_diagrams.py index 3e1d57a53be4..0f54f6e7bdcb 100644 --- a/doc/source/admin/gen_diagrams.py +++ b/doc/source/admin/gen_diagrams.py @@ -10,6 +10,8 @@ AzureFileSourceTemplateConfiguration, DropboxFileSourceConfiguration, DropboxFileSourceTemplateConfiguration, + eLabFTWFileSourceConfiguration, + eLabFTWFileSourceTemplateConfiguration, FileSourceTemplate, FtpFileSourceConfiguration, FtpFileSourceTemplateConfiguration, @@ -61,6 +63,8 @@ FtpFileSourceConfiguration: "file_source_ftp_configuration", WebdavFileSourceTemplateConfiguration: "file_source_webdav_configuration_template", WebdavFileSourceConfiguration: "file_source_webdav_configuration", + eLabFTWFileSourceTemplateConfiguration: "file_source_elabftw_configuration_template", + eLabFTWFileSourceConfiguration: "file_source_elabftw_configuration", } for clazz, diagram_name in class_to_diagram.items(): diff --git a/doc/source/admin/search_for_new_screenshots.py b/doc/source/admin/search_for_new_screenshots.py index ea09034da8d4..7007d3a32ecf 100644 --- a/doc/source/admin/search_for_new_screenshots.py +++ b/doc/source/admin/search_for_new_screenshots.py @@ -23,6 +23,7 @@ "user_file_source_form_full_azure.png", "user_file_source_form_full_ftp.png", "user_file_source_form_full_webdav.png", + "user_file_source_form_full_elabftw.png", ] diff --git a/doc/source/admin/user_file_source_form_full_elabftw.png b/doc/source/admin/user_file_source_form_full_elabftw.png new file mode 100644 index 000000000000..0ecfe98c78bb Binary files /dev/null and b/doc/source/admin/user_file_source_form_full_elabftw.png differ diff --git a/lib/galaxy/files/sources/elabftw.py b/lib/galaxy/files/sources/elabftw.py index bcb0449fbbf5..a3e8439fc73a 100644 --- a/lib/galaxy/files/sources/elabftw.py +++ b/lib/galaxy/files/sources/elabftw.py @@ -6,15 +6,21 @@ overview, try out the live demo [4]. The scope of this implementation is exporting data from and importing data to eLabFTW as file attachments of *already existing* experiments and resources. Each user can configure their preferred eLabFTW instance entering its URL and an API Key. - File sources reference files via a URI, while eLabFTW uses auto-incrementing positive integers. For more details read galaxyproject/galaxy#18665 [5]. This leads to the need to declare a mapping between said identifiers and Galaxy URIs. Those take the form ``elabftw://demo.elabftw.net/entity_type/entity_id/attachment_id``, where: + - ``entity_type`` is either 'experiments' or 'resources' - ``entity_id`` is the id (an integer in string form) of an experiment or resource - ``attachment_id`` is the id (an integer in string form) of an attachment +For the user-defined file sources use case (when users configure one or more instances of the FilesSource via a file +source template), Galaxy URIs have a different scheme and authority, taking the form ``gxuserfiles://file_source_id/ +entity_type/entity_id/attachment_id``, where: + +- ``file_source_id`` is the file source identifier assigned by Galaxy + This implementation uses both ``aiohttp`` and the ``requests`` libraries as underlying mechanisms to communicate with eLabFTW via its REST API [6]. A significant limitation of the implementation is that, due to the fact that the API does not have an endpoint that can list attachments for several experiments and/or resources with a single request, when @@ -27,6 +33,7 @@ unknown. References: + - [1] https://www.elabftw.net/ - [2] https://doc.elabftw.net/user-guide.html#experiments - [3] https://doc.elabftw.net/user-guide.html#resources @@ -47,6 +54,7 @@ from textwrap import dedent from time import time from typing import ( + Any, AsyncIterator, cast, Dict, @@ -78,6 +86,7 @@ from galaxy.files.sources import ( AnyRemoteEntry, BaseFilesSource, + DEFAULT_SCHEME, FilesSourceOptions, FilesSourceProperties, PluginKind, @@ -169,14 +178,29 @@ def __init__(self, *args, **kwargs: Unpack[eLabFTWFilesSourceProperties]): self._endpoint = kwargs["endpoint"] # meant to be accessed only from `_get_endpoint()` self._api_key = kwargs["api_key"] # meant to be accessed only from `_create_session()` - def get_prefix(self) -> Optional[str]: - return None + def get_prefix(self, user_context: OptionalUserContext = None) -> Optional[str]: + endpoint: ParseResult = self._get_endpoint(user_context=user_context) + return self.id if self.scheme not in {"elabftw", DEFAULT_SCHEME} else (endpoint.netloc or None) + # it would make better sense to return + # `self.id if self.scheme == USER_FILE_SOURCES_SCHEME else (endpoint.netloc or None)`, where + # `USER_FILE_SOURCES_SCHEME` comes from `galaxy.managers.file_source_instances`; however, that would lead to a + # circular import (maybe `USER_FILE_SOURCES_SCHEME` should be moved to a module in a layer deeper than + # `galaxy.managers`) def get_scheme(self) -> str: - return "elabftw" + return self.scheme if self.scheme and self.scheme != DEFAULT_SCHEME else "elabftw" + # it would make better sense to return `self.scheme if self.scheme == USER_FILE_SOURCES_SCHEME else "elabftw"`, + # but the same circular import issue as above arises - def get_uri_root(self) -> str: - return super().get_uri_root() + def score_url_match(self, url: str) -> int: + parsed_url = urlparse(url) + return sum( + int(check) + for check in ( + parsed_url.scheme == self.get_scheme(), + parsed_url.netloc == self.get_prefix(), + ) + ) def to_relative_path(self, url: str) -> str: parsed_url = urlparse(url) @@ -225,9 +249,11 @@ def _get_session_headers( Meant to be used only by `_create_session()` and `_create_session_async()`. """ - props = dict( - **(options.extra_props if options and options.extra_props else {}), - **self._serialization_props(user_context), + props = {} + props.update(self._props) + props.update(options.extra_props if options and options.extra_props else {}) + props.update( + {key: value for key, value in self._serialization_props(user_context).items() if value is not None} ) headers = { "Authorization": props.get("api_key", self._api_key), @@ -243,9 +269,11 @@ def _get_endpoint( """ Retrieve the endpoint from the constructor, or override it via a :class:`FileSourceOptions` object. """ - props = dict( - **(options.extra_props if options and options.extra_props else {}), - **self._serialization_props(user_context), + props = {} + props.update(self._props) + props.update(options.extra_props if options and options.extra_props else {}) + props.update( + {key: value for key, value in self._serialization_props(user_context).items() if value is not None} ) endpoint = props.get("endpoint", self._endpoint) # given that `options.extra_props` is of `eLabFTWFilesSourceProperties` type, it should be a string @@ -254,9 +282,14 @@ def _get_endpoint( return urlparse(endpoint) def _serialization_props(self, user_context: OptionalUserContext = None) -> eLabFTWFilesSourceProperties: - effective_props = {} + effective_props: Dict[str, Any] = {} for key, val in self._props.items(): + if key in {"api_key", "endpoint"} and user_context is None: + # prevent exception while expanding `${user.user_vault.read_secret('preferences/elabftw/api_key')}` or + # `${user.preferences['elabftw|endpoint']}` without `user_context` + effective_props[key] = None + continue effective_props[key] = self._evaluate_prop(val, user_context=user_context) return cast(eLabFTWFilesSourceProperties, effective_props) @@ -383,6 +416,7 @@ async def collect_async_iterator(async_iter: AsyncIterator) -> list: self._yield_entity_types( endpoint, session, + user_context=user_context, ) ) ) @@ -423,6 +457,7 @@ async def collect_async_iterator(async_iter: AsyncIterator) -> list: else None ), writable=self.writable, + user_context=user_context, ) ) ) @@ -453,6 +488,7 @@ async def collect_async_iterator(async_iter: AsyncIterator) -> list: cast(str, wrapped_entity.entity_id) if retrieve_entities else cast(str, entity_id), endpoint, session, + user_context=user_context, ) ) ) @@ -530,9 +566,11 @@ async def collect_async_iterator(async_iter: AsyncIterator) -> list: # always matches such value. return (entries := [wrapped_entry.entry for wrapped_entry in wrapped_entries]), len(entries) - @staticmethod async def _yield_entity_types( - endpoint: ParseResult, session: aiohttp.ClientSession + self, + endpoint: ParseResult, + session: aiohttp.ClientSession, + user_context: OptionalUserContext = None, ) -> AsyncIterator[eLabFTWRemoteEntryWrapper[RemoteDirectory]]: """ List the root directory, i.e. "/". @@ -563,7 +601,7 @@ async def _yield_entity_types( RemoteDirectory( **{ "name": "Experiments", - "uri": f"elabftw://{endpoint.netloc}/experiments", + "uri": f"{self.get_scheme()}://{self.get_prefix(user_context=user_context)}/experiments", "path": "/experiments", "class": "Directory", } @@ -573,7 +611,7 @@ async def _yield_entity_types( RemoteDirectory( **{ "name": "Resources", - "uri": f"elabftw://{endpoint.netloc}/resources", + "uri": f"{self.get_scheme()}://{self.get_prefix(user_context=user_context)}/resources", "path": "/resources", "class": "Directory", } @@ -583,8 +621,8 @@ async def _yield_entity_types( yield experiments yield resources - @staticmethod async def _yield_entities( + self, entity_type: str, endpoint: ParseResult, session: aiohttp.ClientSession, @@ -593,6 +631,7 @@ async def _yield_entities( query: Optional[str] = None, order: Optional[str] = None, writable: bool = False, + user_context: OptionalUserContext = None, ) -> AsyncIterator[eLabFTWRemoteEntryWrapper[RemoteDirectory]]: """List an entity type, i.e. either "/experiments" or "/resources".""" url = urljoin( @@ -659,7 +698,10 @@ def validate_and_register_entity(item, mapping: Dict[int, dict]) -> Literal[True RemoteDirectory( **{ "name": entity["title"], - "uri": f"elabftw://{endpoint.netloc}/{entity_type}/{entity['id']}", + "uri": ( + f"{self.get_scheme()}://{self.get_prefix(user_context=user_context)}" + f"/{entity_type}/{entity['id']}" + ), "path": f"/{entity_type}/{entity['id']}", "class": "Directory", } @@ -672,12 +714,13 @@ def validate_and_register_entity(item, mapping: Dict[int, dict]) -> Literal[True if timeout: raise aiohttp.ServerTimeoutError - @staticmethod async def _yield_attachments( + self, entity_type: str, entity_id: str, endpoint: ParseResult, session: aiohttp.ClientSession, + user_context: OptionalUserContext = None, ) -> AsyncIterator[eLabFTWRemoteEntryWrapper[RemoteFile]]: """List attachments of a specific entity, e.g. "/resources/48".""" url = urljoin( @@ -712,7 +755,10 @@ async def _yield_attachments( RemoteFile( **{ "name": upload["real_name"], - "uri": f"elabftw://{endpoint.netloc}/{entity_type}/{entity_id}/{upload['id']}", + "uri": ( + f"{self.get_scheme()}://{self.get_prefix(user_context=user_context)}" + f"/{entity_type}/{entity_id}/{upload['id']}" + ), "path": f"/{entity_type}/{entity_id}/{upload['id']}", "class": "File", "size": upload["filesize"], diff --git a/lib/galaxy/files/templates/examples/production_elabftw.yaml b/lib/galaxy/files/templates/examples/production_elabftw.yaml new file mode 100644 index 000000000000..4523466e8927 --- /dev/null +++ b/lib/galaxy/files/templates/examples/production_elabftw.yaml @@ -0,0 +1,35 @@ +- id: elabftw + version: 0 + name: eLabFTW + description: | + eLabFTW is a free and open source electronic lab notebook from Deltablot. It can keep track of experiments, + equipment, and materials from a research lab. Each lab can either host their own installation or go for Deltablot's + hosted solution. This template configuration allows you to connect to an eLabFTW instance of your choice. + variables: + endpoint: + label: eLabFTW instance endpoint (e.g. https://demo.elabftw.net) + type: string + help: | + The endpoint of the eLabFTW server you are connecting to. This should be the full URL including the protocol + (http or https) and the domain name. + writable: + label: Allow Galaxy to export data to eLabFTW? + type: boolean + default: true + help: | + Allow Galaxy to write data to this eLabFTW instance. Set it to "Yes" if you want to export data from Galaxy to + eLabFTW, set it to "No" if you only need to import data from eLabFTW to Galaxy. Keep in mind that your API key + must have matching permissions. + secrets: + api_key: + label: API Key + help: | + The API key to use to connect to the eLabFTW server. Navigate to the _Settings_ page on your eLabFTW server and + go to the _API Keys_ tab to generate a new key. Choose "Read/Write" permissions to enable both importing and + exporting data. "Read Only" API keys still work for importing data to Galaxy, but they will cause Galaxy to + error out when exporting data to eLabFTW. + configuration: + type: elabftw + endpoint: "{{ variables.endpoint }}" + api_key: "{{ secrets.api_key }}" + writable: "{{ variables.writable }}" diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py index 967d83ea0894..6752f0e4d5ea 100644 --- a/lib/galaxy/files/templates/models.py +++ b/lib/galaxy/files/templates/models.py @@ -30,7 +30,9 @@ UserDetailsDict, ) -FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata", "webdav", "dropbox", "googledrive"] +FileSourceTemplateType = Literal[ + "ftp", "posix", "s3fs", "azure", "onedata", "webdav", "dropbox", "googledrive", "elabftw" +] class PosixFileSourceTemplateConfiguration(StrictModel): @@ -195,6 +197,22 @@ class WebdavFileSourceConfiguration(StrictModel): writable: bool = False +class eLabFTWFileSourceTemplateConfiguration(StrictModel): # noqa + type: Literal["elabftw"] + endpoint: Union[str, TemplateExpansion] + api_key: Union[str, TemplateExpansion] + writable: Union[bool, TemplateExpansion] = True + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class eLabFTWFileSourceConfiguration(StrictModel): # noqa + type: Literal["elabftw"] + endpoint: str + api_key: str + writable: bool = True + + FileSourceTemplateConfiguration = Union[ PosixFileSourceTemplateConfiguration, S3FSFileSourceTemplateConfiguration, @@ -204,6 +222,7 @@ class WebdavFileSourceConfiguration(StrictModel): WebdavFileSourceTemplateConfiguration, DropboxFileSourceTemplateConfiguration, GoogleDriveFileSourceTemplateConfiguration, + eLabFTWFileSourceTemplateConfiguration, ] FileSourceConfiguration = Union[ PosixFileSourceConfiguration, @@ -214,6 +233,7 @@ class WebdavFileSourceConfiguration(StrictModel): WebdavFileSourceConfiguration, DropboxFileSourceConfiguration, GoogleDriveFileSourceConfiguration, + eLabFTWFileSourceConfiguration, ] @@ -284,6 +304,7 @@ def template_to_configuration( "webdav": WebdavFileSourceConfiguration, "dropbox": DropboxFileSourceConfiguration, "googledrive": GoogleDriveFileSourceConfiguration, + "elabftw": eLabFTWFileSourceConfiguration, }