Integrate the iRODS file source to support a new remote repository connection

lib/galaxy/files/sources/irods.py

@@ -0,0 +1,177 @@
+import ssl
+from fnmatch import fnmatch
+import os
+from typing import (
+    Optional,
+    Union,
+)
+
+import fs
+import fs.errors
+
+from galaxy.files.models import (
+    AnyRemoteEntry,
+    BaseFileSourceConfiguration,
+    BaseFileSourceTemplateConfiguration,
+    FilesSourceRuntimeContext,
+)
+from galaxy.exceptions import (
+    AuthenticationRequired,
+    MessageException,
+)
+from galaxy.util.config_templates import TemplateExpansion
+from ._pyfilesystem2 import PyFilesystem2FilesSource
+
+try:
+    from fs_irods import iRODSFS
+except ImportError:
+    iRODSFS = None
+
+try:
+    from irods.session import iRODSSession
+except ImportError:
+    iRODSSession = None
+
+
+class IrodsFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
+    host: Union[str, TemplateExpansion]
+    port: Union[int, TemplateExpansion] = 1247
+    username: Union[str, TemplateExpansion]
+    password: Union[str, TemplateExpansion]
+    zone: Union[str, TemplateExpansion]
+    root: Optional[Union[str, TemplateExpansion]] = None
+    timeout: Union[int, TemplateExpansion] = 30
+    refresh_time: Union[int, TemplateExpansion] = 300
+    client_server_negotiation: Optional[Union[str, TemplateExpansion]] = None
+    client_server_policy: Optional[Union[str, TemplateExpansion]] = None
+    encryption_algorithm: Optional[Union[str, TemplateExpansion]] = None
+    encryption_key_size: Optional[Union[int, TemplateExpansion]] = None
+    encryption_num_hash_rounds: Optional[Union[int, TemplateExpansion]] = None
+    encryption_salt_size: Optional[Union[int, TemplateExpansion]] = None
+    ssl_verify_server: Optional[Union[str, TemplateExpansion]] = None
+    ssl_ca_certificate_file: Optional[Union[str, TemplateExpansion]] = None
+    resource: Optional[Union[str, TemplateExpansion]] = None
+
+
+class IrodsFileSourceConfiguration(BaseFileSourceConfiguration):
+    host: str
+    port: int = 1247
+    username: str
+    password: str
+    zone: str
+    root: Optional[str] = None
+    timeout: int = 30
+    refresh_time: int = 300
+    client_server_negotiation: Optional[str] = None
+    client_server_policy: Optional[str] = None
+    encryption_algorithm: Optional[str] = None
+    encryption_key_size: Optional[int] = None
+    encryption_num_hash_rounds: Optional[int] = None
+    encryption_salt_size: Optional[int] = None
+    ssl_verify_server: Optional[str] = None
+    ssl_ca_certificate_file: Optional[str] = None
+    resource: Optional[str] = None
+
+
+class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfiguration, IrodsFileSourceConfiguration]):
+    plugin_type = "irods"
+    required_module = iRODSFS
+    required_package = "fs-irods"
+
+    template_config_class = IrodsFileSourceTemplateConfiguration
+    resolved_config_class = IrodsFileSourceConfiguration
+
+    def _iter_directory_entries(self, fs_handle, parent_path: str, normalized_query: Optional[str] = None):
+        for raw_name in fs_handle.listdir(parent_path):
+            name = os.path.basename(str(raw_name).rstrip("/"))
+            if not name:
+                continue
+            if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"):
+                continue
+            entry_path = fs.path.join(parent_path, name)
+            info = fs_handle.getinfo(entry_path, namespaces=["details"])
+            yield entry_path, info
+
+    def _list_recursive(self, fs_handle, path: str) -> tuple[list[AnyRemoteEntry], int]:
+        result: list[AnyRemoteEntry] = []
+        pending = [path]
+        while pending:
+            current_path = pending.pop(0)
+            for entry_path, info in self._iter_directory_entries(fs_handle, current_path):
+                result.append(self._resource_info_to_dict(current_path, info))
+                if info.is_dir:
+                    pending.append(entry_path)
+        return result, len(result)
+
+    def _list_non_recursive(
+        self,
+        fs_handle,
+        path: str,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        query: Optional[str] = None,
+    ) -> tuple[list[AnyRemoteEntry], int]:
+        normalized_query = query.lower() if query else None
+        entries = []
+        for _, info in self._iter_directory_entries(fs_handle, path, normalized_query):
+            entries.append(self._resource_info_to_dict(path, info))
+        count = len(entries)
+        page = self._to_page(limit, offset)
+        if page is not None:
+            entries = entries[page[0] : page[1]]
+        return entries, count
+
+    def _list(
+        self,
+        context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration],
+        path="/",
+        recursive=False,
+        write_intent: bool = False,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        query: Optional[str] = None,
+        sort_by: Optional[str] = None,
+    ) -> tuple[list[AnyRemoteEntry], int]:
+        try:
+            with self._open_fs(context) as fs_handle:
+                if recursive:
+                    return self._list_recursive(fs_handle, path)
+                return self._list_non_recursive(fs_handle, path, limit, offset, query)
+        except fs.errors.PermissionDenied as e:
+            raise AuthenticationRequired(
+                f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}."
+            ) from e
+        except fs.errors.FSError as e:
+            raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e
+
+    def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration]):
+        if iRODSFS is None or iRODSSession is None:
+            raise self.required_package_exception
+
+        config = context.config
+        session_kwargs = {
+            "host": config.host,
+            "port": config.port,
+            "user": config.username,
+            "password": config.password,
+            "zone": config.zone,
+            "refresh_time": config.refresh_time,
+            "client_server_negotiation": config.client_server_negotiation,
+            "client_server_policy": config.client_server_policy,
+            "encryption_algorithm": config.encryption_algorithm,
+            "encryption_key_size": config.encryption_key_size,
+            "encryption_num_hash_rounds": config.encryption_num_hash_rounds,
+            "encryption_salt_size": config.encryption_salt_size,
+            "ssl_verify_server": config.ssl_verify_server,
+            "ssl_ca_certificate_file": config.ssl_ca_certificate_file,
+            "ssl_context": ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH),
+        }
+        session = iRODSSession(**session_kwargs)
+        session.connection_timeout = config.timeout
+        if config.resource:
+            session.default_resource = config.resource
+
+        return iRODSFS(session=session, root=config.root)
+
+
+__all__ = ("IrodsFilesSource",)

lib/galaxy/files/templates/examples/irods.yml

@@ -0,0 +1,61 @@
+- id: irods
+  version: 0
+  name: iRODS
+  description: |
+    Use this template to connect an iRODS collection as a Galaxy file source for
+    importing and exporting datasets.
+
+    You need valid iRODS credentials and the target root collection path.
+  variables:
+    host:
+      label: iRODS host
+      type: string
+      help: Hostname of the iRODS server.
+    port:
+      label: iRODS port
+      type: integer
+      help: Port of the iRODS server.
+      default: 1247
+    username:
+      label: iRODS user
+      type: string
+      help: Username used to authenticate against iRODS.
+    zone:
+      label: iRODS zone
+      type: string
+      help: Zone used for authentication (for example tempZone).
+    root:
+      label: Root collection path
+      type: string
+      help: |
+        Collection path exposed in Galaxy, for example /tempZone/home/rods.
+    timeout:
+      label: Connection timeout (seconds)
+      type: integer
+      help: Timeout in seconds for iRODS connection attempts.
+      default: 30
+    refresh_time:
+      label: Session refresh time (seconds)
+      type: integer
+      help: Seconds between refresh checks for active iRODS sessions.
+      default: 300
+    writable:
+      label: Writable?
+      type: boolean
+      help: Allow writing datasets back to iRODS.
+      default: false
+  secrets:
+    password:
+      label: Password
+      help: Password used to authenticate against iRODS.
+  configuration:
+    type: irods
+    host: "{{ variables.host }}"
+    port: "{{ variables.port }}"
+    username: "{{ variables.username }}"
+    password: "{{ secrets.password }}"
+    zone: "{{ variables.zone }}"
+    root: "{{ variables.root }}"
+    timeout: "{{ variables.timeout }}"
+    refresh_time: "{{ variables.refresh_time }}"
+    writable: "{{ variables.writable }}"

lib/galaxy/files/templates/models.py

@@ -37,6 +37,7 @@
     "s3fs",
     "azure",
     "azureflat",
+    "irods",
     "onedata",
     "webdav",
     "dropbox",
@@ -193,6 +194,34 @@ class AzureFlatFileSourceConfiguration(StrictModel):
     writable: bool = False
 
 
+class IrodsFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["irods"]
+    host: Union[str, TemplateExpansion]
+    port: Union[int, TemplateExpansion] = 1247
+    username: Union[str, TemplateExpansion]
+    password: Union[str, TemplateExpansion]
+    zone: Union[str, TemplateExpansion]
+    root: Optional[Union[str, TemplateExpansion]] = None
+    timeout: Union[int, TemplateExpansion] = 30
+    refresh_time: Union[int, TemplateExpansion] = 300
+    writable: Union[bool, TemplateExpansion] = False
+    template_start: Optional[str] = None
+    template_end: Optional[str] = None
+
+
+class IrodsFileSourceConfiguration(StrictModel):
+    type: Literal["irods"]
+    host: str
+    port: int = 1247
+    username: str
+    password: str
+    zone: str
+    root: Optional[str] = None
+    timeout: int = 30
+    refresh_time: int = 300
+    writable: bool = False
+
+
 class OnedataFileSourceTemplateConfiguration(StrictModel):
     type: Literal["onedata"]
     access_token: Union[str, TemplateExpansion]
@@ -358,6 +387,7 @@ class OmeroFileSourceConfiguration(StrictModel):
         FtpFileSourceTemplateConfiguration,
         AzureFileSourceTemplateConfiguration,
         AzureFlatFileSourceTemplateConfiguration,
+        IrodsFileSourceTemplateConfiguration,
         OnedataFileSourceTemplateConfiguration,
         WebdavFileSourceTemplateConfiguration,
         DropboxFileSourceTemplateConfiguration,
@@ -380,6 +410,7 @@ class OmeroFileSourceConfiguration(StrictModel):
         FtpFileSourceConfiguration,
         AzureFileSourceConfiguration,
         AzureFlatFileSourceConfiguration,
+        IrodsFileSourceConfiguration,
         OnedataFileSourceConfiguration,
         WebdavFileSourceConfiguration,
         DropboxFileSourceConfiguration,
@@ -460,6 +491,7 @@ def template_to_configuration(
     "s3fs": S3FSFileSourceConfiguration,
     "azure": AzureFileSourceConfiguration,
     "azureflat": AzureFlatFileSourceConfiguration,
+    "irods": IrodsFileSourceConfiguration,
     "onedata": OnedataFileSourceConfiguration,
     "webdav": WebdavFileSourceConfiguration,
     "dropbox": DropboxFileSourceConfiguration,