From fbde66e8efda8cb200a302d7e9d9d03ecd1e5c1d Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Sun, 15 Feb 2026 15:12:14 +0100 Subject: [PATCH 1/4] Test the sparse checkout syntax --- reframe/core/pipeline.py | 30 ++++++++++++++++++++++-------- reframe/utility/osext.py | 10 ++++++++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 1ced2d6a14..763b91b7a1 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -413,7 +413,7 @@ def pipeline_hooks(cls): #: .. versionchanged:: 3.0 #: Default value is now conditionally set to either ``'src'`` or #: :class:`None`. - sourcesdir = variable(str, type(None), value='src') + sourcesdir = variable(str, typ.Dict, type(None), value='src') #: .. versionadded:: 2.14 #: @@ -1949,11 +1949,12 @@ def _copy_to_stagedir(self, path): else: self._mark_stagedir() - def _clone_to_stagedir(self, url): + def _clone_to_stagedir(self, url, files=None): self.logger.debug(f'Cloning URL {url} into stage directory') osext.git_clone( - self.sourcesdir, self._stagedir, - timeout=rt.runtime().get_option('general/0/git_timeout') + url, self._stagedir, + timeout=rt.runtime().get_option('general/0/git_timeout'), + files=files ) self._mark_stagedir() @@ -1983,7 +1984,7 @@ def compile(self): try: commonpath = os.path.commonpath([self.sourcesdir, self.sourcepath]) - except ValueError: + except (ValueError, TypeError): commonpath = None if commonpath: @@ -1994,11 +1995,24 @@ def compile(self): ) if self._requires_stagedir_contents(): - if osext.is_url(self.sourcesdir): - self._clone_to_stagedir(self.sourcesdir) + srcdir = self.sourcesdir + if not srcdir: + return + if isinstance(srcdir, dict): + url = next(iter(srcdir)) + if not url: + raise ReframeError(f'The {srcdir} misses the url as key') + elif not osext.is_url(url): + raise ReframeError(f'The {srcdir} syntax only supports ' + 'git repositories') + self._clone_to_stagedir(url, + files=srcdir[url]['files'] if 'files' + in srcdir[url] else None) + elif osext.is_url(srcdir): + self._clone_to_stagedir(srcdir) else: self._copy_to_stagedir(os.path.join(self._prefix, - self.sourcesdir)) + srcdir)) # Set executable (only if hasn't been provided) if not hasattr(self, 'executable'): diff --git a/reframe/utility/osext.py b/reframe/utility/osext.py index 544c47900b..bd804db66f 100644 --- a/reframe/utility/osext.py +++ b/reframe/utility/osext.py @@ -692,7 +692,7 @@ def is_url(s): return parsed.scheme != '' and parsed.netloc != '' -def git_clone(url, targetdir=None, opts=None, timeout=5): +def git_clone(url, targetdir=None, opts=None, timeout=5, files=None): '''Clone a git repository from a URL. :arg url: The URL to clone from. @@ -708,7 +708,13 @@ def git_clone(url, targetdir=None, opts=None, timeout=5): targetdir = targetdir or '' opts = ' '.join(opts) if opts is not None else '' - run_command(f'git clone {opts} {url} {targetdir}', check=True) + if not files: + run_command(f'git clone {opts} {url} {targetdir}', check=True) + else: + run_command(f'git clone --no-checkout --depth=1 {opts} {url} {targetdir}', check=True) + run_command(f'git sparse-checkout set --no-cone {" ".join(files)}', check=True, cwd=targetdir) + run_command('git checkout', check=True, cwd=targetdir) + def git_repo_exists(url, timeout=5): From e6cd68cb537d91b0d558910de5db03dc3998a721 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 23 Feb 2026 14:01:52 +0100 Subject: [PATCH 2/4] Change the syntax and support opts --- reframe/core/pipeline.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 763b91b7a1..2d4ae495c0 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1999,14 +1999,17 @@ def compile(self): if not srcdir: return if isinstance(srcdir, dict): - url = next(iter(srcdir)) - if not url: - raise ReframeError(f'The {srcdir} misses the url as key') - elif not osext.is_url(url): + if 'url' not in srcdir: + raise ReframeError(f'The {srcdir} misses the url key') + + url = srcdir['url'] + if not osext.is_url(url): raise ReframeError(f'The {srcdir} syntax only supports ' 'git repositories') self._clone_to_stagedir(url, files=srcdir[url]['files'] if 'files' + in srcdir[url] else None, + opts=srcdir[url]['opts'] if 'opts' in srcdir[url] else None) elif osext.is_url(srcdir): self._clone_to_stagedir(srcdir) From 44f25790161be56732006fde5ecafcf3a42a79f4 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 23 Feb 2026 14:29:11 +0100 Subject: [PATCH 3/4] Update error message --- reframe/core/pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 2d4ae495c0..b5959a4eb5 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -2004,8 +2004,9 @@ def compile(self): url = srcdir['url'] if not osext.is_url(url): - raise ReframeError(f'The {srcdir} syntax only supports ' + raise ReframeError(f'The dictionary syntax only supports ' 'git repositories') + self._clone_to_stagedir(url, files=srcdir[url]['files'] if 'files' in srcdir[url] else None, From ea7a888f05a08298e3c0b5f739b7336bb6d3120d Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 23 Feb 2026 14:38:19 +0100 Subject: [PATCH 4/4] Remove srcdir bug --- reframe/core/pipeline.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index b5959a4eb5..a6e5e72a28 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1996,11 +1996,9 @@ def compile(self): if self._requires_stagedir_contents(): srcdir = self.sourcesdir - if not srcdir: - return if isinstance(srcdir, dict): if 'url' not in srcdir: - raise ReframeError(f'The {srcdir} misses the url key') + raise ReframeError(f'{srcdir} misses the url key') url = srcdir['url'] if not osext.is_url(url):