Merge pull request #1146 from xylar/fix-ncclimo-file-list

Put symlinks for ncclimo in unique directories

Author: xylar

ci/recipe/recipe.yaml

@@ -40,7 +40,7 @@ requirements:
     - mache >=1.11.0
     - matplotlib-base >=3.9.0
     - mpas_tools >=1.3.0,<2.0.0
-    - nco >=4.8.1,!=5.2.6
+    - nco >=4.8.1,!=5.2.6,!=5.3.7
     - netcdf4
     - numpy >=2.0,<3.0
     - pandas

dev-spec.txt

@@ -16,7 +16,7 @@ lxml
 mache >=1.11.0
 matplotlib-base >=3.9.0
 mpas_tools >=1.3.0,<2.0.0
-nco >=4.8.1,!=5.2.6
+nco >=4.8.1,!=5.2.6,!=5.3.7
 netcdf4
 numpy >=2.0,<3.0
 pandas

mpas_analysis/shared/climatology/mpas_climatology_task.py

@@ -414,8 +414,10 @@ def _create_symlinks(self):
 
         climatologyOpDirectory = get_climatology_op_directory(config, self.op)
 
-        symlinkDirectory = '{}/source_symlinks'.format(
-            climatologyOpDirectory)
+        symlinkDirectory = (
+            f'{climatologyOpDirectory}/source_symlinks_'
+            f'{self.ncclimoModel}_{self.startYear:04d}-{self.endYear:04d}'
+        )
 
         make_directories(symlinkDirectory)
 
@@ -425,6 +427,8 @@ def _create_symlinks(self):
                 f'timeSeriesStatsMonthly.{year:04d}-{month:02d}-01.nc'
 
             try:
+                if os.path.lexists(outFileName):
+                    os.remove(outFileName)
                 os.symlink(inFileName, outFileName)
             except OSError:
                 pass

mpas_analysis/test/test_mpas_climatology_task.py

@@ -23,7 +23,7 @@
 
 from mpas_analysis.test import TestCase, loaddatadir
 from mpas_analysis.shared.climatology import MpasClimatologyTask, \
-    RemapMpasClimatologySubtask
+    RefYearMpasClimatologyTask, RemapMpasClimatologySubtask
 from mpas_analysis.shared import AnalysisTask
 from mpas_analysis.shared.analysis_task import \
     update_time_bounds_from_file_names
@@ -171,6 +171,36 @@ def test_update_climatology_bounds_and_create_symlinks(self):
             update_time_bounds_from_file_names(config, 'climatology', 'ocean',
                                                allow_cache=False)
 
+    def test_create_symlinks_isolates_reference_year_files(self):
+        mpasClimatologyTask = self.setup_task()
+
+        refYearTask = RefYearMpasClimatologyTask(
+            config=mpasClimatologyTask.config, componentName='ocean')
+        refYearTask.historyStreams = mpasClimatologyTask.historyStreams
+        refYearTask.startYear = 1
+        refYearTask.endYear = 1
+        refYearTask.inputFiles = []
+
+        for month in range(1, 13):
+            fileName = os.path.join(
+                self.test_dir,
+                f'mpaso.hist.am.timeSeriesStatsMonthly.0001-{month:02d}-01.nc')
+            with open(fileName, 'w'):
+                pass
+            refYearTask.inputFiles.append(fileName)
+
+        refSymlinkDirectory = refYearTask._create_symlinks()
+        mainSymlinkDirectory = mpasClimatologyTask._create_symlinks()
+
+        assert(refSymlinkDirectory != mainSymlinkDirectory)
+
+        mainSymlinkFiles = sorted(os.listdir(mainSymlinkDirectory))
+
+        assert(len(mainSymlinkFiles) == 12)
+        for fileName in mainSymlinkFiles:
+            assert(fileName.startswith(
+                'mpaso.hist.am.timeSeriesStatsMonthly.0002-'))
+
     def test_subtask_run_analysis(self):
         mpasClimatologyTask = self.setup_task()
         self.add_variables(mpasClimatologyTask)

pixi.toml

@@ -17,7 +17,7 @@ lxml = "*"
 mache = ">=1.11.0"
 matplotlib-base = ">=3.9.0"
 mpas_tools = ">=1.3.0,<2.0.0"
-nco = ">=4.8.1,!=5.2.6"
+nco = ">=4.8.1,!=5.2.6,!=5.3.7"
 netcdf4 = "*"
 numpy = ">=2.0,<3.0"
 pandas = "*"